diff --git a/.github/workflows/command-tests.yml b/.github/workflows/command-tests.yml index c64e103..56cb289 100644 --- a/.github/workflows/command-tests.yml +++ b/.github/workflows/command-tests.yml @@ -16,6 +16,8 @@ on: - "tests/unit_lib_github.sh" - "tests/unit_pasarguard.sh" - "tests/unit_pgnode.sh" + - "tests/test_node_service_transaction_guard.sh" + - "tests/test_node_service_readiness_tls.sh" - "tests/unit_pgnode_service.sh" - "tests/unit_restore_archive_safety.sh" - "tests/unit_lib_system.sh" @@ -32,6 +34,8 @@ on: - "tests/unit_lib_system.sh" - "tests/unit_pasarguard.sh" - "tests/unit_pgnode.sh" + - "tests/test_node_service_transaction_guard.sh" + - "tests/test_node_service_readiness_tls.sh" - "tests/unit_pgnode_service.sh" - "tests/unit_restore_archive_safety.sh" workflow_dispatch: @@ -57,6 +61,10 @@ jobs: run: bash tests/unit_pasarguard.sh - name: Run pg-node.sh unit tests run: bash tests/unit_pgnode.sh + - name: Run node service transaction guard tests + run: bash tests/test_node_service_transaction_guard.sh + - name: Run node service DNS-SAN readiness test + run: bash tests/test_node_service_readiness_tls.sh - name: Run pg-node-service.sh unit tests run: bash tests/unit_pgnode_service.sh - name: Run restore archive-safety unit tests diff --git a/lib/common.sh b/lib/common.sh index 1a216f1..f89da65 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -62,7 +62,7 @@ temp_root_dir() { root="/var/lib/pasarguard-scripts/tmp" fi - mkdir -p "$root" + mkdir -p "$root" || return 1 printf '%s\n' "$root" } @@ -72,7 +72,7 @@ create_temp_dir() { local candidate="" local attempt=0 - root=$(temp_root_dir) + root=$(temp_root_dir) || return 1 while [ "$attempt" -lt 20 ]; do candidate="${root}/${prefix}-$$-${RANDOM}-${attempt}" if mkdir "$candidate" 2>/dev/null; then @@ -90,7 +90,7 @@ create_temp_file() { local suffix="${2:-}" local root="" - root=$(temp_root_dir) + root=$(temp_root_dir) || return 1 create_temp_file_in_dir "$root" "$prefix" "$suffix" } @@ -101,7 +101,7 @@ create_temp_file_in_dir() { local candidate="" local attempt=0 - mkdir -p "$dir" + mkdir -p "$dir" || return 1 while [ "$attempt" -lt 20 ]; do candidate="${dir}/${prefix}-$$-${RANDOM}-${attempt}${suffix}" if (set -C; : >"$candidate") 2>/dev/null; then diff --git a/lib/github.sh b/lib/github.sh index b73b627..8b634ed 100644 --- a/lib/github.sh +++ b/lib/github.sh @@ -18,7 +18,7 @@ github_download_file() { backup_scripts() { local backup_dir="" - backup_dir=$(create_temp_dir "scripts-backup") + backup_dir=$(create_temp_dir "scripts-backup") || return 1 # Backup main scripts [ -f "/usr/local/bin/pasarguard" ] && cp "/usr/local/bin/pasarguard" "$backup_dir/" @@ -110,7 +110,7 @@ install_shared_libs_from_repo() { local tmp_dir="" local lib_name="" - tmp_dir=$(create_temp_dir "shared-libs") + tmp_dir=$(create_temp_dir "shared-libs") || return 1 mkdir -p "$SHARED_LIB_INSTALL_DIR" for lib_name in "$@"; do diff --git a/pg-node.sh b/pg-node.sh index f9a5499..9019d3a 100755 --- a/pg-node.sh +++ b/pg-node.sh @@ -168,6 +168,30 @@ FETCH_REPO="PasarGuard/scripts" NODE_SERVICE_REPO="PasarGuard/node-serviced" NODE_SERVICE_RELEASE_API="https://api.github.com/repos/${NODE_SERVICE_REPO}/releases/latest" NODE_SERVICE_BINARY_NAME="node-serviced" +NODE_SERVICE_BACKUP_PATH="" +NODE_SERVICE_HAD_PREVIOUS=false +NODE_SERVICE_STAGED_PATH="" +NODE_SERVICE_DOWNLOAD_TMP_DIR="" +NODE_SERVICE_UPDATE_LOCK_HELD=false +NODE_SERVICE_TRANSACTION_ACTIVE=false +NODE_SERVICE_TRANSACTION_COMMITTED=false +NODE_SERVICE_TRANSACTION_STATE_CAPTURED=false +NODE_SERVICE_TRANSACTION_MUTATION_STARTED=false +NODE_SERVICE_TRANSACTION_HAD_SERVICE=false +NODE_SERVICE_TRANSACTION_WAS_ACTIVE=false +NODE_SERVICE_TRANSACTION_WAS_ENABLED=false +NODE_SERVICE_EXTERNAL_CHILD_PID="" +NODE_SERVICE_EXTERNAL_CHILD_PGID="" +NODE_SERVICE_EXTERNAL_WATCHDOG_PID="" +NODE_SERVICE_UNIT_LIST_TMP="" +NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="" +declare -a NODE_SERVICE_TRANSACTION_PATHS=() +declare -a NODE_SERVICE_TRANSACTION_HAD_PATH=() +declare -a NODE_SERVICE_TRANSACTION_BACKUPS=() +declare -a NODE_SERVICE_TRANSACTION_LINK_TARGETS=() +declare -a NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET=() +declare -a NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS=() +declare -a NODE_SERVICE_TRANSACTION_RESTORED=() set_service_paths() { SERVICE_NAME="${APP_NAME}-service" SERVICE_BINARY_PATH="/usr/local/bin/${SERVICE_NAME}" @@ -180,18 +204,46 @@ require_systemd() { fi } service_installed() { + local query_status=0 + if ! command -v systemctl >/dev/null 2>&1; then return 1 fi set_service_paths - if [ -f "$SERVICE_UNIT" ] || systemctl list-unit-files | grep -q "^${SERVICE_NAME}.service"; then + if [ -f "$SERVICE_UNIT" ]; then return 0 fi - return 1 + NODE_SERVICE_UNIT_LIST_TMP=$(create_temp_file "node-service-unit-list" ".txt") || return 2 + if run_node_service_systemctl list-unit-files "${SERVICE_NAME}.service" --no-legend --no-pager \ + >"$NODE_SERVICE_UNIT_LIST_TMP" 2>/dev/null; then + query_status=0 + else + query_status=$? + rm -f "$NODE_SERVICE_UNIT_LIST_TMP" + NODE_SERVICE_UNIT_LIST_TMP="" + return "$query_status" + fi + if grep -q "^${SERVICE_NAME}.service[[:space:]]" "$NODE_SERVICE_UNIT_LIST_TMP"; then + query_status=0 + else + query_status=1 + fi + rm -f "$NODE_SERVICE_UNIT_LIST_TMP" + NODE_SERVICE_UNIT_LIST_TMP="" + return "$query_status" } restart_service_if_installed() { - if ! service_installed; then - return + local service_query_status=0 + + if service_installed; then + : + else + service_query_status=$? + if [ "$service_query_status" -eq 1 ]; then + return 0 + fi + colorized_echo red "Failed to inspect $SERVICE_NAME before restarting it." + return "$service_query_status" fi if [ "$(id -u)" != "0" ]; then colorized_echo yellow "$SERVICE_NAME is installed; run as root to restart it." @@ -200,18 +252,746 @@ restart_service_if_installed() { systemctl restart "$SERVICE_NAME" colorized_echo blue "$SERVICE_NAME service restarted." } -update_service_if_installed() { - if ! service_installed; then - return + +require_node_service_installed() { + local service_query_status=0 + + if service_installed; then + return 0 + else + service_query_status=$? fi + if [ "$service_query_status" -eq 1 ]; then + colorized_echo red "Service not installed. Run service-install first." + else + colorized_echo red "Failed to inspect $SERVICE_NAME installation state." + fi + return "$service_query_status" +} +update_service_if_installed() ( + local report_missing="${1:-false}" + + set_service_paths if [ "$(id -u)" != "0" ]; then colorized_echo yellow "$SERVICE_NAME is installed; run as root to update/restart it." return fi - install_node_service_script - systemctl daemon-reload - systemctl restart "$SERVICE_NAME" + if ! acquire_node_serviced_update_lock; then + return 1 + fi + # Arm the guard before querying systemd. A wedged systemctl must not hold + # the mutation lock without bounded cleanup and rollback protection. + if ! begin_node_service_transaction; then + colorized_echo red "Failed to snapshot or inspect $SERVICE_NAME before updating it." + return 1 + fi + if [ "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" != true ]; then + commit_node_service_transaction + if [ "$report_missing" = true ]; then + colorized_echo red "Service not installed. Run service-install first." + return 1 + fi + return + fi + if ! install_node_service_script false; then + abort_node_service_transaction + return 1 + fi + if ! run_node_service_systemctl daemon-reload || ! run_node_service_systemctl restart "$SERVICE_NAME" || ! wait_for_node_service_ready; then + colorized_echo red "$SERVICE_NAME failed to become ready after update; restoring the previous binary." + if ! abort_node_service_transaction; then + colorized_echo red "Failed to restore the previous $SERVICE_NAME installation." + fi + return 1 + fi + commit_node_service_transaction colorized_echo blue "$SERVICE_NAME service updated and restarted." +) + +acquire_node_serviced_update_lock() { + local target_dir target_name lock_path + + if [ "$NODE_SERVICE_UPDATE_LOCK_HELD" = true ]; then + return 0 + fi + if ! command -v flock >/dev/null 2>&1; then + colorized_echo red "flock is required to update $SERVICE_NAME safely." + return 1 + fi + target_dir=$(dirname "$SERVICE_BINARY_PATH") + target_name=$(basename "$SERVICE_BINARY_PATH") + lock_path="${NODE_SERVICE_UPDATE_LOCK_PATH:-${target_dir}/.${target_name}.update.lock}" + if ! exec 9>"$lock_path"; then + colorized_echo red "Failed to open the $SERVICE_NAME update lock at $lock_path." + return 1 + fi + if ! flock -n 9; then + exec 9>&- + colorized_echo yellow "Another $SERVICE_NAME update is already in progress." + return 1 + fi + NODE_SERVICE_UPDATE_LOCK_HELD=true +} + +release_node_serviced_update_lock() { + if [ "$NODE_SERVICE_UPDATE_LOCK_HELD" != true ]; then + return 0 + fi + flock -u 9 >/dev/null 2>&1 || true + exec 9>&- + NODE_SERVICE_UPDATE_LOCK_HELD=false +} + +cleanup_node_service_transaction_backups() { + local backup + + for backup in "${NODE_SERVICE_TRANSACTION_BACKUPS[@]}" "${NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS[@]}"; do + [ -z "$backup" ] || rm -f "$backup" + done + NODE_SERVICE_TRANSACTION_PATHS=() + NODE_SERVICE_TRANSACTION_HAD_PATH=() + NODE_SERVICE_TRANSACTION_BACKUPS=() + NODE_SERVICE_TRANSACTION_LINK_TARGETS=() + NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET=() + NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS=() + NODE_SERVICE_TRANSACTION_RESTORED=() +} + +cleanup_node_serviced_temporary_files() { + [ -z "$NODE_SERVICE_STAGED_PATH" ] || rm -f "$NODE_SERVICE_STAGED_PATH" + [ -z "$NODE_SERVICE_DOWNLOAD_TMP_DIR" ] || rm -rf "$NODE_SERVICE_DOWNLOAD_TMP_DIR" + [ -z "$NODE_SERVICE_UNIT_LIST_TMP" ] || rm -f "$NODE_SERVICE_UNIT_LIST_TMP" + NODE_SERVICE_STAGED_PATH="" + NODE_SERVICE_DOWNLOAD_TMP_DIR="" + NODE_SERVICE_UNIT_LIST_TMP="" +} + +signal_node_service_process() { + local signal="$1" + local child_pid="$2" + local child_pgid="$3" + + if [ -n "$child_pgid" ]; then + kill -"$signal" -- "-$child_pgid" >/dev/null 2>&1 || true + elif [ -n "$child_pid" ]; then + kill -"$signal" "$child_pid" >/dev/null 2>&1 || true + fi +} + +terminate_node_service_external_child() { + local child_pid="$NODE_SERVICE_EXTERNAL_CHILD_PID" + local child_pgid="$NODE_SERVICE_EXTERNAL_CHILD_PGID" + local watchdog_pid="$NODE_SERVICE_EXTERNAL_WATCHDOG_PID" + local attempt=0 + + NODE_SERVICE_EXTERNAL_CHILD_PID="" + NODE_SERVICE_EXTERNAL_CHILD_PGID="" + NODE_SERVICE_EXTERNAL_WATCHDOG_PID="" + [ -z "$watchdog_pid" ] || kill -KILL "$watchdog_pid" >/dev/null 2>&1 || true + if [ -n "$child_pid" ] && kill -0 "$child_pid" >/dev/null 2>&1; then + signal_node_service_process TERM "$child_pid" "$child_pgid" + while kill -0 "$child_pid" >/dev/null 2>&1 && [ "$attempt" -lt 20 ]; do + sleep 0.05 + attempt=$((attempt + 1)) + done + if kill -0 "$child_pid" >/dev/null 2>&1; then + signal_node_service_process KILL "$child_pid" "$child_pgid" + fi + wait "$child_pid" >/dev/null 2>&1 || true + fi + [ -z "$watchdog_pid" ] || wait "$watchdog_pid" >/dev/null 2>&1 || true +} + +run_node_service_external() { + local timeout_seconds="$1" + shift + local child_pid child_pgid="" watchdog_pid status + + # Unit tests replace external programs with shell functions. Keep those + # synchronous; production commands use a separate process group below. + if declare -F "${1:-}" >/dev/null 2>&1; then + local function_status=0 + "$@" || function_status=$? + return "$function_status" + fi + if ! [[ "$timeout_seconds" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + colorized_echo red "Invalid node service external-command timeout: $timeout_seconds" + return 1 + fi + if ! command -v setsid >/dev/null 2>&1; then + colorized_echo red "setsid is required to supervise $1 safely." + return 1 + fi + setsid --wait bash -c 'trap - INT TERM; exec "$@"' bash "$@" & + child_pid=$! + child_pgid="$child_pid" + NODE_SERVICE_EXTERNAL_CHILD_PID="$child_pid" + NODE_SERVICE_EXTERNAL_CHILD_PGID="$child_pgid" + ( + trap - INT TERM + sleep "$timeout_seconds" + if kill -0 "$child_pid" >/dev/null 2>&1; then + signal_node_service_process TERM "$child_pid" "$child_pgid" + sleep "${NODE_SERVICE_EXTERNAL_KILL_AFTER_SECONDS:-2}" + if kill -0 "$child_pid" >/dev/null 2>&1; then + signal_node_service_process KILL "$child_pid" "$child_pgid" + fi + fi + ) & + watchdog_pid=$! + NODE_SERVICE_EXTERNAL_WATCHDOG_PID="$watchdog_pid" + if wait "$child_pid"; then + status=0 + else + status=$? + fi + NODE_SERVICE_EXTERNAL_CHILD_PID="" + NODE_SERVICE_EXTERNAL_CHILD_PGID="" + NODE_SERVICE_EXTERNAL_WATCHDOG_PID="" + kill -KILL "$watchdog_pid" >/dev/null 2>&1 || true + wait "$watchdog_pid" >/dev/null 2>&1 || true + return "$status" +} + +run_node_service_systemctl() { + run_node_service_external "${NODE_SERVICE_SYSTEMCTL_TIMEOUT_SECONDS:-30}" systemctl "$@" +} + +snapshot_node_service_path() { + local source_path="$1" + local index="${#NODE_SERVICE_TRANSACTION_PATHS[@]}" + local source_dir source_name backup_path link_target="" link_target_backup="" + local had_path=false had_link_target=false + + source_dir=$(dirname "$source_path") + source_name=$(basename "$source_path") + backup_path="${source_dir}/.${source_name}.transaction.$$.$RANDOM" + if [ -e "$source_path" ] || [ -L "$source_path" ]; then + had_path=true + NODE_SERVICE_TRANSACTION_PATHS[index]="$source_path" + NODE_SERVICE_TRANSACTION_HAD_PATH[index]="$had_path" + NODE_SERVICE_TRANSACTION_BACKUPS[index]="$backup_path" + NODE_SERVICE_TRANSACTION_LINK_TARGETS[index]="" + NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET[index]=false + NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS[index]="" + NODE_SERVICE_TRANSACTION_RESTORED[index]=false + cp -a "$source_path" "$backup_path" || return 1 + if [ -L "$source_path" ]; then + link_target=$(readlink -f "$source_path" 2>/dev/null || true) + if [ -n "$link_target" ] && { [ -e "$link_target" ] || [ -L "$link_target" ]; }; then + had_link_target=true + link_target_backup="$(dirname "$link_target")/.$(basename "$link_target").transaction-target.$$.$RANDOM" + NODE_SERVICE_TRANSACTION_LINK_TARGETS[index]="$link_target" + NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET[index]="$had_link_target" + NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS[index]="$link_target_backup" + cp -a "$link_target" "$link_target_backup" || return 1 + fi + fi + else + backup_path="" + NODE_SERVICE_TRANSACTION_PATHS[index]="$source_path" + NODE_SERVICE_TRANSACTION_HAD_PATH[index]="$had_path" + NODE_SERVICE_TRANSACTION_BACKUPS[index]="" + NODE_SERVICE_TRANSACTION_LINK_TARGETS[index]="" + NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET[index]=false + NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS[index]="" + NODE_SERVICE_TRANSACTION_RESTORED[index]=false + fi +} + +restore_node_service_backup_copy() { + local backup="$1" + local destination="$2" + local destination_dir destination_name restore_path + + destination_dir=$(dirname "$destination") + destination_name=$(basename "$destination") + restore_path=$(create_temp_file_in_dir "$destination_dir" ".${destination_name}.restore" "") || return 1 + rm -f "$restore_path" || return 1 + if ! cp -a "$backup" "$restore_path" || ! mv -f "$restore_path" "$destination"; then + rm -f "$restore_path" + return 1 + fi +} + +restore_node_service_snapshot_entry() { + local index="$1" + local path="${NODE_SERVICE_TRANSACTION_PATHS[index]}" + local backup="${NODE_SERVICE_TRANSACTION_BACKUPS[index]}" + local link_target="${NODE_SERVICE_TRANSACTION_LINK_TARGETS[index]}" + local link_target_backup="${NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS[index]}" + + [ "${NODE_SERVICE_TRANSACTION_RESTORED[index]}" != true ] || return 0 + if [ "${NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET[index]}" = true ] && [ -n "$link_target_backup" ]; then + # Keep the snapshot until the complete rollback, including restoration + # of systemd state, has succeeded. A later systemctl failure otherwise + # leaves an orphaned first-install unit with no recovery copy. + if ! restore_node_service_backup_copy "$link_target_backup" "$link_target"; then + return 1 + fi + fi + if [ "${NODE_SERVICE_TRANSACTION_HAD_PATH[index]}" = true ]; then + if [ -z "$backup" ] || ! restore_node_service_backup_copy "$backup" "$path"; then + return 1 + fi + elif ! rm -f "$path"; then + return 1 + fi + NODE_SERVICE_TRANSACTION_RESTORED[index]=true +} + +rollback_node_service_transaction() { + local restore_failed=false index + + if [ "$NODE_SERVICE_TRANSACTION_MUTATION_STARTED" != true ]; then + return 0 + fi + for ((index = ${#NODE_SERVICE_TRANSACTION_PATHS[@]} - 1; index >= 0; index--)); do + restore_node_service_snapshot_entry "$index" || restore_failed=true + done + discard_node_serviced_backup + + if ! run_node_service_systemctl daemon-reload; then + restore_failed=true + fi + if [ "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" = true ]; then + if [ "$NODE_SERVICE_TRANSACTION_WAS_ENABLED" = true ]; then + run_node_service_systemctl enable "$SERVICE_NAME" >/dev/null 2>&1 || restore_failed=true + else + run_node_service_systemctl disable "$SERVICE_NAME" >/dev/null 2>&1 || restore_failed=true + fi + if [ "$NODE_SERVICE_TRANSACTION_WAS_ACTIVE" = true ]; then + if ! run_node_service_systemctl restart "$SERVICE_NAME" || ! wait_for_node_service_ready; then + restore_failed=true + fi + else + run_node_service_systemctl stop "$SERVICE_NAME" >/dev/null 2>&1 || restore_failed=true + fi + else + if ! run_node_service_systemctl disable --now "$SERVICE_NAME" >/dev/null 2>&1; then + colorized_echo red "Failed to disable the newly created $SERVICE_NAME unit; it may remain loaded or active." + restore_failed=true + fi + fi + [ "$restore_failed" = false ] +} + +finish_node_service_transaction() { + local preserve_backups="${1:-false}" + + trap - EXIT + trap '' INT TERM + if [ "$preserve_backups" != true ]; then + cleanup_node_service_transaction_backups + fi + cleanup_node_serviced_temporary_files + discard_node_serviced_backup + NODE_SERVICE_TRANSACTION_ACTIVE=false + NODE_SERVICE_TRANSACTION_COMMITTED=false + if [ "$preserve_backups" != true ]; then + NODE_SERVICE_TRANSACTION_MUTATION_STARTED=false + fi + release_node_serviced_update_lock + trap - INT TERM +} + +report_retained_node_service_backups() { + local backup + + for backup in "${NODE_SERVICE_TRANSACTION_BACKUPS[@]}" "${NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS[@]}"; do + [ -z "$backup" ] || colorized_echo red "Retained transaction backup: $backup" + done +} + +node_service_transaction_guard() { + local event="$1" + local status="$2" + local signal_status=0 + + local rollback_failed=false + + trap - EXIT + trap '' INT TERM + terminate_node_service_external_child + if [ "$NODE_SERVICE_TRANSACTION_ACTIVE" = true ] && [ "$NODE_SERVICE_TRANSACTION_COMMITTED" != true ]; then + rollback_node_service_transaction || { + rollback_failed=true + colorized_echo red "Failed to fully restore the interrupted $SERVICE_NAME transaction." + } + fi + if [ "$rollback_failed" != true ]; then + cleanup_node_service_transaction_backups + else + colorized_echo red "Transaction backups were retained for manual recovery." + report_retained_node_service_backups + fi + cleanup_node_serviced_temporary_files + discard_node_serviced_backup + NODE_SERVICE_TRANSACTION_ACTIVE=false + if [ "$rollback_failed" != true ]; then + NODE_SERVICE_TRANSACTION_MUTATION_STARTED=false + fi + release_node_serviced_update_lock + + case "$event" in + INT) signal_status=130 ;; + TERM) signal_status=143 ;; + EXIT) return "$status" ;; + esac + exit "$signal_status" +} + +begin_node_service_transaction() { + local service_query_status=0 state_query_status=0 + + NODE_SERVICE_TRANSACTION_ACTIVE=true + NODE_SERVICE_TRANSACTION_COMMITTED=false + NODE_SERVICE_TRANSACTION_STATE_CAPTURED=false + NODE_SERVICE_TRANSACTION_MUTATION_STARTED=false + NODE_SERVICE_TRANSACTION_HAD_SERVICE=false + NODE_SERVICE_TRANSACTION_WAS_ACTIVE=false + NODE_SERVICE_TRANSACTION_WAS_ENABLED=false + NODE_SERVICE_TRANSACTION_PATHS=() + NODE_SERVICE_TRANSACTION_HAD_PATH=() + NODE_SERVICE_TRANSACTION_BACKUPS=() + NODE_SERVICE_TRANSACTION_LINK_TARGETS=() + NODE_SERVICE_TRANSACTION_HAD_LINK_TARGET=() + NODE_SERVICE_TRANSACTION_LINK_TARGET_BACKUPS=() + NODE_SERVICE_TRANSACTION_RESTORED=() + trap 'node_service_transaction_guard EXIT "$?"' EXIT + trap 'node_service_transaction_guard INT "$?"' INT + trap 'node_service_transaction_guard TERM "$?"' TERM + + snapshot_node_service_path "$ENV_FILE" || return 1 + snapshot_node_service_path "$SERVICE_UNIT" || return 1 + snapshot_node_service_path "$SERVICE_BINARY_PATH" || return 1 + if service_installed; then + NODE_SERVICE_TRANSACTION_HAD_SERVICE=true + else + service_query_status=$? + [ "$service_query_status" -eq 1 ] || return "$service_query_status" + fi + if [ "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" = true ]; then + if run_node_service_systemctl is-active --quiet "$SERVICE_NAME" >/dev/null 2>&1; then + NODE_SERVICE_TRANSACTION_WAS_ACTIVE=true + else + state_query_status=$? + # systemctl documents 3 as the normal inactive result. Any other + # status is an unknown/failed query and must not arm rollback. + [ "$state_query_status" -eq 3 ] || return "$state_query_status" + fi + if run_node_service_systemctl is-enabled --quiet "$SERVICE_NAME" >/dev/null 2>&1; then + NODE_SERVICE_TRANSACTION_WAS_ENABLED=true + else + state_query_status=$? + # A known disabled unit returns 1. Timeouts, signals, and other + # inspection failures remain fatal before any mutation starts. + [ "$state_query_status" -eq 1 ] || return "$state_query_status" + fi + fi + NODE_SERVICE_TRANSACTION_STATE_CAPTURED=true +} + +mark_node_service_transaction_mutation_started() { + if [ "$NODE_SERVICE_TRANSACTION_ACTIVE" != true ]; then + return 0 + fi + [ "$NODE_SERVICE_TRANSACTION_STATE_CAPTURED" = true ] || return 1 + NODE_SERVICE_TRANSACTION_MUTATION_STARTED=true +} + +abort_node_service_transaction() { + local status=0 + + trap - EXIT + trap '' INT TERM + rollback_node_service_transaction || status=1 + if [ "$status" -ne 0 ]; then + colorized_echo red "Transaction backups were retained for retry or manual recovery." + report_retained_node_service_backups + finish_node_service_transaction true + else + finish_node_service_transaction false + fi + return "$status" +} + +commit_node_service_transaction() { + NODE_SERVICE_TRANSACTION_COMMITTED=true + finish_node_service_transaction +} + +wait_for_node_service_ready() { + local max_attempts="${NODE_SERVICE_READINESS_ATTEMPTS:-10}" + local stable_required="${NODE_SERVICE_READINESS_STABLE_CHECKS:-3}" + local delay_seconds="${NODE_SERVICE_READINESS_DELAY_SECONDS:-1}" + local attempt=0 stable=0 + + while [ "$attempt" -lt "$max_attempts" ]; do + if run_node_service_systemctl is-active --quiet "$SERVICE_NAME" && node_service_api_ready; then + stable=$((stable + 1)) + if [ "$stable" -ge "$stable_required" ]; then + return 0 + fi + else + stable=0 + fi + attempt=$((attempt + 1)) + if [ "$attempt" -lt "$max_attempts" ]; then + sleep "$delay_seconds" + fi + done + return 1 +} + +read_node_service_env_value() { + local key="$1" + + [ -r "$ENV_FILE" ] || return 1 + # node-serviced uses godotenv.Overload. Parse only data (never source/eval) + # while matching the relevant godotenv rules: export, last duplicate wins, + # comments, single/double quotes, double-quote escapes and prior-key + # expansion. This keeps readiness pointed at the configuration the daemon + # actually loaded. + awk -v wanted="$key" ' + function ltrim(value) { sub(/^[ \t\v\f\r]+/, "", value); return value } + function rtrim(value) { sub(/[ \t\v\f\r]+$/, "", value); return value } + function expand_vars(value, out, i, j, c, name, braced) { + out = "" + for (i = 1; i <= length(value); i++) { + c = substr(value, i, 1) + if (c == "\\" && substr(value, i + 1, 1) == "$") { + out = out "$" + i++ + continue + } + if (c != "$") { + out = out c + continue + } + j = i + 1 + braced = substr(value, j, 1) == "{" + if (braced) j++ + name = "" + while (j <= length(value) && substr(value, j, 1) ~ /[A-Z0-9_]/) { + name = name substr(value, j, 1) + j++ + } + if (name == "") { + out = out c + continue + } + if (braced && substr(value, j, 1) == "}") j++ + out = out vars[name] + i = j - 1 + } + return out + } + function decode_double(value, out, i, c, nextc) { + out = "" + for (i = 1; i <= length(value); i++) { + c = substr(value, i, 1) + if (c != "\\" || i == length(value)) { + out = out c + continue + } + nextc = substr(value, i + 1, 1) + if (nextc == "n") out = out "\n" + else if (nextc == "r") out = out "\r" + else if (nextc == "$") out = out "\\$" + else out = out nextc + i++ + } + return expand_vars(out) + } + { + line = $0 + sub(/\r$/, "", line) + line = ltrim(line) + if (line == "" || substr(line, 1, 1) == "#") next + if (line ~ /^export[ \t\v\f\r]/) { + sub(/^export[ \t\v\f\r]+/, "", line) + } + delimiter = match(line, /[=:]/) + if (!delimiter) { parse_error = 1; next } + name = rtrim(substr(line, 1, delimiter - 1)) + if (name !~ /^[A-Za-z0-9_.]+$/) { parse_error = 1; next } + value = ltrim(substr(line, delimiter + 1)) + quote = substr(value, 1, 1) + if (quote == "\"" || quote == "\047") { + closing = 0 + for (i = 2; i <= length(value); i++) { + if (substr(value, i, 1) == quote && substr(value, i - 1, 1) != "\\") { + closing = i + break + } + } + if (!closing) { parse_error = 1; next } + remainder = ltrim(substr(value, closing + 1)) + if (remainder != "" && substr(remainder, 1, 1) != "#") { + parse_error = 1 + next + } + value = substr(value, 2, closing - 2) + if (quote == "\"") value = decode_double(value) + } else { + comment = 0 + for (i = 2; i <= length(value); i++) { + if (substr(value, i, 1) == "#" && substr(value, i - 1, 1) ~ /[ \t\v\f\r]/) { + comment = i + break + } + } + if (comment) value = substr(value, 1, comment - 1) + value = expand_vars(rtrim(value)) + } + vars[name] = value + } + END { + if (parse_error || !(wanted in vars)) exit 1 + printf "%s", vars[wanted] + } + ' "$ENV_FILE" +} + +node_service_certificate_identity() { + local ssl_cert="$1" + local cert_output cn_output="" token dns_identity="" wildcard_identity="" ip_identity="" san_present=false + local wildcard_suffix subject + + NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="" + cert_output=$(create_temp_file "node-service-cert" ".txt") || return 1 + if ! run_node_service_external "${NODE_SERVICE_OPENSSL_TIMEOUT_SECONDS:-5}" \ + openssl x509 -in "$ssl_cert" -noout -ext subjectAltName >"$cert_output" 2>/dev/null; then + rm -f "$cert_output" + return 1 + fi + while IFS= read -r token; do + token="${token#"${token%%[![:space:]]*}"}" + token="${token%"${token##*[![:space:]]}"}" + case "$token" in + *"Subject Alternative Name:"*) + san_present=true + ;; + DNS:*) + token="${token#DNS:}" + if [ -z "$dns_identity" ] && [[ "$token" != *'*'* ]] && + [[ "$token" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then + dns_identity="$token" + elif [ -z "$wildcard_identity" ] && [[ "$token" == \*.* ]]; then + wildcard_suffix="${token#*.}" + if [[ "$wildcard_suffix" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then + # A fixed single label is covered by a left-most wildcard + # and avoids DNS while retaining valid SNI/Host identity. + wildcard_identity="node-serviced-health.${wildcard_suffix}" + fi + fi + ;; + "IP Address:"*) + token="${token#IP Address:}" + if [ -z "$ip_identity" ] && is_ip_address "$token"; then + ip_identity="$token" + fi + ;; + esac + done < <(tr ',' '\n' <"$cert_output") + rm -f "$cert_output" + if [ -n "$dns_identity" ]; then + NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="$dns_identity" + elif [ -n "$wildcard_identity" ]; then + NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="$wildcard_identity" + elif [ -n "$ip_identity" ]; then + NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="$ip_identity" + elif [ "$san_present" = true ]; then + return 1 + else + # Preserve compatibility with legacy SAN-less certificates. CN is + # considered only when the SAN extension is absent, matching TLS name + # verification precedence. + cn_output=$(create_temp_file "node-service-cert-cn" ".txt") || return 1 + if ! run_node_service_external "${NODE_SERVICE_OPENSSL_TIMEOUT_SECONDS:-5}" \ + openssl x509 -in "$ssl_cert" -noout -subject -nameopt RFC2253 >"$cn_output" 2>/dev/null; then + rm -f "$cn_output" + return 1 + fi + subject=$(<"$cn_output") + rm -f "$cn_output" + subject="${subject#subject=}" + subject="${subject#subject =}" + while IFS= read -r token; do + token="${token#CN=}" + if [[ "$token" == \*.* ]]; then + wildcard_suffix="${token#*.}" + if [[ "$wildcard_suffix" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then + NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="node-serviced-health.${wildcard_suffix}" + return 0 + fi + elif [[ "$token" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then + NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT="$token" + return 0 + fi + done < <(printf '%s\n' "$subject" | tr ',' '\n' | grep '^CN=') + return 1 + fi +} + +node_service_api_ready() { + local api_port api_key ssl_cert tls_identity url_host curl_config curl_status + + api_port=$(read_node_service_env_value API_PORT) + api_key=$(read_node_service_env_value API_KEY) + ssl_cert=$(read_node_service_env_value SSL_CERT_FILE) + # Match node-serviced's own default when an older environment omits + # API_PORT. New installations still write the script-selected port. + api_port="${api_port:-3000}" + ssl_cert="${ssl_cert:-$SSL_CERT_FILE}" + + if ! [[ "$api_port" =~ ^[0-9]+$ && "$api_port" -ge 1 && "$api_port" -le 65535 ]] || + [ -z "$api_key" ] || [[ "$api_key" == *$'\n'* || "$api_key" == *$'\r'* ]] || + [ ! -r "$ssl_cert" ]; then + colorized_echo red "$SERVICE_NAME readiness configuration is incomplete in $ENV_FILE." + return 1 + fi + if ! node_service_certificate_identity "$ssl_cert"; then + colorized_echo red "$SERVICE_NAME certificate has no usable DNS/IP certificate identity." + return 1 + fi + tls_identity="$NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT" + url_host="$tls_identity" + if [[ "$url_host" == *:* ]]; then + url_host="[$url_host]" + fi + + # Keep the API key out of the process argument list. The temporary curl + # configuration is owner-readable only and is removed after the probe. + curl_config=$(create_temp_file "node-service-readiness" ".curl") || return 1 + harden_secret_file "$curl_config" || { + rm -f "$curl_config" + return 1 + } + if ! { + printf 'silent\nshow-error\nfail\n' + printf 'noproxy = "*"\n' + printf 'max-time = "%s"\n' "${NODE_SERVICE_READINESS_TIMEOUT_SECONDS:-5}" + printf 'connect-timeout = "%s"\n' "${NODE_SERVICE_READINESS_CONNECT_TIMEOUT_SECONDS:-2}" + printf 'cacert = "%s"\n' "$(printf '%s' "$ssl_cert" | sed 's/[\\"]/\\&/g')" + printf 'header = "x-api-key: %s"\n' "$(printf '%s' "$api_key" | sed 's/[\\"]/\\&/g')" + # Preserve the certificate identity for TLS SNI and the HTTP Host + # header while forcing the TCP connection to the local service. Unlike + # DNS resolution, connect-to is deterministic during recovery. + # This curl invocation has exactly one URL, so wildcard source fields + # avoid IPv6 host-matching ambiguity while still changing only the TCP + # destination. TLS verification and HTTP Host continue to use the URL. + printf 'connect-to = "::127.0.0.1:%s"\n' "$api_port" + printf 'url = "https://%s:%s/"\n' "$url_host" "$api_port" + } >"$curl_config"; then + rm -f "$curl_config" + return 1 + fi + run_node_service_external "${NODE_SERVICE_READINESS_TIMEOUT_SECONDS:-5}" curl --config "$curl_config" >/dev/null + curl_status=$? + rm -f "$curl_config" + return "$curl_status" } detect_node_serviced_platform() { local arch os platform @@ -286,46 +1066,210 @@ install_node_script() { exit 1 fi } +verify_node_serviced_binary() { + local binary_path="$1" + local magic="" + + if [ -L "$binary_path" ] || [ ! -f "$binary_path" ] || [ ! -s "$binary_path" ]; then + return 1 + fi + chmod 755 "$binary_path" || return 1 + [ -x "$binary_path" ] || return 1 + + # node-serviced releases are Linux ELF executables. Checking the magic + # catches HTML/error bodies and truncated/otherwise invalid extracted files + # even when their executable bit is set. + magic=$(LC_ALL=C od -An -tx1 -N4 "$binary_path" 2>/dev/null | tr -d ' \n') + [ "$magic" = "7f454c46" ] +} + +verify_node_serviced_checksum() { + local checksum_url="$1" + local asset_name="$2" + local archive_path="$3" + local tmp_dir="$4" + local checksum_path expected actual + + if [ -z "$checksum_url" ] || [ "$checksum_url" = "null" ]; then + case "${NODE_SERVICE_REQUIRE_CHECKSUM:-true}" in + false | 0 | no) + colorized_echo yellow "No checksums.txt asset is available; continuing because checksum verification was explicitly disabled." + return 0 + ;; + *) + colorized_echo red "No checksums.txt asset is available and checksum verification is required." + return 1 + ;; + esac + fi + + checksum_path="${tmp_dir}/checksums.txt" + if ! run_node_service_external "${NODE_SERVICE_DOWNLOAD_TIMEOUT_SECONDS:-120}" \ + curl -fL --connect-timeout 10 --max-time 90 --retry 3 --retry-delay 1 "$checksum_url" -o "$checksum_path"; then + colorized_echo red "Failed to download node-serviced checksums from $checksum_url" + return 1 + fi + expected=$(awk -v name="$asset_name" '$2 == name || $2 == "*" name { print $1; exit }' "$checksum_path") + if [[ ! "$expected" =~ ^[0-9a-fA-F]{64}$ ]]; then + colorized_echo red "No valid SHA-256 checksum found for $asset_name" + return 1 + fi + if command -v sha256sum >/dev/null 2>&1; then + actual=$(sha256sum "$archive_path" | awk '{print $1}') + elif command -v shasum >/dev/null 2>&1; then + actual=$(shasum -a 256 "$archive_path" | awk '{print $1}') + else + colorized_echo red "SHA-256 verification is required, but sha256sum/shasum is unavailable." + return 1 + fi + if [ "${actual,,}" != "${expected,,}" ]; then + colorized_echo red "SHA-256 checksum mismatch for $asset_name" + return 1 + fi +} + +activate_node_serviced_binary() { + local candidate_path="$1" + local keep_backup="${2:-false}" + local target_dir target_name staged_path backup_path + + target_dir=$(dirname "$SERVICE_BINARY_PATH") + target_name=$(basename "$SERVICE_BINARY_PATH") + staged_path=$(create_temp_file_in_dir "$target_dir" ".${target_name}.new" "") + NODE_SERVICE_STAGED_PATH="$staged_path" + backup_path="${target_dir}/.${target_name}.backup.$$.$RANDOM" + NODE_SERVICE_BACKUP_PATH="" + NODE_SERVICE_HAD_PREVIOUS=false + + if ! install -m 755 "$candidate_path" "$staged_path" || ! verify_node_serviced_binary "$staged_path"; then + rm -f "$staged_path" + NODE_SERVICE_STAGED_PATH="" + return 1 + fi + if [ -e "$SERVICE_BINARY_PATH" ] || [ -L "$SERVICE_BINARY_PATH" ]; then + if ! cp -a "$SERVICE_BINARY_PATH" "$backup_path"; then + rm -f "$staged_path" + NODE_SERVICE_STAGED_PATH="" + return 1 + fi + NODE_SERVICE_BACKUP_PATH="$backup_path" + NODE_SERVICE_HAD_PREVIOUS=true + fi + if ! mark_node_service_transaction_mutation_started; then + rm -f "$NODE_SERVICE_BACKUP_PATH" "$staged_path" + NODE_SERVICE_BACKUP_PATH="" + NODE_SERVICE_HAD_PREVIOUS=false + NODE_SERVICE_STAGED_PATH="" + return 1 + fi + if ! mv "$staged_path" "$SERVICE_BINARY_PATH"; then + rm -f "$NODE_SERVICE_BACKUP_PATH" + NODE_SERVICE_BACKUP_PATH="" + NODE_SERVICE_HAD_PREVIOUS=false + rm -f "$staged_path" + NODE_SERVICE_STAGED_PATH="" + return 1 + fi + NODE_SERVICE_STAGED_PATH="" + if [ "$keep_backup" != true ]; then + discard_node_serviced_backup + fi +} + +rollback_node_serviced_binary() { + local service_label="${SERVICE_NAME:-node-serviced}" + + if [ "$NODE_SERVICE_HAD_PREVIOUS" = true ] && [ -n "$NODE_SERVICE_BACKUP_PATH" ]; then + # Both files are in the target directory, so this restores the backup + # with one atomic rename over the failed replacement. + mv -f "$NODE_SERVICE_BACKUP_PATH" "$SERVICE_BINARY_PATH" || return 1 + else + colorized_echo yellow "No previous $service_label binary is available; removing the failed replacement." + rm -f "$SERVICE_BINARY_PATH" || return 1 + fi + NODE_SERVICE_BACKUP_PATH="" + NODE_SERVICE_HAD_PREVIOUS=false +} + +discard_node_serviced_backup() { + if [ -n "$NODE_SERVICE_BACKUP_PATH" ]; then + rm -f "$NODE_SERVICE_BACKUP_PATH" + fi + NODE_SERVICE_BACKUP_PATH="" + NODE_SERVICE_HAD_PREVIOUS=false +} + install_node_service_script() { + local keep_backup="${1:-false}" set_service_paths + local platform release_json release_json_path latest_tag latest_version asset_name asset_url checksum_url tmp_dir archive_path binary_path + tmp_dir=$(create_temp_dir "node-serviced") || { + colorized_echo red "Failed to create a temporary directory for the node-serviced release." + return 1 + } + NODE_SERVICE_DOWNLOAD_TMP_DIR="$tmp_dir" if ! command -v jq >/dev/null 2>&1; then detect_os install_package jq fi colorized_echo blue "Installing node-serviced binary" - local platform release_json latest_tag latest_version asset_name asset_url tmp_dir archive_path platform=$(detect_node_serviced_platform) - if ! release_json=$(curl -fsSL "$NODE_SERVICE_RELEASE_API"); then + release_json_path="${tmp_dir}/release.json" + if ! run_node_service_external "${NODE_SERVICE_DOWNLOAD_TIMEOUT_SECONDS:-120}" \ + curl -fsSL --connect-timeout 10 --max-time 90 --retry 3 --retry-delay 1 \ + "$NODE_SERVICE_RELEASE_API" -o "$release_json_path"; then colorized_echo red "Failed to query latest node-serviced release from $NODE_SERVICE_RELEASE_API" - exit 1 + return 1 fi + release_json=$(<"$release_json_path") latest_tag=$(echo "$release_json" | jq -r '.tag_name // empty') latest_version="${latest_tag#v}" if [ -z "$latest_version" ] || [ "$latest_version" = "null" ]; then colorized_echo red "Failed to resolve latest node-serviced version from $NODE_SERVICE_RELEASE_API" - exit 1 + return 1 fi asset_name="${NODE_SERVICE_BINARY_NAME}_${latest_version}_${platform}.tar.gz" asset_url=$(echo "$release_json" | jq -r --arg name "$asset_name" '.assets[]? | select(.name==$name) | .browser_download_url' | head -n 1) if [ -z "$asset_url" ] || [ "$asset_url" = "null" ]; then colorized_echo red "node-serviced asset not found for platform $platform (expected $asset_name)" - exit 1 + return 1 fi - tmp_dir=$(create_temp_dir "node-serviced") + checksum_url=$(echo "$release_json" | jq -r '.assets[]? | select(.name=="checksums.txt") | .browser_download_url' | head -n 1) archive_path="${tmp_dir}/${asset_name}" colorized_echo cyan " Downloading ${asset_name}..." - if ! curl -sSL "$asset_url" -o "$archive_path"; then + if ! run_node_service_external "${NODE_SERVICE_DOWNLOAD_TIMEOUT_SECONDS:-120}" \ + curl -fL --connect-timeout 10 --max-time 90 --retry 3 --retry-delay 1 "$asset_url" -o "$archive_path"; then colorized_echo red "Failed to download node-serviced from $asset_url" rm -rf "$tmp_dir" - exit 1 + return 1 + fi + if [ ! -f "$archive_path" ] || [ ! -s "$archive_path" ]; then + colorized_echo red "Downloaded node-serviced archive is empty or not a regular file." + rm -rf "$tmp_dir" + return 1 + fi + if ! verify_node_serviced_checksum "$checksum_url" "$asset_name" "$archive_path" "$tmp_dir"; then + rm -rf "$tmp_dir" + return 1 fi colorized_echo cyan " Extracting node-serviced..." - if ! tar -xzf "$archive_path" -C "$tmp_dir" "$NODE_SERVICE_BINARY_NAME" 2>/dev/null; then + if ! run_node_service_external "${NODE_SERVICE_EXTRACT_TIMEOUT_SECONDS:-30}" \ + tar -xzf "$archive_path" -C "$tmp_dir" "$NODE_SERVICE_BINARY_NAME" 2>/dev/null; then colorized_echo red "Failed to extract node-serviced binary from archive." rm -rf "$tmp_dir" - exit 1 + return 1 + fi + binary_path="${tmp_dir}/${NODE_SERVICE_BINARY_NAME}" + if ! verify_node_serviced_binary "$binary_path"; then + colorized_echo red "Extracted node-serviced is not a non-empty Linux executable." + rm -rf "$tmp_dir" + return 1 + fi + if ! activate_node_serviced_binary "$binary_path" "$keep_backup"; then + colorized_echo red "Failed to atomically install node-serviced at $SERVICE_BINARY_PATH" + rm -rf "$tmp_dir" + return 1 fi - install -m 755 "${tmp_dir}/${NODE_SERVICE_BINARY_NAME}" "$SERVICE_BINARY_PATH" rm -rf "$tmp_dir" colorized_echo green "node-serviced installed successfully at $SERVICE_BINARY_PATH (v${latest_version})" } @@ -984,6 +1928,9 @@ sync_env_ssl_paths() { local current_cert current_key updated=false current_cert=$(grep -E '^[[:space:]]*SSL_CERT_FILE[[:space:]]*=' "$ENV_FILE" | head -n1 | sed "s/^[[:space:]]*SSL_CERT_FILE[[:space:]]*=[[:space:]]*//;s/[\"']//g") current_key=$(grep -E '^[[:space:]]*SSL_KEY_FILE[[:space:]]*=' "$ENV_FILE" | head -n1 | sed "s/^[[:space:]]*SSL_KEY_FILE[[:space:]]*=[[:space:]]*//;s/[\"']//g") + if [[ -z "$current_cert" || "$current_cert" =~ /var/lib/pg-node/ || -z "$current_key" || "$current_key" =~ /var/lib/pg-node/ ]]; then + mark_node_service_transaction_mutation_started || return 1 + fi if [[ -z "$current_cert" || "$current_cert" =~ /var/lib/pg-node/ ]]; then sed -i "s|^[[:space:]]*SSL_CERT_FILE[[:space:]]*=.*|SSL_CERT_FILE= ${desired_cert}|" "$ENV_FILE" grep -q '^[[:space:]]*SSL_CERT_FILE[[:space:]]*=' "$ENV_FILE" || echo "SSL_CERT_FILE= ${desired_cert}" >>"$ENV_FILE" @@ -1183,7 +2130,7 @@ install_command() { read -p "Do you want to install and start the systemd service for $APP_NAME? (Y/n): " install_service_choice fi if [[ -z "$install_service_choice" || "$install_service_choice" =~ ^[Yy]$ ]]; then - install_service_command + install_service_command || return 1 else colorized_echo yellow "Skipped installing systemd service for $APP_NAME." fi @@ -1215,9 +2162,7 @@ uninstall_command() { if is_node_up; then down_node fi - if service_installed; then - uninstall_service_command - fi + uninstall_service_command true || return 1 uninstall_completion uninstall_node_script uninstall_node @@ -1338,7 +2283,82 @@ restart_command() { follow_node_logs fi } -install_service_command() { +write_node_service_unit() { + local unit_dir unit_temp + + unit_dir=$(dirname "$SERVICE_UNIT") + unit_temp=$(create_temp_file_in_dir "$unit_dir" ".${SERVICE_NAME}.unit" "") || return 1 + if ! cat >"$unit_temp" <>"$ENV_FILE" +} + +persist_node_service_api_port() { + local api_port="$1" + local api_port_comment="$2" + + if grep -q '^API_PORT[[:space:]]*=' "$ENV_FILE"; then + replace_node_service_api_port "$api_port" "$api_port_comment" + else + append_node_service_api_port "$api_port" "$api_port_comment" + fi +} + +install_service_command() ( check_running_as_root require_systemd set_service_paths @@ -1363,31 +2383,42 @@ install_service_command() { colorized_echo red "node not installed! Install it before setting up the service." exit 1 fi + # Serialize the complete installation transaction with service-update. + # The lock must be held before any backup or mutation so a later rollback + # can never overwrite a successfully completed concurrent transaction. + if ! acquire_node_serviced_update_lock; then + return 1 + fi + if ! begin_node_service_transaction; then + colorized_echo red "Failed to snapshot $SERVICE_NAME before installation." + return 1 + fi + ensure_env_exists - sync_env_ssl_paths + sync_env_ssl_paths || return 1 get_occupied_ports local api_port existing_api_port="" local default_api_port=62051 - if existing_api_port=$(grep -E '^API_PORT[[:space:]]*=' "$ENV_FILE" | head -n1 | sed 's/^API_PORT[[:space:]]*=[[:space:]]*//'); then - existing_api_port=$(echo "$existing_api_port" | tr -d '"'\') - fi + existing_api_port=$(read_node_service_env_value API_PORT 2>/dev/null || true) if [[ "$existing_api_port" =~ ^[0-9]+$ ]] && [ "$existing_api_port" -ge 1 ] && [ "$existing_api_port" -le 65535 ]; then colorized_echo blue "Existing API_PORT found in $ENV_FILE: $existing_api_port" default_api_port="$existing_api_port" fi if [ -n "${INSTALL_API_PORT:-}" ]; then api_port="$INSTALL_API_PORT" - if is_port_occupied "$api_port"; then - colorized_echo red "Port $api_port is already in use." - exit 1 - fi if ! [[ "$api_port" =~ ^[0-9]+$ && "$api_port" -ge 1 && "$api_port" -le 65535 ]]; then colorized_echo red "Invalid port. Please enter a port between 1 and 65535." exit 1 fi + if is_port_occupied "$api_port" && + ! can_reuse_node_service_api_port "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" "$api_port" "$existing_api_port"; then + colorized_echo red "Port $api_port is already in use." + exit 1 + fi elif [ "$AUTO_CONFIRM" = true ]; then api_port="$default_api_port" - if is_port_occupied "$api_port"; then + if is_port_occupied "$api_port" && + ! can_reuse_node_service_api_port "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" "$api_port" "$existing_api_port"; then colorized_echo red "Port $api_port is already in use. Run without -y to choose another port." exit 1 fi @@ -1398,7 +2429,8 @@ install_service_command() { api_port="$default_api_port" fi if [[ "$api_port" =~ ^[0-9]+$ && "$api_port" -ge 1 && "$api_port" -le 65535 ]]; then - if is_port_occupied "$api_port"; then + if is_port_occupied "$api_port" && + ! can_reuse_node_service_api_port "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" "$api_port" "$existing_api_port"; then colorized_echo red "Port $api_port is already in use. Please enter another port." else break @@ -1409,80 +2441,108 @@ install_service_command() { done fi local api_port_comment="# API_PORT is used by the node service API ($APP_NAME)" - if grep -q '^API_PORT[[:space:]]*=' "$ENV_FILE"; then - sed -i "s/^API_PORT[[:space:]]*=.*/API_PORT= ${api_port}/" "$ENV_FILE" - if ! grep -q '^# *API_PORT' "$ENV_FILE"; then - sed -i "/^API_PORT[[:space:]]*=.*/i ${api_port_comment}" "$ENV_FILE" - fi + mark_node_service_transaction_mutation_started || return 1 + if persist_node_service_api_port "$api_port" "$api_port_comment"; then + : else - { - echo "" - echo "$api_port_comment" - echo "API_PORT= ${api_port}" - } >>"$ENV_FILE" + colorized_echo red "Failed to save API_PORT in $ENV_FILE; restoring the previous service configuration." + abort_node_service_transaction || + colorized_echo red "Failed to restore the previous $SERVICE_NAME configuration." + return 1 fi colorized_echo magenta "API_PORT selected: ${api_port}" configure_firewall_for_port "$api_port" "tcp" - install_node_service_script + if ! install_node_service_script false; then + colorized_echo red "Failed to install $SERVICE_NAME binary; restoring the previous service configuration." + abort_node_service_transaction || + colorized_echo red "Failed to restore the previous $SERVICE_NAME configuration." + return 1 + fi colorized_echo blue "Creating systemd unit at $SERVICE_UNIT" - cat >"$SERVICE_UNIT" </dev/null 2>&1; then + return + fi require_systemd - if ! service_installed; then - colorized_echo yellow "Service not installed; nothing to uninstall." + set_service_paths + if ! acquire_node_serviced_update_lock; then + return 1 + fi + if ! begin_node_service_transaction; then + colorized_echo red "Failed to snapshot or inspect $SERVICE_NAME before uninstalling it." + return 1 + fi + if [ "$NODE_SERVICE_TRANSACTION_HAD_SERVICE" != true ]; then + commit_node_service_transaction + if [ "$quiet_if_missing" != true ]; then + colorized_echo yellow "Service not installed; nothing to uninstall." + fi return fi - systemctl stop "$SERVICE_NAME" >/dev/null 2>&1 || true - systemctl disable "$SERVICE_NAME" >/dev/null 2>&1 || true + if [ "$NODE_SERVICE_TRANSACTION_WAS_ACTIVE" = true ]; then + mark_node_service_transaction_mutation_started || return 1 + if ! run_node_service_systemctl stop "$SERVICE_NAME" >/dev/null 2>&1; then + colorized_echo red "Failed to stop $SERVICE_NAME; keeping the installed service intact." + abort_node_service_transaction || + colorized_echo red "Failed to fully restore the previous $SERVICE_NAME state." + return 1 + fi + fi + if [ "$NODE_SERVICE_TRANSACTION_WAS_ENABLED" = true ]; then + mark_node_service_transaction_mutation_started || return 1 + if ! run_node_service_systemctl disable "$SERVICE_NAME" >/dev/null 2>&1; then + colorized_echo red "Failed to disable $SERVICE_NAME; restoring its previous state." + abort_node_service_transaction || + colorized_echo red "Failed to fully restore the previous $SERVICE_NAME state." + return 1 + fi + fi + mark_node_service_transaction_mutation_started || return 1 if [ -f "$SERVICE_UNIT" ]; then colorized_echo yellow "Removing systemd unit $SERVICE_UNIT" - rm "$SERVICE_UNIT" + rm "$SERVICE_UNIT" || { + abort_node_service_transaction || + colorized_echo red "Failed to fully restore the previous $SERVICE_NAME installation." + return 1 + } + fi + if ! uninstall_node_service_script || ! run_node_service_systemctl daemon-reload; then + colorized_echo red "Failed to remove $SERVICE_NAME cleanly; restoring the previous installation." + abort_node_service_transaction || + colorized_echo red "Failed to fully restore the previous $SERVICE_NAME installation." + return 1 fi - uninstall_node_service_script - systemctl daemon-reload + commit_node_service_transaction colorized_echo green "$SERVICE_NAME service uninstalled." -} +) service_start_command() { check_running_as_root require_systemd - if ! service_installed; then - colorized_echo red "Service not installed. Run service-install first." - exit 1 - fi + require_node_service_installed || return 1 systemctl start "$SERVICE_NAME" colorized_echo green "$SERVICE_NAME service started." } service_stop_command() { check_running_as_root require_systemd - if ! service_installed; then - colorized_echo red "Service not installed. Run service-install first." - exit 1 - fi + require_node_service_installed || return 1 systemctl stop "$SERVICE_NAME" colorized_echo green "$SERVICE_NAME service stopped." } @@ -1490,22 +2550,12 @@ service_stop_command() { service_update_command() { check_running_as_root require_systemd - if ! service_installed; then - colorized_echo red "Service not installed. Run service-install first." - exit 1 - fi - install_node_service_script - systemctl daemon-reload - systemctl restart "$SERVICE_NAME" - colorized_echo green "$SERVICE_NAME service updated and restarted." + update_service_if_installed true } service_logs_command() { require_systemd - if ! service_installed; then - colorized_echo red "Service not installed. Run service-install first." - exit 1 - fi + require_node_service_installed || return 1 local no_follow=false while [[ "$#" -gt 0 ]]; do case "$1" in @@ -1535,18 +2585,12 @@ service_logs_command() { restart_service_command() { check_running_as_root require_systemd - if ! service_installed; then - colorized_echo red "Service not installed. Run service-install first." - exit 1 - fi + require_node_service_installed || return 1 restart_service_if_installed } status_service_command() { require_systemd - if ! service_installed; then - colorized_echo red "Service not installed. Run service-install first." - exit 1 - fi + require_node_service_installed || return 1 systemctl status --no-pager "$SERVICE_NAME" } status_command() { @@ -2267,10 +3311,10 @@ pg_node_main() { ;; service-install) shift - install_service_command "$@" + install_service_command "$@" || return 1 ;; service-uninstall) - uninstall_service_command + uninstall_service_command || return 1 ;; service-restart) restart_service_command diff --git a/tests/test_node_service_readiness_tls.sh b/tests/test_node_service_readiness_tls.sh new file mode 100644 index 0000000..fbfcf29 --- /dev/null +++ b/tests/test_node_service_readiness_tls.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +WORK_DIR="$(mktemp -d)" +SERVER_PID="" +cleanup() { + [ -z "$SERVER_PID" ] || kill "$SERVER_PID" >/dev/null 2>&1 || true + [ -z "$SERVER_PID" ] || wait "$SERVER_PID" >/dev/null 2>&1 || true + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +export APP_TMP_DIR="$WORK_DIR/tmp" +export APP_NAME="readiness-tls-test" +export APP_DIR="$WORK_DIR/app" +export DATA_DIR="$WORK_DIR/data" +mkdir -p "$APP_TMP_DIR" "$APP_DIR" "$DATA_DIR" + +curl() { printf '\n'; } +export -f curl +export PG_NODE_SOURCE_ONLY=true +# shellcheck source=pg-node.sh +source "$ROOT_DIR/pg-node.sh" +unset -f curl + +CERT_FILE="$WORK_DIR/dns-only-cert.pem" +KEY_FILE="$WORK_DIR/dns-only-key.pem" +ENV_FILE="$WORK_DIR/service.env" +SERVICE_NAME="readiness-tls-test-service" +PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') + +openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj '/CN=node.example.test' \ + -addext 'subjectAltName=DNS:node.example.test' \ + -keyout "$KEY_FILE" -out "$CERT_FILE" >/dev/null 2>&1 +printf 'API_PORT= %s\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$PORT" "$CERT_FILE" > "$ENV_FILE" + +openssl s_server -accept "127.0.0.1:$PORT" -cert "$CERT_FILE" -key "$KEY_FILE" -www \ + >"$WORK_DIR/server.log" 2>&1 & +SERVER_PID=$! +for _ in {1..100}; do + if (exec 7<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + exec 7>&- + break + fi + sleep 0.02 +done + +set +e +curl --silent --show-error --fail --noproxy '*' --cacert "$CERT_FILE" \ + "https://127.0.0.1:$PORT/" >/dev/null 2>&1 +baseline_status=$? +set -e +if [ "$baseline_status" -eq 60 ]; then + printf '✓ baseline: DNS-only certificate rejects https://127.0.0.1 with rc60\n' +else + printf '✗ baseline: expected curl rc60, got %s\n' "$baseline_status" + exit 1 +fi + +if node_service_api_ready; then + printf '✓ readiness: DNS SAN keeps Host/SNI while TCP connects to loopback without DNS\n' +else + printf '✗ readiness: DNS-only certificate probe failed\n' + exit 1 +fi + +kill "$SERVER_PID" >/dev/null 2>&1 || true +wait "$SERVER_PID" >/dev/null 2>&1 || true +SERVER_PID="" +CERT_FILE="$WORK_DIR/wildcard-cert.pem" +KEY_FILE="$WORK_DIR/wildcard-key.pem" +PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj '/CN=*.example.test' \ + -addext 'subjectAltName=DNS:*.example.test' \ + -keyout "$KEY_FILE" -out "$CERT_FILE" >/dev/null 2>&1 +printf 'API_PORT= %s\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$PORT" "$CERT_FILE" > "$ENV_FILE" +openssl s_server -accept "127.0.0.1:$PORT" -cert "$CERT_FILE" -key "$KEY_FILE" -www \ + >"$WORK_DIR/wildcard-server.log" 2>&1 & +SERVER_PID=$! +for _ in {1..100}; do + if (exec 7<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + exec 7>&- + break + fi + sleep 0.02 +done +if node_service_api_ready; then + printf '✓ readiness: wildcard DNS SAN uses a covered synthetic host over loopback\n' +else + printf '✗ readiness: wildcard DNS SAN probe failed\n' + exit 1 +fi + +kill "$SERVER_PID" >/dev/null 2>&1 || true +wait "$SERVER_PID" >/dev/null 2>&1 || true +SERVER_PID="" +CERT_FILE="$WORK_DIR/cn-only-cert.pem" +KEY_FILE="$WORK_DIR/cn-only-key.pem" +PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -config /dev/null \ + -subj '/CN=legacy.example.test' \ + -keyout "$KEY_FILE" -out "$CERT_FILE" >/dev/null 2>&1 +if openssl x509 -in "$CERT_FILE" -noout -ext subjectAltName 2>/dev/null | grep -q 'Subject Alternative Name'; then + printf '✗ setup: CN-only certificate unexpectedly carries a SAN\n' + exit 1 +fi +printf 'API_PORT= %s\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$PORT" "$CERT_FILE" > "$ENV_FILE" +openssl s_server -accept "127.0.0.1:$PORT" -cert "$CERT_FILE" -key "$KEY_FILE" -www \ + >"$WORK_DIR/cn-only-server.log" 2>&1 & +SERVER_PID=$! +for _ in {1..100}; do + if (exec 7<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + exec 7>&- + break + fi + sleep 0.02 +done +if node_service_api_ready; then + printf '✓ readiness: SAN-less legacy CN remains supported over loopback\n' +else + printf '✗ readiness: SAN-less legacy CN probe failed\n' + exit 1 +fi + +kill "$SERVER_PID" >/dev/null 2>&1 || true +wait "$SERVER_PID" >/dev/null 2>&1 || true +SERVER_PID="" +CERT_FILE="$WORK_DIR/unusable-san-cert.pem" +KEY_FILE="$WORK_DIR/unusable-san-key.pem" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj '/CN=must-not-fallback.example.test' \ + -addext 'subjectAltName=email:health@example.test' \ + -keyout "$KEY_FILE" -out "$CERT_FILE" >/dev/null 2>&1 +printf 'API_PORT= 3000\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$CERT_FILE" > "$ENV_FILE" +if node_service_api_ready >/dev/null 2>&1; then + printf '✗ readiness: unusable SAN incorrectly fell back to CN\n' + exit 1 +else + printf '✓ readiness: SAN without DNS/IP identity fails closed\n' +fi + +CERT_FILE="$WORK_DIR/ipv6-san-cert.pem" +KEY_FILE="$WORK_DIR/ipv6-san-key.pem" +PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj '/CN=unused.example.test' \ + -addext 'subjectAltName=IP:::1' \ + -keyout "$KEY_FILE" -out "$CERT_FILE" >/dev/null 2>&1 +printf 'API_PORT= %s\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$PORT" "$CERT_FILE" > "$ENV_FILE" +# Deliberately bind IPv4 loopback only. A broken connect-to source match tries +# ::1 and fails; readiness succeeds only when curl is forced to 127.0.0.1 while +# TLS still verifies the IPv6 IP SAN from the URL identity. +openssl s_server -accept "127.0.0.1:$PORT" -cert "$CERT_FILE" -key "$KEY_FILE" -www \ + >"$WORK_DIR/ipv6-san-server.log" 2>&1 & +SERVER_PID=$! +for _ in {1..100}; do + if (exec 7<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + exec 7>&- + break + fi + sleep 0.02 +done +if node_service_api_ready; then + printf '✓ readiness: IPv6 IP SAN keeps TLS identity while TCP connects to IPv4 loopback\n' +else + printf '✗ readiness: IPv6 IP SAN probe was not forced to IPv4 loopback\n' + exit 1 +fi diff --git a/tests/test_node_service_transaction_guard.sh b/tests/test_node_service_transaction_guard.sh new file mode 100644 index 0000000..666ef77 --- /dev/null +++ b/tests/test_node_service_transaction_guard.sh @@ -0,0 +1,633 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +export APP_TMP_DIR="$WORK_DIR/tmp" +export APP_NAME="transaction-test" +export APP_DIR="$WORK_DIR/app" +export DATA_DIR="$WORK_DIR/data" +mkdir -p "$APP_TMP_DIR" "$APP_DIR" "$DATA_DIR" + +curl() { printf '\n'; } +export -f curl +export PG_NODE_SOURCE_ONLY=true +# shellcheck source=pg-node.sh +source "$ROOT_DIR/pg-node.sh" +original_service_installed_definition=$(declare -f service_installed) +original_uninstall_node_service_script_definition=$(declare -f uninstall_node_service_script) +original_append_node_service_api_port_definition=$(declare -f append_node_service_api_port) + +PASS=0 +FAIL=0 +pass() { printf '✓ %s\n' "$1"; PASS=$((PASS + 1)); } +fail() { printf '✗ %s\n' "$1"; FAIL=$((FAIL + 1)); } +assert_true() { local label="$1"; shift; if "$@"; then pass "$label"; else fail "$label"; fi; } +assert_false() { local label="$1"; shift; if ! "$@"; then pass "$label"; else fail "$label"; fi; } +assert_file() { + local path="$1" expected="$2" label="$3" + if [ -f "$path" ] && [ "$(cat "$path")" = "$expected" ]; then pass "$label"; else fail "$label"; fi +} +assert_no_backups() { + local label="$1" + if find "$CASE_DIR" -type f \( -name '.*.transaction.*' -o -name '.*.transaction-target.*' -o -name '.*.backup.*' \) -print -quit | grep -q .; then + fail "$label" + else + pass "$label" + fi +} +assert_lock_released() { + local label="$1" + if flock -n "$NODE_SERVICE_UPDATE_LOCK_PATH" -c true; then pass "$label"; else fail "$label"; fi +} + +setup_case() { + local name="$1" + CASE_DIR="$WORK_DIR/$name" + mkdir -p "$CASE_DIR" + APP_DIR="$CASE_DIR" + ENV_FILE="$CASE_DIR/.env" + SERVICE_NAME="transaction-test-service" + SERVICE_UNIT="$CASE_DIR/$SERVICE_NAME.service" + SERVICE_BINARY_PATH="$CASE_DIR/$SERVICE_NAME" + NODE_SERVICE_UPDATE_LOCK_PATH="$CASE_DIR/update.lock" + VALID_BINARY="$CASE_DIR/candidate" + ACTIVATED_MARKER="$CASE_DIR/activated" + BLOCK_MARKER="$CASE_DIR/blocking" + TRANSACTION_PID_FILE="$CASE_DIR/transaction-pid" + RELEASE_MARKER="$CASE_DIR/release" + RESTART_COUNT="$CASE_DIR/restarts" + SYSTEMCTL_MUTATION_LOG="$CASE_DIR/systemctl-mutations" + printf 'old-env\n' > "$ENV_FILE" + printf 'old-unit\n' > "$SERVICE_UNIT" + printf '\177ELFold-binary\n' > "$SERVICE_BINARY_PATH" + printf '\177ELFnew-binary\n' > "$VALID_BINARY" + printf '0\n' > "$RESTART_COUNT" + : > "$SYSTEMCTL_MUTATION_LOG" + NODE_SERVICE_UPDATE_LOCK_HELD=false + unset STOP_FAILURE DISABLE_FAILURE SERVICE_INACTIVE SERVICE_DISABLED +} + +set_service_paths() { :; } +service_installed() { return 0; } +id() { printf '0\n'; } +wait_for_node_service_ready() { return 0; } +systemctl() { + case "${1:-}" in + is-active) + [ "${SERVICE_INACTIVE:-false}" = true ] && return 3 + return 0 + ;; + is-enabled) + [ "${SERVICE_DISABLED:-false}" = true ] && return 1 + return 0 + ;; + stop) + printf 'stop\n' >> "$SYSTEMCTL_MUTATION_LOG" + [ "${STOP_FAILURE:-false}" != true ] + ;; + disable) + printf 'disable\n' >> "$SYSTEMCTL_MUTATION_LOG" + [ "${DISABLE_FAILURE:-false}" != true ] + ;; + enable | daemon-reload) + printf '%s\n' "$1" >> "$SYSTEMCTL_MUTATION_LOG" + return 0 + ;; + start) + printf 'start\n' >> "$SYSTEMCTL_MUTATION_LOG" + return 0 + ;; + restart) + printf 'restart\n' >> "$SYSTEMCTL_MUTATION_LOG" + local count + count=$(cat "$RESTART_COUNT") + count=$((count + 1)) + printf '%s\n' "$count" > "$RESTART_COUNT" + if [ "${BLOCK_FIRST_RESTART:-false}" = true ] && [ "$count" -eq 1 ]; then + printf '%s\n' "$BASHPID" > "$TRANSACTION_PID_FILE" + : > "$BLOCK_MARKER" + while [ ! -e "$RELEASE_MARKER" ]; do sleep 0.02; done + fi + return 0 + ;; + *) return 0 ;; + esac +} + +install_node_service_script() { + activate_node_serviced_binary "$VALID_BINARY" false + printf 'new-env\n' > "$ENV_FILE" + printf 'new-unit\n' > "$SERVICE_UNIT" + : > "$ACTIVATED_MARKER" +} +transaction_test_install_node_service_script_definition=$(declare -f install_node_service_script) + +echo "=== test_node_service_transaction_guard.sh ===" + +# Deliver a real TERM after binary activation. The public command runs the +# guard in a subshell, so its traps cannot replace the caller's traps. +setup_case signal +BLOCK_FIRST_RESTART=true +caller_traps_before=$(trap -p EXIT INT TERM) +update_service_if_installed >/dev/null 2>&1 & +transaction_pid=$! +for _ in {1..200}; do + [ -e "$BLOCK_MARKER" ] && break + sleep 0.01 +done +if [ -e "$BLOCK_MARKER" ]; then pass "signal: replacement reached activation"; else fail "signal: replacement reached activation"; fi +guarded_pid=$(cat "$TRANSACTION_PID_FILE") +kill -TERM "$guarded_pid" +set +e +wait "$transaction_pid" +signal_status=$? +set -e +[ "$signal_status" -eq 143 ] && pass "signal: TERM status is preserved" || fail "signal: TERM status is preserved" +assert_file "$ENV_FILE" "old-env" "signal: environment snapshot restored" +assert_file "$SERVICE_UNIT" "old-unit" "signal: unit snapshot restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "signal: binary snapshot restored" || fail "signal: binary snapshot restored" +assert_no_backups "signal: candidate backups removed" +assert_lock_released "signal: flock released" +caller_traps_after=$(trap -p EXIT INT TERM) +[ "$caller_traps_after" = "$caller_traps_before" ] && pass "signal: caller traps preserved" || fail "signal: caller traps preserved" +unset BLOCK_FIRST_RESTART + +# A checksum/download failure happens before activation and therefore must only +# discard snapshots/temp state. It must not restart, enable, disable, or reload +# a service whose files and runtime state were never changed. +setup_case premutation-download-failure +install_node_service_script() { return 64; } +set +e +update_service_if_installed >/dev/null 2>&1 +premutation_status=$? +set -e +[ "$premutation_status" -ne 0 ] && pass "pre-mutation failure: failure propagated" || fail "pre-mutation failure: failure propagated" +[ ! -s "$SYSTEMCTL_MUTATION_LOG" ] && pass "pre-mutation failure: no service mutation" || fail "pre-mutation failure: no service mutation" +assert_file "$ENV_FILE" "old-env" "pre-mutation failure: environment unchanged" +assert_file "$SERVICE_UNIT" "old-unit" "pre-mutation failure: unit unchanged" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "pre-mutation failure: binary unchanged" || fail "pre-mutation failure: binary unchanged" +assert_no_backups "pre-mutation failure: snapshots removed" +assert_lock_released "pre-mutation failure: flock released" +eval "$transaction_test_install_node_service_script_definition" + +# Public service commands must distinguish a confirmed absent unit (status 1) +# from an inspection failure. A timeout/error cannot fall through to start. +setup_case service-inspection-failure +check_running_as_root() { :; } +require_systemd() { :; } +service_installed() { return 124; } +set +e +service_start_command >/dev/null 2>&1 +inspection_status=$? +set -e +[ "$inspection_status" -ne 0 ] && pass "service inspection failure: failure propagated" || fail "service inspection failure: failure propagated" +if grep -qx 'start' "$SYSTEMCTL_MUTATION_LOG"; then fail "service inspection failure: start not attempted"; else pass "service inspection failure: start not attempted"; fi +service_installed() { return 0; } + +# Exercise an unanticipated errexit in a subprocess, without an explicit +# `if ! command` failure branch. +setup_case errexit +unexpected_failure() ( + acquire_node_serviced_update_lock + begin_node_service_transaction + activate_node_serviced_binary "$VALID_BINARY" false + printf 'new-env\n' > "$ENV_FILE" + printf 'new-unit\n' > "$SERVICE_UNIT" + false + commit_node_service_transaction +) +unexpected_failure >/dev/null 2>&1 & +failure_pid=$! +set +e +wait "$failure_pid" +failure_status=$? +set -e +[ "$failure_status" -ne 0 ] && pass "errexit: failure propagated" || fail "errexit: failure propagated" +assert_file "$ENV_FILE" "old-env" "errexit: environment snapshot restored" +assert_file "$SERVICE_UNIT" "old-unit" "errexit: unit snapshot restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "errexit: binary snapshot restored" || fail "errexit: binary snapshot restored" +assert_no_backups "errexit: backups removed" +assert_lock_released "errexit: flock released" + +# Uninstall is guarded too: an unexpected failure after binary removal must +# restore both the binary and unit instead of committing a partial uninstall. +setup_case uninstall +uninstall_node_service_script() { + rm -f "$SERVICE_BINARY_PATH" + return 42 +} +uninstall_service_command >/dev/null 2>&1 & +uninstall_pid=$! +set +e +wait "$uninstall_pid" +uninstall_status=$? +set -e +[ "$uninstall_status" -ne 0 ] && pass "uninstall: failure propagated" || fail "uninstall: failure propagated" +assert_file "$SERVICE_UNIT" "old-unit" "uninstall: unit restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "uninstall: binary restored" || fail "uninstall: binary restored" +assert_no_backups "uninstall: backups removed" +assert_lock_released "uninstall: flock released" + +# stop/disable are part of the uninstall transaction. A failed request may +# have changed systemd despite its status, so fail closed and roll back without +# deleting either installed artifact. +setup_case uninstall-stop-failure +eval "$original_uninstall_node_service_script_definition" +STOP_FAILURE=true +set +e +uninstall_service_command >/dev/null 2>&1 +uninstall_stop_status=$? +set -e +[ "$uninstall_stop_status" -ne 0 ] && pass "uninstall stop failure: failure propagated" || fail "uninstall stop failure: failure propagated" +assert_file "$SERVICE_UNIT" "old-unit" "uninstall stop failure: unit retained" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "uninstall stop failure: binary retained" || fail "uninstall stop failure: binary retained" +assert_no_backups "uninstall stop failure: backups removed" +assert_lock_released "uninstall stop failure: flock released" + +setup_case uninstall-disable-failure +eval "$original_uninstall_node_service_script_definition" +DISABLE_FAILURE=true +set +e +uninstall_service_command >/dev/null 2>&1 +uninstall_disable_status=$? +set -e +[ "$uninstall_disable_status" -ne 0 ] && pass "uninstall disable failure: failure propagated" || fail "uninstall disable failure: failure propagated" +assert_file "$SERVICE_UNIT" "old-unit" "uninstall disable failure: unit retained" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "uninstall disable failure: binary retained" || fail "uninstall disable failure: binary retained" +assert_no_backups "uninstall disable failure: backups removed" +assert_lock_released "uninstall disable failure: flock released" + +setup_case uninstall-inactive-disabled +eval "$original_uninstall_node_service_script_definition" +SERVICE_INACTIVE=true +SERVICE_DISABLED=true +set +e +uninstall_service_command >/dev/null 2>&1 +uninstall_idempotent_status=$? +set -e +[ "$uninstall_idempotent_status" -eq 0 ] && pass "uninstall inactive/disabled: succeeds" || fail "uninstall inactive/disabled: succeeds" +[ ! -e "$SERVICE_UNIT" ] && pass "uninstall inactive/disabled: unit removed" || fail "uninstall inactive/disabled: unit removed" +[ ! -e "$SERVICE_BINARY_PATH" ] && pass "uninstall inactive/disabled: binary removed" || fail "uninstall inactive/disabled: binary removed" +if grep -Eq '^(stop|disable)$' "$SYSTEMCTL_MUTATION_LOG"; then fail "uninstall inactive/disabled: no redundant stop/disable"; else pass "uninstall inactive/disabled: no redundant stop/disable"; fi +assert_no_backups "uninstall inactive/disabled: backups removed" +assert_lock_released "uninstall inactive/disabled: flock released" + +# Reinstall snapshots are armed before env mutation and binary activation. A +# later unit-write failure must restore every artifact through the same guard. +setup_case install +check_running_as_root() { :; } +require_systemd() { :; } +detect_os() { :; } +install_package() { :; } +is_node_installed() { return 0; } +ensure_env_exists() { printf 'generated-env\n' > "$ENV_FILE"; } +sync_env_ssl_paths() { :; } +get_occupied_ports() { OCCUPIED_PORTS=""; } +is_port_occupied() { return 1; } +configure_firewall_for_port() { :; } +write_node_service_unit() { + printf 'partial-unit\n' > "$SERVICE_UNIT" + return 1 +} +AUTO_CONFIRM=true +INSTALL_API_PORT=62051 +install_service_command >/dev/null 2>&1 & +install_pid=$! +set +e +wait "$install_pid" +install_status=$? +set -e +[ "$install_status" -ne 0 ] && pass "install: failure propagated" || fail "install: failure propagated" +assert_file "$ENV_FILE" "old-env" "install: environment restored" +assert_file "$SERVICE_UNIT" "old-unit" "install: unit restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "install: binary restored" || fail "install: binary restored" +assert_no_backups "install: backups removed" +assert_lock_released "install: flock released" + +# Both the replace and append paths for API_PORT are transactional. A write +# failure must stop before binary activation and restore the original env. +setup_case install-api-port-replace-failure +printf 'API_PORT= 61000\n' > "$ENV_FILE" +ensure_env_exists() { :; } +sed() { + if [ "${1:-}" = "-i" ]; then return 74; fi + command sed "$@" +} +rm -f "$ACTIVATED_MARKER" +set +e +install_service_command >/dev/null 2>&1 +api_port_replace_status=$? +set -e +unset -f sed +[ "$api_port_replace_status" -ne 0 ] && pass "API_PORT replace failure: failure propagated" || fail "API_PORT replace failure: failure propagated" +assert_file "$ENV_FILE" "API_PORT= 61000" "API_PORT replace failure: environment restored" +[ ! -e "$ACTIVATED_MARKER" ] && pass "API_PORT replace failure: binary activation skipped" || fail "API_PORT replace failure: binary activation skipped" +assert_no_backups "API_PORT replace failure: snapshots removed" +assert_lock_released "API_PORT replace failure: flock released" + +setup_case install-api-port-append-failure +append_node_service_api_port() { return 75; } +rm -f "$ACTIVATED_MARKER" +set +e +install_service_command >/dev/null 2>&1 +api_port_append_status=$? +set -e +eval "$original_append_node_service_api_port_definition" +[ "$api_port_append_status" -ne 0 ] && pass "API_PORT append failure: failure propagated" || fail "API_PORT append failure: failure propagated" +assert_file "$ENV_FILE" "old-env" "API_PORT append failure: environment restored" +[ ! -e "$ACTIVATED_MARKER" ] && pass "API_PORT append failure: binary activation skipped" || fail "API_PORT append failure: binary activation skipped" +assert_no_backups "API_PORT append failure: snapshots removed" +assert_lock_released "API_PORT append failure: flock released" + +# Snapshot both the symlink inode and its referent. Rollback must restore the +# exact relative link text as well as referent content changed through the link. +setup_case symlink +mkdir -p "$CASE_DIR/targets" +mv "$ENV_FILE" "$CASE_DIR/targets/env" +mv "$SERVICE_UNIT" "$CASE_DIR/targets/unit" +mv "$SERVICE_BINARY_PATH" "$CASE_DIR/targets/binary" +ln -s targets/env "$ENV_FILE" +ln -s targets/unit "$SERVICE_UNIT" +ln -s targets/binary "$SERVICE_BINARY_PATH" +acquire_node_serviced_update_lock +begin_node_service_transaction +mark_node_service_transaction_mutation_started +printf 'changed-target-env\n' > "$CASE_DIR/targets/env" +printf 'changed-target-unit\n' > "$CASE_DIR/targets/unit" +printf '\177ELFchanged-target-binary\n' > "$CASE_DIR/targets/binary" +rm -f "$ENV_FILE" "$SERVICE_UNIT" "$SERVICE_BINARY_PATH" +printf 'replacement-env\n' > "$ENV_FILE" +printf 'replacement-unit\n' > "$SERVICE_UNIT" +printf '\177ELFreplacement-binary\n' > "$SERVICE_BINARY_PATH" +assert_true "symlink: first rollback succeeds" rollback_node_service_transaction +assert_true "symlink: repeated rollback is idempotent" rollback_node_service_transaction +[ -L "$ENV_FILE" ] && [ "$(readlink "$ENV_FILE")" = targets/env ] && pass "symlink: environment link topology restored" || fail "symlink: environment link topology restored" +[ -L "$SERVICE_UNIT" ] && [ "$(readlink "$SERVICE_UNIT")" = targets/unit ] && pass "symlink: unit link topology restored" || fail "symlink: unit link topology restored" +[ -L "$SERVICE_BINARY_PATH" ] && [ "$(readlink "$SERVICE_BINARY_PATH")" = targets/binary ] && pass "symlink: binary link topology restored" || fail "symlink: binary link topology restored" +assert_file "$CASE_DIR/targets/env" "old-env" "symlink: environment referent restored" +assert_file "$CASE_DIR/targets/unit" "old-unit" "symlink: unit referent restored" +grep -q 'old-binary' "$CASE_DIR/targets/binary" && pass "symlink: binary referent restored" || fail "symlink: binary referent restored" +finish_node_service_transaction +assert_no_backups "symlink: backups removed after successful retry-safe rollback" +assert_lock_released "symlink: flock released" + +# A failed restore must retain its backup handle. A second rollback retries only +# unfinished entries and succeeds instead of returning failure permanently. +setup_case retry +acquire_node_serviced_update_lock +begin_node_service_transaction +mark_node_service_transaction_mutation_started +printf 'new-env\n' > "$ENV_FILE" +printf 'new-unit\n' > "$SERVICE_UNIT" +printf '\177ELFnew-binary\n' > "$SERVICE_BINARY_PATH" +MV_FAILURE_MARKER="$CASE_DIR/mv-failed-once" +ENV_BACKUP_TO_FAIL="${NODE_SERVICE_TRANSACTION_BACKUPS[0]}" +cp() { + if [ "${2:-}" = "$ENV_BACKUP_TO_FAIL" ] && [ ! -e "$MV_FAILURE_MARKER" ]; then + : > "$MV_FAILURE_MARKER" + return 1 + fi + command cp "$@" +} +assert_false "retry: first rollback reports restore failure" rollback_node_service_transaction +env_snapshot_index=0 +env_backup_path="${NODE_SERVICE_TRANSACTION_BACKUPS[env_snapshot_index]}" +[ -n "$env_backup_path" ] && [ -e "$env_backup_path" ] && pass "retry: failed restore retains backup handle" || fail "retry: failed restore retains backup handle" +assert_true "retry: second rollback succeeds" rollback_node_service_transaction +assert_file "$ENV_FILE" "old-env" "retry: environment restored on second attempt" +assert_file "$SERVICE_UNIT" "old-unit" "retry: completed unit restore remains intact" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "retry: completed binary restore remains intact" || fail "retry: completed binary restore remains intact" +unset -f cp +finish_node_service_transaction +assert_no_backups "retry: backups removed after successful retry" +assert_lock_released "retry: flock released" + +# On a first install, failing `systemctl disable --now` during rollback means +# the unit can remain loaded/active. Report the orphan risk and retain the +# filesystem snapshots for manual recovery instead of declaring success. +setup_case first-install-disable-failure +rm -f "$SERVICE_UNIT" "$SERVICE_BINARY_PATH" +service_installed() { return 1; } +acquire_node_serviced_update_lock +begin_node_service_transaction +mark_node_service_transaction_mutation_started +printf 'new-env\n' > "$ENV_FILE" +printf 'new-unit\n' > "$SERVICE_UNIT" +printf '\177ELFnew-binary\n' > "$SERVICE_BINARY_PATH" +DISABLE_FAILURE=true +set +e +first_install_rollback_output=$(abort_node_service_transaction 2>&1) +first_install_rollback_status=$? +set -e +[ "$first_install_rollback_status" -ne 0 ] && pass "first-install disable failure: rollback reports failure" || fail "first-install disable failure: rollback reports failure" +assert_file "$ENV_FILE" "old-env" "first-install disable failure: environment restored" +[ ! -e "$SERVICE_UNIT" ] && [ ! -e "$SERVICE_BINARY_PATH" ] && pass "first-install disable failure: new artifacts removed" || fail "first-install disable failure: new artifacts removed" +if [[ "$first_install_rollback_output" == *"may remain loaded or active"* ]]; then pass "first-install disable failure: orphaned unit is reported"; else fail "first-install disable failure: orphaned unit is reported"; fi +if find "$CASE_DIR" -type f -name '.*.transaction.*' -print -quit | grep -q .; then pass "first-install disable failure: snapshots retained"; else fail "first-install disable failure: snapshots retained"; fi +assert_lock_released "first-install disable failure: flock released" +cleanup_node_service_transaction_backups +service_installed() { return 0; } + +# Production uses an external systemctl process group. TERM sent to the +# transaction shell must promptly reach the external child and its descendant, +# then rollback and release the lock without waiting for the child timeout. +setup_case external-signal +EXTERNAL_BIN="$CASE_DIR/bin" +EXTERNAL_CHILD_PID_FILE="$CASE_DIR/external-child-pid" +mkdir -p "$EXTERNAL_BIN" +cat > "$EXTERNAL_BIN/systemctl" <<'EOF' +#!/usr/bin/env bash +case "${1:-}" in +is-active|is-enabled|daemon-reload|enable|disable|stop) exit 0 ;; +restart) + count=$(cat "$RESTART_COUNT") + count=$((count + 1)) + printf '%s\n' "$count" > "$RESTART_COUNT" + if [ "$count" -eq 1 ]; then + printf '%s\n' "$PPID" > "$TRANSACTION_PID_FILE" + ( + trap 'exit 143' TERM INT + printf '%s\n' "$BASHPID" > "$EXTERNAL_CHILD_PID_FILE" + : > "$BLOCK_MARKER" + while :; do sleep 1; done + ) & + wait "$!" + fi + exit 0 + ;; +*) exit 0 ;; +esac +EOF +chmod +x "$EXTERNAL_BIN/systemctl" +export RESTART_COUNT TRANSACTION_PID_FILE EXTERNAL_CHILD_PID_FILE BLOCK_MARKER +external_started=$(date +%s) +( + unset -f systemctl + PATH="$EXTERNAL_BIN:$PATH" + update_service_if_installed >/dev/null 2>&1 & + outer_pid=$! + for _ in {1..200}; do + [ -e "$BLOCK_MARKER" ] && break + sleep 0.01 + done + kill -TERM "$(cat "$TRANSACTION_PID_FILE")" + set +e + wait "$outer_pid" + printf '%s\n' "$?" > "$CASE_DIR/external-status" + set -e +) +external_elapsed=$(( $(date +%s) - external_started )) +[ "$(cat "$CASE_DIR/external-status")" -eq 143 ] && pass "external signal: TERM status preserved" || fail "external signal: TERM status preserved" +[ "$external_elapsed" -lt 5 ] && pass "external signal: child process group terminated promptly" || fail "external signal: child process group terminated promptly" +if [ ! -s "$EXTERNAL_CHILD_PID_FILE" ]; then + fail "external signal: descendant recorded its pid" +elif kill -0 "$(cat "$EXTERNAL_CHILD_PID_FILE")" >/dev/null 2>&1; then + fail "external signal: descendant terminated" +else + pass "external signal: descendant terminated" +fi +assert_file "$ENV_FILE" "old-env" "external signal: environment restored" +assert_file "$SERVICE_UNIT" "old-unit" "external signal: unit restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "external signal: binary restored" || fail "external signal: binary restored" +assert_no_backups "external signal: backups removed" +assert_lock_released "external signal: flock released" + +# A hung external mutation is bounded even without an incoming signal. The +# watchdog terminates its process group, then the failure path rolls back. +setup_case external-timeout +EXTERNAL_BIN="$CASE_DIR/bin" +TIMEOUT_ONCE_MARKER="$CASE_DIR/timed-out-once" +mkdir -p "$EXTERNAL_BIN" +cat > "$EXTERNAL_BIN/systemctl" <<'EOF' +#!/usr/bin/env bash +case "${1:-}" in +is-active|is-enabled|enable|disable|stop|restart) exit 0 ;; +daemon-reload) + if [ -e "$ACTIVATED_MARKER" ] && [ ! -e "$TIMEOUT_ONCE_MARKER" ]; then + : > "$TIMEOUT_ONCE_MARKER" + (while :; do sleep 1; done) & + wait "$!" + fi + exit 0 + ;; +*) exit 0 ;; +esac +EOF +chmod +x "$EXTERNAL_BIN/systemctl" +export ACTIVATED_MARKER TIMEOUT_ONCE_MARKER +timeout_started=$(date +%s) +set +e +( + unset -f systemctl + PATH="$EXTERNAL_BIN:$PATH" + NODE_SERVICE_SYSTEMCTL_TIMEOUT_SECONDS=1 + NODE_SERVICE_EXTERNAL_KILL_AFTER_SECONDS=0.2 + update_service_if_installed >/dev/null 2>&1 +) +timeout_status=$? +set -e +timeout_elapsed=$(( $(date +%s) - timeout_started )) +[ "$timeout_status" -ne 0 ] && pass "external timeout: failure propagated" || fail "external timeout: failure propagated" +[ "$timeout_elapsed" -lt 5 ] && pass "external timeout: bounded policy enforced" || fail "external timeout: bounded policy enforced" +assert_file "$ENV_FILE" "old-env" "external timeout: environment restored" +assert_file "$SERVICE_UNIT" "old-unit" "external timeout: unit restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "external timeout: binary restored" || fail "external timeout: binary restored" +assert_no_backups "external timeout: backups removed" +assert_lock_released "external timeout: flock released" + +# Service discovery itself is an external systemctl call. The mutation lock +# and transaction traps must already be armed when it wedges, so a real TERM +# cannot strand the lock, snapshots, temporary output, or the process group. +setup_case service-query-signal +QUERY_BIN="$CASE_DIR/bin" +QUERY_BLOCK_MARKER="$CASE_DIR/query-blocking" +QUERY_DESCENDANT_PID_FILE="$CASE_DIR/query-descendant-pid" +mkdir -p "$QUERY_BIN" +cat > "$QUERY_BIN/systemctl" <<'EOF' +#!/usr/bin/env bash +case "${1:-}" in +list-unit-files) + printf '%s\n' "$PPID" > "$TRANSACTION_PID_FILE" + trap '' TERM INT + ( + trap '' TERM INT + printf '%s\n' "$BASHPID" > "$QUERY_DESCENDANT_PID_FILE" + : > "$QUERY_BLOCK_MARKER" + while :; do sleep 1; done + ) & + wait "$!" + ;; +*) printf '%s\n' "$1" >> "$SYSTEMCTL_MUTATION_LOG"; exit 0 ;; +esac +EOF +chmod +x "$QUERY_BIN/systemctl" +rm -f "$SERVICE_UNIT" +export TRANSACTION_PID_FILE QUERY_BLOCK_MARKER QUERY_DESCENDANT_PID_FILE SYSTEMCTL_MUTATION_LOG +query_started=$(date +%s) +( + unset -f service_installed systemctl + eval "$original_service_installed_definition" + PATH="$QUERY_BIN:$PATH" + NODE_SERVICE_SYSTEMCTL_TIMEOUT_SECONDS=30 + NODE_SERVICE_EXTERNAL_KILL_AFTER_SECONDS=0.2 + update_service_if_installed >/dev/null 2>&1 & + outer_pid=$! + for _ in {1..200}; do + [ -e "$QUERY_BLOCK_MARKER" ] && break + sleep 0.01 + done + kill -TERM "$(cat "$TRANSACTION_PID_FILE")" + set +e + wait "$outer_pid" + printf '%s\n' "$?" > "$CASE_DIR/query-status" + set -e +) +query_elapsed=$(( $(date +%s) - query_started )) +[ "$(cat "$CASE_DIR/query-status")" -eq 143 ] && pass "service query signal: TERM status preserved" || fail "service query signal: TERM status preserved" +[ "$query_elapsed" -lt 5 ] && pass "service query signal: TERM-ignoring process group terminated promptly" || fail "service query signal: TERM-ignoring process group terminated promptly" +if [ ! -s "$QUERY_DESCENDANT_PID_FILE" ]; then + fail "service query signal: descendant recorded its pid" +elif kill -0 "$(cat "$QUERY_DESCENDANT_PID_FILE")" >/dev/null 2>&1; then + fail "service query signal: descendant terminated" +else + pass "service query signal: descendant terminated" +fi +assert_file "$ENV_FILE" "old-env" "service query signal: environment snapshot restored" +grep -q 'old-binary' "$SERVICE_BINARY_PATH" && pass "service query signal: binary snapshot restored" || fail "service query signal: binary snapshot restored" +[ ! -e "$SERVICE_UNIT" ] && pass "service query signal: absent unit remains absent" || fail "service query signal: absent unit remains absent" +[ ! -s "$SYSTEMCTL_MUTATION_LOG" ] && pass "service query signal: no service mutation with unknown state" || fail "service query signal: no service mutation with unknown state" +assert_no_backups "service query signal: backups removed" +if find "$APP_TMP_DIR" -type f -name 'node-service-unit-list-*' -print -quit | grep -q .; then fail "service query signal: temporary query output removed"; else pass "service query signal: temporary query output removed"; fi +assert_lock_released "service query signal: flock released" + +# Once committed, a later non-zero exit only finalizes cleanup and must not +# restore the old snapshot over the committed installation. +setup_case commit +committed_then_failed() ( + acquire_node_serviced_update_lock + begin_node_service_transaction + mark_node_service_transaction_mutation_started + printf 'committed-env\n' > "$ENV_FILE" + printf 'committed-unit\n' > "$SERVICE_UNIT" + install -m 755 "$VALID_BINARY" "$SERVICE_BINARY_PATH" + commit_node_service_transaction + return 77 +) +committed_then_failed >/dev/null 2>&1 & +commit_pid=$! +set +e +wait "$commit_pid" +commit_status=$? +set -e +[ "$commit_status" -eq 77 ] && pass "commit: later failure propagated" || fail "commit: later failure propagated" +assert_file "$ENV_FILE" "committed-env" "commit: environment not rolled back" +assert_file "$SERVICE_UNIT" "committed-unit" "commit: unit not rolled back" +grep -q 'new-binary' "$SERVICE_BINARY_PATH" && pass "commit: binary not rolled back" || fail "commit: binary not rolled back" +assert_no_backups "commit: backups removed" +assert_lock_released "commit: flock released" + +printf '\nResults: %d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/unit_lib_common.sh b/tests/unit_lib_common.sh index e3043de..478f1b6 100644 --- a/tests/unit_lib_common.sh +++ b/tests/unit_lib_common.sh @@ -38,6 +38,20 @@ d2=$(create_temp_dir) if [ -d "$d2" ]; then pass "create_temp_dir: default prefix works"; else fail "create_temp_dir: default prefix works"; fi rm -rf "$d2" +original_temp_root_dir_definition=$(declare -f temp_root_dir) +temp_root_dir() { return 73; } +if ! create_temp_dir "must-not-reach-root" >/dev/null 2>&1; then + pass "create_temp_dir: temp-root failure is propagated before path construction" +else + fail "create_temp_dir: temp-root failure is propagated before path construction" +fi +if ! create_temp_file "must-not-reach-root" ".tmp" >/dev/null 2>&1; then + pass "create_temp_file: temp-root failure is propagated before path construction" +else + fail "create_temp_file: temp-root failure is propagated before path construction" +fi +eval "$original_temp_root_dir_definition" + # --- create_temp_file --- f=$(create_temp_file "mypfx" ".sh") if [ -f "$f" ]; then pass "create_temp_file: file exists"; else fail "create_temp_file: file exists"; fi diff --git a/tests/unit_lib_github.sh b/tests/unit_lib_github.sh index d3f3831..85bc11b 100644 --- a/tests/unit_lib_github.sh +++ b/tests/unit_lib_github.sh @@ -12,6 +12,8 @@ mkdir -p "$APP_TMP_DIR" "$SHARED_LIB_INSTALL_DIR" source "$ROOT_DIR/lib/common.sh" source "$ROOT_DIR/lib/github.sh" +original_backup_scripts_definition=$(declare -f backup_scripts) +original_install_shared_libs_from_repo_definition=$(declare -f install_shared_libs_from_repo) PASS=0 FAIL=0 @@ -26,6 +28,24 @@ assert_eq() { echo "=== unit_lib_github.sh ===" +# Production callers must propagate allocation failures before constructing +# child paths or touching installation targets. +original_create_temp_dir_definition=$(declare -f create_temp_dir) +create_temp_dir() { return 73; } +if ! backup_scripts >/dev/null 2>&1; then + pass "backup_scripts: temp-dir failure is propagated" +else + fail "backup_scripts: temp-dir failure is propagated" +fi +if ! install_shared_libs_from_repo PasarGuard/scripts common.sh >/dev/null 2>&1; then + pass "install_shared_libs_from_repo: temp-dir failure is propagated" +else + fail "install_shared_libs_from_repo: temp-dir failure is propagated" +fi +eval "$original_create_temp_dir_definition" +eval "$original_backup_scripts_definition" +eval "$original_install_shared_libs_from_repo_definition" + # ----------------------------------------------------------------------- # github_raw_url # ----------------------------------------------------------------------- @@ -48,7 +68,7 @@ mkdir -p "$MOCK_BIN" # Override the functions to use mock paths instead of /usr/local/bin backup_scripts() { local backup_dir - backup_dir=$(create_temp_dir "scripts-backup") + backup_dir=$(create_temp_dir "scripts-backup") || return 1 [ -f "$MOCK_BIN/pasarguard" ] && cp "$MOCK_BIN/pasarguard" "$backup_dir/" [ -f "$MOCK_BIN/pg-node" ] && cp "$MOCK_BIN/pg-node" "$backup_dir/" if [ -d "$SHARED_LIB_INSTALL_DIR" ] && [ "$(ls -A "$SHARED_LIB_INSTALL_DIR" 2>/dev/null)" ]; then diff --git a/tests/unit_pgnode.sh b/tests/unit_pgnode.sh index ab25ea8..4a14ae5 100644 --- a/tests/unit_pgnode.sh +++ b/tests/unit_pgnode.sh @@ -202,6 +202,383 @@ assert_eq "$(detect_node_serviced_platform)" "Linux_arm64" \ "detect_node_serviced_platform: identifies arm64" unset -f uname +# ----------------------------------------------------------------------- +# node-serviced binary validation and atomic replacement +# ----------------------------------------------------------------------- +service_test_dir="$WORK_DIR/node-serviced" +mkdir -p "$service_test_dir" +valid_binary="$service_test_dir/valid" +empty_binary="$service_test_dir/empty" +invalid_binary="$service_test_dir/invalid" +printf '\177ELFtest-binary\n' > "$valid_binary" +: > "$empty_binary" +printf 'download failed\n' > "$invalid_binary" + +assert_true "verify_node_serviced_binary: accepts non-empty ELF" \ + verify_node_serviced_binary "$valid_binary" +assert_false "verify_node_serviced_binary: rejects empty file" \ + verify_node_serviced_binary "$empty_binary" +assert_false "verify_node_serviced_binary: rejects non-ELF content" \ + verify_node_serviced_binary "$invalid_binary" + +checksum_archive="$service_test_dir/archive.tar.gz" +checksum_dir="$service_test_dir/checksum" +checksum_asset="node-serviced_test_Linux_x86_64.tar.gz" +mkdir -p "$checksum_dir" +printf 'archive-content\n' > "$checksum_archive" +checksum_value=$(sha256sum "$checksum_archive" | awk '{print $1}') +curl() { + local output="" + while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then + output="$2" + shift 2 + else + shift + fi + done + printf '%s %s\n' "$checksum_value" "$checksum_asset" > "$output" +} +assert_true "verify_node_serviced_checksum: accepts matching release checksum" \ + verify_node_serviced_checksum "https://example.invalid/checksums.txt" "$checksum_asset" "$checksum_archive" "$checksum_dir" +checksum_value="$(printf '0%.0s' {1..64})" +assert_false "verify_node_serviced_checksum: rejects mismatch" \ + verify_node_serviced_checksum "https://example.invalid/checksums.txt" "$checksum_asset" "$checksum_archive" "$checksum_dir" +curl() { echo ""; return 0; } + +assert_false "verify_node_serviced_checksum: rejects missing checksum by default" \ + verify_node_serviced_checksum "" "$checksum_asset" "$checksum_archive" "$checksum_dir" +NODE_SERVICE_REQUIRE_CHECKSUM=false +assert_true "verify_node_serviced_checksum: allows missing checksum only when explicitly disabled" \ + verify_node_serviced_checksum "" "$checksum_asset" "$checksum_archive" "$checksum_dir" +unset NODE_SERVICE_REQUIRE_CHECKSUM + +# A failed temporary-directory allocation must stop before constructing +# /release.json or attempting any release API request. +original_create_temp_dir_definition=$(declare -f create_temp_dir) +original_run_node_service_external_definition=$(declare -f run_node_service_external) +release_request_attempted=false +release_request_target="" +create_temp_dir() { return 73; } +run_node_service_external() { + release_request_attempted=true + release_request_target="${*: -1}" + return 1 +} +assert_false "install_node_service_script: temp-dir failure is fatal" \ + install_node_service_script false +assert_eq "$release_request_attempted" "false" \ + "install_node_service_script: temp-dir failure skips release request" +assert_false "install_node_service_script: temp-dir failure never targets /release.json" \ + test "$release_request_target" = "/release.json" +eval "$original_create_temp_dir_definition" +eval "$original_run_node_service_external_definition" + +SERVICE_BINARY_PATH="$service_test_dir/pg-node-service" +printf '\177ELFold-binary\n' > "$SERVICE_BINARY_PATH" +assert_true "activate_node_serviced_binary: stages a rollback candidate" \ + activate_node_serviced_binary "$valid_binary" true +if grep -q 'test-binary' "$SERVICE_BINARY_PATH"; then + pass "activate_node_serviced_binary: activates staged binary" +else + fail "activate_node_serviced_binary: activates staged binary" +fi +rollback_node_serviced_binary +if grep -q 'old-binary' "$SERVICE_BINARY_PATH"; then + pass "rollback_node_serviced_binary: restores previous binary" +else + fail "rollback_node_serviced_binary: restores previous binary" +fi +NODE_SERVICE_HAD_PREVIOUS=false +NODE_SERVICE_BACKUP_PATH="" +SERVICE_NAME="" +rollback_output=$(rollback_node_serviced_binary 2>&1) +if [[ "$rollback_output" == *"No previous node-serviced binary is available; removing the failed replacement."* ]]; then + pass "rollback_node_serviced_binary: reports no-backup removal" +else + fail "rollback_node_serviced_binary: reports no-backup removal" +fi +assert_false "rollback_node_serviced_binary: removes failed replacement without backup" \ + test -e "$SERVICE_BINARY_PATH" +printf '\177ELFold-binary\n' > "$SERVICE_BINARY_PATH" +assert_false "activate_node_serviced_binary: rejects invalid candidate" \ + activate_node_serviced_binary "$invalid_binary" true +if grep -q 'old-binary' "$SERVICE_BINARY_PATH"; then + pass "activate_node_serviced_binary: preserves target after rejected candidate" +else + fail "activate_node_serviced_binary: preserves target after rejected candidate" +fi + +ready_cert="$service_test_dir/ssl-cert.pem" +ready_env="$service_test_dir/node-service.env" +: > "$ready_cert" +printf 'API_PORT= 62051\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$ready_cert" > "$ready_env" +ENV_FILE="$ready_env" +printf 'API_KEY= "unit#test-key" # deployment note\n' > "$ready_env" +assert_eq "$(read_node_service_env_value API_KEY)" "unit#test-key" \ + "read_node_service_env_value: preserves a quoted hash before a trailing comment" +printf 'API_KEY= unit-test-key # deployment note\n' > "$ready_env" +assert_eq "$(read_node_service_env_value API_KEY)" "unit-test-key" \ + "read_node_service_env_value: removes comments from unquoted values" +cat > "$ready_env" <<'EOF' +API_PORT= 61000 +export API_PORT = "62051" +PREFIX=unit +API_KEY="${PREFIX}\\nkey-\$literal" +API_KEY='last-${PREFIX}-value' +EOF +assert_eq "$(read_node_service_env_value API_PORT)" "62051" \ + "read_node_service_env_value: matches Overload export and last duplicate wins" +assert_eq "$(read_node_service_env_value API_KEY)" 'last-${PREFIX}-value' \ + "read_node_service_env_value: matches Overload single-quote literal semantics" +cat > "$ready_env" <<'EOF' +PREFIX=unit +API_KEY="${PREFIX}\nkey-\$literal" +EOF +assert_eq "$(read_node_service_env_value API_KEY)" $'unit\nkey-$literal' \ + "read_node_service_env_value: matches Overload expansion and double-quote escapes" +printf 'API_KEY="valid-prefix" trailing-garbage\n' > "$ready_env" +assert_false "read_node_service_env_value: rejects garbage after a quoted value" \ + read_node_service_env_value API_KEY +printf 'API_PORT= 62051\nAPI_KEY= "unit#test-key" # deployment note\nSSL_CERT_FILE= %s\n' "$ready_cert" > "$ready_env" +SERVICE_NAME="pg-node-test-service" +NODE_SERVICE_READINESS_ATTEMPTS=1 +NODE_SERVICE_READINESS_STABLE_CHECKS=1 +NODE_SERVICE_READINESS_DELAY_SECONDS=0 +api_ready_calls=0 +systemctl() { [ "${1:-}" = "is-active" ]; } +openssl() { + case "${NODE_SERVICE_CERT_TEST_MODE:-dns}" in + wildcard) printf 'X509v3 Subject Alternative Name:\n DNS:*.example.test\n' ;; + cn) + if [[ " $* " == *' -subject '* ]]; then + printf 'subject=CN=legacy.example.test\n' + else + printf 'No extensions in certificate\n' + fi + ;; + *) printf 'X509v3 Subject Alternative Name:\n DNS:node.example.test\n' ;; + esac +} +curl() { + api_ready_config=$(cat "$2") + api_ready_config_path="$2" + api_ready_calls=$((api_ready_calls + 1)) + [ "${NODE_SERVICE_API_READY:-true}" = true ] +} +assert_true "wait_for_node_service_ready: requires authenticated API success" \ + wait_for_node_service_ready +assert_eq "$api_ready_calls" "1" \ + "wait_for_node_service_ready: checks the service API after systemd" +if [[ "$api_ready_config" == *'header = "x-api-key: unit#test-key"'* ]]; then + pass "wait_for_node_service_ready: preserves a quoted API key before a trailing comment" +else + fail "wait_for_node_service_ready: preserves a quoted API key before a trailing comment" +fi +if [[ "$api_ready_config" == *'connect-to = "::127.0.0.1:62051"'* ]] && + [[ "$api_ready_config" == *'url = "https://node.example.test:62051/"'* ]]; then + pass "node_service_api_ready: preserves DNS SAN for SNI while connecting to loopback" +else + fail "node_service_api_ready: preserves DNS SAN for SNI while connecting to loopback" +fi +assert_false "wait_for_node_service_ready: removes the temporary curl config" \ + test -e "$api_ready_config_path" +printf 'API_PORT= 62051\nAPI_KEY= '\''unit\\key"quote'\''\nSSL_CERT_FILE= %s\n' "$ready_cert" > "$ready_env" +assert_true "node_service_api_ready: accepts API keys containing quote and backslash" \ + node_service_api_ready +expected_curl_header='header = "x-api-key: unit\\key\"quote"' +if grep -Fqx -- "$expected_curl_header" <<<"$api_ready_config"; then + pass "node_service_api_ready: curl config escapes quote and backslash once" +else + fail "node_service_api_ready: curl config escapes quote and backslash once" +fi +printf 'API_PORT= 62051\nAPI_KEY= "unit#test-key" # deployment note\nSSL_CERT_FILE= %s\n' "$ready_cert" > "$ready_env" +NODE_SERVICE_CERT_TEST_MODE=wildcard +assert_true "node_service_certificate_identity: accepts wildcard DNS SAN" \ + node_service_certificate_identity "$ready_cert" +assert_eq "$NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT" "node-serviced-health.example.test" \ + "node_service_certificate_identity: synthesizes one wildcard label" +NODE_SERVICE_CERT_TEST_MODE=cn +assert_true "node_service_certificate_identity: supports SAN-less CN fallback" \ + node_service_certificate_identity "$ready_cert" +assert_eq "$NODE_SERVICE_CERTIFICATE_IDENTITY_RESULT" "legacy.example.test" \ + "node_service_certificate_identity: uses CN only without SAN" +unset NODE_SERVICE_CERT_TEST_MODE +printf 'API_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$ready_cert" > "$ready_env" +assert_true "node_service_api_ready: uses node-serviced default port when API_PORT is absent" \ + node_service_api_ready +if [[ "$api_ready_config" == *'url = "https://node.example.test:3000/"'* ]]; then + pass "node_service_api_ready: probes node-serviced default port 3000" +else + fail "node_service_api_ready: probes node-serviced default port 3000" +fi +printf 'API_PORT= 62051\nAPI_KEY= unit-test-key\nSSL_CERT_FILE= %s\n' "$ready_cert" > "$ready_env" +NODE_SERVICE_API_READY=false +assert_false "wait_for_node_service_ready: rejects active service with unavailable API" \ + wait_for_node_service_ready +unset NODE_SERVICE_API_READY + +# The lock is held for the entire update transaction so a second updater +# cannot create a stale backup and later overwrite a successful install. +NODE_SERVICE_UPDATE_LOCK_PATH="$service_test_dir/update.lock" +flock() { + case "${1:-}" in + -n) [ "${NODE_SERVICE_UPDATE_LOCK_BUSY:-false}" != true ] ;; + -u) return 0 ;; + *) return 0 ;; + esac +} +NODE_SERVICE_UPDATE_LOCK_BUSY=true +assert_false "acquire_node_serviced_update_lock: rejects a concurrent updater" \ + acquire_node_serviced_update_lock +NODE_SERVICE_UPDATE_LOCK_BUSY=false +assert_true "acquire_node_serviced_update_lock: acquires an available lock" \ + acquire_node_serviced_update_lock +assert_true "release_node_serviced_update_lock: releases the transaction lock" \ + release_node_serviced_update_lock +assert_eq "$NODE_SERVICE_UPDATE_LOCK_HELD" "false" \ + "release_node_serviced_update_lock: clears lock state" +unset -f flock +unset NODE_SERVICE_UPDATE_LOCK_BUSY NODE_SERVICE_UPDATE_LOCK_PATH + +# Exercise kernel-level contention with a separate process. This is the +# inter-process boundary shared by service-install/update/uninstall. +NODE_SERVICE_UPDATE_LOCK_PATH="$service_test_dir/process-update.lock" +lock_ready="$service_test_dir/process-lock-ready" +lock_release="$service_test_dir/process-lock-release" +( + exec 8>"$NODE_SERVICE_UPDATE_LOCK_PATH" + flock 8 + : > "$lock_ready" + while [ ! -e "$lock_release" ]; do + sleep 0.01 + done +) & +lock_holder_pid=$! +lock_wait_attempt=0 +while [ ! -e "$lock_ready" ] && [ "$lock_wait_attempt" -lt 100 ]; do + sleep 0.01 + lock_wait_attempt=$((lock_wait_attempt + 1)) +done +assert_true "node-serviced update lock: separate process acquired the lock" \ + test -e "$lock_ready" +assert_false "node-serviced update lock: blocks a concurrent transaction" \ + acquire_node_serviced_update_lock +: > "$lock_release" +wait "$lock_holder_pid" +assert_true "node-serviced update lock: becomes available after transaction exit" \ + acquire_node_serviced_update_lock +release_node_serviced_update_lock +unset NODE_SERVICE_UPDATE_LOCK_PATH + +assert_true "can_reuse_node_service_api_port: permits the current service port" \ + can_reuse_node_service_api_port true 62051 62051 +assert_false "can_reuse_node_service_api_port: rejects a different occupied port" \ + can_reuse_node_service_api_port true 62052 62051 +assert_false "can_reuse_node_service_api_port: rejects a port without an installed service" \ + can_reuse_node_service_api_port false 62051 62051 + +# Exercise the complete update failure path: the replacement is installed, +# the first restart fails, rollback restores the old binary, and the second +# restart attempts to bring the old version back online. +set_service_paths() { + SERVICE_NAME="pg-node-test-service" + SERVICE_BINARY_PATH="$service_test_dir/pg-node-service" + SERVICE_UNIT="$service_test_dir/pg-node-test-service.service" +} +service_installed() { + [ "$NODE_SERVICE_UPDATE_LOCK_HELD" = true ] +} +id() { echo 0; } +update_activation_marker="$service_test_dir/update-activated" +install_node_service_script() { + activate_node_serviced_binary "$valid_binary" true + : > "$update_activation_marker" +} +NODE_SERVICE_UPDATE_LOCK_PATH="$service_test_dir/transaction-update.lock" +NODE_SERVICE_READINESS_ATTEMPTS=1 +NODE_SERVICE_READINESS_STABLE_CHECKS=1 +NODE_SERVICE_READINESS_DELAY_SECONDS=0 +restart_attempts_file="$service_test_dir/restart-attempts" +printf '0\n' > "$restart_attempts_file" +systemctl() { + if [ "${1:-}" = "daemon-reload" ]; then + return 0 + fi + if [ "${1:-}" = "restart" ]; then + local attempts + attempts=$(cat "$restart_attempts_file") + attempts=$((attempts + 1)) + printf '%s\n' "$attempts" > "$restart_attempts_file" + [ "$attempts" -gt 1 ] + return + fi + return 0 +} +if (update_service_if_installed >/dev/null 2>&1); then + fail "update_service_if_installed: reports restart failure" +else + pass "update_service_if_installed: reports restart failure" +fi +if grep -q 'old-binary' "$SERVICE_BINARY_PATH"; then + pass "update_service_if_installed: rolls back failed replacement" +else + fail "update_service_if_installed: rolls back failed replacement" +fi +assert_eq "$(cat "$restart_attempts_file")" "2" \ + "update_service_if_installed: restarts restored binary" + +# A successful `systemctl restart` is not sufficient: systemd can return zero +# before a crashing Type=simple process exits. Readiness failure must retain and +# restore the backup just like an immediate restart failure. +rm -f "$update_activation_marker" +printf '\177ELFold-binary\n' > "$SERVICE_BINARY_PATH" +printf '0\n' > "$restart_attempts_file" +readiness_attempts_file="$service_test_dir/readiness-attempts" +printf '0\n' > "$readiness_attempts_file" +systemctl() { + case "${1:-}" in + daemon-reload) + return 0 + ;; + restart) + local restarts + restarts=$(cat "$restart_attempts_file") + printf '%s\n' "$((restarts + 1))" > "$restart_attempts_file" + return 0 + ;; + is-active) + local checks + if [ ! -e "$update_activation_marker" ]; then + return 0 + fi + checks=$(cat "$readiness_attempts_file") + printf '%s\n' "$((checks + 1))" > "$readiness_attempts_file" + return 1 + ;; + esac + return 0 +} +sleep() { return 0; } +if (update_service_if_installed >/dev/null 2>&1); then + fail "update_service_if_installed: reports inactive service after restart=0" +else + pass "update_service_if_installed: reports inactive service after restart=0" +fi +if grep -q 'old-binary' "$SERVICE_BINARY_PATH"; then + pass "update_service_if_installed: rolls back restart=0 but inactive replacement" +else + fail "update_service_if_installed: rolls back restart=0 but inactive replacement" +fi +assert_eq "$(cat "$restart_attempts_file")" "2" \ + "update_service_if_installed: retries restored binary after readiness failure" +assert_eq "$(cat "$readiness_attempts_file")" "2" \ + "update_service_if_installed: readiness-checks replacement and restored binary" +unset -f set_service_paths service_installed id install_node_service_script systemctl sleep openssl +rm -f "$update_activation_marker" +unset NODE_SERVICE_READINESS_ATTEMPTS NODE_SERVICE_READINESS_STABLE_CHECKS NODE_SERVICE_READINESS_DELAY_SECONDS +unset NODE_SERVICE_UPDATE_LOCK_PATH + # ----------------------------------------------------------------------- # is_port_occupied # -----------------------------------------------------------------------