fix(backup): prevent stale and incomplete database restores

lib/common.sh

@@ -35,6 +35,62 @@ die() {
     exit 1
 }
 
+# Normalize redundant leading and trailing slashes without requiring the path
+# to exist. This keeps legacy SQLite URLs containing five slashes compatible
+# while producing the same path used by normal four-slash absolute URLs.
+normalize_posix_path() {
+    local path="$1"
+
+    while [[ "$path" == //* ]]; do
+        path="${path#/}"
+    done
+    while [[ "$path" != "/" && "$path" == */ ]]; do
+        path="${path%/}"
+    done
+
+    printf '%s\n' "$path"
+}
+
+# Return the filesystem path represented by a SQLAlchemy SQLite URL.
+#   sqlite:///relative.db       -> relative.db
+#   sqlite:////absolute/db      -> /absolute/db
+#   sqlite://///absolute/db     -> /absolute/db (legacy installer output)
+sqlite_database_path_from_url() {
+    local url="$1"
+    local url_part=""
+    local path=""
+
+    [[ "$url" =~ ^sqlite[^:]*:// ]] || return 1
+
+    url_part="${url#*://}"
+    url_part="${url_part%%\?*}"
+    url_part="${url_part%%#*}"
+
+    if [[ "$url_part" == //* ]]; then
+        path="/${url_part#//}"
+    elif [[ "$url_part" == /* ]]; then
+        path="${url_part#/}"
+    else
+        path="$url_part"
+    fi
+
+    normalize_posix_path "$path"
+}
+
+# Build a SQLAlchemy URL for an absolute SQLite database path. Stripping the
+# path's leading slash before adding the URL prefix guarantees exactly four
+# slashes after the scheme separator.
+sqlite_absolute_database_url() {
+    local driver="$1"
+    local path=""
+
+    path=$(normalize_posix_path "$2")
+    [[ "$driver" =~ ^sqlite([+][A-Za-z0-9_]+)?$ ]] || return 1
+    [[ "$path" == /* ]] || return 1
+
+    printf '%s:////%s\n' "$driver" "${path#/}"
+}
+
 # Ensure a secret-bearing file (e.g. .env, TLS private key) is only readable by
 # its owner. Creates the file with 0600 if it is missing so callers can harden
 # it *before* writing secrets; tightens it to 0600 if it already exists. A

lib/pasarguard-restore.sh

@@ -35,7 +35,27 @@ postgres_dump_looks_restorable() {
     local dump_file="$1"
     [ -s "$dump_file" ] || return 1
     # Require at least one real schema/data statement, not just comments/SET.
-    grep -qiE '^[[:space:]]*(CREATE|COPY|INSERT|ALTER)[[:space:]]' "$dump_file"
+    grep -qiE '^[[:space:]]*(CREATE|COPY|INSERT|ALTER)[[:space:]]' "$dump_file" || return 1
+    # pg_dump writes this only after completing the output. Requiring it keeps a
+    # dump truncated by a full disk or interrupted process from being accepted.
+    grep -qE '^-- PostgreSQL database dump complete([[:space:]]*)$' "$dump_file"
+}
+
+# pg_dumpall uses a different completion marker for the globals-only file.
+postgres_globals_dump_looks_complete() {
+    local dump_file="$1"
+    [ -s "$dump_file" ] || return 1
+    grep -qE '^-- PostgreSQL database cluster dump complete([[:space:]]*)$' "$dump_file"
+}
+
+# Both mysqldump and mariadb-dump emit a completion marker after a successful
+# plain-SQL dump. This accepts either tool while rejecting empty and truncated
+# files before they can be archived or restored.
+mysql_dump_looks_restorable() {
+    local dump_file="$1"
+    [ -s "$dump_file" ] || return 1
+    grep -qE '^-- (MySQL|MariaDB) dump ' "$dump_file" || return 1
+    grep -qE '^-- Dump completed on ' "$dump_file"
 }
 
 # Detect the dump layout inside an extracted backup directory.
@@ -253,6 +273,10 @@ restore_command() {
     local current_sqlalchemy_url=""
     local current_mysql_root_password=""
     local sqlite_basename=""
+    local sqlite_backup_source=""
+    local sqlite_safety_backup=""
+    local restore_timestamp=""
+    restore_timestamp=$(date +%Y%m%d%H%M%S)
 
     redact_database_url() {
         local url="$1"
@@ -684,16 +708,11 @@ restore_command() {
     if [[ "$SQLALCHEMY_DATABASE_URL" =~ ^sqlite ]]; then
         db_type="sqlite"
         colorized_echo green "✓ Detected SQLite database"
-        local sqlite_url_part="${SQLALCHEMY_DATABASE_URL#*://}"
-        sqlite_url_part="${sqlite_url_part%%\?*}"
-        sqlite_url_part="${sqlite_url_part%%#*}"
-
-        if [[ "$sqlite_url_part" =~ ^//(.*)$ ]]; then
-            sqlite_file="/${BASH_REMATCH[1]}"
-        elif [[ "$sqlite_url_part" =~ ^/(.*)$ ]]; then
-            sqlite_file="/${BASH_REMATCH[1]}"
-        else
-            sqlite_file="$sqlite_url_part"
+        if ! sqlite_file=$(sqlite_database_path_from_url "$SQLALCHEMY_DATABASE_URL") || [ -z "$sqlite_file" ]; then
+            colorized_echo red "Invalid SQLite SQLALCHEMY_DATABASE_URL in backup; expected sqlite[+driver]:// followed by a database path."
+            echo "Invalid SQLite SQLALCHEMY_DATABASE_URL: $(redact_database_url "$SQLALCHEMY_DATABASE_URL")" >>"$log_file"
+            rm -rf "$temp_restore_dir"
+            exit 1
         fi
         colorized_echo blue "Database file: $sqlite_file"
     elif [[ "$SQLALCHEMY_DATABASE_URL" =~ ^(mysql|mariadb|postgresql)[^:]*:// ]]; then
@@ -816,39 +835,53 @@ restore_command() {
     case $db_type in
     sqlite)
         sqlite_basename=$(basename "$sqlite_file")
-        local backup_source=""
 
         if [ -f "$temp_restore_dir/$sqlite_basename" ]; then
-            backup_source="$temp_restore_dir/$sqlite_basename"
+            sqlite_backup_source="$temp_restore_dir/$sqlite_basename"
         elif [ -f "$temp_restore_dir/db_backup.sqlite" ]; then
-            backup_source="$temp_restore_dir/db_backup.sqlite"
+            sqlite_backup_source="$temp_restore_dir/db_backup.sqlite"
         fi
 
-        if [ -z "$backup_source" ]; then
+        if [ -z "$sqlite_backup_source" ]; then
             colorized_echo red "SQLite backup file not found in backup archive (looked for $sqlite_basename or db_backup.sqlite)."
             rm -rf "$temp_restore_dir"
             exit 1
         fi
 
-        rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" 2>>"$log_file" || true
-
-        if [ -f "$sqlite_file" ]; then
-            cp "$sqlite_file" "${sqlite_file}.backup.$(date +%Y%m%d%H%M%S)" 2>>"$log_file"
+        if ! command -v sqlite3 >/dev/null 2>&1; then
+            detect_os
+            try_install_package sqlite3 || true
         fi
-
-        if cp "$backup_source" "$sqlite_file" 2>>"$log_file"; then
-            colorized_echo green "SQLite database restored successfully."
-        else
-            colorized_echo red "Failed to restore SQLite database."
-            echo "SQLite restore failed" >>"$log_file"
+        if ! command -v sqlite3 >/dev/null 2>&1; then
+            colorized_echo red "sqlite3 is required to validate the SQLite snapshot before restore. Install sqlite3 and run the restore again."
+            echo "sqlite3 unavailable; cannot validate $sqlite_backup_source" >>"$log_file"
             rm -rf "$temp_restore_dir"
             exit 1
         fi
+        if ! sqlite_snapshot_looks_restorable "$sqlite_backup_source"; then
+            colorized_echo red "SQLite backup is corrupt or incomplete; aborting before replacing the current database."
+            echo "SQLite snapshot validation failed for $sqlite_backup_source" >>"$log_file"
+            rm -rf "$temp_restore_dir"
+            exit 1
+        fi
+
+        if [ -f "$sqlite_file" ]; then
+            sqlite_safety_backup="$backup_dir/sqlite_before_restore_${restore_timestamp}_${sqlite_basename}"
+            if ! sqlite3 "$sqlite_file" ".backup '$sqlite_safety_backup'" >>"$log_file" 2>&1; then
+                colorized_echo red "Failed to create a safety snapshot of the current SQLite database; restore aborted."
+                echo "SQLite safety snapshot failed: $sqlite_file -> $sqlite_safety_backup" >>"$log_file"
+                rm -f "$sqlite_safety_backup"
+                rm -rf "$temp_restore_dir"
+                exit 1
+            fi
+            colorized_echo blue "Current SQLite database saved to $sqlite_safety_backup"
+        fi
         ;;
 
     mariadb|mysql)
-        if [ ! -f "$temp_restore_dir/db_backup.sql" ]; then
-            colorized_echo red "Database backup file not found in backup archive."
+        if ! mysql_dump_looks_restorable "$temp_restore_dir/db_backup.sql"; then
+            colorized_echo red "Database backup is missing, truncated, or invalid; aborting before restore."
+            echo "MySQL/MariaDB dump validation failed for $temp_restore_dir/db_backup.sql" >>"$log_file"
             rm -rf "$temp_restore_dir"
             exit 1
         fi
@@ -968,6 +1001,13 @@ restore_command() {
             exit 1
         fi
 
+        if [ "$pg_layout" = "multi" ] && ! postgres_backup_looks_restorable "$temp_restore_dir" "$db_name"; then
+            colorized_echo red "Multi-database backup is incomplete or does not contain the configured database; aborting before restore."
+            echo "Multi-database dump validation failed for $temp_restore_dir/pg_dump" >>"$log_file"
+            rm -rf "$temp_restore_dir"
+            exit 1
+        fi
+
         if [ "$pg_layout" = "single" ]; then
             # Verify backup file is not empty and is readable
             if [ ! -s "$temp_restore_dir/db_backup.sql" ]; then
@@ -1155,13 +1195,29 @@ restore_command() {
             exit 1
         fi
         if [ "$db_type" = "sqlite" ] && [ -n "${sqlite_file:-}" ]; then
-            rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" 2>>"$log_file" || true
+            rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" "${sqlite_file}-journal" 2>>"$log_file" || true
         fi
         colorized_echo green "Data directory restored to $DATA_DIR."
     else
         colorized_echo yellow "No pasarguard_data directory found in backup. Skipping data restore."
     fi
 
+    # The data directory in legacy archives may contain a raw SQLite main file
+    # and WAL. Apply the consistent snapshot only after that directory has been
+    # restored so the raw copy can never overwrite the authoritative backup.
+    if [ "$db_type" = "sqlite" ]; then
+        mkdir -p "$(dirname "$sqlite_file")"
+        rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" "${sqlite_file}-journal" 2>>"$log_file" || true
+        if cp "$sqlite_backup_source" "$sqlite_file" 2>>"$log_file"; then
+            colorized_echo green "SQLite database restored successfully."
+        else
+            colorized_echo red "Failed to restore SQLite database."
+            echo "SQLite restore failed" >>"$log_file"
+            rm -rf "$temp_restore_dir"
+            exit 1
+        fi
+    fi
+
     # Restore app directory files (full app backup support)
     colorized_echo blue "Restoring app directory files..."
     if [ -d "$temp_restore_dir" ]; then

pasarguard.sh

@@ -1067,7 +1067,8 @@ install_pasarguard() {
             db_driver_scheme="sqlite"
         fi
 
-        sed -i "s~\(SQLALCHEMY_DATABASE_URL = \).*~\1\"${db_driver_scheme}:////${DATA_DIR}/db.sqlite3\"~" "$APP_DIR/.env"
+        SQLALCHEMY_DATABASE_URL=$(sqlite_absolute_database_url "$db_driver_scheme" "$DATA_DIR/db.sqlite3")
+        sed -i "s~\(SQLALCHEMY_DATABASE_URL = \).*~\1\"${SQLALCHEMY_DATABASE_URL}\"~" "$APP_DIR/.env"
 
     fi
 

tests/backup_restore_roundtrip.sh

@@ -57,9 +57,21 @@ EXTRACTED_BACKUP_DIR=""
 COMBINED_BACKUP_ARCHIVE=""
 MULTIPART_SPLIT_SIZE_BYTES=2048
 MULTIPART_SPLIT_THRESHOLD_BYTES=3072
+SQLITE_HOLDER_PID=""
+SQLITE_HOLDER_READY="$WORK_DIR/sqlite-holder.ready"
+SQLITE_HOLDER_STOP="$WORK_DIR/sqlite-holder.stop"
+
+stop_sqlite_holder() {
+    if [ -n "$SQLITE_HOLDER_PID" ] && kill -0 "$SQLITE_HOLDER_PID" 2>/dev/null; then
+        touch "$SQLITE_HOLDER_STOP"
+        wait "$SQLITE_HOLDER_PID"
+    fi
+    SQLITE_HOLDER_PID=""
+}
 
 cleanup() {
     local exit_code=$?
+    stop_sqlite_holder
     if [ "$exit_code" -ne 0 ] && [ -d "$WORK_DIR" ]; then
         while IFS= read -r log_path; do
             [ -n "$log_path" ] || continue
@@ -180,6 +192,7 @@ write_sqlite_env() {
     cat >"$ENV_FILE" <<EOF
 BACKUP_SERVICE_ENABLED=false
 RESTORE_TEST_FLAG=$EXPECTED_ENV_FLAG
+# Keep the legacy five-slash URL here to verify existing installations.
 SQLALCHEMY_DATABASE_URL="sqlite:////$DATA_DIR/db.sqlite3"
 EOF
 }
@@ -275,10 +288,36 @@ record_original_file_hashes() {
 }
 
 setup_sqlite_db() {
-    sqlite3 "$DATA_DIR/db.sqlite3" <<EOF
-CREATE TABLE ci_roundtrip (id INTEGER PRIMARY KEY, value TEXT NOT NULL);
-INSERT INTO ci_roundtrip (id, value) VALUES (1, '$EXPECTED_DB_VALUE');
-EOF
+    python3 - "$DATA_DIR/db.sqlite3" "$SQLITE_HOLDER_READY" "$SQLITE_HOLDER_STOP" "$EXPECTED_DB_VALUE" <<'PY' &
+import sqlite3
+import sys
+import time
+from pathlib import Path
+
+db_path, ready_path, stop_path, expected_value = sys.argv[1:]
+connection = sqlite3.connect(db_path)
+connection.execute("PRAGMA journal_mode=WAL")
+connection.execute("PRAGMA wal_autocheckpoint=0")
+connection.execute("CREATE TABLE ci_roundtrip (id INTEGER PRIMARY KEY, value TEXT NOT NULL)")
+connection.execute("INSERT INTO ci_roundtrip (id, value) VALUES (1, 'old-checkpoint')")
+connection.commit()
+connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+connection.execute("UPDATE ci_roundtrip SET value = ? WHERE id = 1", (expected_value,))
+connection.commit()
+Path(ready_path).touch()
+while not Path(stop_path).exists():
+    time.sleep(0.05)
+connection.close()
+PY
+    SQLITE_HOLDER_PID=$!
+    wait_for_command 50 test -f "$SQLITE_HOLDER_READY"
+
+    # A valid SQLite file with the same basename in APP_DIR used to overwrite
+    # the authoritative snapshot while app files were copied into staging.
+    # Keep this collision valid (not garbage) so integrity checks alone cannot
+    # distinguish it from the live DATA_DIR database.
+    sqlite3 "$APP_DIR/db.sqlite3" \
+        "CREATE TABLE ci_roundtrip (id INTEGER PRIMARY KEY, value TEXT NOT NULL); INSERT INTO ci_roundtrip VALUES (1, 'stale-app-dir-copy');"
 }
 
 sqlite_query() {
@@ -287,6 +326,7 @@ sqlite_query() {
 
 mutate_sqlite_db() {
     sqlite3 "$DATA_DIR/db.sqlite3" "UPDATE ci_roundtrip SET value = 'mutated' WHERE id = 1;"
+    printf 'stale rollback journal\n' >"$DATA_DIR/db.sqlite3-journal"
 }
 
 setup_mysql_container() {
@@ -447,11 +487,7 @@ verify_backup_archive_contents() {
         expected_files=$'.env\ndb_backup.sql\ndocker-compose.yml\npasarguard_data/\npasarguard_data/payload.bin\npasarguard_data/sentinel.txt'
     fi
 
-    if [ "$DB_TYPE" = "sqlite" ]; then
-        assert_zip_contains_required_files "$archive_to_verify" "$expected_files"
-    else
-        assert_zip_contains_exact_files "$archive_to_verify" "$expected_files"
-    fi
+    assert_zip_contains_exact_files "$archive_to_verify" "$expected_files"
     assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/.env" | awk '{print $1}')" "$ORIGINAL_ENV_SHA" "Backed up .env contents changed."
     assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/docker-compose.yml" | awk '{print $1}')" "$ORIGINAL_COMPOSE_SHA" "Backed up docker-compose.yml contents changed."
     assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/pasarguard_data/sentinel.txt" | awk '{print $1}')" "$ORIGINAL_SENTINEL_SHA" "Backed up sentinel.txt contents changed."
@@ -460,9 +496,12 @@ verify_backup_archive_contents() {
     if [ "$DB_TYPE" = "sqlite" ]; then
         assert_sqlite_integrity "$EXTRACTED_BACKUP_DIR/$sqlite_basename"
         assert_equals "$(sqlite_dump_sha "$EXTRACTED_BACKUP_DIR/$sqlite_basename")" "$ORIGINAL_SQLITE_DUMP_SHA" "Backed up SQLite database logical contents changed."
-        if [ -f "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" ]; then
-            assert_sqlite_integrity "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename"
-            assert_equals "$(sqlite_dump_sha "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename")" "$ORIGINAL_SQLITE_DUMP_SHA" "Archived SQLite data-dir database logical contents changed."
+        if [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" ] || \
+            [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-wal" ] || \
+            [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-shm" ] || \
+            [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-journal" ]; then
+            printf 'SQLite database or WAL/SHM/journal leaked into the raw data-directory copy.\n' >&2
+            exit 1
         fi
     elif [ "$DB_TYPE" = "postgresql" ] || [ "$DB_TYPE" = "timescaledb" ]; then
         assert_file_contains "$EXTRACTED_BACKUP_DIR/pg_dump/db-001.sql" "ci_roundtrip"
@@ -484,6 +523,20 @@ verify_restored_files() {
     if [ "$DB_TYPE" = "sqlite" ]; then
         assert_sqlite_integrity "$DATA_DIR/db.sqlite3"
         assert_equals "$(sqlite_dump_sha "$DATA_DIR/db.sqlite3")" "$ORIGINAL_SQLITE_DUMP_SHA" "SQLite database logical contents were not restored from backup."
+        if [ -e "$DATA_DIR/db.sqlite3-journal" ]; then
+            printf 'A stale SQLite rollback journal survived the restore.\n' >&2
+            exit 1
+        fi
+
+        local sqlite_safety_backup=""
+        sqlite_safety_backup=$(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'sqlite_before_restore_*_db.sqlite3' | sort | tail -n 1)
+        if [ -z "$sqlite_safety_backup" ]; then
+            printf 'The pre-restore SQLite safety snapshot did not survive the data-directory restore.\n' >&2
+            exit 1
+        fi
+        assert_sqlite_integrity "$sqlite_safety_backup"
+        assert_equals "$(sqlite3 "$sqlite_safety_backup" 'SELECT value FROM ci_roundtrip WHERE id = 1;')" \
+            "mutated" "Pre-restore SQLite safety snapshot did not preserve the replaced database."
     fi
 }
 
@@ -631,6 +684,7 @@ main() {
     prepare_case
     backup_command
     verify_backup_created
+    stop_sqlite_holder
     mutate_database_after_backup
     mutate_files_after_backup
     run_restore