diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml index dcc0ebe..9daf3ad 100644 --- a/.github/workflows/backup-restore.yml +++ b/.github/workflows/backup-restore.yml @@ -14,6 +14,7 @@ on: - "lib/env.sh" - "lib/system.sh" - "tests/backup_restore_roundtrip.sh" + - "tests/unit_pasarguard.sh" pull_request: paths: - ".github/workflows/backup-restore.yml" @@ -24,6 +25,7 @@ on: - "lib/env.sh" - "lib/system.sh" - "tests/backup_restore_roundtrip.sh" + - "tests/unit_pasarguard.sh" workflow_dispatch: jobs: @@ -46,6 +48,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false - name: Install test dependencies run: | @@ -54,3 +58,25 @@ jobs: - name: Run backup/restore round-trip run: bash tests/backup_restore_roundtrip.sh "${{ matrix.database }}" "${{ matrix.archive_mode }}" + + timescaledb-version-upgrade: + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install test dependencies + run: | + sudo apt-get update + sudo apt-get install -y rsync zip unzip sqlite3 + + - name: Restore TimescaleDB 2.27 backup into 2.28 destination + env: + TIMESCALE_SOURCE_IMAGE: timescale/timescaledb:2.27.2-pg17 + TIMESCALE_TARGET_IMAGE: timescale/timescaledb:2.28.3-pg17 + TIMESCALEDB_COMPAT_IMAGE: timescale/timescaledb-ha:pg17-ts2.28-all + run: bash tests/backup_restore_roundtrip.sh timescaledb single diff --git a/lib/common.sh b/lib/common.sh index 1a216f1..9794881 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -35,6 +35,65 @@ die() { exit 1 } +# Normalize redundant leading and trailing slashes without requiring the path +# to exist. This keeps legacy SQLite URLs containing five slashes compatible +# while producing the same path used by normal four-slash absolute URLs. +normalize_posix_path() { + local path="$1" + + while [[ "$path" == *//* ]]; do + path="${path//\/\//\/}" + done + while [[ "$path" == //* ]]; do + path="${path#/}" + done + while [[ "$path" != "/" && "$path" == */ ]]; do + path="${path%/}" + done + + printf '%s\n' "$path" +} + +# Return the filesystem path represented by a SQLAlchemy SQLite URL. +# sqlite:///relative.db -> relative.db +# sqlite:////absolute/db -> /absolute/db +# sqlite://///absolute/db -> /absolute/db (legacy installer output) +sqlite_database_path_from_url() { + local url="$1" + local url_part="" + local path="" + + [[ "$url" =~ ^sqlite[^:]*:// ]] || return 1 + + url_part="${url#*://}" + url_part="${url_part%%\?*}" + url_part="${url_part%%#*}" + + if [[ "$url_part" == //* ]]; then + path="/${url_part#//}" + elif [[ "$url_part" == /* ]]; then + path="${url_part#/}" + else + path="$url_part" + fi + + normalize_posix_path "$path" +} + +# Build a SQLAlchemy URL for an absolute SQLite database path. Stripping the +# path's leading slash before adding the URL prefix guarantees exactly four +# slashes after the scheme separator. +sqlite_absolute_database_url() { + local driver="$1" + local path="" + + path=$(normalize_posix_path "$2") + [[ "$driver" =~ ^sqlite([+][A-Za-z0-9_]+)?$ ]] || return 1 + [[ "$path" == /* ]] || return 1 + + printf '%s:////%s\n' "$driver" "${path#/}" +} + # Ensure a secret-bearing file (e.g. .env, TLS private key) is only readable by # its owner. Creates the file with 0600 if it is missing so callers can harden # it *before* writing secrets; tightens it to 0600 if it already exists. A diff --git a/lib/pasarguard-backup.sh b/lib/pasarguard-backup.sh index 61cef82..8dfa499 100644 --- a/lib/pasarguard-backup.sh +++ b/lib/pasarguard-backup.sh @@ -144,7 +144,94 @@ pg_manifest_encode() { printf '%s\t%s\t%s\t%s\t%s' "$dbname" "$owner" "$has_ts" "$filename" "$ts_version" } +# Validate the complete database artifact that will be put in an archive. This +# is intentionally independent of a dump command's exit status: a process can +# be interrupted after writing a plausible-looking prefix, or its output can be +# truncated by a full filesystem. +postgres_backup_looks_restorable() { + local temp_dir="$1" + local expected_database="$2" + local layout="" + layout=$(pg_backup_layout "$temp_dir") + + if [ "$layout" = "single" ]; then + postgres_dump_looks_restorable "$temp_dir/db_backup.sql" + return + fi + [ "$layout" = "multi" ] || return 1 + + local dump_dir="$temp_dir/pg_dump" + local manifest="$dump_dir/manifest.tsv" + postgres_globals_dump_looks_complete "$dump_dir/globals.sql" || return 1 + [ -s "$manifest" ] || return 1 + + local manifest_line="" + local dbname="" + local has_timescaledb="" + local filename="" + local field_count=0 + local manifest_count=0 + local expected_found=false + while IFS= read -r manifest_line || [ -n "$manifest_line" ]; do + [ -n "$manifest_line" ] || return 1 + field_count=$(awk -F '\t' '{ print NF }' <<<"$manifest_line") + [ "$field_count" -eq 5 ] || return 1 + + dbname="${manifest_line%%$'\t'*}" + has_timescaledb=$(cut -f3 <<<"$manifest_line") + filename=$(cut -f4 <<<"$manifest_line") + [ -n "$dbname" ] || return 1 + [[ "$has_timescaledb" =~ ^[01]$ ]] || return 1 + [[ "$filename" =~ ^db-[0-9]{3}\.sql$ ]] || return 1 + postgres_dump_looks_restorable "$dump_dir/$filename" || return 1 + + manifest_count=$((manifest_count + 1)) + if [ "$dbname" = "$expected_database" ]; then + expected_found=true + fi + done <"$manifest" + + local dump_count=0 + dump_count=$(find "$dump_dir" -maxdepth 1 -type f -name 'db-[0-9][0-9][0-9].sql' | wc -l | awk '{ print $1 }') + [ "$manifest_count" -gt 0 ] && [ "$dump_count" -eq "$manifest_count" ] && [ "$expected_found" = true ] +} + +sqlite_snapshot_looks_restorable() { + local snapshot_file="$1" + [ -s "$snapshot_file" ] || return 1 + command -v sqlite3 >/dev/null 2>&1 || return 1 + + local integrity="" + integrity=$(sqlite3 "$snapshot_file" 'PRAGMA quick_check;' 2>/dev/null) || return 1 + [ "$integrity" = "ok" ] +} + +database_backup_looks_restorable() { + local db_type="$1" + local temp_dir="$2" + local expected_database="${3:-}" + local sqlite_file="${4:-}" + + case "$db_type" in + mysql | mariadb) + mysql_dump_looks_restorable "$temp_dir/db_backup.sql" + ;; + postgresql | timescaledb) + postgres_backup_looks_restorable "$temp_dir" "$expected_database" + ;; + sqlite) + [ -n "$sqlite_file" ] || return 1 + sqlite_snapshot_looks_restorable "$temp_dir/$(basename "$sqlite_file")" + ;; + *) + return 1 + ;; + esac +} + send_backup_to_telegram() { + local requested_timestamp="${1:-}" + if [ -f "$ENV_FILE" ]; then while IFS='=' read -r key value; do if [[ -z "$key" || "$key" =~ ^# ]]; then @@ -195,9 +282,19 @@ send_backup_to_telegram() { fi local backup_dir="$APP_DIR/backup" local latest_backup="" - latest_backup=$(find "$backup_dir" -maxdepth 1 -type f \ - \( -name 'backup_*.tar.gz' -o -name 'backup_*.zip' -o -name 'backup_*.z[0-9][0-9]' -o -name 'backup_*.part[0-9][0-9].zip' \) \ - -printf '%T@ %f\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-) + if [ -n "$requested_timestamp" ]; then + if [[ ! "$requested_timestamp" =~ ^[0-9]{14}$ ]]; then + colorized_echo red "Invalid backup timestamp requested for upload." + return 1 + fi + latest_backup=$(find "$backup_dir" -maxdepth 1 -type f \ + \( -name "backup_${requested_timestamp}.zip" -o -name "backup_${requested_timestamp}.z[0-9][0-9]" -o -name "backup_${requested_timestamp}.part[0-9][0-9].zip" \) \ + -printf '%T@ %f\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-) + else + latest_backup=$(find "$backup_dir" -maxdepth 1 -type f \ + \( -name 'backup_*.tar.gz' -o -name 'backup_*.zip' -o -name 'backup_*.z[0-9][0-9]' -o -name 'backup_*.part[0-9][0-9].zip' \) \ + -printf '%T@ %f\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-) + fi if [ -z "$latest_backup" ]; then colorized_echo red "No backups found to send." @@ -851,14 +948,17 @@ remove_backup_service() { # PostgreSQL/TimescaleDB container into /pg_dump/. # Layout: globals.sql, db-NNN.sql per database, manifest.tsv # (dbnameownerhas_timescaledbfilenamets_version). -# Returns 0 when at least one database was dumped; otherwise removes the -# pg_dump dir and returns 1 so the caller can fall back to single-database mode. +# The configured application database is passed as argument 6. Returns 0 only +# when every enumerated database was dumped and that application database is in +# the manifest; otherwise removes the pg_dump directory and returns 1 so the +# caller can fall back to a validated single-database dump. pg_dump_all_user_databases() { local container_name="$1" local backup_user="$2" local backup_password="$3" local temp_dir="$4" local log_file="$5" + local expected_database="$6" local out_dir="$temp_dir/pg_dump" local manifest="$out_dir/manifest.tsv" @@ -871,13 +971,22 @@ pg_dump_all_user_databases() { rm -rf "$out_dir" return 1 fi + if ! postgres_globals_dump_looks_complete "$out_dir/globals.sql"; then + echo "pg_dumpall globals output failed completion validation" >>"$log_file" + rm -rf "$out_dir" + return 1 + fi # Enumerate real user databases (skip templates and the postgres maintenance DB). local databases="" - databases=$(docker exec -e PGPASSWORD="$backup_password" "$container_name" \ + if ! databases=$(docker exec -e PGPASSWORD="$backup_password" "$container_name" \ psql -U "$backup_user" -d postgres -At \ -c "SELECT datname FROM pg_database WHERE datistemplate = false AND datname <> 'postgres';" \ - 2>>"$log_file") + 2>>"$log_file"); then + echo "Could not enumerate user databases for multi-DB backup" >>"$log_file" + rm -rf "$out_dir" + return 1 + fi if [ -z "$databases" ]; then echo "No user databases enumerated for multi-DB backup" >>"$log_file" rm -rf "$out_dir" @@ -887,6 +996,7 @@ pg_dump_all_user_databases() { : >"$manifest" local index=0 local dumped=0 + local expected_found=false local dbname="" while IFS= read -r dbname; do [ -n "$dbname" ] || continue @@ -894,15 +1004,17 @@ pg_dump_all_user_databases() { local filename filename=$(pg_dump_index_filename "$index") - # Owner of this database (empty if it can't be determined; restore - # falls back to the admin role). + # Owner of this database. If it cannot be determined, the manifest + # would be incomplete, so abort and let the caller fall back to a + # validated single-database dump. local owner="" if ! owner=$(docker exec -e PGPASSWORD="$backup_password" "$container_name" \ psql -U "$backup_user" -d postgres -At \ -c "SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_database WHERE datname = '${dbname//\'/\'\'}';" \ 2>>"$log_file") || [ -z "$owner" ]; then - echo "Could not determine owner for database '$dbname'; restore will fall back to the admin role" >>"$log_file" - owner="" + echo "Could not determine owner for database '$dbname'; multi-DB backup is incomplete" >>"$log_file" + rm -rf "$out_dir" + return 1 fi # TimescaleDB presence + version for this database. One query gives both: @@ -921,41 +1033,84 @@ pg_dump_all_user_databases() { ts_version="$ext_check" fi else - echo "Could not check timescaledb extension for database '$dbname'; assuming not present" >>"$log_file" + echo "Could not check timescaledb extension for database '$dbname'; multi-DB backup is incomplete" >>"$log_file" + rm -rf "$out_dir" + return 1 fi # Dump this database. if ! docker exec -e PGPASSWORD="$backup_password" "$container_name" \ pg_dump -U "$backup_user" -d "$dbname" --clean --if-exists >"$out_dir/$filename" 2>>"$log_file"; then echo "pg_dump failed for database '$dbname'" >>"$log_file" - rm -f "$out_dir/$filename" - continue + rm -rf "$out_dir" + return 1 fi # Never trust an empty/garbage dump. if ! postgres_dump_looks_restorable "$out_dir/$filename"; then - echo "Dump for database '$dbname' failed content validation; skipping" >>"$log_file" - rm -f "$out_dir/$filename" - continue + echo "Dump for database '$dbname' failed completion/content validation" >>"$log_file" + rm -rf "$out_dir" + return 1 fi local line if ! line=$(pg_manifest_encode "$dbname" "$owner" "$has_ts" "$filename" "$ts_version"); then - echo "Database name '$dbname' is not manifest-safe; skipping" >>"$log_file" - rm -f "$out_dir/$filename" - continue + echo "Database name '$dbname' is not manifest-safe; multi-DB backup is incomplete" >>"$log_file" + rm -rf "$out_dir" + return 1 fi printf '%s\n' "$line" >>"$manifest" dumped=$((dumped + 1)) + if [ "$dbname" = "$expected_database" ]; then + expected_found=true + fi done <<<"$databases" - if [ "$dumped" -eq 0 ]; then + if [ "$dumped" -eq 0 ] || [ "$dumped" -ne "$index" ] || [ "$expected_found" != true ]; then + echo "Multi-DB backup did not include every database and the configured database '$expected_database'" >>"$log_file" rm -rf "$out_dir" return 1 fi return 0 } +# Record the extension version beside a legacy single-database TimescaleDB +# dump. pg_dump itself does not preserve which extension version must be +# created first, so this small sidecar is required for automatic cross-version +# restore when the all-databases layout is unavailable. +write_timescaledb_single_dump_version() { + local container_name="$1" + local backup_user="$2" + local backup_password="$3" + local database_name="$4" + local temp_dir="$5" + local log_file="$6" + local source_version="" + + if ! source_version=$(docker exec -e PGPASSWORD="$backup_password" "$container_name" \ + psql -X -U "$backup_user" -d "$database_name" -At \ + -c "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';" \ + 2>>"$log_file"); then + echo "Could not query TimescaleDB version for single-database dump" >>"$log_file" + return 1 + fi + + if [ -z "$source_version" ]; then + # The compose image can be TimescaleDB while this particular database + # does not use the extension. Preserve that fact so restore can use the + # normal PostgreSQL path instead of requiring a fake version. + printf '%s\n' 'none' >"$temp_dir/db_backup.timescaledb-version" + return 0 + fi + + if [[ ! "$source_version" =~ ^[0-9]+[.][0-9]+[.][0-9]+([-.][A-Za-z0-9]+)*$ ]]; then + echo "Could not record TimescaleDB version for single-database dump" >>"$log_file" + return 1 + fi + + printf '%s\n' "$source_version" >"$temp_dir/db_backup.timescaledb-version" +} + backup_command() { colorized_echo blue "Starting backup process..." @@ -974,6 +1129,7 @@ backup_command() { local split_threshold_bytes="${BACKUP_SPLIT_THRESHOLD_BYTES:-$split_size_bytes}" local staging_root="" local temp_dir="" + local sqlite_snapshot_dir="" local log_file="" # Keep the lock with the backup artifacts so it is not affected by sticky-dir # protections on /tmp when different users invoke the command. @@ -1053,6 +1209,9 @@ backup_command() { cleanup_backup_command() { rm -rf "$temp_dir" + if [ -n "$sqlite_snapshot_dir" ]; then + rm -rf "$sqlite_snapshot_dir" + fi if [ "$keep_log_file" != true ] && [ -n "$log_file" ]; then rm -f "$log_file" fi @@ -1130,26 +1289,11 @@ backup_command() { # Extract database type from scheme if [[ "$SQLALCHEMY_DATABASE_URL" =~ ^sqlite ]]; then - db_type="sqlite" - # Extract SQLite file path - # SQLite URLs: sqlite:///relative/path or sqlite:////absolute/path - local sqlite_url_part="${SQLALCHEMY_DATABASE_URL#*://}" - sqlite_url_part="${sqlite_url_part%%\?*}" - sqlite_url_part="${sqlite_url_part%%#*}" - - # SQLite URL format: - # sqlite:////absolute/path (4 slashes = absolute path /path) - # After removing 'sqlite://', //absolute/path remains, convert to /absolute/path - if [[ "$sqlite_url_part" =~ ^//(.*)$ ]]; then - # Absolute path: sqlite:////absolute/path -> /absolute/path - sqlite_file="/${BASH_REMATCH[1]}" - elif [[ "$sqlite_url_part" =~ ^/(.*)$ ]]; then - # Could be absolute (sqlite:///path) or relative depending on context - # In practice, treat as absolute since SQLAlchemy uses 4 slashes for absolute - sqlite_file="/${BASH_REMATCH[1]}" - else - # Relative path (no leading slash) - sqlite_file="$sqlite_url_part" + db_type="sqlite" + if ! sqlite_file=$(sqlite_database_path_from_url "$SQLALCHEMY_DATABASE_URL") || [ -z "$sqlite_file" ]; then + sqlite_file="" + echo "Invalid SQLite SQLALCHEMY_DATABASE_URL: ${safe_sqlalchemy_url}" >>"$log_file" + error_messages+=("SQLite database URL is malformed; expected sqlite[+driver]:// followed by a database path.") fi elif [[ "$SQLALCHEMY_DATABASE_URL" =~ ^(mysql|mariadb|postgresql)[^:]*:// ]]; then # Extract scheme to determine type @@ -1251,11 +1395,23 @@ backup_command() { # Try root user with MYSQL_ROOT_PASSWORD first for all databases backup if [ -n "${MYSQL_ROOT_PASSWORD:-}" ]; then colorized_echo blue "Backing up all MariaDB databases from container: $container_name (using root user)" - if docker exec -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$container_name" mariadb-dump -u root --all-databases --ignore-database=mysql --ignore-database=performance_schema --ignore-database=information_schema --ignore-database=sys --events --triggers >"$temp_dir/db_backup.sql" 2>>"$log_file"; then + local mariadb_databases="" + local configured_db_found=false + mariadb_databases=$(docker exec -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$container_name" \ + mariadb -N -s -u root -e "SHOW DATABASES;" 2>>"$log_file") || mariadb_databases="" + while IFS= read -r _db_line; do + [ -n "$_db_line" ] || continue + if [ "$_db_line" = "$db_name" ]; then + configured_db_found=true + break + fi + done <<<"$mariadb_databases" + + if [ "$configured_db_found" = true ] && docker exec -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$container_name" mariadb-dump -u root --all-databases --ignore-database=mysql --ignore-database=performance_schema --ignore-database=information_schema --ignore-database=sys --events --triggers >"$temp_dir/db_backup.sql" 2>>"$log_file"; then colorized_echo green "MariaDB backup completed successfully (all databases)" else # Fallback to SQL URL credentials for specific database - colorized_echo yellow "Root backup failed, falling back to app user for specific database" + colorized_echo yellow "Root backup failed or did not include '$db_name'; falling back to app user for the configured database" local backup_user="${db_user:-${DB_USER:-}}" local backup_password="${db_password:-${DB_PASSWORD:-}}" @@ -1339,11 +1495,17 @@ backup_command() { # Collect DB names into an array so each is passed as one argument # (a name containing a space must not be word-split). local -a database_list=() + local configured_db_found=false while IFS= read -r _db_line; do - [ -n "$_db_line" ] && database_list+=("$_db_line") + if [ -n "$_db_line" ]; then + database_list+=("$_db_line") + if [ "$_db_line" = "$db_name" ]; then + configured_db_found=true + fi + fi done <<<"$databases" - if [ -z "$databases" ]; then - colorized_echo yellow "No user databases found, falling back to specific database backup" + if [ -z "$databases" ] || [ "$configured_db_found" != true ]; then + colorized_echo yellow "The configured database '$db_name' was not found in the root database list; falling back to a specific database backup" # Fallback to SQL URL credentials local backup_user="${db_user:-${DB_USER:-}}" local backup_password="${db_password:-${DB_PASSWORD:-}}" @@ -1441,7 +1603,7 @@ backup_command() { error_messages+=("PostgreSQL database name not found.") else colorized_echo blue "Backing up all PostgreSQL databases from container: $container_name (using user: $backup_user)" - if pg_dump_all_user_databases "$container_name" "$backup_user" "$backup_password" "$temp_dir" "$log_file"; then + if pg_dump_all_user_databases "$container_name" "$backup_user" "$backup_password" "$temp_dir" "$log_file" "$db_name"; then colorized_echo green "PostgreSQL backup completed successfully (all databases)" else colorized_echo yellow "Multi-database backup unavailable; falling back to single database '$db_name'." @@ -1512,13 +1674,16 @@ backup_command() { error_messages+=("TimescaleDB database name not found.") else colorized_echo blue "Backing up all TimescaleDB databases from container: $container_name (using user: $backup_user)" - if pg_dump_all_user_databases "$container_name" "$backup_user" "$backup_password" "$temp_dir" "$log_file"; then + if pg_dump_all_user_databases "$container_name" "$backup_user" "$backup_password" "$temp_dir" "$log_file" "$db_name"; then colorized_echo green "TimescaleDB backup completed successfully (all databases)" else colorized_echo yellow "Multi-database backup unavailable; falling back to single database '$db_name'." if ! docker exec -e PGPASSWORD="$backup_password" "$container_name" pg_dump -U "$backup_user" -d "$db_name" --clean --if-exists >"$temp_dir/db_backup.sql" 2>>"$log_file"; then colorized_echo red "TimescaleDB dump failed. Check log file for details: $log_file" error_messages+=("TimescaleDB dump failed for database '$db_name'.") + elif ! write_timescaledb_single_dump_version "$container_name" "$backup_user" "$backup_password" "$db_name" "$temp_dir" "$log_file"; then + colorized_echo red "TimescaleDB version metadata could not be created." + error_messages+=("TimescaleDB version metadata could not be created for '$db_name'.") else colorized_echo green "TimescaleDB backup completed successfully" fi @@ -1533,7 +1698,10 @@ backup_command() { fi ;; sqlite) - if [ -f "$sqlite_file" ]; then + if [ -z "$sqlite_file" ]; then + # URL parsing already recorded the actionable error above. + : + elif [ -f "$sqlite_file" ]; then if ! command -v sqlite3 >/dev/null 2>&1; then detect_os # Best-effort: if sqlite3 can't be installed, continue (the @@ -1544,11 +1712,13 @@ backup_command() { local sqlite_basename=$(basename "$sqlite_file") if command -v sqlite3 >/dev/null 2>&1; then - if ! sqlite3 "$sqlite_file" ".backup '$temp_dir/$sqlite_basename'" >>"$log_file" 2>&1; then + if ! sqlite_snapshot_dir=$(mktemp -d "${staging_root}/pasarguard_sqlite_snapshot.XXXXXX"); then + error_messages+=("Failed to create protected SQLite snapshot staging directory.") + elif ! sqlite3 "$sqlite_file" ".backup '$sqlite_snapshot_dir/$sqlite_basename'" >>"$log_file" 2>&1; then error_messages+=("Failed to create SQLite backup snapshot.") fi - elif ! cp "$sqlite_file" "$temp_dir/$sqlite_basename" 2>>"$log_file"; then - error_messages+=("Failed to copy SQLite database.") + else + error_messages+=("sqlite3 is required to create a consistent SQLite backup snapshot.") fi else error_messages+=("SQLite database file not found at $sqlite_file.") @@ -1563,8 +1733,29 @@ backup_command() { colorized_echo blue "Copying app directory..." if [ -d "$APP_DIR" ]; then - # Use rsync to copy the entire app directory, excluding the backup folder - if ! rsync -av --exclude 'backup' "$APP_DIR/" "$temp_dir/" >>"$log_file" 2>&1; then + # Database artifacts are staged before app files. Old restore versions + # could leave dumps in APP_DIR; copying those over staging would replace + # today's fresh dump with a complete-but-stale dump that still passes + # structural validation. + local app_rsync_args=( + -av + --exclude 'backup' + --exclude 'db_backup.sql' + --exclude 'db_backup.sqlite' + --exclude 'db_backup.timescaledb-version' + --exclude 'pg_dump' + --exclude 'timescaledb-compatible' + --exclude 'pasarguard_restore_error.log' + ) + if [ "$db_type" = "sqlite" ] && [ -n "$sqlite_file" ]; then + local app_sqlite_basename="" + app_sqlite_basename=$(basename "$sqlite_file") + app_rsync_args+=(--exclude "$app_sqlite_basename") + app_rsync_args+=(--exclude "${app_sqlite_basename}-wal") + app_rsync_args+=(--exclude "${app_sqlite_basename}-shm") + app_rsync_args+=(--exclude "${app_sqlite_basename}-journal") + fi + if ! rsync "${app_rsync_args[@]}" "$APP_DIR/" "$temp_dir/" >>"$log_file" 2>&1; then error_messages+=("Failed to copy app directory.") echo "Failed to copy app directory" >>"$log_file" fi @@ -1582,23 +1773,27 @@ backup_command() { # Ensure destination directory exists and is empty (already cleaned above, but be explicit) if [ -d "$DATA_DIR" ]; then local rsync_args=(-av --exclude 'xray-core' --exclude 'mysql' --exclude 'mariadb' --exclude 'postgresql' --exclude 'timescaledb') + local normalized_data_dir="" + normalized_data_dir=$(normalize_posix_path "$DATA_DIR") - if [ "$db_type" = "sqlite" ] && [ -n "$sqlite_file" ] && [[ "$sqlite_file" == "$DATA_DIR/"* ]]; then - local sqlite_relative_path="${sqlite_file#$DATA_DIR/}" + if [ "$db_type" = "sqlite" ] && [ -n "$sqlite_file" ] && [[ "$sqlite_file" == "$normalized_data_dir/"* ]]; then + local sqlite_relative_path="${sqlite_file#"$normalized_data_dir"/}" rsync_args+=(--exclude "$sqlite_relative_path") rsync_args+=(--exclude "${sqlite_relative_path}-wal") rsync_args+=(--exclude "${sqlite_relative_path}-shm") + rsync_args+=(--exclude "${sqlite_relative_path}-journal") echo "Excluding SQLite database from data directory copy: $sqlite_relative_path" >>"$log_file" fi if ! rsync "${rsync_args[@]}" "$DATA_DIR/" "$temp_dir/pasarguard_data/" >>"$log_file" 2>&1; then error_messages+=("Failed to copy data directory.") echo "Failed to copy data directory" >>"$log_file" - elif [ "$db_type" = "sqlite" ] && [ -n "$sqlite_file" ] && [[ "$sqlite_file" == "$DATA_DIR/"* ]]; then - local sqlite_relative_path="${sqlite_file#$DATA_DIR/}" + elif [ "$db_type" = "sqlite" ] && [ -n "$sqlite_file" ] && [[ "$sqlite_file" == "$normalized_data_dir/"* ]]; then + local sqlite_relative_path="${sqlite_file#"$normalized_data_dir"/}" rm -f "$temp_dir/pasarguard_data/$sqlite_relative_path" \ "$temp_dir/pasarguard_data/${sqlite_relative_path}-wal" \ - "$temp_dir/pasarguard_data/${sqlite_relative_path}-shm" 2>>"$log_file" || true + "$temp_dir/pasarguard_data/${sqlite_relative_path}-shm" \ + "$temp_dir/pasarguard_data/${sqlite_relative_path}-journal" 2>>"$log_file" || true fi else colorized_echo yellow "Data directory $DATA_DIR does not exist. Skipping data directory backup." @@ -1618,9 +1813,35 @@ backup_command() { fi fi + # Refuse to archive a database artifact unless the final staged copy is + # complete and contains the configured application database. Keep this + # gate after app/data copying because those operations also write into the + # staging directory and must not replace an already-validated artifact. + if [ -n "$db_type" ] && [ ${#error_messages[@]} -eq 0 ]; then + if [ "$db_type" = "sqlite" ] && [ -n "$sqlite_file" ]; then + local final_sqlite_basename="" + final_sqlite_basename=$(basename "$sqlite_file") + if [ -z "$sqlite_snapshot_dir" ] || [ ! -f "$sqlite_snapshot_dir/$final_sqlite_basename" ]; then + error_messages+=("Protected SQLite snapshot is missing before final archive staging.") + elif ! mv -f "$sqlite_snapshot_dir/$final_sqlite_basename" "$temp_dir/$final_sqlite_basename" 2>>"$log_file"; then + error_messages+=("Failed to move the protected SQLite snapshot into final archive staging.") + fi + rm -f "$temp_dir/${final_sqlite_basename}-wal" \ + "$temp_dir/${final_sqlite_basename}-shm" \ + "$temp_dir/${final_sqlite_basename}-journal" 2>>"$log_file" || true + fi + if [ ${#error_messages[@]} -eq 0 ] && ! database_backup_looks_restorable "$db_type" "$temp_dir" "$db_name" "$sqlite_file"; then + colorized_echo red "Database backup artifact failed completeness validation." + echo "Final staged database artifact failed completeness validation for $db_type" >>"$log_file" + error_messages+=("$db_type backup artifact is missing, truncated, corrupt, or does not contain the configured database.") + fi + fi + colorized_echo blue "Creating backup archive..." # Verify temp_dir exists and has content before creating archive - if [ ! -d "$temp_dir" ] || [ -z "$(ls -A "$temp_dir" 2>/dev/null)" ]; then + if [ ${#error_messages[@]} -gt 0 ]; then + echo "Skipping archive creation because the backup has errors." >>"$log_file" + elif [ ! -d "$temp_dir" ] || [ -z "$(ls -A "$temp_dir" 2>/dev/null)" ]; then error_messages+=("Temporary directory is empty or missing. Cannot create archive.") echo "Temporary directory is empty or missing: $temp_dir" >>"$log_file" elif ! (cd "$temp_dir" && zip -rq "$backup_file" .) 2>>"$log_file"; then @@ -1683,6 +1904,10 @@ backup_command() { fi if [ ${#error_messages[@]} -gt 0 ]; then + rm -f "$backup_file" + find "$backup_dir" -maxdepth 1 -type f \ + \( -name "backup_${timestamp}.z[0-9][0-9]" -o -name "backup_${timestamp}.part[0-9][0-9].zip" \) \ + -delete 2>/dev/null || true keep_log_file=true colorized_echo red "Backup completed with errors:" for error in "${error_messages[@]}"; do @@ -1712,7 +1937,7 @@ backup_command() { done fi if [ -f "$ENV_FILE" ]; then - send_backup_to_telegram "$backup_file" + send_backup_to_telegram "$timestamp" fi cleanup_backup_command } diff --git a/lib/pasarguard-restore.sh b/lib/pasarguard-restore.sh index 6187cc2..c3e2485 100644 --- a/lib/pasarguard-restore.sh +++ b/lib/pasarguard-restore.sh @@ -35,7 +35,27 @@ postgres_dump_looks_restorable() { local dump_file="$1" [ -s "$dump_file" ] || return 1 # Require at least one real schema/data statement, not just comments/SET. - grep -qiE '^[[:space:]]*(CREATE|COPY|INSERT|ALTER)[[:space:]]' "$dump_file" + grep -qiE '^[[:space:]]*(CREATE|COPY|INSERT|ALTER)[[:space:]]' "$dump_file" || return 1 + # pg_dump writes this only after completing the output. Requiring it keeps a + # dump truncated by a full disk or interrupted process from being accepted. + grep -qE '^-- PostgreSQL database dump complete([[:space:]]*)$' "$dump_file" +} + +# pg_dumpall uses a different completion marker for the globals-only file. +postgres_globals_dump_looks_complete() { + local dump_file="$1" + [ -s "$dump_file" ] || return 1 + grep -qE '^-- PostgreSQL database cluster dump complete([[:space:]]*)$' "$dump_file" +} + +# Both mysqldump and mariadb-dump emit a completion marker after a successful +# plain-SQL dump. This accepts either tool while rejecting empty and truncated +# files before they can be archived or restored. +mysql_dump_looks_restorable() { + local dump_file="$1" + [ -s "$dump_file" ] || return 1 + grep -qE '^-- (MySQL|MariaDB) dump ' "$dump_file" || return 1 + grep -qE '^-- Dump completed on ' "$dump_file" } # Detect the dump layout inside an extracted backup directory. @@ -53,12 +73,97 @@ pg_backup_layout() { fi } +# Convert a versioned single-database TimescaleDB backup into the same manifest +# layout used by new multi-database backups. Version lookup order supports new +# sidecars, an exact version pinned in the archived compose file, and an +# explicit override for old archives that recorded neither. +pg_promote_timescaledb_single_backup() { + local restore_dir="$1" + local database_name="$2" + local database_owner="$3" + local log_file="$4" + local requested_source_version="${5:-}" + local source_version="" + local has_timescaledb="1" + + if [ -s "$restore_dir/db_backup.timescaledb-version" ]; then + source_version=$(head -n 1 "$restore_dir/db_backup.timescaledb-version" | tr -d '[:space:]') + elif [ -n "$requested_source_version" ]; then + source_version="$requested_source_version" + elif [ -s "$restore_dir/docker-compose.yml" ]; then + source_version=$(sed -nE 's#^[[:space:]]*image:[[:space:]]*timescale/timescaledb:([0-9]+([.][0-9]+){1,3})(-pg[0-9]+.*)?[[:space:]]*$#\1#p' \ + "$restore_dir/docker-compose.yml" | head -n 1) + fi + + if [ "$source_version" = "none" ]; then + has_timescaledb="0" + source_version="" + elif ! timescaledb_version_is_safe "$source_version"; then + echo "Single-database TimescaleDB backup has no safe source-version metadata" >>"$log_file" + return 1 + fi + case "${database_name}${database_owner}" in + *$'\t'* | *$'\n'*) return 1 ;; + esac + [ -n "$database_name" ] && [ -n "$database_owner" ] || return 1 + + local dump_dir="$restore_dir/pg_dump" + mkdir -p "$dump_dir" || return 1 + cp "$restore_dir/db_backup.sql" "$dump_dir/db-001.sql" || return 1 + + local owner_ident="${database_owner//\"/\"\"}" + printf '%s\n' \ + '-- PasarGuard synthetic globals for a legacy single-database backup' \ + "CREATE ROLE \"$owner_ident\";" \ + '-- PostgreSQL database cluster dump complete' >"$dump_dir/globals.sql" + printf '%s\t%s\t%s\tdb-001.sql\t%s\n' \ + "$database_name" "$database_owner" "$has_timescaledb" "$source_version" >"$dump_dir/manifest.tsv" + return 0 +} + # Strip "DROP/CREATE EXTENSION ... timescaledb" statements from a dump on stdin. # These would undo the timescaledb_pre_restore() setup during restore. pg_filter_timescaledb_extension_lines() { grep -v -E '^\s*(DROP|CREATE)\s+EXTENSION\s+(IF\s+(EXISTS|NOT\s+EXISTS)\s+)?timescaledb\b' || true } +# Keep cluster roles and grants from pg_dumpall, but never restore password +# verifiers from another installation. Restoring an archived ALTER ROLE ... +# PASSWORD silently changes the destination password while its .env still has +# the current password, locking the application out after an otherwise +# successful restore. +pg_filter_global_passwords() { + sed -E \ + -e "s/[[:space:]]+PASSWORD[[:space:]]+'([^']|'')*'//g" \ + -e 's/[[:space:]]+PASSWORD[[:space:]]+NULL//g' +} + +# In addition to removing every archived password, keep the destination admin +# role itself entirely unchanged. pg_dumpall emits CREATE ROLE followed by +# ALTER ROLE; the latter succeeds for an existing role and could otherwise +# remove destination privileges or change connection limits. +pg_filter_globals_for_destination() { + local destination_role="$1" + pg_filter_global_passwords | awk -v role="$destination_role" ' + BEGIN { + quoted_role = role + gsub(/"/, "\"\"", quoted_role) + quoted_role = "\"" quoted_role "\"" + } + { + if ($0 ~ /^(CREATE|ALTER)[[:space:]]+ROLE[[:space:]]/) { + rest = $0 + sub(/^(CREATE|ALTER)[[:space:]]+ROLE[[:space:]]+/, "", rest) + if (rest == role || rest == role ";" || index(rest, role " ") == 1 || + rest == quoted_role || rest == quoted_role ";" || index(rest, quoted_role " ") == 1) { + next + } + } + print + } + ' +} + # True (0) when two timescaledb version strings are identical. Restore uses this # to gate a destructive cross-version restore. The caller treats an empty source # version (legacy backup) as "do not gate". @@ -66,6 +171,293 @@ timescaledb_version_matches() { [ "$1" = "$2" ] } +timescaledb_version_is_safe() { + [[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+([-.][A-Za-z0-9]+)*$ ]] +} + +cleanup_timescaledb_compat_container() { + local container_name="$1" + local volume_name="$2" + + [ -n "$container_name" ] && docker rm -f "$container_name" >/dev/null 2>&1 || true + [ -n "$volume_name" ] && docker volume rm "$volume_name" >/dev/null 2>&1 || true +} + +TIMESCALEDB_COMPAT_CONTAINER="" +TIMESCALEDB_COMPAT_VOLUME="" + +handle_timescaledb_compat_signal() { + local exit_code="$1" + trap - INT TERM + cleanup_timescaledb_compat_container "$TIMESCALEDB_COMPAT_CONTAINER" "$TIMESCALEDB_COMPAT_VOLUME" + TIMESCALEDB_COMPAT_CONTAINER="" + TIMESCALEDB_COMPAT_VOLUME="" + if declare -F start_pasarguard_app_services >/dev/null 2>&1; then + start_pasarguard_app_services >/dev/null 2>&1 || true + fi + exit "$exit_code" +} + +arm_timescaledb_compat_cleanup() { + TIMESCALEDB_COMPAT_CONTAINER="$1" + TIMESCALEDB_COMPAT_VOLUME="$2" + trap 'handle_timescaledb_compat_signal 130' INT + trap 'handle_timescaledb_compat_signal 143' TERM +} + +finish_timescaledb_compat_cleanup() { + trap - INT TERM + cleanup_timescaledb_compat_container "$TIMESCALEDB_COMPAT_CONTAINER" "$TIMESCALEDB_COMPAT_VOLUME" + TIMESCALEDB_COMPAT_CONTAINER="" + TIMESCALEDB_COMPAT_VOLUME="" +} + +# Convert cross-version TimescaleDB dumps before the destination is changed. +# The temporary `pgNN-all` image contains historical extension versions: each +# mismatched database is restored at its source version, upgraded to the exact +# target version, and dumped again. The caller may then use the normal restore +# path against a dump whose extension version matches the destination. +# +# On success PG_PREPARED_DUMP_DIR is set to either source_dir (no conversion was +# needed) or output_dir (every mismatched dump converted and validated). +pg_prepare_timescaledb_compatible_dumps() { + local destination_container="$1" + local admin_user="$2" + local admin_password="$3" + local source_dir="$4" + local output_dir="$5" + local log_file="$6" + local expected_database="${7:-}" + local requested_compat_image="${8:-}" + local manifest="$source_dir/manifest.tsv" + + PG_PREPARED_DUMP_DIR="$source_dir" + [ -s "$manifest" ] || return 1 + + local has_versioned_timescale=false + local needs_conversion=false + local dbname owner has_ts filename source_version + while IFS=$'\t' read -r dbname owner has_ts filename source_version; do + [ -n "$dbname" ] || continue + if [ "$has_ts" = "1" ] && [ -z "$source_version" ]; then + echo "TimescaleDB database '$dbname' has no source-version metadata" >>"$log_file" + return 1 + elif [ "$has_ts" = "1" ]; then + has_versioned_timescale=true + if ! timescaledb_version_is_safe "$source_version"; then + echo "Unsafe TimescaleDB version in manifest: $source_version" >>"$log_file" + return 1 + fi + fi + done <"$manifest" + + [ "$has_versioned_timescale" = true ] || return 0 + + local target_version="" + local server_version_num="" + if ! target_version=$(docker exec -e PGPASSWORD="$admin_password" "$destination_container" \ + psql -X -U "$admin_user" -d postgres -At \ + -c "SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb';" \ + 2>>"$log_file") || ! timescaledb_version_is_safe "$target_version"; then + echo "Could not determine a safe destination TimescaleDB version" >>"$log_file" + return 1 + fi + if ! server_version_num=$(docker exec -e PGPASSWORD="$admin_password" "$destination_container" \ + psql -X -U "$admin_user" -d postgres -At -c "SHOW server_version_num;" \ + 2>>"$log_file") || [[ ! "$server_version_num" =~ ^[0-9]+$ ]]; then + echo "Could not determine destination PostgreSQL major version" >>"$log_file" + return 1 + fi + + while IFS=$'\t' read -r dbname owner has_ts filename source_version; do + [ -n "$dbname" ] || continue + if [ "$has_ts" = "1" ] && [ -n "$source_version" ] && [ "$source_version" != "$target_version" ]; then + needs_conversion=true + break + fi + done <"$manifest" + [ "$needs_conversion" = true ] || return 0 + + local pg_major + pg_major=$((server_version_num / 10000)) + local target_series="${target_version%.*}" + local compat_image="${requested_compat_image:-timescale/timescaledb-ha:pg${pg_major}-ts${target_series}-all}" + local compat_pgdata="" + case "$compat_image" in + *timescale/timescaledb-ha:*) + compat_pgdata="/home/postgres/pgdata/data" + ;; + *timescale/timescaledb:*) + if [ "$pg_major" -ge 18 ]; then + compat_pgdata="/var/lib/postgresql/${pg_major}/docker" + else + compat_pgdata="/var/lib/postgresql/data" + fi + ;; + *) + echo "Unsupported TimescaleDB compatibility image: $compat_image" >>"$log_file" + return 1 + ;; + esac + local compat_suffix="${$}-${RANDOM}-$(date +%s)" + local compat_container="pasarguard-ts-compat-${compat_suffix}" + local compat_volume="" + + colorized_echo blue "Preparing TimescaleDB $target_version-compatible dumps before changing the destination..." + colorized_echo blue "Pulling temporary compatibility image: $compat_image" + if ! docker pull "$compat_image" >>"$log_file" 2>&1; then + echo "Could not pull TimescaleDB compatibility image: $compat_image" >>"$log_file" + return 1 + fi + if ! compat_volume=$(docker volume create --label "com.pasarguard.restore=$compat_suffix" 2>>"$log_file"); then + echo "Could not create temporary TimescaleDB compatibility volume" >>"$log_file" + return 1 + fi + if ! docker run -d --name "$compat_container" --restart=no \ + -e POSTGRES_USER="$admin_user" \ + -e POSTGRES_PASSWORD="$admin_password" \ + -e POSTGRES_DB=postgres \ + -e PGDATA="$compat_pgdata" \ + -v "$compat_volume:$compat_pgdata" \ + "$compat_image" >>"$log_file" 2>&1; then + echo "Could not start temporary TimescaleDB compatibility container" >>"$log_file" + cleanup_timescaledb_compat_container "" "$compat_volume" + return 1 + fi + arm_timescaledb_compat_cleanup "$compat_container" "$compat_volume" + + local ready=false + local waited=0 + while [ "$waited" -lt 180 ]; do + if docker exec "$compat_container" pg_isready -q -U "$admin_user" -d postgres >/dev/null 2>&1; then + ready=true + break + fi + sleep 1 + waited=$((waited + 1)) + done + if [ "$ready" != true ]; then + echo "Temporary TimescaleDB compatibility container did not become ready" >>"$log_file" + finish_timescaledb_compat_cleanup + return 1 + fi + + # Verify every required version exists before doing any conversion work. + local required_versions="$target_version" + while IFS=$'\t' read -r dbname owner has_ts filename source_version; do + [ "$has_ts" = "1" ] && [ -n "$source_version" ] || continue + if ! grep -F -x -q "$source_version" <<<"$required_versions"; then + required_versions+=$'\n'"$source_version" + fi + done <"$manifest" + local required_version available_count + while IFS= read -r required_version; do + [ -n "$required_version" ] || continue + available_count=$(docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -U "$admin_user" -d postgres -At \ + -c "SELECT count(*) FROM pg_available_extension_versions WHERE name = 'timescaledb' AND version = '$required_version';" \ + 2>>"$log_file") || available_count="0" + if [ "$available_count" != "1" ]; then + echo "Compatibility image $compat_image does not contain TimescaleDB $required_version" >>"$log_file" + finish_timescaledb_compat_cleanup + return 1 + fi + done <<<"$required_versions" + + if ! mkdir -p "$output_dir"; then + finish_timescaledb_compat_cleanup + return 1 + fi + cp "$source_dir/globals.sql" "$output_dir/globals.sql" 2>>"$log_file" || { + finish_timescaledb_compat_cleanup + rm -rf "$output_dir" + return 1 + } + : >"$output_dir/manifest.tsv" + + local filtered_globals="$output_dir/globals.no-passwords.sql" + if ! pg_filter_globals_for_destination "$admin_user" <"$source_dir/globals.sql" >"$filtered_globals"; then + finish_timescaledb_compat_cleanup + rm -rf "$output_dir" + return 1 + fi + docker exec -i -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -U "$admin_user" -d postgres <"$filtered_globals" >>"$log_file" 2>&1 || true + rm -f "$filtered_globals" + + local index=0 + local converted_ok=true + while IFS=$'\t' read -r dbname owner has_ts filename source_version; do + [ -n "$dbname" ] || continue + index=$((index + 1)) + if [ "$has_ts" != "1" ] || [ -z "$source_version" ] || [ "$source_version" = "$target_version" ]; then + if ! cp "$source_dir/$filename" "$output_dir/$filename" 2>>"$log_file"; then + converted_ok=false + break + fi + if ! printf '%s\t%s\t%s\t%s\t%s\n' "$dbname" "$owner" "$has_ts" "$filename" "$source_version" >>"$output_dir/manifest.tsv"; then + converted_ok=false + break + fi + continue + fi + + local compat_db="pasarguard_restore_${index}" + local filtered_dump="$output_dir/${filename}.source-filtered" + colorized_echo blue "Converting '$dbname' from TimescaleDB $source_version to $target_version..." + if ! docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" -d postgres \ + -c "CREATE DATABASE \"$compat_db\" OWNER \"${admin_user//\"/\"\"}\";" >>"$log_file" 2>&1 || + ! docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" -d "$compat_db" \ + -c "CREATE EXTENSION timescaledb VERSION '$source_version';" >>"$log_file" 2>&1 || + ! docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" -d "$compat_db" \ + -c "SELECT timescaledb_pre_restore();" >>"$log_file" 2>&1; then + converted_ok=false + break + fi + + if ! pg_filter_timescaledb_extension_lines <"$source_dir/$filename" >"$filtered_dump"; then + converted_ok=false + break + fi + if ! docker exec -i -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" -d "$compat_db" <"$filtered_dump" >>"$log_file" 2>&1 || + ! docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" -d "$compat_db" \ + -c "SELECT timescaledb_post_restore();" >>"$log_file" 2>&1 || + ! docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" -d "$compat_db" \ + -c "ALTER EXTENSION timescaledb UPDATE TO '$target_version';" >>"$log_file" 2>&1 || + ! docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + pg_dump -U "$admin_user" -d "$compat_db" --clean --if-exists >"$output_dir/$filename" 2>>"$log_file" || + ! postgres_dump_looks_restorable "$output_dir/$filename"; then + converted_ok=false + rm -f "$filtered_dump" + break + fi + rm -f "$filtered_dump" + if ! printf '%s\t%s\t1\t%s\t%s\n' "$dbname" "$owner" "$filename" "$target_version" >>"$output_dir/manifest.tsv"; then + converted_ok=false + break + fi + docker exec -e PGPASSWORD="$admin_password" "$compat_container" \ + psql -X -U "$admin_user" -d postgres -c "DROP DATABASE \"$compat_db\";" >>"$log_file" 2>&1 || true + done <"$manifest" + + finish_timescaledb_compat_cleanup + if [ "$converted_ok" != true ] || ! postgres_backup_looks_restorable "$(dirname "$output_dir")" "$expected_database"; then + echo "TimescaleDB compatibility conversion failed validation" >>"$log_file" + rm -rf "$output_dir" + return 1 + fi + + PG_PREPARED_DUMP_DIR="$output_dir" + colorized_echo green "TimescaleDB dumps converted to destination version $target_version." + return 0 +} + # Operator-facing guidance shown when a backup's timescaledb version does not # match this server's. Values are filled in so the output is copy-pasteable. # Empty tgt_ver -> "not installed"; empty pg_major -> a pgNN placeholder. @@ -76,20 +468,19 @@ format_timescaledb_mismatch_help() { if [ -z "$pg_major" ]; then tag_suffix="pgNN (replace NN with your PostgreSQL major version)" fi + local target_series="${tgt_ver%.*}" + local compat_tag="${tag_suffix}-all" + if timescaledb_version_is_safe "$tgt_ver"; then + compat_tag="${tag_suffix}-ts${target_series}-all" + fi printf '%s\n' \ "TimescaleDB version mismatch for database '$dbname':" \ " this backup was taken with timescaledb $src_ver" \ " but THIS server has timescaledb $tgt_display" \ +"Automatic compatibility conversion could not be completed." \ "The restore was stopped BEFORE changing anything - your current data is untouched." \ -"" \ -"To fix, on THIS server (the one you are restoring to):" \ -" 1. Run: $app_name edit" \ -" 2. Set the timescaledb image to the backup's version:" \ -" image: timescale/timescaledb:${src_ver}-${tag_suffix}" \ -" 3. Reset ONLY this server's database volume (do NOT run this on your main server):" \ -" rm -rf /var/lib/postgresql/pasarguard" \ -" 4. Restart: $app_name restart" \ -" 5. Run the restore again." +"Required compatibility image: timescale/timescaledb-ha:${compat_tag}" \ +"Check Docker connectivity and the restore log, then run '$app_name restore' again." } # Restore every database listed in /manifest.tsv. Globals are @@ -105,6 +496,9 @@ pg_restore_all_user_databases() { local admin_password="$5" local pg_dump_dir="$6" local log_file="$7" + local source_app_database="${8:-}" + local target_app_database="${9:-}" + local target_app_owner="${10:-}" local manifest="$pg_dump_dir/manifest.tsv" if [ ! -s "$manifest" ]; then @@ -114,8 +508,14 @@ pg_restore_all_user_databases() { if [ -s "$pg_dump_dir/globals.sql" ]; then colorized_echo blue "Restoring global roles and grants..." + local filtered_globals="$pg_dump_dir/globals.no-passwords.sql" + if ! pg_filter_globals_for_destination "$admin_user" <"$pg_dump_dir/globals.sql" >"$filtered_globals"; then + echo "Could not remove archived role passwords from globals.sql" >>"$log_file" + return 1 + fi docker exec -i -e PGPASSWORD="$admin_password" "$container_name" \ - psql -U "$admin_user" -d postgres < "$pg_dump_dir/globals.sql" >>"$log_file" 2>&1 || true + psql -X -U "$admin_user" -d postgres <"$filtered_globals" >>"$log_file" 2>&1 || true + rm -f "$filtered_globals" fi local total=0 ok=0 @@ -131,28 +531,33 @@ pg_restore_all_user_databases() { continue fi - local db_ident="${dbname//\"/\"\"}" - local db_sql="${dbname//\'/\'\'}" - local owner_ident="${owner//\"/\"\"}" + local destination_dbname="$dbname" + local destination_owner="$owner" + if [ -n "$source_app_database" ] && [ "$dbname" = "$source_app_database" ]; then + destination_dbname="${target_app_database:-$dbname}" + destination_owner="${target_app_owner:-$owner}" + fi + + local db_ident="${destination_dbname//\"/\"\"}" + local db_sql="${destination_dbname//\'/\'\'}" + local owner_ident="${destination_owner//\"/\"\"}" [ -n "$owner_ident" ] || owner_ident="$admin_user" # TimescaleDB cross-version safety gate. If this backup recorded a # timescaledb version, refuse to touch the database unless THIS server's # bundled version matches. Runs BEFORE any terminate/DROP so a mismatch - # never wipes or half-restores data. (Empty ts_version = legacy backup = - # no gate; the single-DB legacy path is unaffected.) + # never wipes or half-restores data. The preflight rejects versionless + # Timescale manifests before this function is called. if [ "$has_ts" = "1" ] && [ -n "$ts_version" ]; then # Read THIS server's bundled timescaledb version (read-only). Separate # a probe that FAILED (transient docker/psql error -> non-zero exit) # from one that SUCCEEDED but returned nothing (the target genuinely # has no timescaledb available). Only a successful probe gates the - # restore: a failed probe falls back to best-effort (we cannot gate on - # information we do not have), while a successful empty result is a - # real mismatch (target lacks the extension) and is skipped before any - # destructive step. + # restore: both a failed probe and a successful empty result fail + # closed before any destructive step. local target_ts="" probe_ok=0 if target_ts=$(docker exec -e PGPASSWORD="$admin_password" "$container_name" \ - psql -U "$admin_user" -d postgres -At \ + psql -X -U "$admin_user" -d postgres -At \ -c "SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb';" \ 2>>"$log_file"); then probe_ok=1 @@ -162,19 +567,20 @@ pg_restore_all_user_databases() { if [ "$probe_ok" = "1" ] && ! timescaledb_version_matches "$ts_version" "$target_ts"; then local svn="" pg_major="" svn=$(docker exec -e PGPASSWORD="$admin_password" "$container_name" \ - psql -U "$admin_user" -d postgres -At -c "SHOW server_version_num;" \ + psql -X -U "$admin_user" -d postgres -At -c "SHOW server_version_num;" \ 2>>"$log_file") || svn="" [ -n "$svn" ] && pg_major=$(( svn / 10000 )) colorized_echo red "$(format_timescaledb_mismatch_help "$dbname" "$ts_version" "$target_ts" "$pg_major" "${APP_NAME:-pasarguard}")" echo "TimescaleDB version mismatch for '$dbname' (backup=$ts_version target=${target_ts:-unavailable}); skipped before any destructive change" >>"$log_file" continue elif [ "$probe_ok" != "1" ]; then - colorized_echo yellow "Could not read this server's timescaledb version for '$dbname'; the cross-version safety check was skipped and restore will proceed best-effort." - echo "Could not read target timescaledb version for '$dbname'; proceeding best-effort (probe failed)" >>"$log_file" + colorized_echo red "Could not verify this server's TimescaleDB version for '$dbname'; skipping it before any destructive change." + echo "Could not read target timescaledb version for '$dbname'; skipped before destructive change" >>"$log_file" + continue fi fi - colorized_echo blue "Restoring database '$dbname'..." + colorized_echo blue "Restoring database '$dbname' as '$destination_dbname'..." docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -U "$admin_user" -d postgres \ -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$db_sql' AND pid <> pg_backend_pid();" \ >>"$log_file" 2>&1 @@ -189,32 +595,50 @@ pg_restore_all_user_databases() { local restored=false if [ "$has_ts" = "1" ]; then - docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -U "$admin_user" --dbname="$dbname" \ - -c "CREATE EXTENSION IF NOT EXISTS timescaledb;" >>"$log_file" 2>&1 - docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -U "$admin_user" --dbname="$dbname" \ - -c "SELECT timescaledb_pre_restore();" >>"$log_file" 2>&1 + local create_extension_sql="CREATE EXTENSION IF NOT EXISTS timescaledb;" + if [ -n "$ts_version" ]; then + create_extension_sql="CREATE EXTENSION IF NOT EXISTS timescaledb VERSION '$ts_version';" + fi + if ! docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ + -c "$create_extension_sql" >>"$log_file" 2>&1 || + ! docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ + -c "SELECT timescaledb_pre_restore();" >>"$log_file" 2>&1; then + colorized_echo red "Could not prepare TimescaleDB database '$destination_dbname'." + continue + fi local filtered="$pg_dump_dir/${filename}.filtered" pg_filter_timescaledb_extension_lines < "$dump_path" > "$filtered" 2>>"$log_file" if docker exec -i -e PGPASSWORD="$restore_password" "$container_name" \ - psql -v ON_ERROR_STOP=1 -U "$restore_user" --dbname="$dbname" < "$filtered" >>"$log_file" 2>&1; then + psql -X -v ON_ERROR_STOP=1 -U "$restore_user" --dbname="$destination_dbname" < "$filtered" >>"$log_file" 2>&1; then restored=true elif docker exec -i -e PGPASSWORD="$admin_password" "$container_name" \ - psql -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$dbname" < "$filtered" >>"$log_file" 2>&1; then + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" < "$filtered" >>"$log_file" 2>&1; then restored=true fi rm -f "$filtered" - docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -U "$admin_user" --dbname="$dbname" \ + docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -X -U "$admin_user" --dbname="$destination_dbname" \ -c "SELECT timescaledb_post_restore();" >>"$log_file" 2>&1 else if docker exec -i -e PGPASSWORD="$restore_password" "$container_name" \ - psql -v ON_ERROR_STOP=1 -U "$restore_user" --dbname="$dbname" < "$dump_path" >>"$log_file" 2>&1; then + psql -X -v ON_ERROR_STOP=1 -U "$restore_user" --dbname="$destination_dbname" < "$dump_path" >>"$log_file" 2>&1; then restored=true elif docker exec -i -e PGPASSWORD="$admin_password" "$container_name" \ - psql -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$dbname" < "$dump_path" >>"$log_file" 2>&1; then + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" < "$dump_path" >>"$log_file" 2>&1; then restored=true fi fi + if [ "$restored" = true ] && [ "$destination_owner" != "$owner" ] && [ -n "$owner" ] && [ -n "$destination_owner" ]; then + local source_owner_ident="${owner//\"/\"\"}" + local target_owner_ident="${destination_owner//\"/\"\"}" + if ! docker exec -e PGPASSWORD="$admin_password" "$container_name" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ + -c "REASSIGN OWNED BY \"$source_owner_ident\" TO \"$target_owner_ident\";" >>"$log_file" 2>&1; then + echo "Could not reassign '$destination_dbname' from '$owner' to '$destination_owner'" >>"$log_file" + restored=false + fi + fi + if [ "$restored" = true ]; then colorized_echo green "Database '$dbname' restored." ok=$((ok + 1)) @@ -252,7 +676,13 @@ restore_command() { local current_db_name="" local current_sqlalchemy_url="" local current_mysql_root_password="" + local requested_timescaledb_backup_version="${TIMESCALEDB_BACKUP_VERSION:-}" + local requested_timescaledb_compat_image="${TIMESCALEDB_COMPAT_IMAGE:-}" local sqlite_basename="" + local sqlite_backup_source="" + local sqlite_safety_backup="" + local restore_timestamp="" + restore_timestamp=$(date +%Y%m%d%H%M%S) redact_database_url() { local url="$1" @@ -298,6 +728,7 @@ restore_command() { local backup_dir="$APP_DIR/backup" local restore_staging_root="" local temp_restore_dir="" + local current_compose_snapshot="" # Check if backup directory exists if [ ! -d "$backup_dir" ]; then @@ -318,6 +749,8 @@ restore_command() { exit 1 fi + current_compose_snapshot="$temp_restore_dir/.pasarguard-destination-compose.yml" + local log_file="${temp_restore_dir}/pasarguard_restore_error.log" >"$log_file" echo "Restore Log - $(date)" >>"$log_file" @@ -684,16 +1117,11 @@ restore_command() { if [[ "$SQLALCHEMY_DATABASE_URL" =~ ^sqlite ]]; then db_type="sqlite" colorized_echo green "✓ Detected SQLite database" - local sqlite_url_part="${SQLALCHEMY_DATABASE_URL#*://}" - sqlite_url_part="${sqlite_url_part%%\?*}" - sqlite_url_part="${sqlite_url_part%%#*}" - - if [[ "$sqlite_url_part" =~ ^//(.*)$ ]]; then - sqlite_file="/${BASH_REMATCH[1]}" - elif [[ "$sqlite_url_part" =~ ^/(.*)$ ]]; then - sqlite_file="/${BASH_REMATCH[1]}" - else - sqlite_file="$sqlite_url_part" + if ! sqlite_file=$(sqlite_database_path_from_url "$SQLALCHEMY_DATABASE_URL") || [ -z "$sqlite_file" ]; then + colorized_echo red "Invalid SQLite SQLALCHEMY_DATABASE_URL in backup; expected sqlite[+driver]:// followed by a database path." + echo "Invalid SQLite SQLALCHEMY_DATABASE_URL: $(redact_database_url "$SQLALCHEMY_DATABASE_URL")" >>"$log_file" + rm -rf "$temp_restore_dir" + exit 1 fi colorized_echo blue "Database file: $sqlite_file" elif [[ "$SQLALCHEMY_DATABASE_URL" =~ ^(mysql|mariadb|postgresql)[^:]*:// ]]; then @@ -816,39 +1244,53 @@ restore_command() { case $db_type in sqlite) sqlite_basename=$(basename "$sqlite_file") - local backup_source="" if [ -f "$temp_restore_dir/$sqlite_basename" ]; then - backup_source="$temp_restore_dir/$sqlite_basename" + sqlite_backup_source="$temp_restore_dir/$sqlite_basename" elif [ -f "$temp_restore_dir/db_backup.sqlite" ]; then - backup_source="$temp_restore_dir/db_backup.sqlite" + sqlite_backup_source="$temp_restore_dir/db_backup.sqlite" fi - if [ -z "$backup_source" ]; then + if [ -z "$sqlite_backup_source" ]; then colorized_echo red "SQLite backup file not found in backup archive (looked for $sqlite_basename or db_backup.sqlite)." rm -rf "$temp_restore_dir" exit 1 fi - rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" 2>>"$log_file" || true - - if [ -f "$sqlite_file" ]; then - cp "$sqlite_file" "${sqlite_file}.backup.$(date +%Y%m%d%H%M%S)" 2>>"$log_file" + if ! command -v sqlite3 >/dev/null 2>&1; then + detect_os + try_install_package sqlite3 || true fi - - if cp "$backup_source" "$sqlite_file" 2>>"$log_file"; then - colorized_echo green "SQLite database restored successfully." - else - colorized_echo red "Failed to restore SQLite database." - echo "SQLite restore failed" >>"$log_file" + if ! command -v sqlite3 >/dev/null 2>&1; then + colorized_echo red "sqlite3 is required to validate the SQLite snapshot before restore. Install sqlite3 and run the restore again." + echo "sqlite3 unavailable; cannot validate $sqlite_backup_source" >>"$log_file" + rm -rf "$temp_restore_dir" + exit 1 + fi + if ! sqlite_snapshot_looks_restorable "$sqlite_backup_source"; then + colorized_echo red "SQLite backup is corrupt or incomplete; aborting before replacing the current database." + echo "SQLite snapshot validation failed for $sqlite_backup_source" >>"$log_file" rm -rf "$temp_restore_dir" exit 1 fi + + if [ -f "$sqlite_file" ]; then + sqlite_safety_backup="$backup_dir/sqlite_before_restore_${restore_timestamp}_${sqlite_basename}" + if ! sqlite3 "$sqlite_file" ".backup '$sqlite_safety_backup'" >>"$log_file" 2>&1; then + colorized_echo red "Failed to create a safety snapshot of the current SQLite database; restore aborted." + echo "SQLite safety snapshot failed: $sqlite_file -> $sqlite_safety_backup" >>"$log_file" + rm -f "$sqlite_safety_backup" + rm -rf "$temp_restore_dir" + exit 1 + fi + colorized_echo blue "Current SQLite database saved to $sqlite_safety_backup" + fi ;; mariadb|mysql) - if [ ! -f "$temp_restore_dir/db_backup.sql" ]; then - colorized_echo red "Database backup file not found in backup archive." + if ! mysql_dump_looks_restorable "$temp_restore_dir/db_backup.sql"; then + colorized_echo red "Database backup is missing, truncated, or invalid; aborting before restore." + echo "MySQL/MariaDB dump validation failed for $temp_restore_dir/db_backup.sql" >>"$log_file" rm -rf "$temp_restore_dir" exit 1 fi @@ -885,27 +1327,28 @@ restore_command() { local backup_restore_password="${db_password:-${DB_PASSWORD:-}}" local app_db_target="${current_db_name:-${db_name:-}}" - # Try root password from backup .env first - if [ -n "${MYSQL_ROOT_PASSWORD:-}" ]; then - colorized_echo blue "Trying root user from backup .env..." - if docker exec -i -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$container_name" "$mysql_cmd" -u root < "$temp_restore_dir/db_backup.sql" 2>>"$log_file"; then + # The destination root password is authoritative. The archived + # value belongs to the source server and is only a legacy + # fallback when the current installation did not provide one. + if [ -n "$current_mysql_root_password" ]; then + colorized_echo blue "Trying root user from current installation .env..." + if docker exec -i -e MYSQL_PWD="$current_mysql_root_password" "$container_name" "$mysql_cmd" -u root < "$temp_restore_dir/db_backup.sql" 2>>"$log_file"; then restore_success=true colorized_echo green "$db_type_name database restored successfully." else - colorized_echo yellow "Root restore failed with backup .env credentials, trying fallback..." - echo "$db_type_name restore failed with backup MYSQL_ROOT_PASSWORD" >>"$log_file" + colorized_echo yellow "Root restore failed with current .env credentials, trying fallback..." + echo "$db_type_name restore failed with current MYSQL_ROOT_PASSWORD" >>"$log_file" fi fi - # If root password changed after backup, try current installation value - if [ "$restore_success" = false ] && [ -n "$current_mysql_root_password" ] && [ "$current_mysql_root_password" != "${MYSQL_ROOT_PASSWORD:-}" ]; then - colorized_echo blue "Trying root user from current installation .env..." - if docker exec -i -e MYSQL_PWD="$current_mysql_root_password" "$container_name" "$mysql_cmd" -u root < "$temp_restore_dir/db_backup.sql" 2>>"$log_file"; then + if [ "$restore_success" = false ] && [ -z "$current_mysql_root_password" ] && [ -n "${MYSQL_ROOT_PASSWORD:-}" ]; then + colorized_echo blue "No destination root password was found; trying the backup .env value..." + if docker exec -i -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$container_name" "$mysql_cmd" -u root < "$temp_restore_dir/db_backup.sql" 2>>"$log_file"; then restore_success=true colorized_echo green "$db_type_name database restored successfully." else - colorized_echo yellow "Root restore failed with current .env credentials, trying app user fallback..." - echo "$db_type_name restore failed with current MYSQL_ROOT_PASSWORD" >>"$log_file" + colorized_echo yellow "Backup root credentials failed, trying app user fallback..." + echo "$db_type_name restore failed with backup MYSQL_ROOT_PASSWORD" >>"$log_file" fi fi @@ -962,8 +1405,31 @@ restore_command() { local pg_layout pg_layout=$(pg_backup_layout "$temp_restore_dir") + if [ "$db_type" = "timescaledb" ] && [ "$pg_layout" = "single" ]; then + if ! pg_promote_timescaledb_single_backup \ + "$temp_restore_dir" "$db_name" "${db_user:-${DB_USER:-postgres}}" "$log_file" \ + "$requested_timescaledb_backup_version"; then + colorized_echo red "This TimescaleDB backup does not record its source extension version; restore stopped before changing the current database." + colorized_echo yellow "For a legacy archive, set TIMESCALEDB_BACKUP_VERSION to its exact source version and run restore again." + start_pasarguard_app_services + rm -rf "$temp_restore_dir" + exit 1 + fi + pg_layout="multi" + colorized_echo blue "Prepared versioned TimescaleDB metadata for the single-database backup." + fi + if [ "$pg_layout" = "none" ]; then colorized_echo red "Database backup not found in backup archive." + start_pasarguard_app_services + rm -rf "$temp_restore_dir" + exit 1 + fi + + if [ "$pg_layout" = "multi" ] && ! postgres_backup_looks_restorable "$temp_restore_dir" "$db_name"; then + colorized_echo red "Multi-database backup is incomplete or does not contain the configured database; aborting before restore." + echo "Multi-database dump validation failed for $temp_restore_dir/pg_dump" >>"$log_file" + start_pasarguard_app_services rm -rf "$temp_restore_dir" exit 1 fi @@ -972,6 +1438,7 @@ restore_command() { # Verify backup file is not empty and is readable if [ ! -s "$temp_restore_dir/db_backup.sql" ]; then colorized_echo red "Database backup file is empty or unreadable." + start_pasarguard_app_services rm -rf "$temp_restore_dir" exit 1 fi @@ -982,6 +1449,7 @@ restore_command() { if ! postgres_dump_looks_restorable "$temp_restore_dir/db_backup.sql"; then colorized_echo red "Database backup does not look like a valid SQL dump; aborting before any changes." echo "Dump content validation failed for $temp_restore_dir/db_backup.sql" >>"$log_file" + start_pasarguard_app_services rm -rf "$temp_restore_dir" exit 1 fi @@ -993,12 +1461,14 @@ restore_command() { if [[ "$db_host" == "127.0.0.1" || "$db_host" == "localhost" || "$db_host" == "::1" ]]; then if [ -z "$container_name" ]; then colorized_echo red "Error: Database container not found. Please start the DB container or specify a valid container name." + start_pasarguard_app_services rm -rf "$temp_restore_dir" exit 1 fi local verified_container=$(verify_and_start_container "$container_name" "$db_type") if [ -z "$verified_container" ]; then colorized_echo red "Failed to start database container. Please start it manually." + start_pasarguard_app_services rm -rf "$temp_restore_dir" exit 1 fi @@ -1015,6 +1485,7 @@ restore_command() { if [ -z "$restore_password" ]; then colorized_echo red "No database password found for restore." + start_pasarguard_app_services rm -rf "$temp_restore_dir" exit 1 fi @@ -1022,7 +1493,29 @@ restore_command() { local restore_success=false if [ "$pg_layout" = "multi" ]; then - if pg_restore_all_user_databases "$container_name" "$restore_user" "$restore_password" "$admin_user" "$admin_password" "$temp_restore_dir/pg_dump" "$log_file"; then + local prepared_pg_dump_dir="$temp_restore_dir/pg_dump" + local compat_restore_root="" + if ! compat_restore_root=$(mktemp -d "$temp_restore_dir/pasarguard_ts_compat.XXXXXX"); then + colorized_echo red "Could not create TimescaleDB compatibility staging directory." + start_pasarguard_app_services + rm -rf "$temp_restore_dir" + exit 1 + fi + if ! pg_prepare_timescaledb_compatible_dumps \ + "$container_name" "$admin_user" "$admin_password" \ + "$temp_restore_dir/pg_dump" "$compat_restore_root/pg_dump" \ + "$log_file" "$db_name" "$requested_timescaledb_compat_image"; then + colorized_echo red "TimescaleDB version compatibility preflight failed. The current database was not changed." + colorized_echo yellow "Check log file for details: $log_file" + start_pasarguard_app_services + rm -rf "$temp_restore_dir" + exit 1 + fi + prepared_pg_dump_dir="$PG_PREPARED_DUMP_DIR" + + if pg_restore_all_user_databases \ + "$container_name" "$restore_user" "$restore_password" "$admin_user" "$admin_password" \ + "$prepared_pg_dump_dir" "$log_file" "$db_name" "$restore_db_name" "${current_db_user:-$restore_user}"; then colorized_echo green "All $db_type databases restored successfully." restore_success=true else @@ -1155,16 +1648,45 @@ restore_command() { exit 1 fi if [ "$db_type" = "sqlite" ] && [ -n "${sqlite_file:-}" ]; then - rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" 2>>"$log_file" || true + rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" "${sqlite_file}-journal" 2>>"$log_file" || true fi colorized_echo green "Data directory restored to $DATA_DIR." else colorized_echo yellow "No pasarguard_data directory found in backup. Skipping data restore." fi + # The data directory in legacy archives may contain a raw SQLite main file + # and WAL. Apply the consistent snapshot only after that directory has been + # restored so the raw copy can never overwrite the authoritative backup. + if [ "$db_type" = "sqlite" ]; then + mkdir -p "$(dirname "$sqlite_file")" + rm -f "${sqlite_file}-wal" "${sqlite_file}-shm" "${sqlite_file}-journal" 2>>"$log_file" || true + if cp "$sqlite_backup_source" "$sqlite_file" 2>>"$log_file"; then + colorized_echo green "SQLite database restored successfully." + else + colorized_echo red "Failed to restore SQLite database." + echo "SQLite restore failed" >>"$log_file" + rm -rf "$temp_restore_dir" + exit 1 + fi + fi + # Restore app directory files (full app backup support) colorized_echo blue "Restoring app directory files..." if [ -d "$temp_restore_dir" ]; then + # Capture this only after archive extraction/DB restore and immediately + # before app-file sync. An archive member with the same internal helper + # name therefore cannot spoof the destination snapshot. Infrastructure + # belongs to the destination installation; restoring an old compose + # file could otherwise downgrade TimescaleDB again. + if [[ "$db_type" != "sqlite" ]] && [ -f "$COMPOSE_FILE" ]; then + if ! cp "$COMPOSE_FILE" "$current_compose_snapshot"; then + colorized_echo red "Failed to snapshot destination docker-compose.yml." + start_pasarguard_app_services + rm -rf "$temp_restore_dir" + exit 1 + fi + fi if ! command -v rsync >/dev/null 2>&1; then detect_os install_package rsync @@ -1174,7 +1696,11 @@ restore_command() { colorized_echo blue "Backing up current app directory before restore..." cp -r "$APP_DIR" "$APP_DIR.backup.$(date +%Y%m%d%H%M%S)" 2>>"$log_file" || true fi - if ! rsync -av --exclude 'pasarguard_data' --exclude 'db_backup.sql' --exclude 'db_backup.sqlite' --exclude "$sqlite_basename" \ + if ! rsync -av --exclude 'pasarguard_data' --exclude 'db_backup.sql' --exclude 'db_backup.sqlite' \ + --exclude 'db_backup.timescaledb-version' --exclude 'pg_dump' \ + --exclude '.pasarguard-destination-compose.yml' --exclude 'pasarguard_ts_compat.*' \ + --exclude '*_combined.zip' --exclude 'pasarguard_env_cleaned' \ + --exclude 'pasarguard_restore_error.log' --exclude "$sqlite_basename" \ "$temp_restore_dir/" "$APP_DIR/" >>"$log_file" 2>&1; then colorized_echo red "Failed to restore app directory files." echo "Failed to restore app directory files from $temp_restore_dir to $APP_DIR" >>"$log_file" @@ -1183,22 +1709,12 @@ restore_command() { fi fi - # Perform configuration adjustments (e.g. preserve credentials) + # Keep the destination database identity. Archived credentials describe the + # source server and must never replace credentials already provisioned on + # this installation, even if the literal values happen to compare equal. if [ -f "$APP_DIR/.env" ]; then - local preserve_db_credentials=false if [[ "$db_type" != "sqlite" ]]; then - if [ -n "$current_db_user" ] && [ -n "${DB_USER:-}" ] && [ "$current_db_user" != "$DB_USER" ]; then - preserve_db_credentials=true - elif [ -n "$current_db_name" ] && [ -n "${DB_NAME:-}" ] && [ "$current_db_name" != "$DB_NAME" ]; then - preserve_db_credentials=true - elif [ -n "$current_db_password" ] && [ -n "${DB_PASSWORD:-}" ] && [ "$current_db_password" != "$DB_PASSWORD" ]; then - preserve_db_credentials=true - elif [ -n "$current_mysql_root_password" ] && [ -n "${MYSQL_ROOT_PASSWORD:-}" ] && [ "$current_mysql_root_password" != "$MYSQL_ROOT_PASSWORD" ]; then - preserve_db_credentials=true - fi - fi - if [ "$preserve_db_credentials" = true ]; then - colorized_echo yellow "Database credentials in backup differ from current installation; preserving current database credentials." + colorized_echo blue "Preserving destination database credentials and connection URL." if [ -n "$current_mysql_root_password" ]; then replace_or_append_env_var "MYSQL_ROOT_PASSWORD" "$current_mysql_root_password" true "$ENV_FILE" fi @@ -1217,6 +1733,16 @@ restore_command() { fi fi + if [[ "$db_type" != "sqlite" ]] && [ -s "$current_compose_snapshot" ]; then + if ! cp "$current_compose_snapshot" "$COMPOSE_FILE"; then + colorized_echo red "Failed to preserve the destination docker-compose.yml." + start_pasarguard_app_services + rm -rf "$temp_restore_dir" + exit 1 + fi + colorized_echo blue "Preserved destination docker-compose.yml." + fi + # Clean up rm -rf "$temp_restore_dir" diff --git a/pasarguard.sh b/pasarguard.sh index 76909ff..47c6b67 100755 --- a/pasarguard.sh +++ b/pasarguard.sh @@ -1067,7 +1067,11 @@ install_pasarguard() { db_driver_scheme="sqlite" fi - sed -i "s~\(SQLALCHEMY_DATABASE_URL = \).*~\1\"${db_driver_scheme}:////${DATA_DIR}/db.sqlite3\"~" "$APP_DIR/.env" + if ! SQLALCHEMY_DATABASE_URL=$(sqlite_absolute_database_url "$db_driver_scheme" "$DATA_DIR/db.sqlite3"); then + colorized_echo red "Failed to build the SQLite database URL for $DATA_DIR/db.sqlite3" + exit 1 + fi + sed -i "s~\(SQLALCHEMY_DATABASE_URL = \).*~\1\"${SQLALCHEMY_DATABASE_URL}\"~" "$APP_DIR/.env" fi diff --git a/tests/backup_restore_roundtrip.sh b/tests/backup_restore_roundtrip.sh index 9054ca3..86653bc 100644 --- a/tests/backup_restore_roundtrip.sh +++ b/tests/backup_restore_roundtrip.sh @@ -42,6 +42,7 @@ CONTAINER_NAME="${APP_NAME}-${DB_TYPE}" MYSQL_ROOT_PASSWORD="rootpass" DB_USER="appuser" DB_PASSWORD="apppass" +CURRENT_DB_PASSWORD="currentpass" DB_NAME="appdb" EXPECTED_DB_VALUE="from_backup" EXPECTED_SENTINEL_VALUE="sentinel-before-backup" @@ -52,14 +53,35 @@ ORIGINAL_COMPOSE_SHA="" ORIGINAL_SENTINEL_SHA="" ORIGINAL_PAYLOAD_SHA="" ORIGINAL_SQLITE_DUMP_SHA="" +CURRENT_COMPOSE_SHA="" LATEST_BACKUP="" EXTRACTED_BACKUP_DIR="" COMBINED_BACKUP_ARCHIVE="" MULTIPART_SPLIT_SIZE_BYTES=2048 MULTIPART_SPLIT_THRESHOLD_BYTES=3072 +SQLITE_HOLDER_PID="" +SQLITE_HOLDER_READY="$WORK_DIR/sqlite-holder.ready" +SQLITE_HOLDER_STOP="$WORK_DIR/sqlite-holder.stop" + +stop_sqlite_holder() { + if [ -n "$SQLITE_HOLDER_PID" ] && kill -0 "$SQLITE_HOLDER_PID" 2>/dev/null; then + touch "$SQLITE_HOLDER_STOP" + local waited=0 + while kill -0 "$SQLITE_HOLDER_PID" 2>/dev/null && [ "$waited" -lt 50 ]; do + sleep 0.2 + waited=$((waited + 1)) + done + if kill -0 "$SQLITE_HOLDER_PID" 2>/dev/null; then + kill -TERM "$SQLITE_HOLDER_PID" 2>/dev/null || true + fi + wait "$SQLITE_HOLDER_PID" 2>/dev/null || true + fi + SQLITE_HOLDER_PID="" +} cleanup() { local exit_code=$? + stop_sqlite_holder if [ "$exit_code" -ne 0 ] && [ -d "$WORK_DIR" ]; then while IFS= read -r log_path; do [ -n "$log_path" ] || continue @@ -176,10 +198,43 @@ write_common_files() { fi } +write_stale_database_artifacts() { + case "$DB_TYPE" in + mysql) + printf '%s\n' \ + '-- MySQL dump 10.13 Distrib 8.0, for Linux (x86_64)' \ + 'CREATE TABLE stale_from_previous_restore (id INT);' \ + '-- Dump completed on 2000-01-01 00:00:00' >"$APP_DIR/db_backup.sql" + ;; + mariadb) + printf '%s\n' \ + '-- MariaDB dump 10.19 Distrib 10.11, for debian-linux-gnu (x86_64)' \ + 'CREATE TABLE stale_from_previous_restore (id INT);' \ + '-- Dump completed on 2000-01-01 00:00:00' >"$APP_DIR/db_backup.sql" + ;; + postgresql | timescaledb) + mkdir -p "$APP_DIR/pg_dump" + printf '%s\n' \ + '-- PostgreSQL database cluster dump' \ + '-- PostgreSQL database cluster dump complete' >"$APP_DIR/pg_dump/globals.sql" + printf '%s\n' \ + '-- PostgreSQL database dump' \ + 'CREATE TABLE stale_from_previous_restore (id integer);' \ + '-- PostgreSQL database dump complete' >"$APP_DIR/pg_dump/db-001.sql" + if [ "$DB_TYPE" = "timescaledb" ]; then + printf 'appdb\tappuser\t1\tdb-001.sql\t2.27.2\n' >"$APP_DIR/pg_dump/manifest.tsv" + else + printf 'appdb\tappuser\t0\tdb-001.sql\t\n' >"$APP_DIR/pg_dump/manifest.tsv" + fi + ;; + esac +} + write_sqlite_env() { cat >"$ENV_FILE" <"$DATA_DIR/db.sqlite3-journal" } setup_mysql_container() { @@ -339,29 +421,31 @@ mutate_mariadb_db() { setup_postgresql_container() { local image="$1" + local password="${2:-$DB_PASSWORD}" + local initial_value="${3:-$EXPECTED_DB_VALUE}" docker run -d --name "$CONTAINER_NAME" \ -e POSTGRES_USER="$DB_USER" \ - -e POSTGRES_PASSWORD="$DB_PASSWORD" \ + -e POSTGRES_PASSWORD="$password" \ -e POSTGRES_DB="$DB_NAME" \ "$image" >/dev/null - wait_for_command 30 docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" \ + wait_for_command 30 docker exec -e PGPASSWORD="$password" "$CONTAINER_NAME" \ pg_isready -U "$DB_USER" -d "$DB_NAME" if [ "$DB_TYPE" = "timescaledb" ]; then - docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" \ + docker exec -e PGPASSWORD="$password" "$CONTAINER_NAME" \ psql -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" \ -c "CREATE EXTENSION IF NOT EXISTS timescaledb;" fi - docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" \ + docker exec -e PGPASSWORD="$password" "$CONTAINER_NAME" \ psql -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" \ - -c "CREATE TABLE ci_roundtrip (id INT PRIMARY KEY, value TEXT NOT NULL); INSERT INTO ci_roundtrip (id, value) VALUES (1, '$EXPECTED_DB_VALUE');" + -c "CREATE TABLE ci_roundtrip (id INT PRIMARY KEY, value TEXT NOT NULL); INSERT INTO ci_roundtrip (id, value) VALUES (1, '$initial_value');" } postgres_query() { - docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" \ - psql -At -U "$DB_USER" -d "$DB_NAME" \ + docker exec -e PGPASSWORD="$CURRENT_DB_PASSWORD" "$CONTAINER_NAME" \ + psql -h 127.0.0.1 -At -U "$DB_USER" -d "$DB_NAME" \ -c "SELECT value FROM ci_roundtrip WHERE id = 1;" } @@ -372,16 +456,63 @@ mutate_postgres_db() { } mutate_files_after_backup() { - cat >"$ENV_FILE" <"$ENV_FILE" <"$COMPOSE_FILE" + else + local scheme="$DB_TYPE" + [ "$DB_TYPE" = "timescaledb" ] && scheme="postgresql" + cat >"$ENV_FILE" <"$DATA_DIR/sentinel.txt" printf 'mutated-payload-%s\n' "$DB_TYPE" >"$DATA_DIR/payload.bin" } +rotate_destination_credentials() { + case "$DB_TYPE" in + sqlite) + return 0 + ;; + mysql) + docker exec -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$CONTAINER_NAME" mysql -uroot \ + -e "ALTER USER '$DB_USER'@'%' IDENTIFIED BY '$CURRENT_DB_PASSWORD';" + ;; + mariadb) + docker exec -e MYSQL_PWD="$MYSQL_ROOT_PASSWORD" "$CONTAINER_NAME" mariadb -uroot \ + -e "ALTER USER '$DB_USER'@'%' IDENTIFIED BY '$CURRENT_DB_PASSWORD';" + ;; + postgresql) + docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" psql -U "$DB_USER" -d postgres \ + -c "ALTER ROLE \"$DB_USER\" PASSWORD '$CURRENT_DB_PASSWORD';" + ;; + timescaledb) + if [ -n "${TIMESCALE_TARGET_IMAGE:-}" ]; then + docker rm -f "$CONTAINER_NAME" >/dev/null + setup_postgresql_container "$TIMESCALE_TARGET_IMAGE" "$CURRENT_DB_PASSWORD" "mutated" + else + docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" psql -U "$DB_USER" -d postgres \ + -c "ALTER ROLE \"$DB_USER\" PASSWORD '$CURRENT_DB_PASSWORD';" + fi + ;; + esac +} + run_restore() { printf '1\nyes\n' | restore_command } @@ -447,11 +578,7 @@ verify_backup_archive_contents() { expected_files=$'.env\ndb_backup.sql\ndocker-compose.yml\npasarguard_data/\npasarguard_data/payload.bin\npasarguard_data/sentinel.txt' fi - if [ "$DB_TYPE" = "sqlite" ]; then - assert_zip_contains_required_files "$archive_to_verify" "$expected_files" - else - assert_zip_contains_exact_files "$archive_to_verify" "$expected_files" - fi + assert_zip_contains_exact_files "$archive_to_verify" "$expected_files" assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/.env" | awk '{print $1}')" "$ORIGINAL_ENV_SHA" "Backed up .env contents changed." assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/docker-compose.yml" | awk '{print $1}')" "$ORIGINAL_COMPOSE_SHA" "Backed up docker-compose.yml contents changed." assert_equals "$(sha256sum "$EXTRACTED_BACKUP_DIR/pasarguard_data/sentinel.txt" | awk '{print $1}')" "$ORIGINAL_SENTINEL_SHA" "Backed up sentinel.txt contents changed." @@ -460,9 +587,12 @@ verify_backup_archive_contents() { if [ "$DB_TYPE" = "sqlite" ]; then assert_sqlite_integrity "$EXTRACTED_BACKUP_DIR/$sqlite_basename" assert_equals "$(sqlite_dump_sha "$EXTRACTED_BACKUP_DIR/$sqlite_basename")" "$ORIGINAL_SQLITE_DUMP_SHA" "Backed up SQLite database logical contents changed." - if [ -f "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" ]; then - assert_sqlite_integrity "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" - assert_equals "$(sqlite_dump_sha "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename")" "$ORIGINAL_SQLITE_DUMP_SHA" "Archived SQLite data-dir database logical contents changed." + if [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/$sqlite_basename" ] || \ + [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-wal" ] || \ + [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-shm" ] || \ + [ -e "$EXTRACTED_BACKUP_DIR/pasarguard_data/${sqlite_basename}-journal" ]; then + printf 'SQLite database or WAL/SHM/journal leaked into the raw data-directory copy.\n' >&2 + exit 1 fi elif [ "$DB_TYPE" = "postgresql" ] || [ "$DB_TYPE" = "timescaledb" ]; then assert_file_contains "$EXTRACTED_BACKUP_DIR/pg_dump/db-001.sql" "ci_roundtrip" @@ -477,13 +607,36 @@ verify_restored_files() { restored_env_sha="$(sha256sum "$ENV_FILE" | awk '{print $1}')" restored_compose_sha="$(sha256sum "$COMPOSE_FILE" | awk '{print $1}')" - assert_equals "$restored_env_sha" "$ORIGINAL_ENV_SHA" ".env was not restored from backup." - assert_equals "$restored_compose_sha" "$ORIGINAL_COMPOSE_SHA" "docker-compose.yml was not restored from backup." + if [ "$DB_TYPE" = "sqlite" ]; then + assert_equals "$restored_env_sha" "$ORIGINAL_ENV_SHA" ".env was not restored from backup." + assert_equals "$restored_compose_sha" "$ORIGINAL_COMPOSE_SHA" "docker-compose.yml was not restored from backup." + else + assert_file_contains "$ENV_FILE" "RESTORE_TEST_FLAG=$EXPECTED_ENV_FLAG" + assert_file_contains "$ENV_FILE" "DB_USER=$DB_USER" + assert_file_contains "$ENV_FILE" "DB_PASSWORD=$CURRENT_DB_PASSWORD" + assert_file_contains "$ENV_FILE" "DB_NAME=$DB_NAME" + assert_file_contains "$ENV_FILE" "$DB_USER:$CURRENT_DB_PASSWORD@127.0.0.1" + assert_equals "$restored_compose_sha" "$CURRENT_COMPOSE_SHA" "Destination docker-compose.yml was replaced by the backup." + fi assert_equals "$(sha256sum "$DATA_DIR/sentinel.txt" | awk '{print $1}')" "$ORIGINAL_SENTINEL_SHA" "sentinel.txt was not restored from backup." assert_equals "$(sha256sum "$DATA_DIR/payload.bin" | awk '{print $1}')" "$ORIGINAL_PAYLOAD_SHA" "payload.bin was not restored from backup." if [ "$DB_TYPE" = "sqlite" ]; then assert_sqlite_integrity "$DATA_DIR/db.sqlite3" assert_equals "$(sqlite_dump_sha "$DATA_DIR/db.sqlite3")" "$ORIGINAL_SQLITE_DUMP_SHA" "SQLite database logical contents were not restored from backup." + if [ -e "$DATA_DIR/db.sqlite3-journal" ]; then + printf 'A stale SQLite rollback journal survived the restore.\n' >&2 + exit 1 + fi + + local sqlite_safety_backup="" + sqlite_safety_backup=$(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'sqlite_before_restore_*_db.sqlite3' | sort | tail -n 1) + if [ -z "$sqlite_safety_backup" ]; then + printf 'The pre-restore SQLite safety snapshot did not survive the data-directory restore.\n' >&2 + exit 1 + fi + assert_sqlite_integrity "$sqlite_safety_backup" + assert_equals "$(sqlite3 "$sqlite_safety_backup" 'SELECT value FROM ci_roundtrip WHERE id = 1;')" \ + "mutated" "Pre-restore SQLite safety snapshot did not preserve the replaced database." fi } @@ -571,7 +724,7 @@ prepare_case() { timescaledb) write_postgres_env write_timescaledb_compose - setup_postgresql_container timescale/timescaledb:latest-pg17 + setup_postgresql_container "${TIMESCALE_SOURCE_IMAGE:-timescale/timescaledb:latest-pg17}" ;; *) printf 'Unsupported database type: %s\n' "$DB_TYPE" >&2 @@ -579,6 +732,7 @@ prepare_case() { ;; esac + write_stale_database_artifacts record_original_file_hashes } @@ -601,6 +755,32 @@ verify_restored_database() { esac assert_equals "$restored_value" "$EXPECTED_DB_VALUE" "Database value was not restored from backup." + + case "$DB_TYPE" in + mysql) + docker exec -e MYSQL_PWD="$CURRENT_DB_PASSWORD" "$CONTAINER_NAME" mysql -h 127.0.0.1 -u "$DB_USER" "$DB_NAME" -e "SELECT 1;" >/dev/null + if docker exec -e MYSQL_PWD="$DB_PASSWORD" "$CONTAINER_NAME" mysql -h 127.0.0.1 -u "$DB_USER" "$DB_NAME" -e "SELECT 1;" >/dev/null 2>&1; then + printf 'Archived MySQL password still authenticates after restore.\n' >&2 + exit 1 + fi + ;; + mariadb) + docker exec -e MYSQL_PWD="$CURRENT_DB_PASSWORD" "$CONTAINER_NAME" mariadb -h 127.0.0.1 -u "$DB_USER" "$DB_NAME" -e "SELECT 1;" >/dev/null + if docker exec -e MYSQL_PWD="$DB_PASSWORD" "$CONTAINER_NAME" mariadb -h 127.0.0.1 -u "$DB_USER" "$DB_NAME" -e "SELECT 1;" >/dev/null 2>&1; then + printf 'Archived MariaDB password still authenticates after restore.\n' >&2 + exit 1 + fi + ;; + postgresql | timescaledb) + docker exec -e PGPASSWORD="$CURRENT_DB_PASSWORD" "$CONTAINER_NAME" \ + psql -h 127.0.0.1 -At -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1;" >/dev/null + if docker exec -e PGPASSWORD="$DB_PASSWORD" "$CONTAINER_NAME" \ + psql -h 127.0.0.1 -At -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1;" >/dev/null 2>&1; then + printf 'Archived PostgreSQL password still authenticates after restore.\n' >&2 + exit 1 + fi + ;; + esac } mutate_database_after_backup() { @@ -631,7 +811,9 @@ main() { prepare_case backup_command verify_backup_created + stop_sqlite_holder mutate_database_after_backup + rotate_destination_credentials mutate_files_after_backup run_restore verify_restored_files diff --git a/tests/unit_lib_common.sh b/tests/unit_lib_common.sh index e3043de..b6eaf42 100644 --- a/tests/unit_lib_common.sh +++ b/tests/unit_lib_common.sh @@ -82,6 +82,27 @@ assert_eq "$(stat -c '%a' "$secret_new")" "600" "harden_secret_file: appended co # Empty path is rejected without creating anything. if harden_secret_file ""; then fail "harden_secret_file: rejects empty path"; else pass "harden_secret_file: rejects empty path"; fi +# --- SQLite SQLAlchemy URL helpers --- +assert_eq "$(normalize_posix_path '/var/lib/pasarguard///')" "/var/lib/pasarguard" \ + "normalize_posix_path: removes repeated trailing slashes" +assert_eq "$(normalize_posix_path '//var//lib///pasarguard//db.sqlite3')" "/var/lib/pasarguard/db.sqlite3" \ + "normalize_posix_path: collapses duplicate interior and leading slashes" +assert_eq "$(sqlite_database_path_from_url 'sqlite:///db.sqlite3')" "db.sqlite3" \ + "sqlite_database_path_from_url: relative path" +assert_eq "$(sqlite_database_path_from_url 'sqlite+aiosqlite:////var/lib/pasarguard/db.sqlite3')" \ + "/var/lib/pasarguard/db.sqlite3" "sqlite_database_path_from_url: absolute path" +assert_eq "$(sqlite_database_path_from_url 'sqlite+aiosqlite://///var/lib/pasarguard/db.sqlite3')" \ + "/var/lib/pasarguard/db.sqlite3" "sqlite_database_path_from_url: legacy five-slash path" +assert_eq "$(sqlite_database_path_from_url 'sqlite:////var/lib/pasarguard/db.sqlite3?mode=ro#fragment')" \ + "/var/lib/pasarguard/db.sqlite3" "sqlite_database_path_from_url: strips query and fragment" +if sqlite_database_path_from_url 'sqlite:/var/lib/pasarguard/db.sqlite3' >/dev/null; then + fail "sqlite_database_path_from_url: malformed URL rejected" +else + pass "sqlite_database_path_from_url: malformed URL rejected" +fi +assert_eq "$(sqlite_absolute_database_url 'sqlite+aiosqlite' '/var/lib/pasarguard/db.sqlite3')" \ + "sqlite+aiosqlite:////var/lib/pasarguard/db.sqlite3" "sqlite_absolute_database_url: exactly four slashes" + echo "" echo "Results: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] || exit 1 diff --git a/tests/unit_pasarguard.sh b/tests/unit_pasarguard.sh index 6b8160c..032ecce 100644 --- a/tests/unit_pasarguard.sh +++ b/tests/unit_pasarguard.sh @@ -251,6 +251,147 @@ assert_eq "$_l_ts" "1" "manifest: legacy has_ts preserved" assert_false "manifest: rejects tab in ts_version" pg_manifest_encode "db" "o" "1" "f.sql" "$(printf '2\t7')" assert_false "manifest: rejects newline in ts_version" pg_manifest_encode "db" "o" "1" "f.sql" "$(printf '2\n7')" +# ----------------------------------------------------------------------- +# database backup completeness validation +# ----------------------------------------------------------------------- +DB_VALIDATION_DIR="$WORK_DIR/db-validation" +mkdir -p "$DB_VALIDATION_DIR" + +printf '%s\n' \ + '-- MySQL dump 10.13 Distrib 8.0.43' \ + 'CREATE TABLE users (id int);' \ + '-- Dump completed on 2026-08-01 12:00:00' >"$DB_VALIDATION_DIR/db_backup.sql" +assert_true "backup validation: complete MySQL dump accepted" \ + database_backup_looks_restorable mysql "$DB_VALIDATION_DIR" appdb "" +sed -i '$d' "$DB_VALIDATION_DIR/db_backup.sql" +assert_false "backup validation: truncated MySQL dump rejected" \ + database_backup_looks_restorable mysql "$DB_VALIDATION_DIR" appdb "" + +printf '%s\n' \ + '-- PostgreSQL database dump' \ + 'CREATE TABLE public.users (id integer);' \ + '-- PostgreSQL database dump complete' >"$DB_VALIDATION_DIR/db_backup.sql" +assert_true "backup validation: complete PostgreSQL single dump accepted" \ + database_backup_looks_restorable postgresql "$DB_VALIDATION_DIR" appdb "" +sed -i '$d' "$DB_VALIDATION_DIR/db_backup.sql" +assert_false "backup validation: truncated PostgreSQL single dump rejected" \ + database_backup_looks_restorable timescaledb "$DB_VALIDATION_DIR" appdb "" + +rm -f "$DB_VALIDATION_DIR/db_backup.sql" +mkdir -p "$DB_VALIDATION_DIR/pg_dump" +printf '%s\n' \ + '-- PostgreSQL database cluster dump' \ + 'CREATE ROLE appuser;' \ + '-- PostgreSQL database cluster dump complete' >"$DB_VALIDATION_DIR/pg_dump/globals.sql" +printf '%s\n' \ + '-- PostgreSQL database dump' \ + 'CREATE TABLE public.users (id integer);' \ + '-- PostgreSQL database dump complete' >"$DB_VALIDATION_DIR/pg_dump/db-001.sql" +printf '%s\n' "$(pg_manifest_encode appdb appuser 1 db-001.sql 2.27.2)" >"$DB_VALIDATION_DIR/pg_dump/manifest.tsv" +assert_true "backup validation: complete TimescaleDB multi dump accepted" \ + database_backup_looks_restorable timescaledb "$DB_VALIDATION_DIR" appdb "" +assert_false "backup validation: configured PostgreSQL database must be present" \ + database_backup_looks_restorable postgresql "$DB_VALIDATION_DIR" missingdb "" +printf '%s\n' "$(pg_manifest_encode appdb appuser 0 db-001.sql '')" >"$DB_VALIDATION_DIR/pg_dump/manifest.tsv" +assert_true "backup validation: PostgreSQL manifest with empty ts_version accepted" \ + database_backup_looks_restorable postgresql "$DB_VALIDATION_DIR" appdb "" + +SQLITE_VALIDATION_DIR="$WORK_DIR/sqlite-validation" +mkdir -p "$SQLITE_VALIDATION_DIR" +printf 'sqlite fixture\n' >"$SQLITE_VALIDATION_DIR/app.sqlite3" +sqlite3() { + [ "$2" = "PRAGMA quick_check;" ] || return 1 + printf 'ok\n' +} +assert_true "backup validation: SQLite quick_check success accepted" \ + database_backup_looks_restorable sqlite "$SQLITE_VALIDATION_DIR" "" /source/app.sqlite3 +sqlite3() { printf 'database disk image is malformed\n'; } +assert_false "backup validation: corrupt SQLite snapshot rejected" \ + database_backup_looks_restorable sqlite "$SQLITE_VALIDATION_DIR" "" /source/app.sqlite3 +unset -f sqlite3 + +# A multi-database PostgreSQL/TimescaleDB backup must be atomic. Previously the +# helper returned success as long as any one database dumped successfully. +MOCK_PG_FAIL_DB="" +docker() { + local joined="$*" + case "$joined" in + *" pg_dumpall "*) + printf '%s\n' '-- PostgreSQL database cluster dump' '-- PostgreSQL database cluster dump complete' + ;; + *"SELECT datname FROM pg_database"*) + printf '%s\n' appdb analytics + ;; + *"pg_get_userbyid"*) + printf 'appuser\n' + ;; + *"SELECT extversion FROM pg_extension"*) + return 0 + ;; + *" pg_dump "*) + if [ -n "$MOCK_PG_FAIL_DB" ] && [[ "$joined" == *" -d $MOCK_PG_FAIL_DB "* ]]; then + return 1 + fi + printf '%s\n' \ + '-- PostgreSQL database dump' \ + 'CREATE TABLE public.events (id integer);' \ + '-- PostgreSQL database dump complete' + ;; + *) + return 1 + ;; + esac +} + +PG_ATOMIC_DIR="$WORK_DIR/pg-atomic-success" +mkdir -p "$PG_ATOMIC_DIR" +assert_true "pg multi backup: every database plus configured database succeeds" \ + pg_dump_all_user_databases pg appuser pass "$PG_ATOMIC_DIR" "$WORK_DIR/pg-success.log" appdb +assert_true "pg multi backup: completed artifact passes final validation" \ + postgres_backup_looks_restorable "$PG_ATOMIC_DIR" appdb + +PG_PARTIAL_DIR="$WORK_DIR/pg-atomic-partial" +mkdir -p "$PG_PARTIAL_DIR" +MOCK_PG_FAIL_DB="analytics" +assert_false "pg multi backup: one failed database fails the whole operation" \ + pg_dump_all_user_databases pg appuser pass "$PG_PARTIAL_DIR" "$WORK_DIR/pg-partial.log" appdb +assert_false "pg multi backup: partial dump directory is removed" test -d "$PG_PARTIAL_DIR/pg_dump" + +PG_MISSING_DIR="$WORK_DIR/pg-atomic-missing-app" +mkdir -p "$PG_MISSING_DIR" +MOCK_PG_FAIL_DB="" +assert_false "pg multi backup: missing configured database fails the operation" \ + pg_dump_all_user_databases pg appuser pass "$PG_MISSING_DIR" "$WORK_DIR/pg-missing.log" missingdb +assert_false "pg multi backup: missing-app dump directory is removed" test -d "$PG_MISSING_DIR/pg_dump" +unset -f docker + +docker() { + if [[ "$*" == *"SELECT extversion FROM pg_extension"* ]]; then + printf '2.27.2\n' + return 0 + fi + return 1 +} +PG_SINGLE_META_DIR="$WORK_DIR/pg-single-metadata" +mkdir -p "$PG_SINGLE_META_DIR" +assert_true "single_ts backup: records extension version sidecar" \ + write_timescaledb_single_dump_version pg appuser pass appdb "$PG_SINGLE_META_DIR" "$WORK_DIR/pg-single-meta.log" +assert_eq "$(cat "$PG_SINGLE_META_DIR/db_backup.timescaledb-version")" "2.27.2" "single_ts backup: exact version persisted" +unset -f docker + +docker() { + if [[ "$*" == *"SELECT extversion FROM pg_extension"* ]]; then + return 0 + fi + return 1 +} +PG_SINGLE_NOEXT_DIR="$WORK_DIR/pg-single-no-extension" +mkdir -p "$PG_SINGLE_NOEXT_DIR" +assert_true "single_ts backup: extension absence is recorded" \ + write_timescaledb_single_dump_version pg appuser pass appdb "$PG_SINGLE_NOEXT_DIR" "$WORK_DIR/pg-single-meta.log" +assert_eq "$(cat "$PG_SINGLE_NOEXT_DIR/db_backup.timescaledb-version")" "none" "single_ts backup: no-extension sentinel persisted" +unset -f docker + # ----------------------------------------------------------------------- # get_acme_sh_binary # ----------------------------------------------------------------------- @@ -399,6 +540,35 @@ mkdir -p "$PGL_DIR/pg_dump" touch "$PGL_DIR/pg_dump/manifest.tsv" assert_eq "$(pg_backup_layout "$PGL_DIR")" "multi" "layout: manifest present -> multi (precedence)" +SINGLE_TS_DIR="$WORK_DIR/single-timescale" +mkdir -p "$SINGLE_TS_DIR" +printf '%s\n' \ + '-- PostgreSQL database dump' \ + 'CREATE TABLE metrics (time timestamptz);' \ + '-- PostgreSQL database dump complete' >"$SINGLE_TS_DIR/db_backup.sql" +printf '%s\n' '2.27.2' >"$SINGLE_TS_DIR/db_backup.timescaledb-version" +assert_true "single_ts: versioned dump promoted to manifest" \ + pg_promote_timescaledb_single_backup "$SINGLE_TS_DIR" "appdb" "appuser" "$WORK_DIR/promote.log" +assert_eq "$(cut -f5 "$SINGLE_TS_DIR/pg_dump/manifest.tsv")" "2.27.2" "single_ts: source version recorded" +assert_true "single_ts: promoted artifact validates" postgres_backup_looks_restorable "$SINGLE_TS_DIR" "appdb" + +NOEXT_TS_DIR="$WORK_DIR/single-timescale-no-extension" +mkdir -p "$NOEXT_TS_DIR" +cp "$SINGLE_TS_DIR/db_backup.sql" "$NOEXT_TS_DIR/db_backup.sql" +printf '%s\n' 'none' >"$NOEXT_TS_DIR/db_backup.timescaledb-version" +assert_true "single_ts: explicit no-extension dump promoted" \ + pg_promote_timescaledb_single_backup "$NOEXT_TS_DIR" "appdb" "appuser" "$WORK_DIR/promote.log" +assert_eq "$(cut -f3 "$NOEXT_TS_DIR/pg_dump/manifest.tsv")" "0" "single_ts: no-extension manifest uses PostgreSQL path" +assert_eq "$(cut -f5 "$NOEXT_TS_DIR/pg_dump/manifest.tsv")" "" "single_ts: no fake extension version recorded" +assert_true "single_ts: no-extension artifact validates" postgres_backup_looks_restorable "$NOEXT_TS_DIR" "appdb" + +UNVERSIONED_TS_DIR="$WORK_DIR/unversioned-timescale" +mkdir -p "$UNVERSIONED_TS_DIR" +cp "$SINGLE_TS_DIR/db_backup.sql" "$UNVERSIONED_TS_DIR/db_backup.sql" +unset TIMESCALEDB_BACKUP_VERSION 2>/dev/null || true +assert_false "single_ts: unversioned legacy dump fails safely" \ + pg_promote_timescaledb_single_backup "$UNVERSIONED_TS_DIR" "appdb" "appuser" "$WORK_DIR/promote.log" + # ----------------------------------------------------------------------- # pg_filter_timescaledb_extension_lines # ----------------------------------------------------------------------- @@ -410,12 +580,80 @@ _ts_out=$(printf '%s\n' \ _ts_expected=$(printf '%s\n' "CREATE TABLE foo (id int);" "INSERT INTO foo VALUES (1);") assert_eq "$_ts_out" "$_ts_expected" "ts_filter: removes timescaledb extension lines, keeps rest" +# ----------------------------------------------------------------------- +# pg_filter_global_passwords +# ----------------------------------------------------------------------- +_globals_out=$(printf '%s\n' \ + "CREATE ROLE appuser;" \ + "ALTER ROLE appuser WITH LOGIN PASSWORD 'SCRAM-SHA-256\$4096:salt\$stored:server' VALID UNTIL 'infinity';" \ + "ALTER ROLE disabled PASSWORD NULL NOSUPERUSER;" \ + "GRANT appuser TO postgres;" | pg_filter_global_passwords) +_globals_expected=$(printf '%s\n' \ + "CREATE ROLE appuser;" \ + "ALTER ROLE appuser WITH LOGIN VALID UNTIL 'infinity';" \ + "ALTER ROLE disabled NOSUPERUSER;" \ + "GRANT appuser TO postgres;") +assert_eq "$_globals_out" "$_globals_expected" "global_filter: strips password verifiers and keeps role attributes/grants" +assert_false "global_filter: no PASSWORD keyword remains" grep -qE '[[:space:]]PASSWORD[[:space:]]' <<<"$_globals_out" + +_destination_globals_out=$(printf '%s\n' \ + "CREATE ROLE appuser;" \ + "ALTER ROLE appuser WITH NOSUPERUSER PASSWORD 'old-password';" \ + "CREATE ROLE worker;" \ + "ALTER ROLE worker WITH LOGIN PASSWORD 'worker-password';" \ + "GRANT worker TO appuser;" | pg_filter_globals_for_destination "appuser") +_destination_globals_expected=$(printf '%s\n' \ + "CREATE ROLE worker;" \ + "ALTER ROLE worker WITH LOGIN;" \ + "GRANT worker TO appuser;") +assert_eq "$_destination_globals_out" "$_destination_globals_expected" "global_filter: preserves the destination role definition and grants" + +_quoted_destination_globals_out=$(printf '%s\n' \ + "CREATE ROLE \"app user\";" \ + "ALTER ROLE \"app user\" WITH NOSUPERUSER PASSWORD 'old-password';" \ + "CREATE ROLE worker;" \ + "GRANT worker TO \"app user\";" | pg_filter_globals_for_destination "app user") +_quoted_destination_globals_expected=$(printf '%s\n' \ + "CREATE ROLE worker;" \ + "GRANT worker TO \"app user\";") +assert_eq "$_quoted_destination_globals_out" "$_quoted_destination_globals_expected" "global_filter: preserves quoted destination role with spaces" + # ----------------------------------------------------------------------- # timescaledb_version_matches # ----------------------------------------------------------------------- assert_true "ts_match: equal" timescaledb_version_matches "2.27.2" "2.27.2" assert_false "ts_match: differ" timescaledb_version_matches "2.27.2" "2.15.0" assert_false "ts_match: source vs empty" timescaledb_version_matches "2.27.2" "" +assert_true "ts_version: release is safe" timescaledb_version_is_safe "2.28.3" +assert_true "ts_version: prerelease is safe" timescaledb_version_is_safe "2.28.0-rc1" +assert_false "ts_version: SQL is rejected" timescaledb_version_is_safe "2.28'; DROP DATABASE appdb; --" +assert_false "ts_version: image fragment is rejected" timescaledb_version_is_safe "2.28/evil" + +TS_PREP_DIR="$WORK_DIR/ts-prepare/source/pg_dump" +mkdir -p "$TS_PREP_DIR" +printf '%s\n' '-- PostgreSQL database cluster dump complete' >"$TS_PREP_DIR/globals.sql" +printf '%s\n' 'CREATE TABLE metrics (id int);' '-- PostgreSQL database dump complete' >"$TS_PREP_DIR/db-001.sql" +printf 'appdb\tappuser\t1\tdb-001.sql\t2.28.3\n' >"$TS_PREP_DIR/manifest.tsv" +MOCK_COMPAT_PULL=false +docker() { + case "$*" in + *"default_version FROM pg_available_extensions"*) printf '2.28.3\n' ;; + *"SHOW server_version_num"*) printf '170010\n' ;; + *" pull "*) MOCK_COMPAT_PULL=true; return 1 ;; + *) return 1 ;; + esac +} +assert_true "ts_prepare: matching destination needs no conversion" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +assert_eq "$PG_PREPARED_DUMP_DIR" "$TS_PREP_DIR" "ts_prepare: original validated dump retained" +assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: compatibility image not pulled for matching versions" + +printf 'appdb\tappuser\t1\tdb-001.sql\t\n' >"$TS_PREP_DIR/manifest.tsv" +assert_false "ts_prepare: versionless Timescale manifest fails closed" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +unset -f docker # ----------------------------------------------------------------------- # format_timescaledb_mismatch_help @@ -424,11 +662,10 @@ contains() { [[ "$1" == *"$2"* ]]; } _help=$(format_timescaledb_mismatch_help "pasarguard" "2.27.2" "2.15.0" "17" "pasarguard") assert_true "help: shows source version" contains "$_help" "timescaledb 2.27.2" assert_true "help: shows target version" contains "$_help" "timescaledb 2.15.0" -assert_true "help: exact image tag" contains "$_help" "timescale/timescaledb:2.27.2-pg17" +assert_true "help: compatibility image" contains "$_help" "timescale/timescaledb-ha:pg17-ts2.15-all" assert_true "help: data untouched" contains "$_help" "untouched" -assert_true "help: this-server warning" contains "$_help" "do NOT run this on your main server" -assert_true "help: uses edit subcommand" contains "$_help" "pasarguard edit" -assert_true "help: uses restart subcmd" contains "$_help" "pasarguard restart" +assert_true "help: automatic conversion" contains "$_help" "Automatic compatibility conversion" +assert_true "help: retries restore" contains "$_help" "pasarguard restore" _help2=$(format_timescaledb_mismatch_help "db" "2.27.2" "" "" "pasarguard") assert_true "help: target not installed" contains "$_help2" "not installed" diff --git a/tests/unit_restore_archive_safety.sh b/tests/unit_restore_archive_safety.sh index ee225be..552b17d 100644 --- a/tests/unit_restore_archive_safety.sh +++ b/tests/unit_restore_archive_safety.sh @@ -71,9 +71,14 @@ CREATE TABLE public.users (id integer NOT NULL); COPY public.users (id) FROM stdin; 1 \. +-- PostgreSQL database dump complete EOF assert_true "postgres_dump_looks_restorable: real dump accepted" postgres_dump_looks_restorable "$good_dump" +truncated_dump="$WORK_DIR/truncated.sql" +sed '$d' "$good_dump" >"$truncated_dump" +assert_false "postgres_dump_looks_restorable: missing completion marker rejected" postgres_dump_looks_restorable "$truncated_dump" + empty_dump="$WORK_DIR/empty.sql"; : > "$empty_dump" assert_false "postgres_dump_looks_restorable: empty file rejected" postgres_dump_looks_restorable "$empty_dump" @@ -86,6 +91,22 @@ assert_false "postgres_dump_looks_restorable: no DDL/data rejected" postgres_dum assert_false "postgres_dump_looks_restorable: missing file rejected" postgres_dump_looks_restorable "$WORK_DIR/nope.sql" +good_globals="$WORK_DIR/globals.sql" +printf '%s\n' '-- PostgreSQL database cluster dump' 'CREATE ROLE appuser;' '-- PostgreSQL database cluster dump complete' >"$good_globals" +assert_true "postgres_globals_dump_looks_complete: completed dump accepted" postgres_globals_dump_looks_complete "$good_globals" +sed '$d' "$good_globals" >"$WORK_DIR/truncated-globals.sql" +assert_false "postgres_globals_dump_looks_complete: truncated dump rejected" postgres_globals_dump_looks_complete "$WORK_DIR/truncated-globals.sql" + +good_mysql="$WORK_DIR/mysql.sql" +printf '%s\n' '-- MySQL dump 10.13 Distrib 8.0.43' 'CREATE TABLE users (id int);' '-- Dump completed on 2026-08-01 12:00:00' >"$good_mysql" +assert_true "mysql_dump_looks_restorable: completed MySQL dump accepted" mysql_dump_looks_restorable "$good_mysql" +sed '$d' "$good_mysql" >"$WORK_DIR/truncated-mysql.sql" +assert_false "mysql_dump_looks_restorable: truncated MySQL dump rejected" mysql_dump_looks_restorable "$WORK_DIR/truncated-mysql.sql" + +good_mariadb="$WORK_DIR/mariadb.sql" +printf '%s\n' '-- MariaDB dump 10.19 Distrib 11.8.2-MariaDB' 'CREATE TABLE users (id int);' '-- Dump completed on 2026-08-01 12:00:00' >"$good_mariadb" +assert_true "mysql_dump_looks_restorable: completed MariaDB dump accepted" mysql_dump_looks_restorable "$good_mariadb" + echo "" echo "Results: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] || exit 1