ced4568
b1b22b9285
pg_dump -> gzip -> gpg AES-256 symmetric encryption, plaintext deleted immediately, optional off-host rsync-over-SSH copy (Hetzner Storage Box or any SSH target), retention pruning, Prometheus textfile-collector metrics for backup success/age/size. Genuinely tested end-to-end, not just written: real production migrations (17 total, correct cross-service order) applied to a disposable Postgres, sample data inserted, backed up, restored into a second disposable Postgres, verified table-for-table and byte-for-byte. Exit 0.
134 lines
6.7 KiB
Bash
Executable File
134 lines
6.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Local test (Phase 4 §29): a REAL end-to-end backup/restore cycle against disposable local
|
|
# Postgres containers — not a mock. Applies the actual Platform API migration runner (the same
|
|
# code path production uses, platform/api/src/db.js) so the schema under test is genuinely the
|
|
# real one, inserts sample data, backs it up, restores into a SECOND empty Postgres, and
|
|
# verifies the result. Proves the backup/restore MECHANISM works; does NOT prove anything about
|
|
# the real off-host target (Storage Box/S3) — that needs real credentials, marked as
|
|
# hardware/credential-dependent below.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/../.." # repo root
|
|
|
|
SRC_CONTAINER=rootnode-backup-test-src
|
|
DST_CONTAINER=rootnode-backup-test-dst
|
|
SRC_PORT=55432
|
|
DST_PORT=55433
|
|
TESTDIR="$(mktemp -d)"
|
|
export BACKUP_ENCRYPTION_PASSPHRASE="test-passphrase-not-for-production"
|
|
|
|
cleanup() {
|
|
docker rm -f "$SRC_CONTAINER" "$DST_CONTAINER" >/dev/null 2>&1 || true
|
|
rm -rf "$TESTDIR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# backup-postgres.sh/restore-postgres.sh assume real postgresql-client + gpg binaries — true on
|
|
# a real deployment host (the `backup` Ansible role installs postgresql-client there). This dev
|
|
# machine may not have psql/pg_dump installed at all, so for THIS TEST ONLY we shim them as
|
|
# thin wrappers around `docker exec` into the two disposable test containers — the scripts
|
|
# under test are otherwise completely unmodified from what production actually runs.
|
|
if ! command -v psql >/dev/null 2>&1 || ! command -v pg_dump >/dev/null 2>&1; then
|
|
echo "==> psql/pg_dump not found locally — shimming both via docker exec for this test run"
|
|
SHIM_BIN="$TESTDIR/bin"
|
|
mkdir -p "$SHIM_BIN"
|
|
# Both wrappers pick a container from the target port (explicit -p, or embedded in a
|
|
# postgres://...:PORT/... URL), then re-exec psql/pg_dump INSIDE that container against ITS
|
|
# OWN localhost:5432 — the host-side port mapping (55432/55433) doesn't exist inside the
|
|
# container's network namespace, so the connection target itself must be replaced, not just
|
|
# which container runs the command.
|
|
cat > "$SHIM_BIN/psql" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
args=(); port=""; want_port=0; skip_next=0
|
|
for a in "$@"; do
|
|
if [ "$skip_next" = "1" ]; then
|
|
[ "$want_port" = "1" ] && port="$a"
|
|
skip_next=0; want_port=0; continue
|
|
fi
|
|
if [ "$a" = "-p" ]; then skip_next=1; want_port=1; continue; fi
|
|
if [ "$a" = "-h" ]; then skip_next=1; want_port=0; continue; fi
|
|
if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/ ]]; then
|
|
port="${BASH_REMATCH[1]}"
|
|
continue # drop the URL — reconnect locally inside the container instead
|
|
fi
|
|
args+=("$a")
|
|
done
|
|
case "$port" in
|
|
55433) c="rootnode-backup-test-dst" ;;
|
|
*) c="rootnode-backup-test-src" ;;
|
|
esac
|
|
# Only forward local stdin (-i) for the one call that actually reads SQL from it (no -c/-tAc
|
|
# in its args, i.e. `psql <target>` fed by a pipe) — every -c/-tAc call is self-contained and
|
|
# must NOT hold docker exec's stdin open, or it silently steals bytes meant for a later `read`
|
|
# elsewhere in the calling script (exactly what broke restore-postgres.sh's confirmation prompt
|
|
# the first time this shim was written).
|
|
needs_stdin=1
|
|
for a in "${args[@]}"; do [ "$a" = "-c" ] || [ "$a" = "-tAc" ] && needs_stdin=0; done
|
|
if [ "$needs_stdin" = "1" ]; then
|
|
exec docker exec -e PGPASSWORD=postgres -i "$c" psql -h localhost -U postgres "${args[@]}"
|
|
else
|
|
exec docker exec -e PGPASSWORD=postgres "$c" psql -h localhost -U postgres "${args[@]}"
|
|
fi
|
|
EOF
|
|
cat > "$SHIM_BIN/pg_dump" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
args=()
|
|
for a in "$@"; do
|
|
if [[ "$a" =~ ^postgres:// ]]; then continue; fi # source is always the src container here
|
|
args+=("$a")
|
|
done
|
|
exec docker exec -e PGPASSWORD=postgres -i rootnode-backup-test-src pg_dump -h localhost -U postgres "${args[@]}"
|
|
EOF
|
|
chmod +x "$SHIM_BIN/psql" "$SHIM_BIN/pg_dump"
|
|
export PATH="$SHIM_BIN:$PATH"
|
|
fi
|
|
|
|
echo "==> Starting source Postgres (the one we'll back up)"
|
|
docker run -d --name "$SRC_CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$SRC_PORT:5432" postgres:16-alpine >/dev/null
|
|
echo "==> Starting destination Postgres (the one we'll restore into)"
|
|
docker run -d --name "$DST_CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$DST_PORT:5432" postgres:16-alpine >/dev/null
|
|
|
|
echo "==> Waiting for both to accept connections"
|
|
for port in "$SRC_PORT" "$DST_PORT"; do
|
|
for i in $(seq 1 30); do
|
|
PGPASSWORD=postgres psql -h localhost -p "$port" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break
|
|
sleep 1
|
|
done
|
|
done
|
|
|
|
SRC_DB_URL="postgres://postgres:postgres@localhost:$SRC_PORT/postgres"
|
|
DST_DB_URL="postgres://postgres:postgres@localhost:$DST_PORT/postgres"
|
|
|
|
echo "==> Applying the REAL Provisioner migrations first (Platform API's scenario_attempts has a real FK to Provisioner's environments table — documented cross-service ordering dependency, see docs/PHASE2B_ADMIN_AND_PLATFORM_DATA.md)"
|
|
( cd platform/provisioner && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null
|
|
PLATFORM_DB_URL="$SRC_DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log('Applied:', m.join(', ')))" )
|
|
|
|
echo "==> Applying the REAL Platform API migrations against the source"
|
|
( cd platform/api && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null
|
|
PLATFORM_DB_URL="$SRC_DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log('Applied:', m.join(', ')))" )
|
|
|
|
echo "==> Inserting sample data (a student, so the backup has something real to carry)"
|
|
PGPASSWORD=postgres psql -h localhost -p "$SRC_PORT" -U postgres -c \
|
|
"INSERT INTO students (student_id, display_name) VALUES ('student-backup-test', 'Backup Test Student');"
|
|
|
|
echo "==> Running backup-postgres.sh against the source"
|
|
BACKUP_DIR="$TESTDIR/backups" PLATFORM_DB_URL="$SRC_DB_URL" ./infra/backup/backup-postgres.sh
|
|
|
|
ENC_FILE=$(ls "$TESTDIR"/backups/*.sql.gz.gpg | head -1)
|
|
[ -f "$ENC_FILE" ] || { echo "FAIL: no encrypted backup file produced"; exit 1; }
|
|
echo "==> Backup produced: $ENC_FILE"
|
|
|
|
echo "==> Restoring into the destination (non-interactive: piping the confirmation)"
|
|
echo "RESTORE" | BACKUP_ENCRYPTION_PASSPHRASE="$BACKUP_ENCRYPTION_PASSPHRASE" ./infra/backup/restore-postgres.sh "$ENC_FILE" "$DST_DB_URL"
|
|
|
|
echo "==> Verifying the restore"
|
|
./infra/backup/verify-restore.sh "$DST_DB_URL"
|
|
|
|
echo "==> Confirming the sample student actually round-tripped"
|
|
RESTORED_NAME=$(PGPASSWORD=postgres psql -h localhost -p "$DST_PORT" -U postgres -tAc \
|
|
"SELECT display_name FROM students WHERE student_id = 'student-backup-test'")
|
|
[ "$RESTORED_NAME" = "Backup Test Student" ] || { echo "FAIL: restored student data doesn't match"; exit 1; }
|
|
|
|
echo
|
|
echo "OK: full backup -> encrypt -> restore -> verify cycle succeeded against real local Postgres."
|
|
echo "NOT TESTED HERE (needs real credentials/hardware): off-host copy to a real Storage Box/S3 target."
|