#!/usr/bin/env bash # Phase 4.1 §20 — the DESTRUCTIVE disaster-recovery drill, one level up from # test-backup-restore.sh's mechanism-only test: seeds realistic academy data through the REAL # Platform API (owner, admin instructor, cohort, 3 students, progress, attempts, an assessment # definition, achievements), backs it up, DESTROYS the disposable Postgres storage entirely # (docker rm — not just "restore into a second container," the actual source is gone), # recreates Postgres from nothing, restores, and boots the REAL Platform API again against the # restored database to prove the whole stack — not just the SQL — comes back. # # Never touches the developer's real persistent database — every Postgres here is a disposable # container on a throwaway port, created and destroyed entirely within this script. set -euo pipefail cd "$(dirname "$0")/../.." # repo root CONTAINER=rootnode-drill-postgres PORT=55434 API_PORT=4390 TESTDIR="$(mktemp -d)" # curriculum.js defaults to platform/api/scenarios/{data,bundles} — only present inside the # Docker image build (COPY scenarios/ ./scenarios/). Running src/server.js directly from # source (as this drill does, to avoid needing a full container build) needs these pointed at # the repo's real scenario content instead. export SCENARIOS_DIR="$PWD/scenarios/data" export BUNDLES_DIR="$PWD/scenarios/bundles" export CURRICULUM_PATH_FILE="$PWD/scenarios/rootnode-path.json" export BACKUP_ENCRYPTION_PASSPHRASE="drill-passphrase-not-for-production" export PROVISIONER_SHARED_SECRET="drill-shared-secret-not-for-production" API_PID="" cleanup() { [ -n "$API_PID" ] && kill "$API_PID" 2>/dev/null || true docker rm -f "$CONTAINER" >/dev/null 2>&1 || true rm -rf "$TESTDIR" } trap cleanup EXIT if ! command -v psql >/dev/null 2>&1 || ! command -v pg_dump >/dev/null 2>&1; then echo "==> psql/pg_dump not found locally — reusing the same docker-exec shim as test-backup-restore.sh" SHIM_BIN="$TESTDIR/bin"; mkdir -p "$SHIM_BIN" # Same shim as test-backup-restore.sh (see that file's comment for the full "why") — the URL # is DROPPED, not kept, since the host-side port it names doesn't exist inside the # container's own network namespace; -tAc counts as a self-contained call (needs no stdin) # the same way -c does, or the admin-verification queries below would hang waiting on stdin. cat > "$SHIM_BIN/psql" <<'EOF' #!/usr/bin/env bash args=(); skip_next=0 for a in "$@"; do if [ "$skip_next" = "1" ]; then skip_next=0; continue; fi if [ "$a" = "-p" ] || [ "$a" = "-h" ]; then skip_next=1; continue; fi if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/ ]]; then continue; fi # drop the URL — reconnect locally inside the container instead args+=("$a") done container="rootnode-drill-postgres" needs_stdin=1 for a in "${args[@]}"; do { [ "$a" = "-c" ] || [ "$a" = "-tAc" ]; } && needs_stdin=0; done if [ "$needs_stdin" = "1" ]; then exec docker exec -e PGPASSWORD=postgres -i "$container" psql -h localhost -U postgres "${args[@]}" else exec docker exec -e PGPASSWORD=postgres "$container" psql -h localhost -U postgres "${args[@]}" fi EOF cat > "$SHIM_BIN/pg_dump" <<'EOF' #!/usr/bin/env bash args=() for a in "$@"; do if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/(.*)$ ]]; then args+=("postgres://postgres:postgres@localhost:5432/${BASH_REMATCH[2]}"); continue; fi args+=("$a") done exec docker exec -e PGPASSWORD=postgres rootnode-drill-postgres pg_dump "${args[@]}" EOF chmod +x "$SHIM_BIN/psql" "$SHIM_BIN/pg_dump" export PATH="$SHIM_BIN:$PATH" fi DB_URL="postgres://postgres:postgres@localhost:$PORT/postgres" echo "==> [1/9] Starting disposable Postgres" docker run -d --name "$CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$PORT:5432" postgres:16-alpine >/dev/null for i in $(seq 1 30); do PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break; sleep 1; done echo "==> [2/9] Applying real Provisioner + Platform API migrations" ( cd platform/provisioner && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null PLATFORM_DB_URL="$DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log(' Provisioner:', m.join(', ')))" ) ( cd platform/api && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null PLATFORM_DB_URL="$DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log(' Platform API:', m.join(', ')))" ) echo "==> [3/9] Bootstrapping the drill owner and starting a REAL Platform API" ( cd platform/api && PLATFORM_DB_URL="$DB_URL" node bootstrap-owner.js drill-owner@rootnode.test "Drill Owner" 'DrillOwnerPass!2026' ) ( cd platform/api && PLATFORM_DB_URL="$DB_URL" PROVISIONER_URL="http://localhost:1" PLATFORM_API_PORT="$API_PORT" node src/server.js > "$TESTDIR/api.log" 2>&1 & echo $! > "$TESTDIR/api.pid" ) API_PID=$(cat "$TESTDIR/api.pid") for i in $(seq 1 30); do curl -sf "http://localhost:$API_PORT/healthz" >/dev/null 2>&1 && break; sleep 1; done curl -sf "http://localhost:$API_PORT/healthz" >/dev/null || { echo "FAIL: Platform API never came up"; cat "$TESTDIR/api.log"; exit 1; } echo " Platform API up on :$API_PORT" echo "==> [4/9] Seeding realistic academy data through the real API (owner/admin/cohort/3 students/progress/attempts/assessment/achievements)" ( cd infra/backup && PLATFORM_API_URL="http://localhost:$API_PORT" PLATFORM_DB_URL="$DB_URL" node seed-realistic-data.js ) echo "==> [5/9] Backing up (real backup-postgres.sh)" BACKUP_DIR="$TESTDIR/backups" PLATFORM_DB_URL="$DB_URL" ./infra/backup/backup-postgres.sh ENC_FILE=$(ls "$TESTDIR"/backups/*.sql.gz.gpg | head -1) [ -f "$ENC_FILE" ] || { echo "FAIL: no encrypted backup produced"; exit 1; } echo "==> [6/9] DESTROYING the Postgres storage entirely (this is the destructive part — not restoring into a second container, the original is gone)" kill "$API_PID" 2>/dev/null || true; API_PID="" docker rm -f "$CONTAINER" >/dev/null echo " source container + its storage are gone" echo "==> [7/9] Recreating Postgres from nothing" docker run -d --name "$CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$PORT:5432" postgres:16-alpine >/dev/null for i in $(seq 1 30); do PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break; sleep 1; done echo "==> [8/9] Restoring from the encrypted backup (schema + data both come from the dump — no separate migration step)" echo "RESTORE" | BACKUP_ENCRYPTION_PASSPHRASE="$BACKUP_ENCRYPTION_PASSPHRASE" ./infra/backup/restore-postgres.sh "$ENC_FILE" "$DB_URL" ./infra/backup/verify-restore.sh "$DB_URL" echo "==> [9/9] Booting the REAL Platform API again against the RESTORED database" ( cd platform/api && PLATFORM_DB_URL="$DB_URL" PROVISIONER_URL="http://localhost:1" PLATFORM_API_PORT="$API_PORT" node src/server.js > "$TESTDIR/api2.log" 2>&1 & echo $! > "$TESTDIR/api2.pid" ) API_PID=$(cat "$TESTDIR/api2.pid") for i in $(seq 1 30); do curl -sf "http://localhost:$API_PORT/healthz" >/dev/null 2>&1 && break; sleep 1; done curl -sf "http://localhost:$API_PORT/healthz" >/dev/null || { echo "FAIL: Platform API did not come back up against the restored DB"; cat "$TESTDIR/api2.log"; exit 1; } echo " Platform API is back up, against the RESTORED database — no fresh migration run needed" echo "==> Verifying accounts/RBAC/cohorts/students/progress/attempts/achievements survived" FAIL=0 check() { local desc="$1" want="$2" got="$3"; if [ "$want" = "$got" ]; then echo " ok — $desc"; else echo " FAIL — $desc (want $want, got $got)"; FAIL=1; fi; } check "owner account (role) restored" "owner" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT role FROM instructors WHERE email='drill-owner@rootnode.test'" | tr -d '[:space:]')" check "admin instructor restored" "admin" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT role FROM instructors WHERE email='drill-admin@rootnode.test'" | tr -d '[:space:]')" check "cohort restored" "1" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM cohorts WHERE cohort_id='drill-cohort-01'" | tr -d '[:space:]')" check "3 students restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM cohort_enrollments WHERE cohort_id='drill-cohort-01'" | tr -d '[:space:]')" check "3 completed attempts restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM scenario_attempts WHERE result='passed' AND scenario_id='pod-basics-mcq'" | tr -d '[:space:]')" check "3 achievements restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM achievements WHERE stage_id='foundation'" | tr -d '[:space:]')" check "assessment definition restored" "1" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM assessment_definitions WHERE assessment_definition_id='drill-exam-01'" | tr -d '[:space:]')" echo "==> Verifying RBAC still works against the restored DB (real login through the real API)" LOGIN_STATUS=$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://localhost:$API_PORT/api/auth/instructor/login" \ -H 'Content-Type: application/json' -d '{"email":"drill-admin@rootnode.test","password":"DrillAdminPass!2026"}') check "restored admin instructor can log in" "200" "$LOGIN_STATUS" if [ "$FAIL" = "1" ]; then echo; echo "DRILL FAILED — see above."; exit 1 fi echo echo "OK: full destructive restore drill passed — owner/admin/cohort/3 students/progress/attempts/" echo "assessment/achievements all survived a real Postgres storage loss + recreation + restore, and" echo "the real Platform API booted and authenticated against the restored database with no fresh" echo "migration run required." echo "NOT TESTED HERE (needs real credentials/hardware): the actual off-host Storage Box/S3 copy."