ced4568
f5110d3c1b
infra/backup/test-destructive-restore-drill.sh - realistic academy data (owner, admin instructor, cohort, 3 students, progress, attempts, an assessment definition, achievements) seeded through the REAL Platform API, encrypted-backed-up, the source Postgres container genuinely destroyed (docker rm, not just left running alongside a second one), a fresh Postgres created from nothing, restored, and the real Platform API booted again against the restored database and used to log in as the restored admin instructor over HTTP. Never touches the developer's real database. Confirmed passing.
150 lines
9.7 KiB
Bash
Executable File
150 lines
9.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Phase 4.1 §20 — the DESTRUCTIVE disaster-recovery drill, one level up from
|
|
# test-backup-restore.sh's mechanism-only test: seeds realistic academy data through the REAL
|
|
# Platform API (owner, admin instructor, cohort, 3 students, progress, attempts, an assessment
|
|
# definition, achievements), backs it up, DESTROYS the disposable Postgres storage entirely
|
|
# (docker rm — not just "restore into a second container," the actual source is gone),
|
|
# recreates Postgres from nothing, restores, and boots the REAL Platform API again against the
|
|
# restored database to prove the whole stack — not just the SQL — comes back.
|
|
#
|
|
# Never touches the developer's real persistent database — every Postgres here is a disposable
|
|
# container on a throwaway port, created and destroyed entirely within this script.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/../.." # repo root
|
|
|
|
CONTAINER=rootnode-drill-postgres
|
|
PORT=55434
|
|
API_PORT=4390
|
|
TESTDIR="$(mktemp -d)"
|
|
# curriculum.js defaults to platform/api/scenarios/{data,bundles} — only present inside the
|
|
# Docker image build (COPY scenarios/ ./scenarios/). Running src/server.js directly from
|
|
# source (as this drill does, to avoid needing a full container build) needs these pointed at
|
|
# the repo's real scenario content instead.
|
|
export SCENARIOS_DIR="$PWD/scenarios/data"
|
|
export BUNDLES_DIR="$PWD/scenarios/bundles"
|
|
export CURRICULUM_PATH_FILE="$PWD/scenarios/rootnode-path.json"
|
|
export BACKUP_ENCRYPTION_PASSPHRASE="drill-passphrase-not-for-production"
|
|
export PROVISIONER_SHARED_SECRET="drill-shared-secret-not-for-production"
|
|
|
|
API_PID=""
|
|
cleanup() {
|
|
[ -n "$API_PID" ] && kill "$API_PID" 2>/dev/null || true
|
|
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
|
rm -rf "$TESTDIR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if ! command -v psql >/dev/null 2>&1 || ! command -v pg_dump >/dev/null 2>&1; then
|
|
echo "==> psql/pg_dump not found locally — reusing the same docker-exec shim as test-backup-restore.sh"
|
|
SHIM_BIN="$TESTDIR/bin"; mkdir -p "$SHIM_BIN"
|
|
# Same shim as test-backup-restore.sh (see that file's comment for the full "why") — the URL
|
|
# is DROPPED, not kept, since the host-side port it names doesn't exist inside the
|
|
# container's own network namespace; -tAc counts as a self-contained call (needs no stdin)
|
|
# the same way -c does, or the admin-verification queries below would hang waiting on stdin.
|
|
cat > "$SHIM_BIN/psql" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
args=(); skip_next=0
|
|
for a in "$@"; do
|
|
if [ "$skip_next" = "1" ]; then skip_next=0; continue; fi
|
|
if [ "$a" = "-p" ] || [ "$a" = "-h" ]; then skip_next=1; continue; fi
|
|
if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/ ]]; then continue; fi # drop the URL — reconnect locally inside the container instead
|
|
args+=("$a")
|
|
done
|
|
container="rootnode-drill-postgres"
|
|
needs_stdin=1
|
|
for a in "${args[@]}"; do { [ "$a" = "-c" ] || [ "$a" = "-tAc" ]; } && needs_stdin=0; done
|
|
if [ "$needs_stdin" = "1" ]; then
|
|
exec docker exec -e PGPASSWORD=postgres -i "$container" psql -h localhost -U postgres "${args[@]}"
|
|
else
|
|
exec docker exec -e PGPASSWORD=postgres "$container" psql -h localhost -U postgres "${args[@]}"
|
|
fi
|
|
EOF
|
|
cat > "$SHIM_BIN/pg_dump" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
args=()
|
|
for a in "$@"; do
|
|
if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/(.*)$ ]]; then args+=("postgres://postgres:postgres@localhost:5432/${BASH_REMATCH[2]}"); continue; fi
|
|
args+=("$a")
|
|
done
|
|
exec docker exec -e PGPASSWORD=postgres rootnode-drill-postgres pg_dump "${args[@]}"
|
|
EOF
|
|
chmod +x "$SHIM_BIN/psql" "$SHIM_BIN/pg_dump"
|
|
export PATH="$SHIM_BIN:$PATH"
|
|
fi
|
|
|
|
DB_URL="postgres://postgres:postgres@localhost:$PORT/postgres"
|
|
|
|
echo "==> [1/9] Starting disposable Postgres"
|
|
docker run -d --name "$CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$PORT:5432" postgres:16-alpine >/dev/null
|
|
for i in $(seq 1 30); do PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break; sleep 1; done
|
|
|
|
echo "==> [2/9] Applying real Provisioner + Platform API migrations"
|
|
( cd platform/provisioner && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null
|
|
PLATFORM_DB_URL="$DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log(' Provisioner:', m.join(', ')))" )
|
|
( cd platform/api && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null
|
|
PLATFORM_DB_URL="$DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log(' Platform API:', m.join(', ')))" )
|
|
|
|
echo "==> [3/9] Bootstrapping the drill owner and starting a REAL Platform API"
|
|
( cd platform/api && PLATFORM_DB_URL="$DB_URL" node bootstrap-owner.js [email protected] "Drill Owner" 'DrillOwnerPass!2026' )
|
|
( cd platform/api && PLATFORM_DB_URL="$DB_URL" PROVISIONER_URL="http://localhost:1" PLATFORM_API_PORT="$API_PORT" node src/server.js > "$TESTDIR/api.log" 2>&1 & echo $! > "$TESTDIR/api.pid" )
|
|
API_PID=$(cat "$TESTDIR/api.pid")
|
|
for i in $(seq 1 30); do curl -sf "http://localhost:$API_PORT/healthz" >/dev/null 2>&1 && break; sleep 1; done
|
|
curl -sf "http://localhost:$API_PORT/healthz" >/dev/null || { echo "FAIL: Platform API never came up"; cat "$TESTDIR/api.log"; exit 1; }
|
|
echo " Platform API up on :$API_PORT"
|
|
|
|
echo "==> [4/9] Seeding realistic academy data through the real API (owner/admin/cohort/3 students/progress/attempts/assessment/achievements)"
|
|
( cd infra/backup && PLATFORM_API_URL="http://localhost:$API_PORT" PLATFORM_DB_URL="$DB_URL" node seed-realistic-data.js )
|
|
|
|
echo "==> [5/9] Backing up (real backup-postgres.sh)"
|
|
BACKUP_DIR="$TESTDIR/backups" PLATFORM_DB_URL="$DB_URL" ./infra/backup/backup-postgres.sh
|
|
ENC_FILE=$(ls "$TESTDIR"/backups/*.sql.gz.gpg | head -1)
|
|
[ -f "$ENC_FILE" ] || { echo "FAIL: no encrypted backup produced"; exit 1; }
|
|
|
|
echo "==> [6/9] DESTROYING the Postgres storage entirely (this is the destructive part — not restoring into a second container, the original is gone)"
|
|
kill "$API_PID" 2>/dev/null || true; API_PID=""
|
|
docker rm -f "$CONTAINER" >/dev/null
|
|
echo " source container + its storage are gone"
|
|
|
|
echo "==> [7/9] Recreating Postgres from nothing"
|
|
docker run -d --name "$CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$PORT:5432" postgres:16-alpine >/dev/null
|
|
for i in $(seq 1 30); do PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break; sleep 1; done
|
|
|
|
echo "==> [8/9] Restoring from the encrypted backup (schema + data both come from the dump — no separate migration step)"
|
|
echo "RESTORE" | BACKUP_ENCRYPTION_PASSPHRASE="$BACKUP_ENCRYPTION_PASSPHRASE" ./infra/backup/restore-postgres.sh "$ENC_FILE" "$DB_URL"
|
|
./infra/backup/verify-restore.sh "$DB_URL"
|
|
|
|
echo "==> [9/9] Booting the REAL Platform API again against the RESTORED database"
|
|
( cd platform/api && PLATFORM_DB_URL="$DB_URL" PROVISIONER_URL="http://localhost:1" PLATFORM_API_PORT="$API_PORT" node src/server.js > "$TESTDIR/api2.log" 2>&1 & echo $! > "$TESTDIR/api2.pid" )
|
|
API_PID=$(cat "$TESTDIR/api2.pid")
|
|
for i in $(seq 1 30); do curl -sf "http://localhost:$API_PORT/healthz" >/dev/null 2>&1 && break; sleep 1; done
|
|
curl -sf "http://localhost:$API_PORT/healthz" >/dev/null || { echo "FAIL: Platform API did not come back up against the restored DB"; cat "$TESTDIR/api2.log"; exit 1; }
|
|
echo " Platform API is back up, against the RESTORED database — no fresh migration run needed"
|
|
|
|
echo "==> Verifying accounts/RBAC/cohorts/students/progress/attempts/achievements survived"
|
|
FAIL=0
|
|
check() { local desc="$1" want="$2" got="$3"; if [ "$want" = "$got" ]; then echo " ok — $desc"; else echo " FAIL — $desc (want $want, got $got)"; FAIL=1; fi; }
|
|
|
|
check "owner account (role) restored" "owner" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT role FROM instructors WHERE email='[email protected]'" | tr -d '[:space:]')"
|
|
check "admin instructor restored" "admin" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT role FROM instructors WHERE email='[email protected]'" | tr -d '[:space:]')"
|
|
check "cohort restored" "1" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM cohorts WHERE cohort_id='drill-cohort-01'" | tr -d '[:space:]')"
|
|
check "3 students restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM cohort_enrollments WHERE cohort_id='drill-cohort-01'" | tr -d '[:space:]')"
|
|
check "3 completed attempts restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM scenario_attempts WHERE result='passed' AND scenario_id='pod-basics-mcq'" | tr -d '[:space:]')"
|
|
check "3 achievements restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM achievements WHERE stage_id='foundation'" | tr -d '[:space:]')"
|
|
check "assessment definition restored" "1" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM assessment_definitions WHERE assessment_definition_id='drill-exam-01'" | tr -d '[:space:]')"
|
|
|
|
echo "==> Verifying RBAC still works against the restored DB (real login through the real API)"
|
|
LOGIN_STATUS=$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://localhost:$API_PORT/api/auth/instructor/login" \
|
|
-H 'Content-Type: application/json' -d '{"email":"[email protected]","password":"DrillAdminPass!2026"}')
|
|
check "restored admin instructor can log in" "200" "$LOGIN_STATUS"
|
|
|
|
if [ "$FAIL" = "1" ]; then
|
|
echo; echo "DRILL FAILED — see above."; exit 1
|
|
fi
|
|
|
|
echo
|
|
echo "OK: full destructive restore drill passed — owner/admin/cohort/3 students/progress/attempts/"
|
|
echo "assessment/achievements all survived a real Postgres storage loss + recreation + restore, and"
|
|
echo "the real Platform API booted and authenticated against the restored database with no fresh"
|
|
echo "migration run required."
|
|
echo "NOT TESTED HERE (needs real credentials/hardware): the actual off-host Storage Box/S3 copy."
|