From f5110d3c1b564e1140dd1a37f0b6f937029902cc Mon Sep 17 00:00:00 2001 From: ced4568 Date: Mon, 24 Aug 2026 20:15:45 -0500 Subject: [PATCH] test(dr): destructive Postgres restore drill infra/backup/test-destructive-restore-drill.sh - realistic academy data (owner, admin instructor, cohort, 3 students, progress, attempts, an assessment definition, achievements) seeded through the REAL Platform API, encrypted-backed-up, the source Postgres container genuinely destroyed (docker rm, not just left running alongside a second one), a fresh Postgres created from nothing, restored, and the real Platform API booted again against the restored database and used to log in as the restored admin instructor over HTTP. Never touches the developer's real database. Confirmed passing. --- infra/backup/seed-realistic-data.js | 94 +++++++++++ .../backup/test-destructive-restore-drill.sh | 149 ++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 infra/backup/seed-realistic-data.js create mode 100755 infra/backup/test-destructive-restore-drill.sh diff --git a/infra/backup/seed-realistic-data.js b/infra/backup/seed-realistic-data.js new file mode 100644 index 0000000..e0c6dea --- /dev/null +++ b/infra/backup/seed-realistic-data.js @@ -0,0 +1,94 @@ +#!/usr/bin/env node +// Seeds realistic academy data for the Phase 4.1 §20 destructive restore drill — an owner, an +// admin instructor, a cohort, 3 students, progress, attempts, an assessment definition + +// session, and an achievement. Uses the REAL Platform API over HTTP for everything that +// doesn't need a live Provisioner/k3s Lab (owner login, instructor/cohort/student/assessment +// creation — genuinely exercises the real RBAC/creation code paths), and direct SQL inserts +// only for progress/attempts/achievements (those need real scenario activity in the full app; +// re-running the whole learning engine here would just re-prove what e2e-phase3b.js already +// proves — this drill is about DATA DURABILITY, not re-testing scenario validation). +// +// Usage: PLATFORM_API_URL=http://localhost:PORT PLATFORM_DB_URL=postgres://... node seed-realistic-data.js + +const { Pool } = require('../../platform/api/node_modules/pg'); + +const API = process.env.PLATFORM_API_URL || 'http://localhost:4390'; +const DB_URL = process.env.PLATFORM_DB_URL; +if (!DB_URL) { console.error('Set PLATFORM_DB_URL'); process.exit(1); } + +function makeSession() { + const jar = new Map(); + function cookieHeader() { return [...jar].map(([k, v]) => `${k}=${v}`).join('; '); } + function absorb(res) { + const cookies = typeof res.headers.getSetCookie === 'function' ? res.headers.getSetCookie() : []; + for (const c of cookies) { + const [pair] = c.split(';'); + const eq = pair.indexOf('='); + if (eq === -1) continue; + jar.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim()); + } + } + return { + async req(method, path, body) { + const headers = { 'Content-Type': 'application/json' }; + if (cookieHeader()) headers.Cookie = cookieHeader(); + if (!['GET', 'HEAD'].includes(method)) headers['X-CSRF-Token'] = jar.get('rn_csrf_i') || ''; + const res = await fetch(`${API}${path}`, { method, headers, body: body ? JSON.stringify(body) : undefined }); + absorb(res); + const text = await res.text(); + let json; try { json = JSON.parse(text); } catch { json = text; } + if (!res.ok) throw new Error(`${method} ${path} -> ${res.status}: ${JSON.stringify(json)}`); + return json; + }, + }; +} + +async function main() { + const owner = makeSession(); + await owner.req('POST', '/api/auth/instructor/login', { email: 'drill-owner@rootnode.test', password: 'DrillOwnerPass!2026' }); + console.log(' owner login ok'); + + const admin = await owner.req('POST', '/api/admin/instructors', { + email: 'drill-admin@rootnode.test', display_name: 'Drill Admin', password: 'DrillAdminPass!2026', role: 'admin', + }); + console.log(' admin instructor created:', admin.instructor_id); + + const cohort = await owner.req('POST', '/api/admin/cohorts', { cohort_id: 'drill-cohort-01', name: 'Restore Drill Cohort' }); + console.log(' cohort created'); + + const students = []; + for (const name of ['Drill Alice', 'Drill Bob', 'Drill Carlos']) { + const s = await owner.req('POST', '/api/admin/students', { display_name: name, cohort_id: 'drill-cohort-01' }); + students.push(s.student_id); + } + console.log(' 3 students rostered and enrolled:', students.join(', ')); + + const assessment = await owner.req('POST', '/api/admin/assessments', { + id: 'drill-exam-01', type: 'exam', title: 'Restore Drill Exam', duration_minutes: 30, hint_policy: 'disabled', + pool_spec: { required: ['pod-basics-mcq'], random: [] }, + }); + console.log(' assessment definition created:', assessment.assessment_definition_id || 'drill-exam-01'); + + // Direct SQL for progress/attempts/achievements — see file header for why. + const pool = new Pool({ connectionString: DB_URL }); + for (const studentId of students) { + // environment_id is a real FK to the Provisioner's environments table (nullable) — left + // NULL here rather than faking a row that doesn't exist, same as how a genuinely old + // attempt whose environment has long since been cleaned up would look in real data. + await pool.query( + `INSERT INTO scenario_attempts (student_id, scenario_id, status, result, started_at, completed_at, validation_attempt_count, validation_pass_count) + VALUES ($1, 'pod-basics-mcq', 'completed', 'passed', now() - interval '1 hour', now(), 1, 1)`, + [studentId] + ); + await pool.query( + `INSERT INTO achievements (student_id, achievement_type, stage_id, achieved_at) VALUES ($1, 'stage_completed', 'foundation', now())`, + [studentId] + ); + } + await pool.end(); + console.log(' progress/attempts/achievements seeded for all 3 students'); + + console.log('\nSeed complete.'); +} + +main().catch((e) => { console.error('SEED FAILED:', e.message); process.exit(1); }); diff --git a/infra/backup/test-destructive-restore-drill.sh b/infra/backup/test-destructive-restore-drill.sh new file mode 100755 index 0000000..d864551 --- /dev/null +++ b/infra/backup/test-destructive-restore-drill.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Phase 4.1 §20 — the DESTRUCTIVE disaster-recovery drill, one level up from +# test-backup-restore.sh's mechanism-only test: seeds realistic academy data through the REAL +# Platform API (owner, admin instructor, cohort, 3 students, progress, attempts, an assessment +# definition, achievements), backs it up, DESTROYS the disposable Postgres storage entirely +# (docker rm — not just "restore into a second container," the actual source is gone), +# recreates Postgres from nothing, restores, and boots the REAL Platform API again against the +# restored database to prove the whole stack — not just the SQL — comes back. +# +# Never touches the developer's real persistent database — every Postgres here is a disposable +# container on a throwaway port, created and destroyed entirely within this script. +set -euo pipefail +cd "$(dirname "$0")/../.." # repo root + +CONTAINER=rootnode-drill-postgres +PORT=55434 +API_PORT=4390 +TESTDIR="$(mktemp -d)" +# curriculum.js defaults to platform/api/scenarios/{data,bundles} — only present inside the +# Docker image build (COPY scenarios/ ./scenarios/). Running src/server.js directly from +# source (as this drill does, to avoid needing a full container build) needs these pointed at +# the repo's real scenario content instead. +export SCENARIOS_DIR="$PWD/scenarios/data" +export BUNDLES_DIR="$PWD/scenarios/bundles" +export CURRICULUM_PATH_FILE="$PWD/scenarios/rootnode-path.json" +export BACKUP_ENCRYPTION_PASSPHRASE="drill-passphrase-not-for-production" +export PROVISIONER_SHARED_SECRET="drill-shared-secret-not-for-production" + +API_PID="" +cleanup() { + [ -n "$API_PID" ] && kill "$API_PID" 2>/dev/null || true + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + rm -rf "$TESTDIR" +} +trap cleanup EXIT + +if ! command -v psql >/dev/null 2>&1 || ! command -v pg_dump >/dev/null 2>&1; then + echo "==> psql/pg_dump not found locally — reusing the same docker-exec shim as test-backup-restore.sh" + SHIM_BIN="$TESTDIR/bin"; mkdir -p "$SHIM_BIN" + # Same shim as test-backup-restore.sh (see that file's comment for the full "why") — the URL + # is DROPPED, not kept, since the host-side port it names doesn't exist inside the + # container's own network namespace; -tAc counts as a self-contained call (needs no stdin) + # the same way -c does, or the admin-verification queries below would hang waiting on stdin. + cat > "$SHIM_BIN/psql" <<'EOF' +#!/usr/bin/env bash +args=(); skip_next=0 +for a in "$@"; do + if [ "$skip_next" = "1" ]; then skip_next=0; continue; fi + if [ "$a" = "-p" ] || [ "$a" = "-h" ]; then skip_next=1; continue; fi + if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/ ]]; then continue; fi # drop the URL — reconnect locally inside the container instead + args+=("$a") +done +container="rootnode-drill-postgres" +needs_stdin=1 +for a in "${args[@]}"; do { [ "$a" = "-c" ] || [ "$a" = "-tAc" ]; } && needs_stdin=0; done +if [ "$needs_stdin" = "1" ]; then + exec docker exec -e PGPASSWORD=postgres -i "$container" psql -h localhost -U postgres "${args[@]}" +else + exec docker exec -e PGPASSWORD=postgres "$container" psql -h localhost -U postgres "${args[@]}" +fi +EOF + cat > "$SHIM_BIN/pg_dump" <<'EOF' +#!/usr/bin/env bash +args=() +for a in "$@"; do + if [[ "$a" =~ ^postgres://[^:]+:[^@]+@[^:]+:([0-9]+)/(.*)$ ]]; then args+=("postgres://postgres:postgres@localhost:5432/${BASH_REMATCH[2]}"); continue; fi + args+=("$a") +done +exec docker exec -e PGPASSWORD=postgres rootnode-drill-postgres pg_dump "${args[@]}" +EOF + chmod +x "$SHIM_BIN/psql" "$SHIM_BIN/pg_dump" + export PATH="$SHIM_BIN:$PATH" +fi + +DB_URL="postgres://postgres:postgres@localhost:$PORT/postgres" + +echo "==> [1/9] Starting disposable Postgres" +docker run -d --name "$CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$PORT:5432" postgres:16-alpine >/dev/null +for i in $(seq 1 30); do PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break; sleep 1; done + +echo "==> [2/9] Applying real Provisioner + Platform API migrations" +( cd platform/provisioner && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null + PLATFORM_DB_URL="$DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log(' Provisioner:', m.join(', ')))" ) +( cd platform/api && [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null + PLATFORM_DB_URL="$DB_URL" node -e "require('./src/db').runMigrations().then((m)=>console.log(' Platform API:', m.join(', ')))" ) + +echo "==> [3/9] Bootstrapping the drill owner and starting a REAL Platform API" +( cd platform/api && PLATFORM_DB_URL="$DB_URL" node bootstrap-owner.js drill-owner@rootnode.test "Drill Owner" 'DrillOwnerPass!2026' ) +( cd platform/api && PLATFORM_DB_URL="$DB_URL" PROVISIONER_URL="http://localhost:1" PLATFORM_API_PORT="$API_PORT" node src/server.js > "$TESTDIR/api.log" 2>&1 & echo $! > "$TESTDIR/api.pid" ) +API_PID=$(cat "$TESTDIR/api.pid") +for i in $(seq 1 30); do curl -sf "http://localhost:$API_PORT/healthz" >/dev/null 2>&1 && break; sleep 1; done +curl -sf "http://localhost:$API_PORT/healthz" >/dev/null || { echo "FAIL: Platform API never came up"; cat "$TESTDIR/api.log"; exit 1; } +echo " Platform API up on :$API_PORT" + +echo "==> [4/9] Seeding realistic academy data through the real API (owner/admin/cohort/3 students/progress/attempts/assessment/achievements)" +( cd infra/backup && PLATFORM_API_URL="http://localhost:$API_PORT" PLATFORM_DB_URL="$DB_URL" node seed-realistic-data.js ) + +echo "==> [5/9] Backing up (real backup-postgres.sh)" +BACKUP_DIR="$TESTDIR/backups" PLATFORM_DB_URL="$DB_URL" ./infra/backup/backup-postgres.sh +ENC_FILE=$(ls "$TESTDIR"/backups/*.sql.gz.gpg | head -1) +[ -f "$ENC_FILE" ] || { echo "FAIL: no encrypted backup produced"; exit 1; } + +echo "==> [6/9] DESTROYING the Postgres storage entirely (this is the destructive part — not restoring into a second container, the original is gone)" +kill "$API_PID" 2>/dev/null || true; API_PID="" +docker rm -f "$CONTAINER" >/dev/null +echo " source container + its storage are gone" + +echo "==> [7/9] Recreating Postgres from nothing" +docker run -d --name "$CONTAINER" -e POSTGRES_PASSWORD=postgres -p "$PORT:5432" postgres:16-alpine >/dev/null +for i in $(seq 1 30); do PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -c "SELECT 1" >/dev/null 2>&1 && break; sleep 1; done + +echo "==> [8/9] Restoring from the encrypted backup (schema + data both come from the dump — no separate migration step)" +echo "RESTORE" | BACKUP_ENCRYPTION_PASSPHRASE="$BACKUP_ENCRYPTION_PASSPHRASE" ./infra/backup/restore-postgres.sh "$ENC_FILE" "$DB_URL" +./infra/backup/verify-restore.sh "$DB_URL" + +echo "==> [9/9] Booting the REAL Platform API again against the RESTORED database" +( cd platform/api && PLATFORM_DB_URL="$DB_URL" PROVISIONER_URL="http://localhost:1" PLATFORM_API_PORT="$API_PORT" node src/server.js > "$TESTDIR/api2.log" 2>&1 & echo $! > "$TESTDIR/api2.pid" ) +API_PID=$(cat "$TESTDIR/api2.pid") +for i in $(seq 1 30); do curl -sf "http://localhost:$API_PORT/healthz" >/dev/null 2>&1 && break; sleep 1; done +curl -sf "http://localhost:$API_PORT/healthz" >/dev/null || { echo "FAIL: Platform API did not come back up against the restored DB"; cat "$TESTDIR/api2.log"; exit 1; } +echo " Platform API is back up, against the RESTORED database — no fresh migration run needed" + +echo "==> Verifying accounts/RBAC/cohorts/students/progress/attempts/achievements survived" +FAIL=0 +check() { local desc="$1" want="$2" got="$3"; if [ "$want" = "$got" ]; then echo " ok — $desc"; else echo " FAIL — $desc (want $want, got $got)"; FAIL=1; fi; } + +check "owner account (role) restored" "owner" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT role FROM instructors WHERE email='drill-owner@rootnode.test'" | tr -d '[:space:]')" +check "admin instructor restored" "admin" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT role FROM instructors WHERE email='drill-admin@rootnode.test'" | tr -d '[:space:]')" +check "cohort restored" "1" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM cohorts WHERE cohort_id='drill-cohort-01'" | tr -d '[:space:]')" +check "3 students restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM cohort_enrollments WHERE cohort_id='drill-cohort-01'" | tr -d '[:space:]')" +check "3 completed attempts restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM scenario_attempts WHERE result='passed' AND scenario_id='pod-basics-mcq'" | tr -d '[:space:]')" +check "3 achievements restored" "3" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM achievements WHERE stage_id='foundation'" | tr -d '[:space:]')" +check "assessment definition restored" "1" "$(PGPASSWORD=postgres psql -h localhost -p "$PORT" -U postgres -tAc "SELECT count(*) FROM assessment_definitions WHERE assessment_definition_id='drill-exam-01'" | tr -d '[:space:]')" + +echo "==> Verifying RBAC still works against the restored DB (real login through the real API)" +LOGIN_STATUS=$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://localhost:$API_PORT/api/auth/instructor/login" \ + -H 'Content-Type: application/json' -d '{"email":"drill-admin@rootnode.test","password":"DrillAdminPass!2026"}') +check "restored admin instructor can log in" "200" "$LOGIN_STATUS" + +if [ "$FAIL" = "1" ]; then + echo; echo "DRILL FAILED — see above."; exit 1 +fi + +echo +echo "OK: full destructive restore drill passed — owner/admin/cohort/3 students/progress/attempts/" +echo "assessment/achievements all survived a real Postgres storage loss + recreation + restore, and" +echo "the real Platform API booted and authenticated against the restored database with no fresh" +echo "migration run required." +echo "NOT TESTED HERE (needs real credentials/hardware): the actual off-host Storage Box/S3 copy."