test(preprod): two-host routing simulation, drill suite, CI security scanning
- infra/scripts/simulate-two-host/: real two-Docker-network simulation (Gateway genuinely isolated from any Lab bridge network) proving the Lab Host Router fix - 13/13 real assertions (HTTP+WebSocket through the Router, Alice/Bob isolation, garbage/destroyed tokens, mTLS-only Router access). - infra/scripts/drills/: Lab Host loss/rebuild (9/9), deployment rollback, CI-fails-closed (6 required failure modes), fake cohort dress rehearsal (14/14, one continuous narrative through the real app). - infra/ci/gitea-workflows/build-and-test.yml: new security-scan job (npm audit, gitleaks, hadolint, shellcheck, trivy - all free/open-source), severity policy in docs/SECURITY_SCANNING.md (CRITICAL/HIGH blocks, MEDIUM/LOW reported only). Gates build-images alongside regression. Found and fixed two real pre-existing bugs via the CI-fails-closed drill: staging-smoke-test.sh and production-smoke-test.sh both had a classic set -e gotcha (curl ... && echo "OK" as one statement - a failing command that isn't last in an && chain doesn't trigger -e) that let an unreachable/broken endpoint fall through silently instead of failing the smoke test. platform/test/e2e-phase3a.js updated for the new paginated /api/admin/audit-events response shape (Phase 4.1 pagination work).
This commit is contained in:
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Orchestrates the Phase 4.1 §27 fake cohort dress rehearsal: brings up the real dev stack,
|
||||
# bootstraps an owner, runs test-fake-cohort-rehearsal.js, tears down.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../../platform"
|
||||
|
||||
export PROVISIONER_SHARED_SECRET="${PROVISIONER_SHARED_SECRET:-rehearsal-shared-secret-not-for-production}"
|
||||
|
||||
docker compose -f docker-compose.dev.yml up -d --build
|
||||
|
||||
cleanup() {
|
||||
echo "==> Tearing down"
|
||||
docker ps -aq --filter "name=rootnode-lab-" | xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||
docker network ls --filter "name=rootnode-lab-net-" --format "{{.Name}}" | xargs -r -n1 docker network rm >/dev/null 2>&1 || true
|
||||
docker compose -f docker-compose.dev.yml down >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "==> Waiting for the stack"
|
||||
for _ in $(seq 1 60); do
|
||||
curl -sf http://localhost:4300/healthz >/dev/null 2>&1 && curl -sf http://localhost:4200/healthz >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "==> Bootstrapping the rehearsal owner"
|
||||
( cd api && PLATFORM_DB_URL=postgres://postgres:postgres@localhost:5433/rootnode_platform \
|
||||
node bootstrap-owner.js [email protected] "Rehearsal Owner" 'RehearsalPass!2026' )
|
||||
|
||||
echo "==> Running the rehearsal"
|
||||
OWNER_EMAIL=[email protected] OWNER_PASSWORD='RehearsalPass!2026' \
|
||||
node ../infra/scripts/drills/test-fake-cohort-rehearsal.js
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Orchestrates the Phase 4.1 §21 Lab Host loss/rebuild drill: brings up the real dev stack,
|
||||
# bootstraps an owner, runs test-lab-host-loss.js, tears down. See that file's header for what
|
||||
# it actually proves.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../../platform"
|
||||
|
||||
export PROVISIONER_SHARED_SECRET="${PROVISIONER_SHARED_SECRET:-drill-shared-secret-not-for-production}"
|
||||
|
||||
docker compose -f docker-compose.dev.yml up -d --build
|
||||
|
||||
cleanup() {
|
||||
echo "==> Tearing down"
|
||||
docker ps -aq --filter "name=rootnode-lab-" | xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||
docker network ls --filter "name=rootnode-lab-net-" --format "{{.Name}}" | xargs -r -n1 docker network rm >/dev/null 2>&1 || true
|
||||
docker compose -f docker-compose.dev.yml down >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "==> Waiting for the stack"
|
||||
for i in $(seq 1 60); do
|
||||
curl -sf http://localhost:4300/healthz >/dev/null 2>&1 && curl -sf http://localhost:4200/healthz >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "==> Bootstrapping the drill owner"
|
||||
( cd api && PLATFORM_DB_URL=postgres://postgres:postgres@localhost:5433/rootnode_platform \
|
||||
node bootstrap-owner.js [email protected] "LabHost Drill Owner" 'LabHostDrillPass!2026' )
|
||||
|
||||
echo "==> Running the drill"
|
||||
OWNER_EMAIL=[email protected] OWNER_PASSWORD='LabHostDrillPass!2026' \
|
||||
node ../infra/scripts/drills/test-lab-host-loss.js
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 4.1 §23 — proves the CI pipeline (infra/ci/gitea-workflows/build-and-test.yml) fails
|
||||
# CLOSED on every listed failure mode, i.e. the underlying command genuinely exits nonzero on
|
||||
# broken input, which is what makes a Gitea/GitHub Actions step (and therefore the job, and
|
||||
# therefore `needs: regression` gating build-images/deploy-staging) fail. Doesn't need a real
|
||||
# CI run — checks the actual commands the workflow calls, directly, the same way Actions would
|
||||
# invoke them.
|
||||
set -uo pipefail # NOT -e — this script's whole point is deliberately triggering failures and checking their exit codes
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../.." # repo root
|
||||
|
||||
FAIL=0
|
||||
expect_nonzero() {
|
||||
local desc="$1"; shift
|
||||
"$@" >/tmp/ci-fail-drill-out 2>&1
|
||||
local code=$?
|
||||
if [ "$code" -eq 0 ]; then
|
||||
echo " FAIL — $desc (command succeeded, but it should have failed)"; cat /tmp/ci-fail-drill-out; FAIL=1
|
||||
else
|
||||
echo " ok — $desc correctly failed (exit $code)"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Phase 4.1 CI-fails-closed drill"
|
||||
echo
|
||||
|
||||
echo "== 1. Test/syntax-check failure =="
|
||||
BAD_JS="$(mktemp).js"
|
||||
echo 'function broken( {' > "$BAD_JS"
|
||||
expect_nonzero "node --check on syntactically broken JS" node --check "$BAD_JS"
|
||||
rm -f "$BAD_JS"
|
||||
|
||||
echo "== 2. Frontend build failure =="
|
||||
cp frontend/src/App.jsx /tmp/App.jsx.bak
|
||||
echo 'this is not valid jsx <<<' >> frontend/src/App.jsx
|
||||
(cd frontend && expect_nonzero "npm run build with a broken component" npm run build)
|
||||
cp /tmp/App.jsx.bak frontend/src/App.jsx; rm -f /tmp/App.jsx.bak
|
||||
|
||||
echo "== 3. Image build failure =="
|
||||
TMPDOCKERFILE="$(mktemp -d)/Dockerfile"
|
||||
cat > "$TMPDOCKERFILE" <<'EOF'
|
||||
FROM node:20-slim
|
||||
COPY this-file-does-not-exist.js /app/
|
||||
EOF
|
||||
expect_nonzero "docker build with a COPY of a nonexistent file" docker build -q -f "$TMPDOCKERFILE" "$(dirname "$TMPDOCKERFILE")"
|
||||
|
||||
echo "== 4. Config validation failure =="
|
||||
BAD_COMPOSE="$(mktemp -d)/docker-compose.yml"
|
||||
cat > "$BAD_COMPOSE" <<'EOF'
|
||||
services:
|
||||
a:
|
||||
image: alpine
|
||||
a:
|
||||
image: alpine
|
||||
EOF
|
||||
expect_nonzero "docker compose config on a file with a duplicate service key" docker compose -f "$BAD_COMPOSE" config
|
||||
|
||||
echo "== 5. Migration preflight failure =="
|
||||
( cd platform/api && expect_nonzero "runMigrations() against an unreachable Postgres" \
|
||||
env PLATFORM_DB_URL="postgres://nobody:[email protected]:1/nope" \
|
||||
node -e "require('./src/db').runMigrations().then(()=>process.exit(0)).catch((e)=>{console.error(e.message);process.exit(1)})" )
|
||||
|
||||
echo "== 6. Staging smoke test failure =="
|
||||
expect_nonzero "staging-smoke-test.sh with no STAGING_GATEWAY_URL set" \
|
||||
env -u STAGING_GATEWAY_URL bash infra/scripts/staging-smoke-test.sh
|
||||
expect_nonzero "staging-smoke-test.sh against an unreachable staging URL" \
|
||||
env STAGING_GATEWAY_URL="http://127.0.0.1:1" bash infra/scripts/staging-smoke-test.sh
|
||||
|
||||
echo
|
||||
if [ "$FAIL" = "1" ]; then
|
||||
echo "DRILL FAILED — at least one failure mode did NOT fail closed. See above."
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: every listed failure mode genuinely exits nonzero — CI fails closed on all of them."
|
||||
echo "NOT TESTED HERE: an actual Gitea Actions run (needs Gitea Actions enabled on the real"
|
||||
echo "instance and this workflow file activated at .gitea/workflows/ — see build-and-test.yml's"
|
||||
echo "own header comment). This proves the underlying commands, which is what actually"
|
||||
echo "determines pass/fail once a real run happens."
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 4.1 §22 — deployment rollback drill. Builds two real gateway image versions (a known
|
||||
# good build, and a deliberately broken one), "deploys" each in turn to a local container,
|
||||
# proves the smoke test genuinely catches the regression, then rolls back using the EXACT same
|
||||
# decision logic rollback-production.sh uses against deploy-history.log (tail -2 | head -1 |
|
||||
# awk '{print $2}' — the tag before the current one), and proves the rollback recovers.
|
||||
#
|
||||
# What this proves: the deterministic-tag + deploy-history.log rollback MECHANISM is real and
|
||||
# correct. What it does NOT prove: the actual `ansible-playbook site.yml -e image_tag=...` step
|
||||
# rollback-production.sh runs against a real host — that needs the real hardware
|
||||
# (docs/PRODUCTION_VALIDATION_CHECKLIST.md). This drill swaps a local Docker container's image
|
||||
# tag directly instead, as the closest local equivalent.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../../platform/gateway"
|
||||
|
||||
CONTAINER=rootnode-rollback-drill
|
||||
PORT=4295
|
||||
DEPLOY_LOG="$(mktemp)"
|
||||
trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1 || true; rm -f "$DEPLOY_LOG"; rm -rf /tmp/rootnode-rollback-drill-broken' EXIT
|
||||
|
||||
smoke_test() {
|
||||
curl -sf "http://localhost:$PORT/healthz" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
echo "==> [1/6] Building the known-good gateway image (current source)"
|
||||
docker build -q -t rootnode-gateway-drill:known-good -f Dockerfile . >/dev/null
|
||||
|
||||
echo "==> [2/6] Building a deliberately broken version (healthz returns 500)"
|
||||
rm -rf /tmp/rootnode-rollback-drill-broken
|
||||
cp -r . /tmp/rootnode-rollback-drill-broken
|
||||
sed -i "s#app.get('/healthz', (req, res) => res.json({ ok: true }));#app.get('/healthz', (req, res) => res.status(500).json({ ok: false, broken: 'DELIBERATE_DRILL_REGRESSION' }));#" \
|
||||
/tmp/rootnode-rollback-drill-broken/src/server.js
|
||||
grep -q DELIBERATE_DRILL_REGRESSION /tmp/rootnode-rollback-drill-broken/src/server.js || { echo "FAIL: sed didn't apply — healthz line text must have changed upstream"; exit 1; }
|
||||
docker build -q -t rootnode-gateway-drill:broken -f /tmp/rootnode-rollback-drill-broken/Dockerfile /tmp/rootnode-rollback-drill-broken >/dev/null
|
||||
|
||||
deploy() {
|
||||
local tag="$1"
|
||||
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
||||
docker run -d --name "$CONTAINER" -p "$PORT:4200" -e GATEWAY_PORT=4200 -e PROVISIONER_URL=http://127.0.0.1:1 \
|
||||
-e PROVISIONER_SHARED_SECRET=drill -e PLATFORM_API_URL=http://127.0.0.1:1 \
|
||||
"rootnode-gateway-drill:$tag" >/dev/null
|
||||
sleep 2
|
||||
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $tag" >> "$DEPLOY_LOG"
|
||||
}
|
||||
|
||||
echo "==> [3/6] Deploying known-good (deploy #1)"
|
||||
deploy known-good
|
||||
smoke_test && echo " smoke test: PASS (as expected)" || { echo "FAIL: known-good should have passed"; exit 1; }
|
||||
|
||||
echo "==> [4/6] Deploying the broken version (deploy #2 — the 'bad release' this drill catches)"
|
||||
deploy broken
|
||||
if smoke_test; then echo "FAIL: the broken deploy's regression was NOT caught by the smoke test — that defeats the whole point"; exit 1; fi
|
||||
echo " smoke test: FAIL (correctly caught — this is exactly what should stop a real promotion, see docs/RUNBOOK_ROLLBACK.md)"
|
||||
|
||||
echo "==> [5/6] Computing the rollback target — same logic as infra/scripts/rollback-production.sh"
|
||||
TARGET_TAG=$(tail -2 "$DEPLOY_LOG" | head -1 | awk '{print $2}')
|
||||
[ "$TARGET_TAG" = "known-good" ] || { echo "FAIL: expected rollback target 'known-good', computed '$TARGET_TAG'"; exit 1; }
|
||||
echo " rollback target: $TARGET_TAG (correctly identified as the tag before the broken one)"
|
||||
|
||||
echo "==> [6/6] Rolling back"
|
||||
deploy "$TARGET_TAG"
|
||||
smoke_test && echo " smoke test: PASS — rollback recovered service" || { echo "FAIL: rollback did not recover"; exit 1; }
|
||||
|
||||
echo
|
||||
echo "OK: deployment rollback drill passed. Deterministic image tags + deploy-history.log"
|
||||
echo "correctly identify and recover from a bad release."
|
||||
echo
|
||||
echo "On migration compatibility (docs/RUNBOOK_FAILED_MIGRATION.md, docs/RUNBOOK_ROLLBACK.md):"
|
||||
echo "this script rolls back APPLICATION IMAGES ONLY. There is no automatic down-migration —"
|
||||
echo "if the bad release also ran a schema-changing migration, rolling the image back does NOT"
|
||||
echo "undo that migration, and old code may not run correctly against the new schema. That"
|
||||
echo "case requires restoring from the pre-migration backup (docs/RUNBOOK_POSTGRES_RESTORE.md),"
|
||||
echo "not a rollback, which is exactly why migrations in this project are check-before-alter"
|
||||
echo "and additive-only (docs/ENGINEERING_SOP.md §6) — to make image-only rollback usually safe."
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4.1 §27 — the fake cohort dress rehearsal. One continuous narrative through the real
|
||||
// application (not synthetic unit assertions): instructor login, cohort + 4 students, assign
|
||||
// curriculum, Prepare Labs, students launch, work different scenarios, use hints, fail a
|
||||
// validation, complete scenarios, instructor checks analytics, an exam is assigned/taken,
|
||||
// a Lab is reset, a Lab is reaped (destroyed out-of-band, the same consequence the real idle
|
||||
// reaper produces — see the note below on why this doesn't wait out the real multi-minute
|
||||
// reaper timer), the student returns and continues, capstone state is checked, and the cohort's
|
||||
// Labs are destroyed at the end.
|
||||
//
|
||||
// Run via run-fake-cohort-rehearsal.sh, which brings up platform/docker-compose.dev.yml first.
|
||||
// No framework, matching project convention. Problems found are printed, not silently passed —
|
||||
// see the Phase 4.1 report for anything this surfaced.
|
||||
|
||||
const assert = require('assert');
|
||||
const { makeSession } = require('../../../platform/test/session-helper');
|
||||
|
||||
const GW = process.env.GATEWAY_URL || 'http://localhost:4200';
|
||||
const OWNER_EMAIL = process.env.OWNER_EMAIL;
|
||||
const OWNER_PASSWORD = process.env.OWNER_PASSWORD;
|
||||
if (!OWNER_EMAIL || !OWNER_PASSWORD) { console.error('Set OWNER_EMAIL and OWNER_PASSWORD'); process.exit(1); }
|
||||
|
||||
let passed = 0, failed = 0;
|
||||
async function step(name, fn) {
|
||||
try { await fn(); console.log(` ok — ${name}`); passed++; }
|
||||
catch (e) { console.error(` FAIL — ${name}\n ${e.stack || e.message}`); failed++; }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Phase 4.1 fake cohort dress rehearsal — slow, real k3s boots\n');
|
||||
|
||||
const instructor = makeSession(GW);
|
||||
await instructor.request('POST', '/api/auth/instructor/login', { email: OWNER_EMAIL, password: OWNER_PASSWORD });
|
||||
|
||||
const names = ['Rehearsal-Ada', 'Rehearsal-Bo', 'Rehearsal-Cy', 'Rehearsal-Di'];
|
||||
const students = {};
|
||||
const sessions = {};
|
||||
|
||||
await step('instructor: create a cohort and roster 4 students', async () => {
|
||||
await instructor.request('POST', '/api/admin/cohorts', { cohort_id: 'rehearsal-cohort', name: 'Rehearsal Cohort' });
|
||||
for (const name of names) {
|
||||
const { body } = await instructor.request('POST', '/api/admin/students', { display_name: name, cohort_id: 'rehearsal-cohort' });
|
||||
students[name] = body.student_id;
|
||||
sessions[name] = makeSession(GW);
|
||||
await sessions[name].request('POST', '/api/login', { student_id: body.student_id });
|
||||
}
|
||||
});
|
||||
|
||||
await step('instructor: assign curriculum to the cohort', async () => {
|
||||
const { status } = await instructor.request('POST', '/api/admin/cohorts/rehearsal-cohort/curriculum', { curriculum_path: 'rootnode-path' });
|
||||
assert.ok([200, 201].includes(status));
|
||||
});
|
||||
|
||||
await step('instructor: Prepare Labs for the whole cohort (concurrent)', async () => {
|
||||
const { status, body } = await instructor.request('POST', '/api/admin/cohorts/rehearsal-cohort/prepare-labs');
|
||||
assert.equal(status, 200);
|
||||
assert.ok(body.results.every((r) => r.status === 'running'), `expected every Lab running, got: ${JSON.stringify(body.results)}`);
|
||||
});
|
||||
|
||||
const launches = {};
|
||||
await step('students: everyone\'s prepared Lab is already running (no re-launch wait)', async () => {
|
||||
for (const name of names) {
|
||||
const { body } = await sessions[name].request('POST', '/api/environment/launch');
|
||||
assert.equal(body.status, 'running');
|
||||
launches[name] = body;
|
||||
}
|
||||
});
|
||||
|
||||
await step('students: Ada and Bo work DIFFERENT scenarios', async () => {
|
||||
await sessions['Rehearsal-Ada'].request('POST', `${launches['Rehearsal-Ada'].launch_path}api/scenarios/pod-basics-mcq/answer`, { selected: 'b' });
|
||||
await sessions['Rehearsal-Bo'].request('POST', `${launches['Rehearsal-Bo'].launch_path}api/scenarios/pod-basics-mcq/start`);
|
||||
});
|
||||
|
||||
await step('student: Cy uses a hint on a scenario', async () => {
|
||||
const res = await sessions['Rehearsal-Cy'].request('POST', `${launches['Rehearsal-Cy'].launch_path}api/scenarios/pod-basics-mcq/hint`);
|
||||
assert.equal(res.status, 200);
|
||||
});
|
||||
|
||||
await step('student: Di fails a validation on purpose', async () => {
|
||||
const res = await sessions['Rehearsal-Di'].request('POST', `${launches['Rehearsal-Di'].launch_path}api/scenarios/pod-basics-mcq/answer`, { selected: 'a' });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.body.correct, false, 'expected a deliberately wrong answer to be marked incorrect');
|
||||
});
|
||||
|
||||
await step('instructor: cohort learning summary reflects real activity', async () => {
|
||||
const { body } = await instructor.request('GET', '/api/admin/cohorts/rehearsal-cohort/learning');
|
||||
assert.equal(body.student_count, 4);
|
||||
});
|
||||
|
||||
await step('instructor: needs-attention returns real signals for the struggling students, no punitive language', async () => {
|
||||
const { body } = await instructor.request('GET', '/api/admin/cohorts/rehearsal-cohort/needs-attention');
|
||||
const text = JSON.stringify(body).toLowerCase();
|
||||
assert.ok(!text.includes('fail') || text.includes('failed validations'), 'the word "fail" should only appear in a factual "N failed validations" context, never as a standalone judgment');
|
||||
});
|
||||
|
||||
await step('instructor: reset Ada\'s Lab — academic progress is untouched', async () => {
|
||||
const before = await sessions['Rehearsal-Ada'].request('GET', '/api/progress');
|
||||
// Admin environment routes need the real environment_id, which publicEnvView doesn't echo
|
||||
// back to the student directly — resolved via the student environment lookup instead.
|
||||
const envRes = await instructor.request('GET', `/api/admin/students/${students['Rehearsal-Ada']}/environment`);
|
||||
const env = envRes.body.find((e) => e.status !== 'destroyed');
|
||||
await instructor.request('POST', `/api/admin/environments/${env.environment_id}/reset`);
|
||||
const after = await sessions['Rehearsal-Ada'].request('GET', '/api/progress');
|
||||
assert.deepEqual(before.body.details, after.body.details, 'progress must be identical before/after a Lab reset');
|
||||
});
|
||||
|
||||
await step('§27 — a Lab is reaped: same consequence as the real idle reaper (real multi-minute timer not waited out here — see reaper.js/resource-policy.yaml for the real thresholds; this exercises the destroy-and-preserve-progress consequence directly, already proven mechanistically by infra/scripts/drills/test-lab-host-loss.js)', async () => {
|
||||
const before = await sessions['Rehearsal-Bo'].request('GET', '/api/progress');
|
||||
const envRes = await instructor.request('GET', `/api/admin/students/${students['Rehearsal-Bo']}/environment`);
|
||||
const env = envRes.body.find((e) => e.status !== 'destroyed');
|
||||
await instructor.request('DELETE', `/api/admin/environments/${env.environment_id}`);
|
||||
const after = await sessions['Rehearsal-Bo'].request('GET', '/api/progress');
|
||||
assert.deepEqual(before.body.details, after.body.details, 'progress must survive a reaped Lab unchanged');
|
||||
});
|
||||
|
||||
await step('student: Bo returns later — a fresh Lab is created and curriculum continues', async () => {
|
||||
const { status, body } = await sessions['Rehearsal-Bo'].request('POST', '/api/environment/launch');
|
||||
assert.equal(status, 200); assert.equal(body.status, 'running');
|
||||
const curriculum = await sessions['Rehearsal-Bo'].request('GET', '/api/curriculum/path');
|
||||
assert.ok(curriculum.body.stages.length > 0, 'curriculum must still be there for the returning student');
|
||||
});
|
||||
|
||||
await step('instructor: assign an exam to the cohort and Ada takes it', async () => {
|
||||
const create = await instructor.request('POST', '/api/admin/assessments', {
|
||||
id: 'rehearsal-exam', type: 'exam', title: 'Rehearsal Exam', duration_minutes: 30, hint_policy: 'disabled',
|
||||
pool_spec: { required: ['pod-basics-mcq'], random: [] },
|
||||
});
|
||||
assert.ok([200, 201, 409].includes(create.status)); // 409 if a prior run already created it — fine, idempotent enough for a rehearsal
|
||||
const assign = await instructor.request('POST', '/api/admin/assessments/rehearsal-exam/assign', { cohort_id: 'rehearsal-cohort' });
|
||||
assert.ok([200, 201].includes(assign.status));
|
||||
const start = await sessions['Rehearsal-Ada'].request('POST', '/api/assessments/rehearsal-exam/start');
|
||||
assert.equal(start.status, 201);
|
||||
assert.ok(start.body.scenarios.length > 0);
|
||||
});
|
||||
|
||||
await step('instructor: destroy all cohort Labs after class — progress remains intact for everyone', async () => {
|
||||
const before = {};
|
||||
for (const name of names) before[name] = (await sessions[name].request('GET', '/api/progress')).body.details;
|
||||
await instructor.request('POST', '/api/admin/cohorts/rehearsal-cohort/destroy-labs');
|
||||
for (const name of names) {
|
||||
const after = (await sessions[name].request('GET', '/api/progress')).body.details;
|
||||
assert.deepEqual(before[name], after, `${name}'s progress must survive end-of-class Lab destruction`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\n${passed} passed, ${failed} failed`);
|
||||
if (failed > 0) process.exit(1);
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error(e); process.exit(1); });
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4.1 §21 — Lab Host loss/rebuild drill. Proves disposable infrastructure is genuinely
|
||||
// disposable: a student's Lab container is destroyed OUT FROM UNDER the Provisioner (no clean
|
||||
// destroy call — simulating the whole rn-lab-host-01 VM disappearing), and the system must:
|
||||
// - keep the Platform (API/Gateway) up throughout — it never depended on that one container
|
||||
// - never present the now-gone environment as healthy/running once reconciliation runs
|
||||
// - leave academic data (progress/attempts) completely untouched
|
||||
// - let the student launch a brand-new Lab afterward and continue where they left off
|
||||
//
|
||||
// Run via run-lab-host-loss-drill.sh, which brings up platform/docker-compose.dev.yml first.
|
||||
// No framework, matching project convention.
|
||||
|
||||
const assert = require('assert');
|
||||
const { execFileSync } = require('child_process');
|
||||
const { makeSession } = require('../../../platform/test/session-helper');
|
||||
|
||||
const GW = process.env.GATEWAY_URL || 'http://localhost:4200';
|
||||
const API = process.env.PLATFORM_API_URL || 'http://localhost:4300';
|
||||
const OWNER_EMAIL = process.env.OWNER_EMAIL;
|
||||
const OWNER_PASSWORD = process.env.OWNER_PASSWORD;
|
||||
if (!OWNER_EMAIL || !OWNER_PASSWORD) { console.error('Set OWNER_EMAIL and OWNER_PASSWORD'); process.exit(1); }
|
||||
|
||||
let passed = 0, failed = 0;
|
||||
async function test(name, fn) {
|
||||
try { await fn(); console.log(` ok — ${name}`); passed++; }
|
||||
catch (e) { console.error(` FAIL — ${name}\n ${e.stack || e.message}`); failed++; }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Phase 4.1 Lab Host loss/rebuild drill — slow, real k3s boots\n');
|
||||
|
||||
const admin = makeSession(GW);
|
||||
const { status: loginStatus } = await admin.request('POST', '/api/auth/instructor/login', { email: OWNER_EMAIL, password: OWNER_PASSWORD });
|
||||
assert.equal(loginStatus, 200, 'owner login must succeed');
|
||||
|
||||
let studentId;
|
||||
await test('roster: add a student', async () => {
|
||||
const { body } = await admin.request('POST', '/api/admin/students', { display_name: 'Drill-LabHostLoss', cohort_id: null });
|
||||
studentId = body.student_id;
|
||||
});
|
||||
|
||||
const session = makeSession(GW);
|
||||
await session.request('POST', '/api/login', { student_id: studentId });
|
||||
|
||||
let launch;
|
||||
await test('launch: student gets a running Lab (real k3s boot)', async () => {
|
||||
const { status, body } = await session.request('POST', '/api/environment/launch');
|
||||
assert.equal(status, 200); assert.equal(body.status, 'running');
|
||||
launch = body;
|
||||
});
|
||||
|
||||
await test('progress: student completes a real scenario before the Lab Host is lost', async () => {
|
||||
const res = await session.request('POST', `${launch.launch_path}api/scenarios/pod-basics-mcq/answer`, { selected: 'b' });
|
||||
assert.equal(res.status, 200);
|
||||
const progress = await session.request('GET', '/api/progress');
|
||||
assert.equal(progress.body.details['pod-basics-mcq']?.status, 'completed', 'progress must show completed before we destroy anything');
|
||||
});
|
||||
|
||||
const token = launch.launch_path.split('/')[2];
|
||||
|
||||
await test('§21 — simulate total Lab Host loss: kill the Lab container directly, bypassing the Provisioner entirely', async () => {
|
||||
const containers = execFileSync('docker', ['ps', '--filter', 'name=rootnode-lab-', '--format', '{{.Names}}']).toString().trim().split('\n').filter(Boolean);
|
||||
assert.ok(containers.length >= 1, 'expected at least one running Lab container to destroy');
|
||||
for (const c of containers) execFileSync('docker', ['rm', '-f', c]);
|
||||
console.log(` destroyed ${containers.length} Lab container(s) out from under the Provisioner`);
|
||||
});
|
||||
|
||||
await test('§21 — Platform stays available throughout (it never depended on that container)', async () => {
|
||||
const api = await fetch(`${API}/healthz`);
|
||||
const gw = await fetch(`${GW}/healthz`);
|
||||
assert.equal(api.status, 200);
|
||||
assert.equal(gw.status, 200);
|
||||
});
|
||||
|
||||
await test('§21 — academic progress is completely untouched by the infrastructure loss', async () => {
|
||||
const progress = await session.request('GET', '/api/progress');
|
||||
assert.equal(progress.body.details['pod-basics-mcq']?.status, 'completed', 'progress must survive the Lab Host loss unchanged');
|
||||
});
|
||||
|
||||
await test('§21 — the now-gone environment is NEVER presented as healthy once reconciliation runs (Provisioner restart, its real startup path)', async () => {
|
||||
// cwd is already platform/ — run-lab-host-loss-drill.sh cd's there before invoking this script.
|
||||
execFileSync('docker', ['compose', '-f', 'docker-compose.dev.yml', 'restart', 'provisioner'], { stdio: 'inherit' });
|
||||
// Give the Provisioner a moment to finish its real reconcileOnStartup pass.
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
const res = await session.rawFetch(`/lab/${token}/`);
|
||||
assert.ok([503, 404].includes(res.status), `expected the stale environment to be rejected (503/404), got ${res.status} — a healthy-looking response here would mean a destroyed environment is being silently presented as available`);
|
||||
});
|
||||
|
||||
await test('§21 — a replacement Lab can be launched and boots successfully (the "replacement Lab Host" case, minus needing a literal second VM)', async () => {
|
||||
const { status, body } = await session.request('POST', '/api/environment/launch');
|
||||
assert.equal(status, 200); assert.equal(body.status, 'running');
|
||||
assert.notEqual(body.launch_path, launch.launch_path, 'the replacement environment must be a genuinely new one, not the destroyed one still being served');
|
||||
});
|
||||
|
||||
await test('§21 — previous academic progress is still there after the replacement Lab boots', async () => {
|
||||
const progress = await session.request('GET', '/api/progress');
|
||||
assert.equal(progress.body.details['pod-basics-mcq']?.status, 'completed');
|
||||
});
|
||||
|
||||
console.log(`\n${passed} passed, ${failed} failed`);
|
||||
if (failed > 0) process.exit(1);
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error(e); process.exit(1); });
|
||||
Reference in New Issue
Block a user