rootnode-academy/infra/scripts/drills/test-lab-host-loss.js

#!/usr/bin/env node
// Phase 4.1 §21 — Lab Host loss/rebuild drill. Proves disposable infrastructure is genuinely
// disposable: a student's Lab container is destroyed OUT FROM UNDER the Provisioner (no clean
// destroy call — simulating the whole rn-lab-host-01 VM disappearing), and the system must:
//   - keep the Platform (API/Gateway) up throughout — it never depended on that one container
//   - never present the now-gone environment as healthy/running once reconciliation runs
//   - leave academic data (progress/attempts) completely untouched
//   - let the student launch a brand-new Lab afterward and continue where they left off
//
// Run via run-lab-host-loss-drill.sh, which brings up platform/docker-compose.dev.yml first.
// No framework, matching project convention.

const assert = require('assert');
const { execFileSync } = require('child_process');
const { makeSession } = require('../../../platform/test/session-helper');

const GW = process.env.GATEWAY_URL || 'http://localhost:4200';
const API = process.env.PLATFORM_API_URL || 'http://localhost:4300';
const OWNER_EMAIL = process.env.OWNER_EMAIL;
const OWNER_PASSWORD = process.env.OWNER_PASSWORD;
if (!OWNER_EMAIL || !OWNER_PASSWORD) { console.error('Set OWNER_EMAIL and OWNER_PASSWORD'); process.exit(1); }

let passed = 0, failed = 0;
async function test(name, fn) {
  try { await fn(); console.log(`  ok — ${name}`); passed++; }
  catch (e) { console.error(`  FAIL — ${name}\n    ${e.stack || e.message}`); failed++; }
}

async function main() {
  console.log('Phase 4.1 Lab Host loss/rebuild drill — slow, real k3s boots\n');

  const admin = makeSession(GW);
  const { status: loginStatus } = await admin.request('POST', '/api/auth/instructor/login', { email: OWNER_EMAIL, password: OWNER_PASSWORD });
  assert.equal(loginStatus, 200, 'owner login must succeed');

  let studentId;
  await test('roster: add a student', async () => {
    const { body } = await admin.request('POST', '/api/admin/students', { display_name: 'Drill-LabHostLoss', cohort_id: null });
    studentId = body.student_id;
  });

  const session = makeSession(GW);
  await session.request('POST', '/api/login', { student_id: studentId });

  let launch;
  await test('launch: student gets a running Lab (real k3s boot)', async () => {
    const { status, body } = await session.request('POST', '/api/environment/launch');
    assert.equal(status, 200); assert.equal(body.status, 'running');
    launch = body;
  });

  await test('progress: student completes a real scenario before the Lab Host is lost', async () => {
    const res = await session.request('POST', `${launch.launch_path}api/scenarios/pod-basics-mcq/answer`, { selected: 'b' });
    assert.equal(res.status, 200);
    const progress = await session.request('GET', '/api/progress');
    assert.equal(progress.body.details['pod-basics-mcq']?.status, 'completed', 'progress must show completed before we destroy anything');
  });

  const token = launch.launch_path.split('/')[2];

  await test('§21 — simulate total Lab Host loss: kill the Lab container directly, bypassing the Provisioner entirely', async () => {
    const containers = execFileSync('docker', ['ps', '--filter', 'name=rootnode-lab-', '--format', '{{.Names}}']).toString().trim().split('\n').filter(Boolean);
    assert.ok(containers.length >= 1, 'expected at least one running Lab container to destroy');
    for (const c of containers) execFileSync('docker', ['rm', '-f', c]);
    console.log(`    destroyed ${containers.length} Lab container(s) out from under the Provisioner`);
  });

  await test('§21 — Platform stays available throughout (it never depended on that container)', async () => {
    const api = await fetch(`${API}/healthz`);
    const gw = await fetch(`${GW}/healthz`);
    assert.equal(api.status, 200);
    assert.equal(gw.status, 200);
  });

  await test('§21 — academic progress is completely untouched by the infrastructure loss', async () => {
    const progress = await session.request('GET', '/api/progress');
    assert.equal(progress.body.details['pod-basics-mcq']?.status, 'completed', 'progress must survive the Lab Host loss unchanged');
  });

  await test('§21 — the now-gone environment is NEVER presented as healthy once reconciliation runs (Provisioner restart, its real startup path)', async () => {
    // cwd is already platform/ — run-lab-host-loss-drill.sh cd's there before invoking this script.
    execFileSync('docker', ['compose', '-f', 'docker-compose.dev.yml', 'restart', 'provisioner'], { stdio: 'inherit' });
    // Give the Provisioner a moment to finish its real reconcileOnStartup pass.
    await new Promise((r) => setTimeout(r, 5000));
    const res = await session.rawFetch(`/lab/${token}/`);
    assert.ok([503, 404].includes(res.status), `expected the stale environment to be rejected (503/404), got ${res.status} — a healthy-looking response here would mean a destroyed environment is being silently presented as available`);
  });

  await test('§21 — a replacement Lab can be launched and boots successfully (the "replacement Lab Host" case, minus needing a literal second VM)', async () => {
    const { status, body } = await session.request('POST', '/api/environment/launch');
    assert.equal(status, 200); assert.equal(body.status, 'running');
    assert.notEqual(body.launch_path, launch.launch_path, 'the replacement environment must be a genuinely new one, not the destroyed one still being served');
  });

  await test('§21 — previous academic progress is still there after the replacement Lab boots', async () => {
    const progress = await session.request('GET', '/api/progress');
    assert.equal(progress.body.details['pod-basics-mcq']?.status, 'completed');
  });

  console.log(`\n${passed} passed, ${failed} failed`);
  if (failed > 0) process.exit(1);
}

main().catch((e) => { console.error(e); process.exit(1); });