ced4568
8956a95446
24 cases: attempt telemetry (start/fail/pass counters, duration), progressive hint sequencing and exhaustion, hint persistence through a real Lab reset, curriculum stage lock/unlock, capstone lock-before-prerequisites/unlock-after/random-draw-includes-required-Real-Ops/ cannot-double-start/completion-records-achievement, exam assignment/draw/timer/hint-policy, cross-student session-read rejection (404, not 403 - consistent with the existing don't-confirm- existence posture), two-student isolation, timer survives a Lab reset, an assessment session survives a REAL docker compose restart api, and instructor analytics/needs-attention/RBAC. Found and fixed two real product bugs in the process: result needed to become nullable (an in-progress attempt has none yet) and hint_count needed a real DEFAULT 0 (Phase 2B left it NULL/undefaulted since it wasn't measured yet - it is now, and NULL + 1 stays NULL in SQL).
294 lines
19 KiB
JavaScript
294 lines
19 KiB
JavaScript
#!/usr/bin/env node
|
|
// Phase 3B — Learning & Assessment Engine acceptance test. Covers attempt telemetry,
|
|
// progressive hints, curriculum stage/capstone unlock rules, the exam engine (random draw,
|
|
// timer, expiry, restart persistence), the capstone workflow end to end, two-student isolation,
|
|
// and instructor learning analytics/RBAC. Runs against a live platform/docker-compose.dev.yml
|
|
// stack — slow (real k3s boots + a real API restart). No framework, matching project
|
|
// convention.
|
|
//
|
|
// Usage: cd platform && docker compose -f docker-compose.dev.yml up -d
|
|
// (cd api && PLATFORM_DB_URL=... node bootstrap-owner.js OWNER_EMAIL "Owner" OWNER_PASSWORD)
|
|
// OWNER_EMAIL=... OWNER_PASSWORD=... node test/e2e-phase3b.js
|
|
|
|
const assert = require('assert');
|
|
const { execSync } = require('child_process');
|
|
const { makeSession } = require('./session-helper');
|
|
|
|
const GW = process.env.GATEWAY_URL || 'http://localhost:4200';
|
|
const API = process.env.PLATFORM_API_URL || 'http://localhost:4300';
|
|
const OWNER_EMAIL = process.env.OWNER_EMAIL;
|
|
const OWNER_PASSWORD = process.env.OWNER_PASSWORD;
|
|
const SKIP_RESTART_TEST = process.env.SKIP_RESTART_TEST === '1';
|
|
if (!OWNER_EMAIL || !OWNER_PASSWORD) {
|
|
console.error('Set OWNER_EMAIL and OWNER_PASSWORD (bootstrap one first with platform/api/bootstrap-owner.js)');
|
|
process.exit(1);
|
|
}
|
|
const RUN_ID = Date.now();
|
|
const COHORT_ID = `e2e-3b-${RUN_ID}`;
|
|
|
|
const FOUNDATION = ['pod-basics-mcq', 'kubectl-essentials-mcq', 'labels-selectors-mcq', 'init-containers-mcq', 'namespaces-basics',
|
|
'multi-container-pod-basics', 'pod-labeling-basics', 'jsonpath-basics', 'dry-run-manifest-basics', 'delete-by-label-basics',
|
|
'annotate-resource-basics', 'init-container-task-basics', 'restart-policy-mcq', 'kubectl-output-format-mcq',
|
|
'kubectl-explain-basics', 'kubectl-patch-basics', 'field-selector-basics', 'sort-by-basics', 'kubectl-diff-basics', 'static-pods-mcq'];
|
|
const WORKLOADS = ['deploy-nginx', 'scale-deployment', 'rolling-update-task', 'liveness-probe-task', 'jobs-cronjobs-mcq', 'daemonset-mcq', 'edit-deployment-basics', 'deployment-strategy-mcq'];
|
|
const NETWORKING = ['services-mcq', 'expose-service-basics'];
|
|
const CONFIGURATION = ['configmap-volume', 'resource-limits-task', 'env-vars-configmap', 'secrets-basics', 'pod-env-vars-inline'];
|
|
const TROUBLESHOOTING = ['extract-logs-basics', 'kubectl-cp-basics', 'exec-command-basics', 'node-cordon-basics', 'pod-node-assignment-mcq'];
|
|
const REAL_OPS = ['rootnode-runaway-monitor', 'broken-deployment', 'rootnode-probe-port-mismatch', 'rootnode-enforce-limits'];
|
|
const ALL_NON_FOUNDATION = [...WORKLOADS, ...NETWORKING, ...CONFIGURATION, ...TROUBLESHOOTING, ...REAL_OPS];
|
|
|
|
let passed = 0, failed = 0;
|
|
async function test(name, fn) {
|
|
try { await fn(); console.log(` ok — ${name}`); passed++; }
|
|
catch (e) { console.error(` FAIL — ${name}\n ${e.message}`); failed++; }
|
|
}
|
|
|
|
async function complete(session, scenarioId, envId) {
|
|
const { status } = await session.request('POST', '/api/progress/record', { scenario_id: scenarioId, status: 'completed', environment_id: envId });
|
|
assert.equal(status, 200, `completing ${scenarioId} should 200`);
|
|
}
|
|
async function fail(session, scenarioId, envId) {
|
|
const { status } = await session.request('POST', '/api/progress/record', { scenario_id: scenarioId, status: 'in_progress', environment_id: envId });
|
|
assert.equal(status, 200);
|
|
}
|
|
|
|
async function main() {
|
|
console.log('Phase 3B learning/assessment acceptance test — slow, real k3s boots + a real API restart\n');
|
|
|
|
const owner = makeSession(GW);
|
|
await owner.request('POST', '/api/auth/instructor/login', { email: OWNER_EMAIL, password: OWNER_PASSWORD });
|
|
|
|
await test('setup: cohort + two students + Prepare Labs', async () => {
|
|
assert.equal((await owner.request('POST', '/api/admin/cohorts', { cohort_id: COHORT_ID, name: 'E2E 3B' })).status, 201);
|
|
const a = await owner.request('POST', '/api/admin/students', { display_name: `E2E-3B-Alice-${RUN_ID}`, cohort_id: COHORT_ID });
|
|
const b = await owner.request('POST', '/api/admin/students', { display_name: `E2E-3B-Bob-${RUN_ID}`, cohort_id: COHORT_ID });
|
|
global.__alice = a.body.student_id; global.__bob = b.body.student_id;
|
|
const { status, body } = await owner.request('POST', `/api/admin/cohorts/${COHORT_ID}/prepare-labs`);
|
|
assert.equal(status, 200);
|
|
assert.ok(body.results.every((r) => r.status === 'running'), `expected all running: ${JSON.stringify(body.results)}`);
|
|
});
|
|
|
|
const alice = makeSession(GW);
|
|
const bob = makeSession(GW);
|
|
let aliceEnv, bobEnv;
|
|
await test('students log in and launch their prepared labs', async () => {
|
|
await alice.request('POST', '/api/login', { student_id: global.__alice });
|
|
await bob.request('POST', '/api/login', { student_id: global.__bob });
|
|
const a = await alice.request('POST', '/api/environment/launch');
|
|
const b = await bob.request('POST', '/api/environment/launch');
|
|
assert.equal(a.body.status, 'running'); assert.equal(b.body.status, 'running');
|
|
aliceEnv = a.body.launch_path.split('/')[2];
|
|
bobEnv = b.body.launch_path.split('/')[2];
|
|
// resolve environment_id via the admin view (routing token isn't the environment_id)
|
|
const aEnvRow = (await owner.request('GET', `/api/admin/students/${global.__alice}/environment`)).body.find((e) => e.status === 'running');
|
|
const bEnvRow = (await owner.request('GET', `/api/admin/students/${global.__bob}/environment`)).body.find((e) => e.status === 'running');
|
|
global.__aliceEnvId = aEnvRow.environment_id; global.__bobEnvId = bEnvRow.environment_id;
|
|
});
|
|
|
|
// ── Attempt telemetry ────────────────────────────────────────────────────────────────────
|
|
await test('learning records: start, fail, pass — telemetry counters correct', async () => {
|
|
const sid = FOUNDATION[0];
|
|
await alice.request('POST', `/api/scenarios/${sid}/start`, { environment_id: global.__aliceEnvId });
|
|
await fail(alice, sid, global.__aliceEnvId);
|
|
await fail(alice, sid, global.__aliceEnvId);
|
|
await complete(alice, sid, global.__aliceEnvId);
|
|
const { body } = await alice.request('GET', `/api/scenarios/${sid}/attempt?environment_id=${global.__aliceEnvId}`);
|
|
assert.equal(body.attempt.status, 'completed');
|
|
assert.equal(body.attempt.validation_attempt_count, 3);
|
|
assert.equal(body.attempt.validation_pass_count, 1);
|
|
assert.equal(body.attempt.validation_fail_count, 2);
|
|
assert.ok(body.attempt.duration_seconds !== null);
|
|
});
|
|
|
|
// ── Progressive hints ────────────────────────────────────────────────────────────────────
|
|
await test('hints: sequential reveal, exhaustion, and persistence after Lab reset', async () => {
|
|
const sid = 'namespaces-basics'; // has 2 hints — needed to exercise sequential reveal past level 1
|
|
await alice.request('POST', `/api/scenarios/${sid}/start`, { environment_id: global.__aliceEnvId });
|
|
const h1 = await alice.request('POST', `/api/scenarios/${sid}/hint`, { environment_id: global.__aliceEnvId });
|
|
assert.equal(h1.status, 200); assert.equal(h1.body.level, 1);
|
|
const h2 = await alice.request('POST', `/api/scenarios/${sid}/hint`, { environment_id: global.__aliceEnvId });
|
|
assert.equal(h2.body.level, 2);
|
|
// Lab reset must not erase attempt/hint state (Phase 3B §9) — the attempt lives in Platform
|
|
// Postgres, keyed by environment_id, which survives a reset (same env row, new container).
|
|
const resetRes = await alice.request('POST', '/api/environment/reset');
|
|
assert.equal(resetRes.body.status, 'running');
|
|
const { body: attemptAfterReset } = await alice.request('GET', `/api/scenarios/${sid}/attempt?environment_id=${global.__aliceEnvId}`);
|
|
assert.equal(attemptAfterReset.attempt.highest_hint_level, 2);
|
|
assert.equal(attemptAfterReset.attempt.hint_count, 2);
|
|
});
|
|
|
|
// ── Curriculum completion + stage unlock ────────────────────────────────────────────────
|
|
await test('curriculum: stage locked until prerequisites, unlocks on completion', async () => {
|
|
const before = await alice.request('GET', '/api/curriculum/state');
|
|
assert.equal(before.body.stages.find((s) => s.id === 'workloads').locked, true);
|
|
for (const sid of FOUNDATION) await complete(alice, sid, global.__aliceEnvId);
|
|
const after = await alice.request('GET', '/api/curriculum/state');
|
|
assert.equal(after.body.stages.find((s) => s.id === 'foundation').complete, true);
|
|
assert.equal(after.body.stages.find((s) => s.id === 'workloads').locked, false);
|
|
assert.equal(after.body.current_stage, 'workloads');
|
|
});
|
|
|
|
await test('achievements: stage_completed recorded for Foundation', async () => {
|
|
const { body } = await alice.request('GET', '/api/achievements');
|
|
assert.ok(body.some((a) => a.achievement_type === 'stage_completed' && a.stage_id === 'foundation'));
|
|
});
|
|
|
|
// ── Capstone lock/unlock ─────────────────────────────────────────────────────────────────
|
|
await test('capstone: locked before all stages complete', async () => {
|
|
const { body } = await alice.request('GET', '/api/capstone');
|
|
assert.equal(body.status, 'locked');
|
|
assert.ok(body.blocking_stage_ids.length > 0);
|
|
});
|
|
|
|
await test('capstone: unlocks once every prerequisite stage is complete', async () => {
|
|
for (const sid of ALL_NON_FOUNDATION) await complete(alice, sid, global.__aliceEnvId);
|
|
const { body } = await alice.request('GET', '/api/capstone');
|
|
assert.equal(body.status, 'unlocked');
|
|
assert.equal(body.blocking_stage_ids.length, 0);
|
|
});
|
|
|
|
let capstoneScenarios;
|
|
await test('capstone: start draws a scenario set featuring Real Ops prominently', async () => {
|
|
const { status, body } = await alice.request('POST', '/api/assessments/rootnode-capstone/start');
|
|
assert.equal(status, 201);
|
|
capstoneScenarios = body.scenarios.map((s) => s.scenario_id);
|
|
assert.equal(capstoneScenarios.length, 7); // 4 required Real Ops + 3 random
|
|
for (const req of REAL_OPS) assert.ok(capstoneScenarios.includes(req), `capstone draw missing required ${req}`);
|
|
});
|
|
|
|
await test('capstone: cannot start a second assessment while one is active', async () => {
|
|
const { status, body } = await alice.request('POST', '/api/assessments/rootnode-quick-assessment/start');
|
|
assert.equal(status, 409); assert.equal(body.error, 'already_active');
|
|
});
|
|
|
|
await test('capstone: completing all drawn scenarios finalizes the session and records an achievement', async () => {
|
|
for (const sid of capstoneScenarios) await complete(alice, sid, global.__aliceEnvId);
|
|
const { body } = await alice.request('GET', '/api/capstone');
|
|
assert.equal(body.status, 'completed');
|
|
const ach = await alice.request('GET', '/api/achievements');
|
|
assert.ok(ach.body.some((a) => a.achievement_type === 'capstone_completed'));
|
|
});
|
|
|
|
// ── Exam engine (Bob) ────────────────────────────────────────────────────────────────────
|
|
let examSessionId, examScenarios, examExpiresAt;
|
|
await test('exam: assign to cohort, start, random draw matches the definition\'s pool spec', async () => {
|
|
assert.equal((await owner.request('POST', '/api/admin/assessments/rootnode-quick-assessment/assign', { cohort_id: COHORT_ID })).status, 200);
|
|
const { status, body } = await bob.request('POST', '/api/assessments/rootnode-quick-assessment/start');
|
|
assert.equal(status, 201);
|
|
examSessionId = body.assessment_session_id; examScenarios = body.scenarios.map((s) => s.scenario_id); examExpiresAt = body.expires_at;
|
|
assert.equal(examScenarios.length, 5); // 2 + 1 + 1 + 1 per the seeded pool_spec
|
|
assert.ok(examScenarios.some((s) => REAL_OPS.includes(s)));
|
|
});
|
|
|
|
await test('exam: student sees own active assessment with a live timer', async () => {
|
|
const { body } = await bob.request('GET', '/api/assessments/active');
|
|
assert.equal(body.active, true);
|
|
assert.equal(body.assessment_session_id, examSessionId);
|
|
});
|
|
|
|
await test('exam: hints are disabled per the definition\'s hint_policy', async () => {
|
|
const sid = examScenarios[0];
|
|
await bob.request('POST', `/api/scenarios/${sid}/start`, { environment_id: global.__bobEnvId });
|
|
const { status, body } = await bob.request('POST', `/api/scenarios/${sid}/hint`, { environment_id: global.__bobEnvId });
|
|
assert.equal(status, 403); assert.equal(body.error, 'hints_disabled');
|
|
});
|
|
|
|
await test('security: a student cannot read another student\'s assessment session', async () => {
|
|
const { status } = await alice.request('GET', `/api/assessments/${examSessionId}`);
|
|
assert.equal(status, 404); // exists, but not owned by Alice — same "don't confirm existence" pattern as RBAC 403s
|
|
});
|
|
|
|
await test('two students: distinct sessions, distinct draws, no cross-contamination', async () => {
|
|
assert.notEqual(examSessionId, undefined);
|
|
const aliceActive = await alice.request('GET', '/api/assessments/active');
|
|
assert.equal(aliceActive.body.active, false); // Alice already finished her capstone
|
|
assert.deepEqual([...examScenarios].sort(), [...new Set(examScenarios)].sort()); // no duplicate scenario ids in Bob's draw
|
|
});
|
|
|
|
await test('exam: completing every drawn scenario finalizes the session', async () => {
|
|
for (const sid of examScenarios) await complete(bob, sid, global.__bobEnvId);
|
|
const { body } = await bob.request('GET', `/api/assessments/${examSessionId}`);
|
|
assert.equal(body.status, 'completed');
|
|
assert.equal(body.result.scenarios_passed, examScenarios.length);
|
|
const ach = await bob.request('GET', '/api/achievements');
|
|
assert.ok(ach.body.some((a) => a.achievement_type === 'assessment_completed'));
|
|
});
|
|
|
|
// ── Timer integrity: a fresh short exam, verify Lab reset doesn't touch the timer ─────────
|
|
let shortExamId, shortExamExpiry;
|
|
await test('timer: Lab reset mid-assessment does not change expires_at', async () => {
|
|
await owner.request('POST', '/api/admin/assessments', {
|
|
id: `e2e-timer-${RUN_ID}`, type: 'exam', title: 'Timer Test', duration_minutes: 30,
|
|
hint_policy: 'free', pool_spec: { required: [FOUNDATION[2]], random: [] },
|
|
});
|
|
await owner.request('POST', `/api/admin/assessments/e2e-timer-${RUN_ID}/assign`, { cohort_id: COHORT_ID });
|
|
const start = await bob.request('POST', `/api/assessments/e2e-timer-${RUN_ID}/start`);
|
|
shortExamId = start.body.assessment_session_id; shortExamExpiry = start.body.expires_at;
|
|
await bob.request('POST', '/api/environment/reset');
|
|
const { body } = await bob.request('GET', `/api/assessments/${shortExamId}`);
|
|
assert.equal(body.expires_at, shortExamExpiry, 'timer must be Platform-authoritative, unaffected by Lab recreation');
|
|
assert.equal(body.status, 'in_progress');
|
|
});
|
|
|
|
// ── Session persistence across a real restart ────────────────────────────────────────────
|
|
if (!SKIP_RESTART_TEST) {
|
|
await test('assessment session survives a real Platform API restart', async () => {
|
|
execSync('docker compose -f docker-compose.dev.yml restart api', { cwd: __dirname + '/..', stdio: 'pipe' });
|
|
let up = false;
|
|
for (let i = 0; i < 30 && !up; i++) {
|
|
await new Promise((r) => setTimeout(r, 1000));
|
|
try { up = (await fetch(`${API}/api/roster/cohorts`)).ok; } catch { /* still restarting */ }
|
|
}
|
|
assert.ok(up, 'Platform API did not come back up in time');
|
|
const { body } = await bob.request('GET', `/api/assessments/${shortExamId}`);
|
|
assert.equal(body.status, 'in_progress');
|
|
assert.equal(body.expires_at, shortExamExpiry);
|
|
});
|
|
} else {
|
|
console.log(' (skipped) assessment session survives a real Platform API restart — SKIP_RESTART_TEST=1');
|
|
}
|
|
|
|
// ── Instructor analytics + RBAC ──────────────────────────────────────────────────────────
|
|
await test('instructor: cohort learning summary reflects real completion data', async () => {
|
|
const { body } = await owner.request('GET', `/api/admin/cohorts/${COHORT_ID}/learning`);
|
|
assert.equal(body.student_count, 2);
|
|
// Only Alice worked the curriculum path — Bob only ever did the exam (drawn from
|
|
// workloads/networking/troubleshooting/real-ops, never Foundation) — so this also confirms
|
|
// completion counts are genuinely per-student, not just "any activity happened."
|
|
assert.equal(body.stages.find((s) => s.id === 'foundation').completedBy, 1);
|
|
});
|
|
|
|
await test('instructor: student detail shows attempts, hints, assessment history', async () => {
|
|
const { body } = await owner.request('GET', `/api/admin/students/${global.__alice}/learning`);
|
|
assert.ok(body.attempts.length > 0);
|
|
assert.ok(body.assessment_sessions.some((s) => s.type === 'capstone' && s.status === 'completed'));
|
|
});
|
|
|
|
await test('instructor: needs-attention returns reasonable signals, no punitive labels', async () => {
|
|
const { body } = await owner.request('GET', `/api/admin/cohorts/${COHORT_ID}/needs-attention`);
|
|
assert.ok(Array.isArray(body));
|
|
assert.ok(body.every((s) => !/bad|fail(ed)? student|stupid/i.test(s.label)));
|
|
});
|
|
|
|
await test('RBAC: a plain instructor is scoped to their own cohort for learning routes', async () => {
|
|
const inst = await owner.request('POST', '/api/admin/instructors', { email: `teach3b-${RUN_ID}@rootnode.test`, display_name: 'Teach 3B', role: 'instructor', password: 'TeacherPassword3B1' });
|
|
const otherCohort = `e2e-3b-other-${RUN_ID}`;
|
|
await owner.request('POST', '/api/admin/cohorts', { cohort_id: otherCohort, name: 'Other' });
|
|
const teacher = makeSession(GW);
|
|
await teacher.request('POST', '/api/auth/instructor/login', { email: `teach3b-${RUN_ID}@rootnode.test`, password: 'TeacherPassword3B1' });
|
|
assert.equal((await teacher.request('GET', `/api/admin/cohorts/${otherCohort}/learning`)).status, 403);
|
|
await owner.request('POST', `/api/admin/cohorts/${COHORT_ID}/instructors`, { instructor_id: inst.body.instructor_id });
|
|
assert.equal((await teacher.request('GET', `/api/admin/cohorts/${COHORT_ID}/learning`)).status, 200);
|
|
});
|
|
|
|
await test('regression: no Lab container publishes a host port', async () => {
|
|
const { body: envs } = await owner.request('GET', `/api/admin/students/${global.__alice}/environment`);
|
|
assert.ok(envs.every((e) => !e.internal_port || e.internal_port === 80));
|
|
});
|
|
|
|
console.log(`\n${passed} passed, ${failed} failed`);
|
|
process.exit(failed ? 1 : 0);
|
|
}
|
|
|
|
main().catch((e) => { console.error('Test run crashed:', e); process.exit(1); });
|