feat(assessment): add exam and capstone definition/session engine

New tables: assessment_definitions (reusable template — pool_spec, duration, hint_policy,
completion/unlock rules, a plain incrementing version bumped only on semantic edits),
assessment_cohort_assignments, assessment_sessions (one per student attempt, full snapshot of
what was drawn and under what rules — never affected by a later definition edit),
assessment_session_scenarios (the drawn set, ordered, with a real FK to scenario_attempts).
Deliberately does not touch/reuse exam_sessions (migration 006) — that was already scoped in
Phase 2B as historical-import-only, never the live mechanism; these tables are the real thing.

drawScenarios: every 'required' scenario id included, plus N random picks per pool group,
shuffled together (tiny seeded PRNG, mulberry32 — no new dependency; the seed is recorded for
audit, never shown to the student before the draw). One assessment at a time per student.
Timer is server-authoritative (expires_at fixed at draw time) and lazily expired — checked
whenever a session is touched, no background cron. Ships two seeded defaults: rootnode-capstone
(Real Ops scenarios required, weighted toward operational ability) and rootnode-quick-assessment
(a general example exam matching the spec's category-draw example).
This commit is contained in:
2026-08-18 14:35:11 -05:00
parent 01a862e007
commit b908ade8e0
2 changed files with 274 additions and 0 deletions
@@ -0,0 +1,98 @@
-- Assessment engine: definitions (exam/capstone templates), cohort assignment, live sessions,
-- and the per-session scenario draw. exam_sessions (migration 006) is left untouched — it was
-- already scoped in Phase 2B as historical-import-only from the old Lab-local SQLite exam
-- bookkeeping, never the live mechanism. These tables ARE the live mechanism; duplicating
-- exam_sessions' purpose would mean building the real thing twice, so this doesn't reuse it.
--
-- Definitions are lightly versioned (a plain incrementing integer, not full history) so a
-- later edit to pool_spec/duration/hint_policy never changes what an already-started session is
-- graded against — every session snapshots the fields that matter at draw time.
CREATE TABLE IF NOT EXISTS assessment_definitions (
assessment_definition_id TEXT PRIMARY KEY, -- slug, e.g. 'rootnode-capstone'
type TEXT NOT NULL CHECK (type IN ('exam', 'capstone')),
title TEXT NOT NULL,
description TEXT,
duration_minutes INTEGER NOT NULL,
hint_policy TEXT NOT NULL DEFAULT 'disabled' CHECK (hint_policy IN ('free', 'disabled', 'flagged')),
pool_spec JSONB NOT NULL, -- {"required": [scenario_id,...], "random": [{"count":n,"from":[scenario_id,...]},...]}
completion_rule JSONB NOT NULL DEFAULT '{"type":"all_scenarios_passed"}',
unlock_rule JSONB, -- capstone only: {"type":"stages_complete","stage_ids":[...]}
version INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive')),
created_by UUID REFERENCES instructors(instructor_id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS assessment_cohort_assignments (
assessment_definition_id TEXT NOT NULL REFERENCES assessment_definitions(assessment_definition_id),
cohort_id TEXT NOT NULL REFERENCES cohorts(cohort_id),
assigned_by UUID REFERENCES instructors(instructor_id),
assigned_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (assessment_definition_id, cohort_id)
);
CREATE TABLE IF NOT EXISTS assessment_sessions (
assessment_session_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
assessment_definition_id TEXT NOT NULL REFERENCES assessment_definitions(assessment_definition_id),
definition_version INTEGER NOT NULL, -- snapshot: which version of the definition was drawn against
student_id TEXT NOT NULL REFERENCES students(student_id),
cohort_id TEXT,
type TEXT NOT NULL, -- snapshot
title TEXT NOT NULL, -- snapshot
duration_minutes INTEGER NOT NULL, -- snapshot
hint_policy TEXT NOT NULL, -- snapshot
status TEXT NOT NULL DEFAULT 'not_started'
CHECK (status IN ('not_started', 'in_progress', 'completed', 'expired', 'cancelled')),
started_at TIMESTAMPTZ,
expires_at TIMESTAMPTZ, -- server-computed = started_at + duration_minutes; authoritative, never client-set
submitted_at TIMESTAMPTZ,
random_seed TEXT, -- recorded for audit/reproducibility, not exposed to the student before the draw
result JSONB, -- {"scenarios_total":n,"scenarios_passed":n,"passed":bool}
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS assessment_session_scenarios (
id BIGSERIAL PRIMARY KEY,
assessment_session_id UUID NOT NULL REFERENCES assessment_sessions(assessment_session_id),
scenario_id TEXT NOT NULL,
position INTEGER NOT NULL,
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'in_progress', 'completed', 'failed')),
attempt_id BIGINT REFERENCES scenario_attempts(attempt_id),
UNIQUE (assessment_session_id, scenario_id)
);
DO $$ BEGIN
ALTER TABLE scenario_attempts ADD CONSTRAINT scenario_attempts_assessment_session_fk
FOREIGN KEY (assessment_session_id) REFERENCES assessment_sessions(assessment_session_id);
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;
CREATE INDEX IF NOT EXISTS idx_assessment_sessions_student ON assessment_sessions(student_id, status);
CREATE INDEX IF NOT EXISTS idx_assessment_sessions_definition ON assessment_sessions(assessment_definition_id);
CREATE INDEX IF NOT EXISTS idx_assessment_session_scenarios_session ON assessment_session_scenarios(assessment_session_id);
-- Seed defaults — a working RootNode Capstone and one example exam ship out of the box, per the
-- Phase 3B spec ("ship with a sensible RootNode default"). Scenario ids are a snapshot of
-- scenarios/rootnode-path.json's stage lists as of this migration; edit the definition via the
-- admin API if curriculum content changes later, same as any other admin-owned config.
INSERT INTO assessment_definitions (assessment_definition_id, type, title, description, duration_minutes, hint_policy, pool_spec, completion_rule, unlock_rule)
VALUES (
'rootnode-capstone', 'capstone', 'RootNode Capstone',
'Combines skills from every curriculum stage into one operational assessment, weighted toward RootNode Real Ops incident scenarios. Evaluates operational ability, not memorized kubectl syntax.',
90, 'flagged',
'{"required": ["rootnode-runaway-monitor","broken-deployment","rootnode-probe-port-mismatch","rootnode-enforce-limits"], "random": [{"count":1,"from":["deploy-nginx","scale-deployment","rolling-update-task","liveness-probe-task","edit-deployment-basics"]},{"count":1,"from":["resource-limits-task","env-vars-configmap","secrets-basics"]},{"count":1,"from":["extract-logs-basics","kubectl-cp-basics","exec-command-basics","node-cordon-basics"]}]}',
'{"type":"all_scenarios_passed"}',
'{"type":"stages_complete","stage_ids":["foundation","workloads","networking","configuration","troubleshooting","real-ops"]}'
) ON CONFLICT (assessment_definition_id) DO NOTHING;
INSERT INTO assessment_definitions (assessment_definition_id, type, title, description, duration_minutes, hint_policy, pool_spec, completion_rule)
VALUES (
'rootnode-quick-assessment', 'exam', 'RootNode Kubernetes Assessment',
'A timed, randomized skills check drawn from across the curriculum. Example exam definition — instructors can create their own from the admin portal.',
60, 'disabled',
'{"required": [], "random": [{"count":2,"from":["deploy-nginx","scale-deployment","rolling-update-task","liveness-probe-task","daemonset-mcq"]},{"count":1,"from":["services-mcq","expose-service-basics"]},{"count":1,"from":["extract-logs-basics","kubectl-cp-basics","exec-command-basics","node-cordon-basics"]},{"count":1,"from":["rootnode-runaway-monitor","broken-deployment","rootnode-probe-port-mismatch","rootnode-enforce-limits"]}]}',
'{"type":"all_scenarios_passed"}'
) ON CONFLICT (assessment_definition_id) DO NOTHING;
+176
View File
@@ -0,0 +1,176 @@
// Assessment engine — exam and capstone definitions, cohort assignment, and live sessions.
// The timer and the drawn scenario set are BOTH server-authoritative: expires_at is computed
// once at session start and never recomputed from a client-supplied value, and the drawn
// scenario_ids live only in Postgres (assessment_session_scenarios) — a student can see them
// (they need to know what to work on) but can't set or replace them. See
// docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §19 for the full integrity story.
const crypto = require('crypto');
// Tiny deterministic PRNG (mulberry32) — enough for a reproducible/audit-able draw without a
// new dependency. Seeded from a string via a cheap hash; the seed itself is recorded on the
// session for audit, not exposed to the student before the draw happens.
function mulberry32(seed) {
let a = seed >>> 0;
return function () {
a |= 0; a = (a + 0x6D2B79F5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
function seedToInt(seed) {
let h = 0;
for (let i = 0; i < seed.length; i++) { h = (Math.imul(31, h) + seed.charCodeAt(i)) | 0; }
return h;
}
function shuffle(arr, rng) {
const a = [...arr];
for (let i = a.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[a[i], a[j]] = [a[j], a[i]];
}
return a;
}
// Draws a scenario set from a pool_spec: every id in `required` is included, plus `count`
// randomly-chosen ids from each `random` group's `from` list, all shuffled together into the
// student's presentation order.
function drawScenarios(poolSpec, seed) {
const actualSeed = seed || crypto.randomBytes(8).toString('hex');
const rng = mulberry32(seedToInt(actualSeed));
const required = poolSpec.required || [];
const randomPicks = (poolSpec.random || []).flatMap((group) => shuffle(group.from, rng).slice(0, group.count));
return { scenario_ids: shuffle([...required, ...randomPicks], rng), seed: actualSeed };
}
async function getActiveSessionForStudent(pool, studentId) {
const { rows } = await pool.query(
`SELECT * FROM assessment_sessions WHERE student_id = $1 AND status IN ('not_started', 'in_progress')
ORDER BY created_at DESC LIMIT 1`,
[studentId]
);
const session = rows[0];
if (!session) return null;
return lazyExpire(pool, session);
}
// Lazy expiry — checked whenever a session is read or written, rather than a background cron
// (no multi-host scheduler in this phase, see the Phase 3B non-goals list). A session past its
// server-computed expires_at is transitioned to 'expired' on the next touch, whoever that is.
async function lazyExpire(pool, session) {
if (session.status === 'in_progress' && session.expires_at && new Date(session.expires_at) <= new Date()) {
const { rows } = await pool.query(
`UPDATE assessment_sessions SET status = 'expired', submitted_at = COALESCE(submitted_at, now())
WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`,
[session.assessment_session_id]
);
return rows[0] || session;
}
return session;
}
async function getSessionScenarios(pool, sessionId) {
const { rows } = await pool.query(
`SELECT * FROM assessment_session_scenarios WHERE assessment_session_id = $1 ORDER BY position`, [sessionId]
);
return rows;
}
// Starts a new session for a student against a definition — refuses if they already have a
// live (not_started/in_progress) session for ANY definition, exam or capstone (one assessment
// at a time; keeps the "which Lab scenario am I working on for which purpose" question
// unambiguous for both the student and the hint-policy/attempt-linking logic).
async function startSession(pool, { definition, studentId, cohortId }) {
const active = await getActiveSessionForStudent(pool, studentId);
if (active) { const err = new Error('An assessment is already in progress.'); err.code = 'ALREADY_ACTIVE'; throw err; }
const { scenario_ids, seed } = drawScenarios(definition.pool_spec, null);
const startedAt = new Date();
const expiresAt = new Date(startedAt.getTime() + definition.duration_minutes * 60 * 1000);
const client = await pool.connect();
try {
await client.query('BEGIN');
const { rows } = await client.query(
`INSERT INTO assessment_sessions
(assessment_definition_id, definition_version, student_id, cohort_id, type, title, duration_minutes, hint_policy,
status, started_at, expires_at, random_seed)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'in_progress',$9,$10,$11) RETURNING *`,
[definition.assessment_definition_id, definition.version, studentId, cohortId || null, definition.type,
definition.title, definition.duration_minutes, definition.hint_policy, startedAt, expiresAt, seed]
);
const session = rows[0];
for (let i = 0; i < scenario_ids.length; i++) {
await client.query(
`INSERT INTO assessment_session_scenarios (assessment_session_id, scenario_id, position) VALUES ($1,$2,$3)`,
[session.assessment_session_id, scenario_ids[i], i]
);
}
await client.query('COMMIT');
return session;
} catch (e) {
await client.query('ROLLBACK');
throw e;
} finally {
client.release();
}
}
// Finds the live session (if any) that has THIS scenario_id in its draw for THIS student — the
// server-side signal that a validate/hint call is happening "inside" an assessment, derived
// entirely from what the server itself stored at draw time. A client asserting "I'm in
// assessment mode" is never trusted; this lookup is the only source of truth.
async function findActiveAssessmentContextForScenario(pool, studentId, scenarioId) {
const session = await getActiveSessionForStudent(pool, studentId);
if (!session || session.status !== 'in_progress') return null;
const { rows } = await pool.query(
`SELECT * FROM assessment_session_scenarios WHERE assessment_session_id = $1 AND scenario_id = $2`,
[session.assessment_session_id, scenarioId]
);
if (!rows[0]) return null;
return { session, sessionScenario: rows[0] };
}
// Updates the per-scenario row after a scenario_attempts completion, and finalizes the whole
// session once every drawn scenario is in a terminal state. Returns the finalized session row
// if this call was the one that completed it, otherwise null — the caller uses that to decide
// whether to record an achievement.
async function recordScenarioOutcome(pool, sessionId, scenarioId, attemptId, passed) {
await pool.query(
`UPDATE assessment_session_scenarios SET status = $3, attempt_id = $4
WHERE assessment_session_id = $1 AND scenario_id = $2`,
[sessionId, scenarioId, passed ? 'completed' : 'in_progress', attemptId]
);
if (!passed) return null; // still workable — session stays open until this scenario passes or time runs out
const scenarios = await getSessionScenarios(pool, sessionId);
const allDone = scenarios.every((s) => s.status === 'completed');
if (!allDone) return null;
const total = scenarios.length;
const passedCount = scenarios.filter((s) => s.status === 'completed').length;
const { rows } = await pool.query(
`UPDATE assessment_sessions SET status = 'completed', submitted_at = now(),
result = $2 WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`,
[sessionId, JSON.stringify({ scenarios_total: total, scenarios_passed: passedCount, passed: passedCount === total })]
);
return rows[0] || null;
}
async function submitSession(pool, sessionId) {
const scenarios = await getSessionScenarios(pool, sessionId);
const total = scenarios.length;
const passedCount = scenarios.filter((s) => s.status === 'completed').length;
const { rows } = await pool.query(
`UPDATE assessment_sessions SET status = 'completed', submitted_at = now(), result = $2
WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`,
[sessionId, JSON.stringify({ scenarios_total: total, scenarios_passed: passedCount, passed: passedCount === total })]
);
return rows[0] || null;
}
module.exports = {
drawScenarios, getActiveSessionForStudent, lazyExpire, getSessionScenarios, startSession,
findActiveAssessmentContextForScenario, recordScenarioOutcome, submitSession,
};