From b908ade8e0cc050d4d103f352f4e232074cdded8 Mon Sep 17 00:00:00 2001 From: ced4568 Date: Tue, 18 Aug 2026 14:35:11 -0500 Subject: [PATCH] feat(assessment): add exam and capstone definition/session engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New tables: assessment_definitions (reusable template — pool_spec, duration, hint_policy, completion/unlock rules, a plain incrementing version bumped only on semantic edits), assessment_cohort_assignments, assessment_sessions (one per student attempt, full snapshot of what was drawn and under what rules — never affected by a later definition edit), assessment_session_scenarios (the drawn set, ordered, with a real FK to scenario_attempts). Deliberately does not touch/reuse exam_sessions (migration 006) — that was already scoped in Phase 2B as historical-import-only, never the live mechanism; these tables are the real thing. drawScenarios: every 'required' scenario id included, plus N random picks per pool group, shuffled together (tiny seeded PRNG, mulberry32 — no new dependency; the seed is recorded for audit, never shown to the student before the draw). One assessment at a time per student. Timer is server-authoritative (expires_at fixed at draw time) and lazily expired — checked whenever a session is touched, no background cron. Ships two seeded defaults: rootnode-capstone (Real Ops scenarios required, weighted toward operational ability) and rootnode-quick-assessment (a general example exam matching the spec's category-draw example). --- platform/api/migrations/013_assessments.sql | 98 +++++++++++ platform/api/src/assessments.js | 176 ++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 platform/api/migrations/013_assessments.sql create mode 100644 platform/api/src/assessments.js diff --git a/platform/api/migrations/013_assessments.sql b/platform/api/migrations/013_assessments.sql new file mode 100644 index 0000000..182d752 --- /dev/null +++ b/platform/api/migrations/013_assessments.sql @@ -0,0 +1,98 @@ +-- Assessment engine: definitions (exam/capstone templates), cohort assignment, live sessions, +-- and the per-session scenario draw. exam_sessions (migration 006) is left untouched — it was +-- already scoped in Phase 2B as historical-import-only from the old Lab-local SQLite exam +-- bookkeeping, never the live mechanism. These tables ARE the live mechanism; duplicating +-- exam_sessions' purpose would mean building the real thing twice, so this doesn't reuse it. +-- +-- Definitions are lightly versioned (a plain incrementing integer, not full history) so a +-- later edit to pool_spec/duration/hint_policy never changes what an already-started session is +-- graded against — every session snapshots the fields that matter at draw time. + +CREATE TABLE IF NOT EXISTS assessment_definitions ( + assessment_definition_id TEXT PRIMARY KEY, -- slug, e.g. 'rootnode-capstone' + type TEXT NOT NULL CHECK (type IN ('exam', 'capstone')), + title TEXT NOT NULL, + description TEXT, + duration_minutes INTEGER NOT NULL, + hint_policy TEXT NOT NULL DEFAULT 'disabled' CHECK (hint_policy IN ('free', 'disabled', 'flagged')), + pool_spec JSONB NOT NULL, -- {"required": [scenario_id,...], "random": [{"count":n,"from":[scenario_id,...]},...]} + completion_rule JSONB NOT NULL DEFAULT '{"type":"all_scenarios_passed"}', + unlock_rule JSONB, -- capstone only: {"type":"stages_complete","stage_ids":[...]} + version INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive')), + created_by UUID REFERENCES instructors(instructor_id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS assessment_cohort_assignments ( + assessment_definition_id TEXT NOT NULL REFERENCES assessment_definitions(assessment_definition_id), + cohort_id TEXT NOT NULL REFERENCES cohorts(cohort_id), + assigned_by UUID REFERENCES instructors(instructor_id), + assigned_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (assessment_definition_id, cohort_id) +); + +CREATE TABLE IF NOT EXISTS assessment_sessions ( + assessment_session_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + assessment_definition_id TEXT NOT NULL REFERENCES assessment_definitions(assessment_definition_id), + definition_version INTEGER NOT NULL, -- snapshot: which version of the definition was drawn against + student_id TEXT NOT NULL REFERENCES students(student_id), + cohort_id TEXT, + type TEXT NOT NULL, -- snapshot + title TEXT NOT NULL, -- snapshot + duration_minutes INTEGER NOT NULL, -- snapshot + hint_policy TEXT NOT NULL, -- snapshot + status TEXT NOT NULL DEFAULT 'not_started' + CHECK (status IN ('not_started', 'in_progress', 'completed', 'expired', 'cancelled')), + started_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ, -- server-computed = started_at + duration_minutes; authoritative, never client-set + submitted_at TIMESTAMPTZ, + random_seed TEXT, -- recorded for audit/reproducibility, not exposed to the student before the draw + result JSONB, -- {"scenarios_total":n,"scenarios_passed":n,"passed":bool} + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS assessment_session_scenarios ( + id BIGSERIAL PRIMARY KEY, + assessment_session_id UUID NOT NULL REFERENCES assessment_sessions(assessment_session_id), + scenario_id TEXT NOT NULL, + position INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'in_progress', 'completed', 'failed')), + attempt_id BIGINT REFERENCES scenario_attempts(attempt_id), + UNIQUE (assessment_session_id, scenario_id) +); + +DO $$ BEGIN + ALTER TABLE scenario_attempts ADD CONSTRAINT scenario_attempts_assessment_session_fk + FOREIGN KEY (assessment_session_id) REFERENCES assessment_sessions(assessment_session_id); +EXCEPTION + WHEN duplicate_object THEN NULL; +END $$; + +CREATE INDEX IF NOT EXISTS idx_assessment_sessions_student ON assessment_sessions(student_id, status); +CREATE INDEX IF NOT EXISTS idx_assessment_sessions_definition ON assessment_sessions(assessment_definition_id); +CREATE INDEX IF NOT EXISTS idx_assessment_session_scenarios_session ON assessment_session_scenarios(assessment_session_id); + +-- Seed defaults — a working RootNode Capstone and one example exam ship out of the box, per the +-- Phase 3B spec ("ship with a sensible RootNode default"). Scenario ids are a snapshot of +-- scenarios/rootnode-path.json's stage lists as of this migration; edit the definition via the +-- admin API if curriculum content changes later, same as any other admin-owned config. +INSERT INTO assessment_definitions (assessment_definition_id, type, title, description, duration_minutes, hint_policy, pool_spec, completion_rule, unlock_rule) +VALUES ( + 'rootnode-capstone', 'capstone', 'RootNode Capstone', + 'Combines skills from every curriculum stage into one operational assessment, weighted toward RootNode Real Ops incident scenarios. Evaluates operational ability, not memorized kubectl syntax.', + 90, 'flagged', + '{"required": ["rootnode-runaway-monitor","broken-deployment","rootnode-probe-port-mismatch","rootnode-enforce-limits"], "random": [{"count":1,"from":["deploy-nginx","scale-deployment","rolling-update-task","liveness-probe-task","edit-deployment-basics"]},{"count":1,"from":["resource-limits-task","env-vars-configmap","secrets-basics"]},{"count":1,"from":["extract-logs-basics","kubectl-cp-basics","exec-command-basics","node-cordon-basics"]}]}', + '{"type":"all_scenarios_passed"}', + '{"type":"stages_complete","stage_ids":["foundation","workloads","networking","configuration","troubleshooting","real-ops"]}' +) ON CONFLICT (assessment_definition_id) DO NOTHING; + +INSERT INTO assessment_definitions (assessment_definition_id, type, title, description, duration_minutes, hint_policy, pool_spec, completion_rule) +VALUES ( + 'rootnode-quick-assessment', 'exam', 'RootNode Kubernetes Assessment', + 'A timed, randomized skills check drawn from across the curriculum. Example exam definition — instructors can create their own from the admin portal.', + 60, 'disabled', + '{"required": [], "random": [{"count":2,"from":["deploy-nginx","scale-deployment","rolling-update-task","liveness-probe-task","daemonset-mcq"]},{"count":1,"from":["services-mcq","expose-service-basics"]},{"count":1,"from":["extract-logs-basics","kubectl-cp-basics","exec-command-basics","node-cordon-basics"]},{"count":1,"from":["rootnode-runaway-monitor","broken-deployment","rootnode-probe-port-mismatch","rootnode-enforce-limits"]}]}', + '{"type":"all_scenarios_passed"}' +) ON CONFLICT (assessment_definition_id) DO NOTHING; diff --git a/platform/api/src/assessments.js b/platform/api/src/assessments.js new file mode 100644 index 0000000..b4061bd --- /dev/null +++ b/platform/api/src/assessments.js @@ -0,0 +1,176 @@ +// Assessment engine — exam and capstone definitions, cohort assignment, and live sessions. +// The timer and the drawn scenario set are BOTH server-authoritative: expires_at is computed +// once at session start and never recomputed from a client-supplied value, and the drawn +// scenario_ids live only in Postgres (assessment_session_scenarios) — a student can see them +// (they need to know what to work on) but can't set or replace them. See +// docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §19 for the full integrity story. + +const crypto = require('crypto'); + +// Tiny deterministic PRNG (mulberry32) — enough for a reproducible/audit-able draw without a +// new dependency. Seeded from a string via a cheap hash; the seed itself is recorded on the +// session for audit, not exposed to the student before the draw happens. +function mulberry32(seed) { + let a = seed >>> 0; + return function () { + a |= 0; a = (a + 0x6D2B79F5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} +function seedToInt(seed) { + let h = 0; + for (let i = 0; i < seed.length; i++) { h = (Math.imul(31, h) + seed.charCodeAt(i)) | 0; } + return h; +} +function shuffle(arr, rng) { + const a = [...arr]; + for (let i = a.length - 1; i > 0; i--) { + const j = Math.floor(rng() * (i + 1)); + [a[i], a[j]] = [a[j], a[i]]; + } + return a; +} + +// Draws a scenario set from a pool_spec: every id in `required` is included, plus `count` +// randomly-chosen ids from each `random` group's `from` list, all shuffled together into the +// student's presentation order. +function drawScenarios(poolSpec, seed) { + const actualSeed = seed || crypto.randomBytes(8).toString('hex'); + const rng = mulberry32(seedToInt(actualSeed)); + const required = poolSpec.required || []; + const randomPicks = (poolSpec.random || []).flatMap((group) => shuffle(group.from, rng).slice(0, group.count)); + return { scenario_ids: shuffle([...required, ...randomPicks], rng), seed: actualSeed }; +} + +async function getActiveSessionForStudent(pool, studentId) { + const { rows } = await pool.query( + `SELECT * FROM assessment_sessions WHERE student_id = $1 AND status IN ('not_started', 'in_progress') + ORDER BY created_at DESC LIMIT 1`, + [studentId] + ); + const session = rows[0]; + if (!session) return null; + return lazyExpire(pool, session); +} + +// Lazy expiry — checked whenever a session is read or written, rather than a background cron +// (no multi-host scheduler in this phase, see the Phase 3B non-goals list). A session past its +// server-computed expires_at is transitioned to 'expired' on the next touch, whoever that is. +async function lazyExpire(pool, session) { + if (session.status === 'in_progress' && session.expires_at && new Date(session.expires_at) <= new Date()) { + const { rows } = await pool.query( + `UPDATE assessment_sessions SET status = 'expired', submitted_at = COALESCE(submitted_at, now()) + WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`, + [session.assessment_session_id] + ); + return rows[0] || session; + } + return session; +} + +async function getSessionScenarios(pool, sessionId) { + const { rows } = await pool.query( + `SELECT * FROM assessment_session_scenarios WHERE assessment_session_id = $1 ORDER BY position`, [sessionId] + ); + return rows; +} + +// Starts a new session for a student against a definition — refuses if they already have a +// live (not_started/in_progress) session for ANY definition, exam or capstone (one assessment +// at a time; keeps the "which Lab scenario am I working on for which purpose" question +// unambiguous for both the student and the hint-policy/attempt-linking logic). +async function startSession(pool, { definition, studentId, cohortId }) { + const active = await getActiveSessionForStudent(pool, studentId); + if (active) { const err = new Error('An assessment is already in progress.'); err.code = 'ALREADY_ACTIVE'; throw err; } + + const { scenario_ids, seed } = drawScenarios(definition.pool_spec, null); + const startedAt = new Date(); + const expiresAt = new Date(startedAt.getTime() + definition.duration_minutes * 60 * 1000); + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + const { rows } = await client.query( + `INSERT INTO assessment_sessions + (assessment_definition_id, definition_version, student_id, cohort_id, type, title, duration_minutes, hint_policy, + status, started_at, expires_at, random_seed) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'in_progress',$9,$10,$11) RETURNING *`, + [definition.assessment_definition_id, definition.version, studentId, cohortId || null, definition.type, + definition.title, definition.duration_minutes, definition.hint_policy, startedAt, expiresAt, seed] + ); + const session = rows[0]; + for (let i = 0; i < scenario_ids.length; i++) { + await client.query( + `INSERT INTO assessment_session_scenarios (assessment_session_id, scenario_id, position) VALUES ($1,$2,$3)`, + [session.assessment_session_id, scenario_ids[i], i] + ); + } + await client.query('COMMIT'); + return session; + } catch (e) { + await client.query('ROLLBACK'); + throw e; + } finally { + client.release(); + } +} + +// Finds the live session (if any) that has THIS scenario_id in its draw for THIS student — the +// server-side signal that a validate/hint call is happening "inside" an assessment, derived +// entirely from what the server itself stored at draw time. A client asserting "I'm in +// assessment mode" is never trusted; this lookup is the only source of truth. +async function findActiveAssessmentContextForScenario(pool, studentId, scenarioId) { + const session = await getActiveSessionForStudent(pool, studentId); + if (!session || session.status !== 'in_progress') return null; + const { rows } = await pool.query( + `SELECT * FROM assessment_session_scenarios WHERE assessment_session_id = $1 AND scenario_id = $2`, + [session.assessment_session_id, scenarioId] + ); + if (!rows[0]) return null; + return { session, sessionScenario: rows[0] }; +} + +// Updates the per-scenario row after a scenario_attempts completion, and finalizes the whole +// session once every drawn scenario is in a terminal state. Returns the finalized session row +// if this call was the one that completed it, otherwise null — the caller uses that to decide +// whether to record an achievement. +async function recordScenarioOutcome(pool, sessionId, scenarioId, attemptId, passed) { + await pool.query( + `UPDATE assessment_session_scenarios SET status = $3, attempt_id = $4 + WHERE assessment_session_id = $1 AND scenario_id = $2`, + [sessionId, scenarioId, passed ? 'completed' : 'in_progress', attemptId] + ); + if (!passed) return null; // still workable — session stays open until this scenario passes or time runs out + + const scenarios = await getSessionScenarios(pool, sessionId); + const allDone = scenarios.every((s) => s.status === 'completed'); + if (!allDone) return null; + + const total = scenarios.length; + const passedCount = scenarios.filter((s) => s.status === 'completed').length; + const { rows } = await pool.query( + `UPDATE assessment_sessions SET status = 'completed', submitted_at = now(), + result = $2 WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`, + [sessionId, JSON.stringify({ scenarios_total: total, scenarios_passed: passedCount, passed: passedCount === total })] + ); + return rows[0] || null; +} + +async function submitSession(pool, sessionId) { + const scenarios = await getSessionScenarios(pool, sessionId); + const total = scenarios.length; + const passedCount = scenarios.filter((s) => s.status === 'completed').length; + const { rows } = await pool.query( + `UPDATE assessment_sessions SET status = 'completed', submitted_at = now(), result = $2 + WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`, + [sessionId, JSON.stringify({ scenarios_total: total, scenarios_passed: passedCount, passed: passedCount === total })] + ); + return rows[0] || null; +} + +module.exports = { + drawScenarios, getActiveSessionForStudent, lazyExpire, getSessionScenarios, startSession, + findActiveAssessmentContextForScenario, recordScenarioOutcome, submitSession, +};