feat(assessment): add exam and capstone definition/session engine
New tables: assessment_definitions (reusable template — pool_spec, duration, hint_policy, completion/unlock rules, a plain incrementing version bumped only on semantic edits), assessment_cohort_assignments, assessment_sessions (one per student attempt, full snapshot of what was drawn and under what rules — never affected by a later definition edit), assessment_session_scenarios (the drawn set, ordered, with a real FK to scenario_attempts). Deliberately does not touch/reuse exam_sessions (migration 006) — that was already scoped in Phase 2B as historical-import-only, never the live mechanism; these tables are the real thing. drawScenarios: every 'required' scenario id included, plus N random picks per pool group, shuffled together (tiny seeded PRNG, mulberry32 — no new dependency; the seed is recorded for audit, never shown to the student before the draw). One assessment at a time per student. Timer is server-authoritative (expires_at fixed at draw time) and lazily expired — checked whenever a session is touched, no background cron. Ships two seeded defaults: rootnode-capstone (Real Ops scenarios required, weighted toward operational ability) and rootnode-quick-assessment (a general example exam matching the spec's category-draw example).
This commit is contained in:
@@ -0,0 +1,98 @@
|
|||||||
|
-- Assessment engine: definitions (exam/capstone templates), cohort assignment, live sessions,
|
||||||
|
-- and the per-session scenario draw. exam_sessions (migration 006) is left untouched — it was
|
||||||
|
-- already scoped in Phase 2B as historical-import-only from the old Lab-local SQLite exam
|
||||||
|
-- bookkeeping, never the live mechanism. These tables ARE the live mechanism; duplicating
|
||||||
|
-- exam_sessions' purpose would mean building the real thing twice, so this doesn't reuse it.
|
||||||
|
--
|
||||||
|
-- Definitions are lightly versioned (a plain incrementing integer, not full history) so a
|
||||||
|
-- later edit to pool_spec/duration/hint_policy never changes what an already-started session is
|
||||||
|
-- graded against — every session snapshots the fields that matter at draw time.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS assessment_definitions (
|
||||||
|
assessment_definition_id TEXT PRIMARY KEY, -- slug, e.g. 'rootnode-capstone'
|
||||||
|
type TEXT NOT NULL CHECK (type IN ('exam', 'capstone')),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
duration_minutes INTEGER NOT NULL,
|
||||||
|
hint_policy TEXT NOT NULL DEFAULT 'disabled' CHECK (hint_policy IN ('free', 'disabled', 'flagged')),
|
||||||
|
pool_spec JSONB NOT NULL, -- {"required": [scenario_id,...], "random": [{"count":n,"from":[scenario_id,...]},...]}
|
||||||
|
completion_rule JSONB NOT NULL DEFAULT '{"type":"all_scenarios_passed"}',
|
||||||
|
unlock_rule JSONB, -- capstone only: {"type":"stages_complete","stage_ids":[...]}
|
||||||
|
version INTEGER NOT NULL DEFAULT 1,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive')),
|
||||||
|
created_by UUID REFERENCES instructors(instructor_id),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS assessment_cohort_assignments (
|
||||||
|
assessment_definition_id TEXT NOT NULL REFERENCES assessment_definitions(assessment_definition_id),
|
||||||
|
cohort_id TEXT NOT NULL REFERENCES cohorts(cohort_id),
|
||||||
|
assigned_by UUID REFERENCES instructors(instructor_id),
|
||||||
|
assigned_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (assessment_definition_id, cohort_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS assessment_sessions (
|
||||||
|
assessment_session_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
assessment_definition_id TEXT NOT NULL REFERENCES assessment_definitions(assessment_definition_id),
|
||||||
|
definition_version INTEGER NOT NULL, -- snapshot: which version of the definition was drawn against
|
||||||
|
student_id TEXT NOT NULL REFERENCES students(student_id),
|
||||||
|
cohort_id TEXT,
|
||||||
|
type TEXT NOT NULL, -- snapshot
|
||||||
|
title TEXT NOT NULL, -- snapshot
|
||||||
|
duration_minutes INTEGER NOT NULL, -- snapshot
|
||||||
|
hint_policy TEXT NOT NULL, -- snapshot
|
||||||
|
status TEXT NOT NULL DEFAULT 'not_started'
|
||||||
|
CHECK (status IN ('not_started', 'in_progress', 'completed', 'expired', 'cancelled')),
|
||||||
|
started_at TIMESTAMPTZ,
|
||||||
|
expires_at TIMESTAMPTZ, -- server-computed = started_at + duration_minutes; authoritative, never client-set
|
||||||
|
submitted_at TIMESTAMPTZ,
|
||||||
|
random_seed TEXT, -- recorded for audit/reproducibility, not exposed to the student before the draw
|
||||||
|
result JSONB, -- {"scenarios_total":n,"scenarios_passed":n,"passed":bool}
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS assessment_session_scenarios (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
assessment_session_id UUID NOT NULL REFERENCES assessment_sessions(assessment_session_id),
|
||||||
|
scenario_id TEXT NOT NULL,
|
||||||
|
position INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'in_progress', 'completed', 'failed')),
|
||||||
|
attempt_id BIGINT REFERENCES scenario_attempts(attempt_id),
|
||||||
|
UNIQUE (assessment_session_id, scenario_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
DO $$ BEGIN
|
||||||
|
ALTER TABLE scenario_attempts ADD CONSTRAINT scenario_attempts_assessment_session_fk
|
||||||
|
FOREIGN KEY (assessment_session_id) REFERENCES assessment_sessions(assessment_session_id);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN duplicate_object THEN NULL;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_assessment_sessions_student ON assessment_sessions(student_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_assessment_sessions_definition ON assessment_sessions(assessment_definition_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_assessment_session_scenarios_session ON assessment_session_scenarios(assessment_session_id);
|
||||||
|
|
||||||
|
-- Seed defaults — a working RootNode Capstone and one example exam ship out of the box, per the
|
||||||
|
-- Phase 3B spec ("ship with a sensible RootNode default"). Scenario ids are a snapshot of
|
||||||
|
-- scenarios/rootnode-path.json's stage lists as of this migration; edit the definition via the
|
||||||
|
-- admin API if curriculum content changes later, same as any other admin-owned config.
|
||||||
|
INSERT INTO assessment_definitions (assessment_definition_id, type, title, description, duration_minutes, hint_policy, pool_spec, completion_rule, unlock_rule)
|
||||||
|
VALUES (
|
||||||
|
'rootnode-capstone', 'capstone', 'RootNode Capstone',
|
||||||
|
'Combines skills from every curriculum stage into one operational assessment, weighted toward RootNode Real Ops incident scenarios. Evaluates operational ability, not memorized kubectl syntax.',
|
||||||
|
90, 'flagged',
|
||||||
|
'{"required": ["rootnode-runaway-monitor","broken-deployment","rootnode-probe-port-mismatch","rootnode-enforce-limits"], "random": [{"count":1,"from":["deploy-nginx","scale-deployment","rolling-update-task","liveness-probe-task","edit-deployment-basics"]},{"count":1,"from":["resource-limits-task","env-vars-configmap","secrets-basics"]},{"count":1,"from":["extract-logs-basics","kubectl-cp-basics","exec-command-basics","node-cordon-basics"]}]}',
|
||||||
|
'{"type":"all_scenarios_passed"}',
|
||||||
|
'{"type":"stages_complete","stage_ids":["foundation","workloads","networking","configuration","troubleshooting","real-ops"]}'
|
||||||
|
) ON CONFLICT (assessment_definition_id) DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO assessment_definitions (assessment_definition_id, type, title, description, duration_minutes, hint_policy, pool_spec, completion_rule)
|
||||||
|
VALUES (
|
||||||
|
'rootnode-quick-assessment', 'exam', 'RootNode Kubernetes Assessment',
|
||||||
|
'A timed, randomized skills check drawn from across the curriculum. Example exam definition — instructors can create their own from the admin portal.',
|
||||||
|
60, 'disabled',
|
||||||
|
'{"required": [], "random": [{"count":2,"from":["deploy-nginx","scale-deployment","rolling-update-task","liveness-probe-task","daemonset-mcq"]},{"count":1,"from":["services-mcq","expose-service-basics"]},{"count":1,"from":["extract-logs-basics","kubectl-cp-basics","exec-command-basics","node-cordon-basics"]},{"count":1,"from":["rootnode-runaway-monitor","broken-deployment","rootnode-probe-port-mismatch","rootnode-enforce-limits"]}]}',
|
||||||
|
'{"type":"all_scenarios_passed"}'
|
||||||
|
) ON CONFLICT (assessment_definition_id) DO NOTHING;
|
||||||
@@ -0,0 +1,176 @@
|
|||||||
|
// Assessment engine — exam and capstone definitions, cohort assignment, and live sessions.
|
||||||
|
// The timer and the drawn scenario set are BOTH server-authoritative: expires_at is computed
|
||||||
|
// once at session start and never recomputed from a client-supplied value, and the drawn
|
||||||
|
// scenario_ids live only in Postgres (assessment_session_scenarios) — a student can see them
|
||||||
|
// (they need to know what to work on) but can't set or replace them. See
|
||||||
|
// docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §19 for the full integrity story.
|
||||||
|
|
||||||
|
const crypto = require('crypto');
|
||||||
|
|
||||||
|
// Tiny deterministic PRNG (mulberry32) — enough for a reproducible/audit-able draw without a
|
||||||
|
// new dependency. Seeded from a string via a cheap hash; the seed itself is recorded on the
|
||||||
|
// session for audit, not exposed to the student before the draw happens.
|
||||||
|
function mulberry32(seed) {
|
||||||
|
let a = seed >>> 0;
|
||||||
|
return function () {
|
||||||
|
a |= 0; a = (a + 0x6D2B79F5) | 0;
|
||||||
|
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
||||||
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||||||
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function seedToInt(seed) {
|
||||||
|
let h = 0;
|
||||||
|
for (let i = 0; i < seed.length; i++) { h = (Math.imul(31, h) + seed.charCodeAt(i)) | 0; }
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
function shuffle(arr, rng) {
|
||||||
|
const a = [...arr];
|
||||||
|
for (let i = a.length - 1; i > 0; i--) {
|
||||||
|
const j = Math.floor(rng() * (i + 1));
|
||||||
|
[a[i], a[j]] = [a[j], a[i]];
|
||||||
|
}
|
||||||
|
return a;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Draws a scenario set from a pool_spec: every id in `required` is included, plus `count`
|
||||||
|
// randomly-chosen ids from each `random` group's `from` list, all shuffled together into the
|
||||||
|
// student's presentation order.
|
||||||
|
function drawScenarios(poolSpec, seed) {
|
||||||
|
const actualSeed = seed || crypto.randomBytes(8).toString('hex');
|
||||||
|
const rng = mulberry32(seedToInt(actualSeed));
|
||||||
|
const required = poolSpec.required || [];
|
||||||
|
const randomPicks = (poolSpec.random || []).flatMap((group) => shuffle(group.from, rng).slice(0, group.count));
|
||||||
|
return { scenario_ids: shuffle([...required, ...randomPicks], rng), seed: actualSeed };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getActiveSessionForStudent(pool, studentId) {
|
||||||
|
const { rows } = await pool.query(
|
||||||
|
`SELECT * FROM assessment_sessions WHERE student_id = $1 AND status IN ('not_started', 'in_progress')
|
||||||
|
ORDER BY created_at DESC LIMIT 1`,
|
||||||
|
[studentId]
|
||||||
|
);
|
||||||
|
const session = rows[0];
|
||||||
|
if (!session) return null;
|
||||||
|
return lazyExpire(pool, session);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lazy expiry — checked whenever a session is read or written, rather than a background cron
|
||||||
|
// (no multi-host scheduler in this phase, see the Phase 3B non-goals list). A session past its
|
||||||
|
// server-computed expires_at is transitioned to 'expired' on the next touch, whoever that is.
|
||||||
|
async function lazyExpire(pool, session) {
|
||||||
|
if (session.status === 'in_progress' && session.expires_at && new Date(session.expires_at) <= new Date()) {
|
||||||
|
const { rows } = await pool.query(
|
||||||
|
`UPDATE assessment_sessions SET status = 'expired', submitted_at = COALESCE(submitted_at, now())
|
||||||
|
WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`,
|
||||||
|
[session.assessment_session_id]
|
||||||
|
);
|
||||||
|
return rows[0] || session;
|
||||||
|
}
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getSessionScenarios(pool, sessionId) {
|
||||||
|
const { rows } = await pool.query(
|
||||||
|
`SELECT * FROM assessment_session_scenarios WHERE assessment_session_id = $1 ORDER BY position`, [sessionId]
|
||||||
|
);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Starts a new session for a student against a definition — refuses if they already have a
|
||||||
|
// live (not_started/in_progress) session for ANY definition, exam or capstone (one assessment
|
||||||
|
// at a time; keeps the "which Lab scenario am I working on for which purpose" question
|
||||||
|
// unambiguous for both the student and the hint-policy/attempt-linking logic).
|
||||||
|
async function startSession(pool, { definition, studentId, cohortId }) {
|
||||||
|
const active = await getActiveSessionForStudent(pool, studentId);
|
||||||
|
if (active) { const err = new Error('An assessment is already in progress.'); err.code = 'ALREADY_ACTIVE'; throw err; }
|
||||||
|
|
||||||
|
const { scenario_ids, seed } = drawScenarios(definition.pool_spec, null);
|
||||||
|
const startedAt = new Date();
|
||||||
|
const expiresAt = new Date(startedAt.getTime() + definition.duration_minutes * 60 * 1000);
|
||||||
|
|
||||||
|
const client = await pool.connect();
|
||||||
|
try {
|
||||||
|
await client.query('BEGIN');
|
||||||
|
const { rows } = await client.query(
|
||||||
|
`INSERT INTO assessment_sessions
|
||||||
|
(assessment_definition_id, definition_version, student_id, cohort_id, type, title, duration_minutes, hint_policy,
|
||||||
|
status, started_at, expires_at, random_seed)
|
||||||
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'in_progress',$9,$10,$11) RETURNING *`,
|
||||||
|
[definition.assessment_definition_id, definition.version, studentId, cohortId || null, definition.type,
|
||||||
|
definition.title, definition.duration_minutes, definition.hint_policy, startedAt, expiresAt, seed]
|
||||||
|
);
|
||||||
|
const session = rows[0];
|
||||||
|
for (let i = 0; i < scenario_ids.length; i++) {
|
||||||
|
await client.query(
|
||||||
|
`INSERT INTO assessment_session_scenarios (assessment_session_id, scenario_id, position) VALUES ($1,$2,$3)`,
|
||||||
|
[session.assessment_session_id, scenario_ids[i], i]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await client.query('COMMIT');
|
||||||
|
return session;
|
||||||
|
} catch (e) {
|
||||||
|
await client.query('ROLLBACK');
|
||||||
|
throw e;
|
||||||
|
} finally {
|
||||||
|
client.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finds the live session (if any) that has THIS scenario_id in its draw for THIS student — the
|
||||||
|
// server-side signal that a validate/hint call is happening "inside" an assessment, derived
|
||||||
|
// entirely from what the server itself stored at draw time. A client asserting "I'm in
|
||||||
|
// assessment mode" is never trusted; this lookup is the only source of truth.
|
||||||
|
async function findActiveAssessmentContextForScenario(pool, studentId, scenarioId) {
|
||||||
|
const session = await getActiveSessionForStudent(pool, studentId);
|
||||||
|
if (!session || session.status !== 'in_progress') return null;
|
||||||
|
const { rows } = await pool.query(
|
||||||
|
`SELECT * FROM assessment_session_scenarios WHERE assessment_session_id = $1 AND scenario_id = $2`,
|
||||||
|
[session.assessment_session_id, scenarioId]
|
||||||
|
);
|
||||||
|
if (!rows[0]) return null;
|
||||||
|
return { session, sessionScenario: rows[0] };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Updates the per-scenario row after a scenario_attempts completion, and finalizes the whole
|
||||||
|
// session once every drawn scenario is in a terminal state. Returns the finalized session row
|
||||||
|
// if this call was the one that completed it, otherwise null — the caller uses that to decide
|
||||||
|
// whether to record an achievement.
|
||||||
|
async function recordScenarioOutcome(pool, sessionId, scenarioId, attemptId, passed) {
|
||||||
|
await pool.query(
|
||||||
|
`UPDATE assessment_session_scenarios SET status = $3, attempt_id = $4
|
||||||
|
WHERE assessment_session_id = $1 AND scenario_id = $2`,
|
||||||
|
[sessionId, scenarioId, passed ? 'completed' : 'in_progress', attemptId]
|
||||||
|
);
|
||||||
|
if (!passed) return null; // still workable — session stays open until this scenario passes or time runs out
|
||||||
|
|
||||||
|
const scenarios = await getSessionScenarios(pool, sessionId);
|
||||||
|
const allDone = scenarios.every((s) => s.status === 'completed');
|
||||||
|
if (!allDone) return null;
|
||||||
|
|
||||||
|
const total = scenarios.length;
|
||||||
|
const passedCount = scenarios.filter((s) => s.status === 'completed').length;
|
||||||
|
const { rows } = await pool.query(
|
||||||
|
`UPDATE assessment_sessions SET status = 'completed', submitted_at = now(),
|
||||||
|
result = $2 WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`,
|
||||||
|
[sessionId, JSON.stringify({ scenarios_total: total, scenarios_passed: passedCount, passed: passedCount === total })]
|
||||||
|
);
|
||||||
|
return rows[0] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function submitSession(pool, sessionId) {
|
||||||
|
const scenarios = await getSessionScenarios(pool, sessionId);
|
||||||
|
const total = scenarios.length;
|
||||||
|
const passedCount = scenarios.filter((s) => s.status === 'completed').length;
|
||||||
|
const { rows } = await pool.query(
|
||||||
|
`UPDATE assessment_sessions SET status = 'completed', submitted_at = now(), result = $2
|
||||||
|
WHERE assessment_session_id = $1 AND status = 'in_progress' RETURNING *`,
|
||||||
|
[sessionId, JSON.stringify({ scenarios_total: total, scenarios_passed: passedCount, passed: passedCount === total })]
|
||||||
|
);
|
||||||
|
return rows[0] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
drawScenarios, getActiveSessionForStudent, lazyExpire, getSessionScenarios, startSession,
|
||||||
|
findActiveAssessmentContextForScenario, recordScenarioOutcome, submitSession,
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user