feat(provisioner): add disposable lab lifecycle

Internal-only service (platform/provisioner/) implementing the environment lifecycle from docs/PHASE1_PLATFORM_FOUNDATION.md: provision -> running -> (reset -> running | destroy -> destroyed), plus experimental suspend/resume (decision #3 — never on the guaranteed path, never relied on for correctness). - src/docker.js — the lab-host abstraction (correction D). Resolves a lab_hosts row into a connected dockerode client, unix:// socket or tcp:// with TLS client-cert auth. Refuses to connect to a tcp:// endpoint with no TLS material configured rather than connecting unauthenticated. Validated against both a local socket and a real remote Docker-in-Docker TLS endpoint during testing — same code path either way, only the lab_hosts config differs, which is the entire point. - src/lifecycle.js — provision/destroy/reset/suspend/resume orchestration, wired to docker.js + egress.js + the Postgres registry. Every Lab container is privileged (required for k3s, not avoidable), gets its own Docker network, no published host ports, deterministic naming (rootnode-lab-<short-id> / rootnode-lab-net-<short-id>), and a CSPRNG routing token. Includes reconcileOnStartup(), implementing the "don't trust stale state after a crash/restart" requirement from docs/LAB_ARCHITECTURE.md §8 — verified by killing a container out-of-band and confirming a Provisioner restart flags the mismatch instead of trusting the stale 'running' row. Found and fixed during testing: a provisioning failure (e.g. readiness timeout) left an orphaned container/network/firewall chain behind with only the DB row marked 'error'. provisionEnvironment's catch block now does best-effort teardown of whatever was already created before marking the row error — confirmed by deliberately failing a provision and checking no container/network/chain survived it. - src/server.js — the internal HTTP API (X-Provisioner-Key shared-secret auth on every route, matching the instructor-passcode pattern already used for the roster endpoint). Input validation on student_id/environment_id, structured JSON errors with a request id, routing tokens redacted from every response. - docker-compose.dev.yml — Postgres + egress-proxy + provisioner on one machine (the documented Phase 1 dev exception). provisioner and egress-proxy run with network_mode: host + NET_ADMIN/NET_RAW, which is what lets src/egress.js manipulate the host's DOCKER-USER chain without needing interactive host sudo. Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-08-18 01:21:52 -05:00
parent 62241d26d2
commit c8dd33522f
7 changed files with 922 additions and 0 deletions
@@ -0,0 +1,214 @@
+// RootNode Academy — Phase 1 Provisioner API.
+// Internal-only (requirement #3/#8) — every route below requires X-Provisioner-Key. Nothing
+// here is reachable by a student or the public internet in this phase; there's no frontend/
+// Gateway wiring yet, so this is exercised directly (curl / the test suite in test/run.js).
+//
+// See docs/PHASE1_PLATFORM_FOUNDATION.md §4 for the approved route list this implements.
+
+const express = require('express');
+const { randomUUID } = require('crypto');
+const { getPool, runMigrations } = require('./db');
+const { applyLabHostsSeed, getResourcePolicy, reloadResourcePolicy } = require('./config');
+const { healthCheck: dockerHealthCheck } = require('./docker');
+const lifecycle = require('./lifecycle');
+const { startReaper } = require('./reaper');
+
+const PORT = Number(process.env.PROVISIONER_PORT) || 4100;
+const SHARED_SECRET = process.env.PROVISIONER_SHARED_SECRET;
+const LAB_IMAGE_REF = process.env.LAB_IMAGE_REF || 'rootnode-academy:test';
+// No fixed EGRESS_PROXY_HOST: the proxy runs with host networking and is reached via each
+// environment's own network gateway IP, resolved per-environment in lifecycle.js — a single
+// static hostname wouldn't be reachable from every isolated student network. Only the port
+// is fixed/shared.
+const EGRESS_PROXY_PORT = Number(process.env.EGRESS_PROXY_PORT) || 3128;
+const DNS_RESOLVER = process.env.DNS_RESOLVER || '127.0.0.11'; // Docker's embedded DNS by default
+const NO_PROXY_CIDRS = (process.env.NO_PROXY_CIDRS || '10.42.0.0/16,10.43.0.0/16').split(',');
+
+if (!SHARED_SECRET) {
+  console.error('FATAL: PROVISIONER_SHARED_SECRET is not set. Refusing to start with an unauthenticated internal API.');
+  process.exit(1);
+}
+
+function egressConfig() {
+  return {
+    proxyPort: EGRESS_PROXY_PORT,
+    dnsResolver: DNS_RESOLVER,
+    extraAllowedHosts: [], // none required in Phase 1 — the Lab image doesn't call back to the Platform yet
+    noProxyCidrs: NO_PROXY_CIDRS,
+  };
+}
+
+function resourcePolicyFor() {
+  const policy = getResourcePolicy().default;
+  return {
+    cpuLimit: policy.cpu_limit,
+    memLimitMb: policy.mem_limit_mb,
+    pidsLimit: policy.pids_limit,
+    storageLimitMb: policy.storage_limit_mb,
+  };
+}
+
+function healthOpts() {
+  const hc = getResourcePolicy().health_check || {};
+  return { path: hc.path || '/api/health', timeoutMs: hc.timeout_ms || 120000 };
+}
+
+const app = express();
+app.use(express.json());
+
+// Every request gets a request id for structured error correlation, and every response is
+// JSON — no HTML error pages, no leaking stack traces to a caller.
+app.use((req, res, next) => {
+  req.requestId = randomUUID();
+  next();
+});
+
+function requireAuth(req, res, next) {
+  const key = req.get('X-Provisioner-Key');
+  if (!key || key !== SHARED_SECRET) {
+    return res.status(401).json({ error: 'unauthorized', request_id: req.requestId });
+  }
+  next();
+}
+
+function isValidStudentId(v) {
+  return typeof v === 'string' && v.length > 0 && v.length <= 128 && /^[a-zA-Z0-9_.-]+$/.test(v);
+}
+function isValidUUID(v) {
+  return typeof v === 'string' && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(v);
+}
+
+function asyncRoute(fn) {
+  return (req, res) => fn(req, res).catch((e) => {
+    console.error(`[${req.requestId}]`, e);
+    const status = e.code === 'NOT_FOUND' ? 404 : e.code === 'ALREADY_EXISTS' ? 409 : 500;
+    res.status(status).json({ error: e.message, code: e.code || 'INTERNAL', request_id: req.requestId });
+  });
+}
+
+app.use(requireAuth);
+
+// GET /health — Provisioner's own health: Docker reachable, Postgres reachable.
+app.get('/health', asyncRoute(async (req, res) => {
+  const pool = getPool();
+  const dbOk = await pool.query('SELECT 1').then(() => true).catch(() => false);
+  const labHosts = await pool.query('SELECT * FROM lab_hosts WHERE enabled = true').catch(() => ({ rows: [] }));
+  const hostChecks = await Promise.all(labHosts.rows.map(async (h) => ({ lab_host_id: h.lab_host_id, ...(await dockerHealthCheck(h)) })));
+  const ok = dbOk && hostChecks.every((h) => h.ok);
+  res.status(ok ? 200 : 503).json({ ok, db: dbOk, lab_hosts: hostChecks });
+}));
+
+// POST /environments — { student_id }
+app.post('/environments', asyncRoute(async (req, res) => {
+  const { student_id } = req.body || {};
+  if (!isValidStudentId(student_id)) {
+    return res.status(400).json({ error: 'student_id is required and must be a simple identifier', request_id: req.requestId });
+  }
+  const env = await lifecycle.provisionEnvironment({
+    pool: getPool(),
+    studentId: student_id,
+    resourcePolicy: resourcePolicyFor(),
+    imageRef: LAB_IMAGE_REF,
+    egressConfig: egressConfig(),
+    health: healthOpts(),
+  });
+  res.status(201).json({ environment_id: env.environment_id, status: env.status, routing_token: env.routing_token });
+}));
+
+// GET /environments?student_id=
+app.get('/environments', asyncRoute(async (req, res) => {
+  const { student_id } = req.query;
+  if (!isValidStudentId(student_id)) {
+    return res.status(400).json({ error: 'student_id query param is required', request_id: req.requestId });
+  }
+  const envs = await lifecycle.listEnvironments(getPool(), student_id);
+  res.json(envs.map(redactRow));
+}));
+
+// GET /environments/:id
+app.get('/environments/:id', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.getEnvironment(getPool(), req.params.id);
+  if (!env) return res.status(404).json({ error: 'not found', request_id: req.requestId });
+  res.json(redactRow(env));
+}));
+
+// POST /environments/:id/reset
+app.post('/environments/:id/reset', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.resetEnvironment({
+    pool: getPool(), environmentId: req.params.id,
+    resourcePolicy: resourcePolicyFor(), egressConfig: egressConfig(), health: healthOpts(),
+    actor: req.get('X-Actor') || 'system',
+  });
+  res.json(redactRow(env));
+}));
+
+// DELETE /environments/:id
+app.delete('/environments/:id', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.destroyEnvironment({ pool: getPool(), environmentId: req.params.id, actor: req.get('X-Actor') || 'system' });
+  res.json(redactRow(env));
+}));
+
+// POST /environments/:id/suspend — EXPERIMENTAL, see decision #3 / correction A.
+app.post('/environments/:id/suspend', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.suspendEnvironment({ pool: getPool(), environmentId: req.params.id, actor: req.get('X-Actor') || 'system' });
+  res.json({ ...redactRow(env), experimental: true });
+}));
+
+// POST /environments/:id/resume — EXPERIMENTAL, see decision #3 / correction A.
+app.post('/environments/:id/resume', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.resumeEnvironment({ pool: getPool(), environmentId: req.params.id, health: healthOpts(), actor: req.get('X-Actor') || 'system' });
+  res.json({ ...redactRow(env), experimental: true });
+}));
+
+// GET /environments/:id/events
+app.get('/environments/:id/events', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const { rows } = await getPool().query(
+    'SELECT id, event_type, actor, detail, created_at FROM environment_events WHERE environment_id = $1 ORDER BY created_at ASC',
+    [req.params.id]
+  );
+  res.json(rows);
+}));
+
+function redactRow(env) {
+  const { routing_token, ...rest } = env;
+  return { ...rest, routing_token: routing_token ? '[redacted]' : null };
+}
+
+process.on('SIGHUP', () => {
+  console.log('SIGHUP received, reloading resource-policy.yaml');
+  try { reloadResourcePolicy(); } catch (e) { console.error('reload failed:', e.message); }
+});
+
+async function main() {
+  const pool = getPool();
+  const applied = await runMigrations();
+  if (applied.length) console.log('Applied migrations:', applied.join(', '));
+  const seeded = await applyLabHostsSeed(pool);
+  console.log(`Applied lab_hosts seed (${seeded} row(s))`);
+
+  const reconcileResult = await lifecycle.reconcileOnStartup(pool);
+  console.log(`Startup reconciliation: checked=${reconcileResult.checked} ok=${reconcileResult.ok} flagged=${reconcileResult.flagged}`);
+
+  const policy = getResourcePolicy();
+  const reaper = startReaper({
+    pool,
+    idleThresholdMinutes: policy.idle_threshold_minutes,
+    abandonedThresholdMinutes: policy.abandoned_threshold_minutes,
+    pollIntervalSeconds: policy.reaper_poll_interval_seconds,
+  });
+  process.on('SIGTERM', () => { reaper.stop(); process.exit(0); });
+
+  app.listen(PORT, () => console.log(`Provisioner API listening on :${PORT}`));
+}
+
+if (require.main === module) {
+  main().catch((e) => { console.error('FATAL:', e); process.exit(1); });
+}
+
+module.exports = { app, main };