feat(provisioner): add disposable lab lifecycle

Internal-only service (platform/provisioner/) implementing the environment lifecycle from docs/PHASE1_PLATFORM_FOUNDATION.md: provision -> running -> (reset -> running | destroy -> destroyed), plus experimental suspend/resume (decision #3 — never on the guaranteed path, never relied on for correctness). - src/docker.js — the lab-host abstraction (correction D). Resolves a lab_hosts row into a connected dockerode client, unix:// socket or tcp:// with TLS client-cert auth. Refuses to connect to a tcp:// endpoint with no TLS material configured rather than connecting unauthenticated. Validated against both a local socket and a real remote Docker-in-Docker TLS endpoint during testing — same code path either way, only the lab_hosts config differs, which is the entire point. - src/lifecycle.js — provision/destroy/reset/suspend/resume orchestration, wired to docker.js + egress.js + the Postgres registry. Every Lab container is privileged (required for k3s, not avoidable), gets its own Docker network, no published host ports, deterministic naming (rootnode-lab-<short-id> / rootnode-lab-net-<short-id>), and a CSPRNG routing token. Includes reconcileOnStartup(), implementing the "don't trust stale state after a crash/restart" requirement from docs/LAB_ARCHITECTURE.md §8 — verified by killing a container out-of-band and confirming a Provisioner restart flags the mismatch instead of trusting the stale 'running' row. Found and fixed during testing: a provisioning failure (e.g. readiness timeout) left an orphaned container/network/firewall chain behind with only the DB row marked 'error'. provisionEnvironment's catch block now does best-effort teardown of whatever was already created before marking the row error — confirmed by deliberately failing a provision and checking no container/network/chain survived it. - src/server.js — the internal HTTP API (X-Provisioner-Key shared-secret auth on every route, matching the instructor-passcode pattern already used for the roster endpoint). Input validation on student_id/environment_id, structured JSON errors with a request id, routing tokens redacted from every response. - docker-compose.dev.yml — Postgres + egress-proxy + provisioner on one machine (the documented Phase 1 dev exception). provisioner and egress-proxy run with network_mode: host + NET_ADMIN/NET_RAW, which is what lets src/egress.js manipulate the host's DOCKER-USER chain without needing interactive host sudo. Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-08-18 01:21:52 -05:00
parent 62241d26d2
commit c8dd33522f
7 changed files with 922 additions and 0 deletions
@@ -0,0 +1,66 @@
+// Lab-host abstraction (correction D) — resolves a `lab_hosts` row into a connected dockerode
+// client. Nothing in this module or its callers may assume Docker is local; the endpoint
+// always comes from a `lab_hosts` row, whether that row points at a Unix socket (Phase 1 dev)
+// or a remote TCP+TLS endpoint (production Lab Host VM, per docs/LAB_ARCHITECTURE.md §1).
+
+const fs = require('fs');
+const Docker = require('dockerode');
+
+// Caches one dockerode client per lab_host_id — cheap to keep around, avoids re-parsing TLS
+// material on every call.
+const clientCache = new Map();
+
+function buildClient(labHost) {
+  if (labHost.endpoint.startsWith('unix://')) {
+    return new Docker({ socketPath: labHost.endpoint.replace('unix://', '') });
+  }
+  if (labHost.endpoint.startsWith('tcp://')) {
+    const url = new URL(labHost.endpoint.replace('tcp://', 'http://'));
+    const opts = { host: url.hostname, port: Number(url.port) || 2376, protocol: 'https' };
+    if (labHost.tls_ca_path && labHost.tls_cert_path && labHost.tls_key_path) {
+      opts.ca = fs.readFileSync(labHost.tls_ca_path);
+      opts.cert = fs.readFileSync(labHost.tls_cert_path);
+      opts.key = fs.readFileSync(labHost.tls_key_path);
+    } else {
+      // ponytail: no TLS material configured for a remote endpoint is a real misconfiguration,
+      // not a silently-degraded mode — fail loudly rather than connect unauthenticated to a
+      // remote Docker API (see docs/PHASE1_PLATFORM_FOUNDATION.md §6 on why that cert matters
+      // as much as the Provisioner API's own shared secret).
+      throw new Error(
+        `lab_host '${labHost.lab_host_id}' has a tcp:// endpoint but no TLS cert/key/ca configured — refusing to connect unauthenticated.`
+      );
+    }
+    return new Docker(opts);
+  }
+  throw new Error(`lab_host '${labHost.lab_host_id}' has an unrecognized endpoint scheme: ${labHost.endpoint}`);
+}
+
+function clientFor(labHost) {
+  let client = clientCache.get(labHost.lab_host_id);
+  if (!client) {
+    client = buildClient(labHost);
+    clientCache.set(labHost.lab_host_id, client);
+  }
+  return client;
+}
+
+// Phase 1: no scheduler, just "the one enabled row" (docs/PHASE1_PLATFORM_FOUNDATION.md §4).
+async function selectLabHost(pool) {
+  const { rows } = await pool.query(
+    `SELECT * FROM lab_hosts WHERE enabled = true ORDER BY created_at ASC LIMIT 1`
+  );
+  if (!rows.length) throw new Error('No enabled lab_hosts row configured — cannot provision.');
+  return rows[0];
+}
+
+async function healthCheck(labHost) {
+  try {
+    const client = clientFor(labHost);
+    const info = await client.info();
+    return { ok: true, containers: info.Containers, serverVersion: info.ServerVersion };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+module.exports = { clientFor, selectLabHost, healthCheck };
@@ -0,0 +1,459 @@
+// Environment lifecycle orchestration — the state machine from
+// docs/PHASE1_PLATFORM_FOUNDATION.md §3, wired to docker.js (lab-host abstraction), egress.js
+// (firewall), and db.js (registry + audit).
+//
+// Guaranteed path: requested -> provisioning -> running -> resetting -> running
+//                  (provisioning|running|error) -> destroying -> destroyed
+// Suspend/resume is experimental only (decision #3 / correction A) — see suspendEnvironment/
+// resumeEnvironment below, both explicitly opt-in and never on the guaranteed path.
+
+const crypto = require('crypto');
+const http = require('http');
+const { selectLabHost, clientFor } = require('./docker');
+const { setupEgressChain, teardownEgressChain } = require('./egress');
+
+function shortId(environmentId) {
+  return environmentId.replace(/-/g, '').slice(0, 12);
+}
+
+function generateRoutingToken() {
+  return crypto.randomBytes(32).toString('base64url'); // opaque, CSPRNG, URL-safe
+}
+
+// Never let a routing token reach the audit log in plaintext (requirement #9).
+function redact(detail) {
+  if (!detail || typeof detail !== 'object') return detail;
+  const copy = { ...detail };
+  if (copy.routing_token) copy.routing_token = '[redacted]';
+  return copy;
+}
+
+async function emitEvent(pool, environmentId, eventType, actor, detail = null) {
+  await pool.query(
+    `INSERT INTO environment_events (environment_id, event_type, actor, detail) VALUES ($1, $2, $3, $4)`,
+    [environmentId, eventType, actor, detail ? JSON.stringify(redact(detail)) : null]
+  );
+}
+
+async function getEnvironment(pool, environmentId) {
+  const { rows } = await pool.query('SELECT * FROM environments WHERE environment_id = $1', [environmentId]);
+  return rows[0] || null;
+}
+
+async function listEnvironments(pool, studentId) {
+  const { rows } = await pool.query(
+    'SELECT * FROM environments WHERE student_id = $1 ORDER BY created_at DESC',
+    [studentId]
+  );
+  return rows;
+}
+
+async function getActiveEnvironmentForStudent(pool, studentId) {
+  const { rows } = await pool.query(
+    `SELECT * FROM environments WHERE student_id = $1
+     AND status NOT IN ('destroyed', 'error') ORDER BY created_at DESC LIMIT 1`,
+    [studentId]
+  );
+  return rows[0] || null;
+}
+
+async function pollHealth(ip, port, path, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const ok = await new Promise((resolve) => {
+      const req = http.get({ host: ip, port, path, timeout: 2000 }, (res) => {
+        res.resume();
+        resolve(res.statusCode >= 200 && res.statusCode < 300);
+      });
+      req.on('error', () => resolve(false));
+      req.on('timeout', () => { req.destroy(); resolve(false); });
+    });
+    if (ok) return true;
+    await new Promise((r) => setTimeout(r, 2000));
+  }
+  return false;
+}
+
+/**
+ * provision — the full requested -> provisioning -> running (or error) path.
+ * @param {object} opts
+ * @param {import('pg').Pool} opts.pool
+ * @param {string} opts.studentId          - opaque, not validated against a roster (correction C)
+ * @param {object} opts.resourcePolicy     - { cpuLimit, memLimitMb, pidsLimit, storageLimitMb }
+ * @param {string} opts.imageRef
+ * @param {object} opts.egressConfig       - { proxyHost, proxyPort, dnsResolver, extraAllowedHosts, noProxyCidrs }
+ * @param {object} [opts.health]           - { path='/api/health', timeoutMs=120000 } — overridable for testing
+ */
+async function provisionEnvironment({ pool, studentId, resourcePolicy, imageRef, egressConfig, health = {} }) {
+  const existing = await getActiveEnvironmentForStudent(pool, studentId);
+  if (existing) {
+    throw Object.assign(new Error(`student '${studentId}' already has a non-terminal environment (${existing.environment_id})`), { code: 'ALREADY_EXISTS', existing });
+  }
+
+  const labHost = await selectLabHost(pool);
+  const docker = clientFor(labHost);
+  const routingToken = generateRoutingToken();
+
+  const insertResult = await pool.query(
+    `INSERT INTO environments
+       (student_id, lab_host_id, status, routing_token, image_ref, cpu_limit, mem_limit_mb, pids_limit, storage_limit_mb)
+     VALUES ($1, $2, 'requested', $3, $4, $5, $6, $7, $8)
+     RETURNING *`,
+    [studentId, labHost.lab_host_id, routingToken, imageRef,
+      resourcePolicy.cpuLimit, resourcePolicy.memLimitMb, resourcePolicy.pidsLimit, resourcePolicy.storageLimitMb]
+  );
+  const env = insertResult.rows[0];
+  await emitEvent(pool, env.environment_id, 'provision_requested', 'system', { student_id: studentId });
+  await emitEvent(pool, env.environment_id, 'host_selected', 'system', { lab_host_id: labHost.lab_host_id });
+
+  try {
+    await pool.query(`UPDATE environments SET status = 'provisioning' WHERE environment_id = $1`, [env.environment_id]);
+
+    const id = shortId(env.environment_id);
+    const networkName = `rootnode-lab-net-${id}`;
+    const containerName = `rootnode-lab-${id}`;
+
+    // Network
+    const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' });
+    const netInfo = await network.inspect();
+    const subnet = netInfo.IPAM.Config[0].Subnet;
+    // The egress proxy runs with host networking (see docker-compose.dev.yml), so it's present
+    // on every bridge network's gateway interface — that gateway IP, not a fixed hostname, is
+    // what's actually reachable from *this* environment's private network. Resolving it per
+    // network (rather than one static proxyHost) is what makes the proxy reachable at all once
+    // there's more than one isolated student network.
+    const proxyHost = netInfo.IPAM.Config[0].Gateway;
+    await pool.query(
+      `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3 WHERE environment_id = $4`,
+      [network.id, networkName, subnet, env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'network_created', 'system', { network_name: networkName, subnet });
+
+    // Firewall — installed before the container starts, so there's never a window where the
+    // container has network access but no restriction in place.
+    await setupEgressChain({
+      environmentId: env.environment_id,
+      subnet,
+      egressProxyHost: proxyHost,
+      egressProxyPort: egressConfig.proxyPort,
+      dnsResolver: egressConfig.dnsResolver,
+      extraAllowedHosts: egressConfig.extraAllowedHosts || [],
+    });
+    await emitEvent(pool, env.environment_id, 'firewall_configured', 'system', { subnet });
+
+    // Container
+    const container = await docker.createContainer({
+      name: containerName,
+      Image: imageRef,
+      HostConfig: {
+        Privileged: true, // required for k3s — see docs/LAB_ARCHITECTURE.md §7, not avoidable
+        NetworkMode: networkName,
+        NanoCpus: Math.round(resourcePolicy.cpuLimit * 1e9),
+        Memory: resourcePolicy.memLimitMb * 1024 * 1024,
+        MemorySwap: resourcePolicy.memLimitMb * 1024 * 1024, // pinned equal to Memory — no swap
+        PidsLimit: resourcePolicy.pidsLimit,
+        PublishAllPorts: false, // no public host port mappings — requirement #5
+        StorageOpt: resourcePolicy.storageLimitMb ? { size: `${resourcePolicy.storageLimitMb}m` } : undefined,
+      },
+      Env: [
+        `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+        `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+        `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`,
+      ],
+    });
+    await container.start();
+    const containerInfo = await container.inspect();
+    const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress;
+
+    await pool.query(
+      `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`,
+      [container.id, containerName, internalIp, env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'container_created', 'system', { container_name: containerName, internal_ip: internalIp });
+
+    // Readiness
+    const healthPath = health.path || '/api/health';
+    const timeoutMs = health.timeoutMs || 120000;
+    const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs);
+    if (!ready) {
+      throw new Error(`environment ${env.environment_id} did not become ready within ${timeoutMs}ms`);
+    }
+
+    const { rows } = await pool.query(
+      `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`,
+      [env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'ready', 'system', {});
+    return rows[0];
+  } catch (e) {
+    // Best-effort teardown of whatever got created before the failure — a provisioning
+    // failure must not leave an orphaned container/network/firewall chain running
+    // indefinitely. Found by testing: a readiness timeout left a real container and firewall
+    // chain in place with only the DB row marked 'error', which is exactly the kind of
+    // resource leak requirement #10 (idle reaper) exists to prevent — except this environment
+    // never reaches 'running', so it would never have had a last_active_at to age out on
+    // (see the reaper fix in reaper.js). Cleaning up here directly is more honest than relying
+    // on the reaper to catch a case it wasn't actually able to catch.
+    try {
+      const fresh = await getEnvironment(pool, env.environment_id);
+      const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [fresh.lab_host_id]);
+      const docker = clientFor(labHostRes.rows[0]);
+      if (fresh.container_id) {
+        await docker.getContainer(fresh.container_id).remove({ force: true }).catch(() => {});
+      }
+      if (fresh.network_subnet) {
+        await teardownEgressChain({ environmentId: fresh.environment_id, subnet: fresh.network_subnet }).catch(() => {});
+      }
+      if (fresh.network_id) {
+        await docker.getNetwork(fresh.network_id).remove().catch(() => {});
+      }
+    } catch (cleanupErr) {
+      // Cleanup itself failing is exactly what the reaper's COALESCE(last_active_at,
+      // created_at) fallback exists to catch eventually — log and move on, don't mask the
+      // original provisioning error with a cleanup error.
+      await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'provision_failure_cleanup', message: cleanupErr.message }).catch(() => {});
+    }
+
+    await pool.query(
+      `UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+      [e.message, env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'error', 'system', { message: e.message });
+    throw e;
+  }
+}
+
+/**
+ * destroy — stop+remove container, remove network+firewall chain, mark destroyed.
+ * Never touches academic data — that lives entirely outside the Lab (Platform SQLite today,
+ * see docs/LAB_ARCHITECTURE.md §3) — destroying an environment is purely infrastructure teardown.
+ */
+async function destroyEnvironment({ pool, environmentId, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+  if (env.status === 'destroyed') return env;
+
+  await pool.query(`UPDATE environments SET status = 'destroying' WHERE environment_id = $1`, [environmentId]);
+
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+
+  if (env.container_id) {
+    try {
+      const container = docker.getContainer(env.container_id);
+      await container.stop({ t: 10 }).catch(() => {}); // already stopped/gone is fine
+      await container.remove({ force: true }).catch(() => {});
+    } catch (e) {
+      // Container already gone — proceed with cleanup regardless (failure-safe, per requirement #10).
+    }
+  }
+
+  if (env.network_subnet) {
+    await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch((e) => {
+      // Log via event but don't block destroy on firewall cleanup failure — an orphaned
+      // per-env chain for a destroyed environment is a cleanup task, not a live security hole
+      // (its subnet is no longer bound to any running container).
+      return emitEvent(pool, environmentId, 'error', actor, { stage: 'firewall_teardown', message: e.message });
+    });
+  }
+
+  if (env.network_id) {
+    try {
+      await docker.getNetwork(env.network_id).remove();
+    } catch (e) { /* already gone is fine */ }
+  }
+
+  const { rows } = await pool.query(
+    `UPDATE environments SET status = 'destroyed' WHERE environment_id = $1 RETURNING *`,
+    [environmentId]
+  );
+  await emitEvent(pool, environmentId, 'destroyed', actor, {});
+  return rows[0];
+}
+
+/**
+ * reset — destroy + provision again, same environment_id/routing_token (same DB row).
+ */
+async function resetEnvironment({ pool, environmentId, resourcePolicy, egressConfig, health, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+
+  await pool.query(`UPDATE environments SET status = 'resetting' WHERE environment_id = $1`, [environmentId]);
+  await emitEvent(pool, environmentId, 'reset', actor, {});
+
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+
+  if (env.container_id) {
+    const container = docker.getContainer(env.container_id);
+    await container.stop({ t: 10 }).catch(() => {});
+    await container.remove({ force: true }).catch(() => {});
+  }
+  if (env.network_subnet) {
+    await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch(() => {});
+  }
+  if (env.network_id) {
+    await docker.getNetwork(env.network_id).remove().catch(() => {});
+  }
+
+  // Recreate against the SAME row: same environment_id/routing_token, fresh everything else.
+  const id = shortId(env.environment_id);
+  const networkName = `rootnode-lab-net-${id}`;
+  const containerName = `rootnode-lab-${id}`;
+
+  const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' });
+  const netInfo = await network.inspect();
+  const subnet = netInfo.IPAM.Config[0].Subnet;
+  const proxyHost = netInfo.IPAM.Config[0].Gateway; // see provisionEnvironment for why this is per-network, not static
+  await pool.query(
+    `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3, container_id = NULL, container_name = NULL, internal_ip = NULL, error_reason = NULL WHERE environment_id = $4`,
+    [network.id, networkName, subnet, environmentId]
+  );
+  await emitEvent(pool, environmentId, 'network_created', actor, { network_name: networkName, subnet });
+
+  await setupEgressChain({
+    environmentId, subnet,
+    egressProxyHost: proxyHost, egressProxyPort: egressConfig.proxyPort,
+    dnsResolver: egressConfig.dnsResolver, extraAllowedHosts: egressConfig.extraAllowedHosts || [],
+  });
+  await emitEvent(pool, environmentId, 'firewall_configured', actor, { subnet });
+
+  const container = await docker.createContainer({
+    name: containerName,
+    Image: env.image_ref,
+    HostConfig: {
+      Privileged: true,
+      NetworkMode: networkName,
+      NanoCpus: Math.round(Number(env.cpu_limit) * 1e9),
+      Memory: env.mem_limit_mb * 1024 * 1024,
+      MemorySwap: env.mem_limit_mb * 1024 * 1024,
+      PidsLimit: env.pids_limit,
+      PublishAllPorts: false,
+      StorageOpt: env.storage_limit_mb ? { size: `${env.storage_limit_mb}m` } : undefined,
+    },
+    Env: [
+      `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+      `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+      `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`,
+    ],
+  });
+  await container.start();
+  const containerInfo = await container.inspect();
+  const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress;
+  await pool.query(
+    `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`,
+    [container.id, containerName, internalIp, environmentId]
+  );
+  await emitEvent(pool, environmentId, 'container_created', actor, { container_name: containerName, internal_ip: internalIp });
+
+  const healthPath = (health && health.path) || '/api/health';
+  const timeoutMs = (health && health.timeoutMs) || 120000;
+  const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs);
+  if (!ready) {
+    await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+      [`reset: did not become ready within ${timeoutMs}ms`, environmentId]);
+    await emitEvent(pool, environmentId, 'error', actor, { stage: 'reset_readiness' });
+    throw new Error(`environment ${environmentId} did not become ready after reset`);
+  }
+
+  const { rows } = await pool.query(
+    `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`,
+    [environmentId]
+  );
+  await emitEvent(pool, environmentId, 'ready', actor, {});
+  return rows[0];
+}
+
+// ── Suspend/resume — EXPERIMENTAL ONLY (decision #3 / correction A) ────────────────────────
+// Never called by the reaper or any guaranteed-path logic. Exists so the test plan
+// (docs/PHASE1_PLATFORM_FOUNDATION.md §8 test 7) can measure whether it's viable at all.
+
+async function suspendEnvironment({ pool, environmentId, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+  await docker.getContainer(env.container_id).stop({ t: 10 });
+  const { rows } = await pool.query(`UPDATE environments SET status = 'suspended' WHERE environment_id = $1 RETURNING *`, [environmentId]);
+  await emitEvent(pool, environmentId, 'suspend', actor, { experimental: true });
+  return rows[0];
+}
+
+async function resumeEnvironment({ pool, environmentId, health = {}, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+  await docker.getContainer(env.container_id).start();
+
+  const healthPath = health.path || '/api/health';
+  const timeoutMs = health.timeoutMs || 120000;
+  const ready = await pollHealth(env.internal_ip, env.internal_port, healthPath, timeoutMs);
+  if (!ready) {
+    await pool.query(`UPDATE environments SET status = 'error', error_reason = 'resume: did not become ready' WHERE environment_id = $1`, [environmentId]);
+    await emitEvent(pool, environmentId, 'error', actor, { stage: 'resume_readiness', experimental: true });
+    throw new Error(`environment ${environmentId} did not become ready after resume (suspend/resume is experimental — see decision #3)`);
+  }
+  const { rows } = await pool.query(`UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, [environmentId]);
+  await emitEvent(pool, environmentId, 'resume', actor, { experimental: true });
+  return rows[0];
+}
+
+// ── Reconciliation on startup — docs/LAB_ARCHITECTURE.md §8 ────────────────────────────────
+// "On Provisioner restart, it must reconcile its registry against actual running containers
+// rather than trust stale DB state." Anything the registry claims is running/in-flight that
+// isn't actually running on its lab host gets marked error, not silently left wrong.
+async function reconcileOnStartup(pool) {
+  const { rows } = await pool.query(
+    `SELECT * FROM environments WHERE status IN ('provisioning', 'running', 'resetting', 'destroying', 'suspended')`
+  );
+  const results = { checked: rows.length, flagged: 0, ok: 0 };
+  for (const env of rows) {
+    try {
+      const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+      if (!labHostRes.rows.length) {
+        await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+          ['reconcile: lab_host no longer configured', env.environment_id]);
+        await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'lab_host missing' });
+        results.flagged++;
+        continue;
+      }
+      const docker = clientFor(labHostRes.rows[0]);
+      let actuallyRunning = false;
+      if (env.container_id) {
+        try {
+          const info = await docker.getContainer(env.container_id).inspect();
+          actuallyRunning = info.State && info.State.Running;
+        } catch {
+          actuallyRunning = false; // container doesn't exist on this host at all
+        }
+      }
+      const expectedRunning = env.status === 'running' || env.status === 'suspended';
+      if (expectedRunning && !actuallyRunning) {
+        await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+          ['reconcile: DB claimed running/suspended but container is not actually running', env.environment_id]);
+        await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'container not running' });
+        results.flagged++;
+      } else if (['provisioning', 'resetting', 'destroying'].includes(env.status)) {
+        // Any environment caught mid-transition by a Provisioner crash/restart is not
+        // trustworthy either way — never silently assume it finished.
+        await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+          [`reconcile: caught mid-transition (${env.status}) on restart`, env.environment_id]);
+        await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: `mid-transition:${env.status}` });
+        results.flagged++;
+      } else {
+        results.ok++;
+      }
+    } catch (e) {
+      await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', message: e.message }).catch(() => {});
+      results.flagged++;
+    }
+  }
+  return results;
+}
+
+module.exports = {
+  generateRoutingToken, emitEvent, getEnvironment, listEnvironments, getActiveEnvironmentForStudent,
+  provisionEnvironment, destroyEnvironment, resetEnvironment,
+  suspendEnvironment, resumeEnvironment, pollHealth, shortId, reconcileOnStartup,
+};
@@ -0,0 +1,214 @@
+// RootNode Academy — Phase 1 Provisioner API.
+// Internal-only (requirement #3/#8) — every route below requires X-Provisioner-Key. Nothing
+// here is reachable by a student or the public internet in this phase; there's no frontend/
+// Gateway wiring yet, so this is exercised directly (curl / the test suite in test/run.js).
+//
+// See docs/PHASE1_PLATFORM_FOUNDATION.md §4 for the approved route list this implements.
+
+const express = require('express');
+const { randomUUID } = require('crypto');
+const { getPool, runMigrations } = require('./db');
+const { applyLabHostsSeed, getResourcePolicy, reloadResourcePolicy } = require('./config');
+const { healthCheck: dockerHealthCheck } = require('./docker');
+const lifecycle = require('./lifecycle');
+const { startReaper } = require('./reaper');
+
+const PORT = Number(process.env.PROVISIONER_PORT) || 4100;
+const SHARED_SECRET = process.env.PROVISIONER_SHARED_SECRET;
+const LAB_IMAGE_REF = process.env.LAB_IMAGE_REF || 'rootnode-academy:test';
+// No fixed EGRESS_PROXY_HOST: the proxy runs with host networking and is reached via each
+// environment's own network gateway IP, resolved per-environment in lifecycle.js — a single
+// static hostname wouldn't be reachable from every isolated student network. Only the port
+// is fixed/shared.
+const EGRESS_PROXY_PORT = Number(process.env.EGRESS_PROXY_PORT) || 3128;
+const DNS_RESOLVER = process.env.DNS_RESOLVER || '127.0.0.11'; // Docker's embedded DNS by default
+const NO_PROXY_CIDRS = (process.env.NO_PROXY_CIDRS || '10.42.0.0/16,10.43.0.0/16').split(',');
+
+if (!SHARED_SECRET) {
+  console.error('FATAL: PROVISIONER_SHARED_SECRET is not set. Refusing to start with an unauthenticated internal API.');
+  process.exit(1);
+}
+
+function egressConfig() {
+  return {
+    proxyPort: EGRESS_PROXY_PORT,
+    dnsResolver: DNS_RESOLVER,
+    extraAllowedHosts: [], // none required in Phase 1 — the Lab image doesn't call back to the Platform yet
+    noProxyCidrs: NO_PROXY_CIDRS,
+  };
+}
+
+function resourcePolicyFor() {
+  const policy = getResourcePolicy().default;
+  return {
+    cpuLimit: policy.cpu_limit,
+    memLimitMb: policy.mem_limit_mb,
+    pidsLimit: policy.pids_limit,
+    storageLimitMb: policy.storage_limit_mb,
+  };
+}
+
+function healthOpts() {
+  const hc = getResourcePolicy().health_check || {};
+  return { path: hc.path || '/api/health', timeoutMs: hc.timeout_ms || 120000 };
+}
+
+const app = express();
+app.use(express.json());
+
+// Every request gets a request id for structured error correlation, and every response is
+// JSON — no HTML error pages, no leaking stack traces to a caller.
+app.use((req, res, next) => {
+  req.requestId = randomUUID();
+  next();
+});
+
+function requireAuth(req, res, next) {
+  const key = req.get('X-Provisioner-Key');
+  if (!key || key !== SHARED_SECRET) {
+    return res.status(401).json({ error: 'unauthorized', request_id: req.requestId });
+  }
+  next();
+}
+
+function isValidStudentId(v) {
+  return typeof v === 'string' && v.length > 0 && v.length <= 128 && /^[a-zA-Z0-9_.-]+$/.test(v);
+}
+function isValidUUID(v) {
+  return typeof v === 'string' && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(v);
+}
+
+function asyncRoute(fn) {
+  return (req, res) => fn(req, res).catch((e) => {
+    console.error(`[${req.requestId}]`, e);
+    const status = e.code === 'NOT_FOUND' ? 404 : e.code === 'ALREADY_EXISTS' ? 409 : 500;
+    res.status(status).json({ error: e.message, code: e.code || 'INTERNAL', request_id: req.requestId });
+  });
+}
+
+app.use(requireAuth);
+
+// GET /health — Provisioner's own health: Docker reachable, Postgres reachable.
+app.get('/health', asyncRoute(async (req, res) => {
+  const pool = getPool();
+  const dbOk = await pool.query('SELECT 1').then(() => true).catch(() => false);
+  const labHosts = await pool.query('SELECT * FROM lab_hosts WHERE enabled = true').catch(() => ({ rows: [] }));
+  const hostChecks = await Promise.all(labHosts.rows.map(async (h) => ({ lab_host_id: h.lab_host_id, ...(await dockerHealthCheck(h)) })));
+  const ok = dbOk && hostChecks.every((h) => h.ok);
+  res.status(ok ? 200 : 503).json({ ok, db: dbOk, lab_hosts: hostChecks });
+}));
+
+// POST /environments — { student_id }
+app.post('/environments', asyncRoute(async (req, res) => {
+  const { student_id } = req.body || {};
+  if (!isValidStudentId(student_id)) {
+    return res.status(400).json({ error: 'student_id is required and must be a simple identifier', request_id: req.requestId });
+  }
+  const env = await lifecycle.provisionEnvironment({
+    pool: getPool(),
+    studentId: student_id,
+    resourcePolicy: resourcePolicyFor(),
+    imageRef: LAB_IMAGE_REF,
+    egressConfig: egressConfig(),
+    health: healthOpts(),
+  });
+  res.status(201).json({ environment_id: env.environment_id, status: env.status, routing_token: env.routing_token });
+}));
+
+// GET /environments?student_id=
+app.get('/environments', asyncRoute(async (req, res) => {
+  const { student_id } = req.query;
+  if (!isValidStudentId(student_id)) {
+    return res.status(400).json({ error: 'student_id query param is required', request_id: req.requestId });
+  }
+  const envs = await lifecycle.listEnvironments(getPool(), student_id);
+  res.json(envs.map(redactRow));
+}));
+
+// GET /environments/:id
+app.get('/environments/:id', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.getEnvironment(getPool(), req.params.id);
+  if (!env) return res.status(404).json({ error: 'not found', request_id: req.requestId });
+  res.json(redactRow(env));
+}));
+
+// POST /environments/:id/reset
+app.post('/environments/:id/reset', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.resetEnvironment({
+    pool: getPool(), environmentId: req.params.id,
+    resourcePolicy: resourcePolicyFor(), egressConfig: egressConfig(), health: healthOpts(),
+    actor: req.get('X-Actor') || 'system',
+  });
+  res.json(redactRow(env));
+}));
+
+// DELETE /environments/:id
+app.delete('/environments/:id', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.destroyEnvironment({ pool: getPool(), environmentId: req.params.id, actor: req.get('X-Actor') || 'system' });
+  res.json(redactRow(env));
+}));
+
+// POST /environments/:id/suspend — EXPERIMENTAL, see decision #3 / correction A.
+app.post('/environments/:id/suspend', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.suspendEnvironment({ pool: getPool(), environmentId: req.params.id, actor: req.get('X-Actor') || 'system' });
+  res.json({ ...redactRow(env), experimental: true });
+}));
+
+// POST /environments/:id/resume — EXPERIMENTAL, see decision #3 / correction A.
+app.post('/environments/:id/resume', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const env = await lifecycle.resumeEnvironment({ pool: getPool(), environmentId: req.params.id, health: healthOpts(), actor: req.get('X-Actor') || 'system' });
+  res.json({ ...redactRow(env), experimental: true });
+}));
+
+// GET /environments/:id/events
+app.get('/environments/:id/events', asyncRoute(async (req, res) => {
+  if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId });
+  const { rows } = await getPool().query(
+    'SELECT id, event_type, actor, detail, created_at FROM environment_events WHERE environment_id = $1 ORDER BY created_at ASC',
+    [req.params.id]
+  );
+  res.json(rows);
+}));
+
+function redactRow(env) {
+  const { routing_token, ...rest } = env;
+  return { ...rest, routing_token: routing_token ? '[redacted]' : null };
+}
+
+process.on('SIGHUP', () => {
+  console.log('SIGHUP received, reloading resource-policy.yaml');
+  try { reloadResourcePolicy(); } catch (e) { console.error('reload failed:', e.message); }
+});
+
+async function main() {
+  const pool = getPool();
+  const applied = await runMigrations();
+  if (applied.length) console.log('Applied migrations:', applied.join(', '));
+  const seeded = await applyLabHostsSeed(pool);
+  console.log(`Applied lab_hosts seed (${seeded} row(s))`);
+
+  const reconcileResult = await lifecycle.reconcileOnStartup(pool);
+  console.log(`Startup reconciliation: checked=${reconcileResult.checked} ok=${reconcileResult.ok} flagged=${reconcileResult.flagged}`);
+
+  const policy = getResourcePolicy();
+  const reaper = startReaper({
+    pool,
+    idleThresholdMinutes: policy.idle_threshold_minutes,
+    abandonedThresholdMinutes: policy.abandoned_threshold_minutes,
+    pollIntervalSeconds: policy.reaper_poll_interval_seconds,
+  });
+  process.on('SIGTERM', () => { reaper.stop(); process.exit(0); });
+
+  app.listen(PORT, () => console.log(`Provisioner API listening on :${PORT}`));
+}
+
+if (require.main === module) {
+  main().catch((e) => { console.error('FATAL:', e); process.exit(1); });
+}
+
+module.exports = { app, main };