feat(provisioner): add disposable lab lifecycle

Internal-only service (platform/provisioner/) implementing the environment lifecycle from docs/PHASE1_PLATFORM_FOUNDATION.md: provision -> running -> (reset -> running | destroy -> destroyed), plus experimental suspend/resume (decision #3 — never on the guaranteed path, never relied on for correctness). - src/docker.js — the lab-host abstraction (correction D). Resolves a lab_hosts row into a connected dockerode client, unix:// socket or tcp:// with TLS client-cert auth. Refuses to connect to a tcp:// endpoint with no TLS material configured rather than connecting unauthenticated. Validated against both a local socket and a real remote Docker-in-Docker TLS endpoint during testing — same code path either way, only the lab_hosts config differs, which is the entire point. - src/lifecycle.js — provision/destroy/reset/suspend/resume orchestration, wired to docker.js + egress.js + the Postgres registry. Every Lab container is privileged (required for k3s, not avoidable), gets its own Docker network, no published host ports, deterministic naming (rootnode-lab-<short-id> / rootnode-lab-net-<short-id>), and a CSPRNG routing token. Includes reconcileOnStartup(), implementing the "don't trust stale state after a crash/restart" requirement from docs/LAB_ARCHITECTURE.md §8 — verified by killing a container out-of-band and confirming a Provisioner restart flags the mismatch instead of trusting the stale 'running' row. Found and fixed during testing: a provisioning failure (e.g. readiness timeout) left an orphaned container/network/firewall chain behind with only the DB row marked 'error'. provisionEnvironment's catch block now does best-effort teardown of whatever was already created before marking the row error — confirmed by deliberately failing a provision and checking no container/network/chain survived it. - src/server.js — the internal HTTP API (X-Provisioner-Key shared-secret auth on every route, matching the instructor-passcode pattern already used for the roster endpoint). Input validation on student_id/environment_id, structured JSON errors with a request id, routing tokens redacted from every response. - docker-compose.dev.yml — Postgres + egress-proxy + provisioner on one machine (the documented Phase 1 dev exception). provisioner and egress-proxy run with network_mode: host + NET_ADMIN/NET_RAW, which is what lets src/egress.js manipulate the host's DOCKER-USER chain without needing interactive host sudo. Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-08-18 01:21:52 -05:00
parent 62241d26d2
commit c8dd33522f
7 changed files with 922 additions and 0 deletions
@@ -0,0 +1,459 @@
+// Environment lifecycle orchestration — the state machine from
+// docs/PHASE1_PLATFORM_FOUNDATION.md §3, wired to docker.js (lab-host abstraction), egress.js
+// (firewall), and db.js (registry + audit).
+//
+// Guaranteed path: requested -> provisioning -> running -> resetting -> running
+//                  (provisioning|running|error) -> destroying -> destroyed
+// Suspend/resume is experimental only (decision #3 / correction A) — see suspendEnvironment/
+// resumeEnvironment below, both explicitly opt-in and never on the guaranteed path.
+
+const crypto = require('crypto');
+const http = require('http');
+const { selectLabHost, clientFor } = require('./docker');
+const { setupEgressChain, teardownEgressChain } = require('./egress');
+
+function shortId(environmentId) {
+  return environmentId.replace(/-/g, '').slice(0, 12);
+}
+
+function generateRoutingToken() {
+  return crypto.randomBytes(32).toString('base64url'); // opaque, CSPRNG, URL-safe
+}
+
+// Never let a routing token reach the audit log in plaintext (requirement #9).
+function redact(detail) {
+  if (!detail || typeof detail !== 'object') return detail;
+  const copy = { ...detail };
+  if (copy.routing_token) copy.routing_token = '[redacted]';
+  return copy;
+}
+
+async function emitEvent(pool, environmentId, eventType, actor, detail = null) {
+  await pool.query(
+    `INSERT INTO environment_events (environment_id, event_type, actor, detail) VALUES ($1, $2, $3, $4)`,
+    [environmentId, eventType, actor, detail ? JSON.stringify(redact(detail)) : null]
+  );
+}
+
+async function getEnvironment(pool, environmentId) {
+  const { rows } = await pool.query('SELECT * FROM environments WHERE environment_id = $1', [environmentId]);
+  return rows[0] || null;
+}
+
+async function listEnvironments(pool, studentId) {
+  const { rows } = await pool.query(
+    'SELECT * FROM environments WHERE student_id = $1 ORDER BY created_at DESC',
+    [studentId]
+  );
+  return rows;
+}
+
+async function getActiveEnvironmentForStudent(pool, studentId) {
+  const { rows } = await pool.query(
+    `SELECT * FROM environments WHERE student_id = $1
+     AND status NOT IN ('destroyed', 'error') ORDER BY created_at DESC LIMIT 1`,
+    [studentId]
+  );
+  return rows[0] || null;
+}
+
+async function pollHealth(ip, port, path, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const ok = await new Promise((resolve) => {
+      const req = http.get({ host: ip, port, path, timeout: 2000 }, (res) => {
+        res.resume();
+        resolve(res.statusCode >= 200 && res.statusCode < 300);
+      });
+      req.on('error', () => resolve(false));
+      req.on('timeout', () => { req.destroy(); resolve(false); });
+    });
+    if (ok) return true;
+    await new Promise((r) => setTimeout(r, 2000));
+  }
+  return false;
+}
+
+/**
+ * provision — the full requested -> provisioning -> running (or error) path.
+ * @param {object} opts
+ * @param {import('pg').Pool} opts.pool
+ * @param {string} opts.studentId          - opaque, not validated against a roster (correction C)
+ * @param {object} opts.resourcePolicy     - { cpuLimit, memLimitMb, pidsLimit, storageLimitMb }
+ * @param {string} opts.imageRef
+ * @param {object} opts.egressConfig       - { proxyHost, proxyPort, dnsResolver, extraAllowedHosts, noProxyCidrs }
+ * @param {object} [opts.health]           - { path='/api/health', timeoutMs=120000 } — overridable for testing
+ */
+async function provisionEnvironment({ pool, studentId, resourcePolicy, imageRef, egressConfig, health = {} }) {
+  const existing = await getActiveEnvironmentForStudent(pool, studentId);
+  if (existing) {
+    throw Object.assign(new Error(`student '${studentId}' already has a non-terminal environment (${existing.environment_id})`), { code: 'ALREADY_EXISTS', existing });
+  }
+
+  const labHost = await selectLabHost(pool);
+  const docker = clientFor(labHost);
+  const routingToken = generateRoutingToken();
+
+  const insertResult = await pool.query(
+    `INSERT INTO environments
+       (student_id, lab_host_id, status, routing_token, image_ref, cpu_limit, mem_limit_mb, pids_limit, storage_limit_mb)
+     VALUES ($1, $2, 'requested', $3, $4, $5, $6, $7, $8)
+     RETURNING *`,
+    [studentId, labHost.lab_host_id, routingToken, imageRef,
+      resourcePolicy.cpuLimit, resourcePolicy.memLimitMb, resourcePolicy.pidsLimit, resourcePolicy.storageLimitMb]
+  );
+  const env = insertResult.rows[0];
+  await emitEvent(pool, env.environment_id, 'provision_requested', 'system', { student_id: studentId });
+  await emitEvent(pool, env.environment_id, 'host_selected', 'system', { lab_host_id: labHost.lab_host_id });
+
+  try {
+    await pool.query(`UPDATE environments SET status = 'provisioning' WHERE environment_id = $1`, [env.environment_id]);
+
+    const id = shortId(env.environment_id);
+    const networkName = `rootnode-lab-net-${id}`;
+    const containerName = `rootnode-lab-${id}`;
+
+    // Network
+    const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' });
+    const netInfo = await network.inspect();
+    const subnet = netInfo.IPAM.Config[0].Subnet;
+    // The egress proxy runs with host networking (see docker-compose.dev.yml), so it's present
+    // on every bridge network's gateway interface — that gateway IP, not a fixed hostname, is
+    // what's actually reachable from *this* environment's private network. Resolving it per
+    // network (rather than one static proxyHost) is what makes the proxy reachable at all once
+    // there's more than one isolated student network.
+    const proxyHost = netInfo.IPAM.Config[0].Gateway;
+    await pool.query(
+      `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3 WHERE environment_id = $4`,
+      [network.id, networkName, subnet, env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'network_created', 'system', { network_name: networkName, subnet });
+
+    // Firewall — installed before the container starts, so there's never a window where the
+    // container has network access but no restriction in place.
+    await setupEgressChain({
+      environmentId: env.environment_id,
+      subnet,
+      egressProxyHost: proxyHost,
+      egressProxyPort: egressConfig.proxyPort,
+      dnsResolver: egressConfig.dnsResolver,
+      extraAllowedHosts: egressConfig.extraAllowedHosts || [],
+    });
+    await emitEvent(pool, env.environment_id, 'firewall_configured', 'system', { subnet });
+
+    // Container
+    const container = await docker.createContainer({
+      name: containerName,
+      Image: imageRef,
+      HostConfig: {
+        Privileged: true, // required for k3s — see docs/LAB_ARCHITECTURE.md §7, not avoidable
+        NetworkMode: networkName,
+        NanoCpus: Math.round(resourcePolicy.cpuLimit * 1e9),
+        Memory: resourcePolicy.memLimitMb * 1024 * 1024,
+        MemorySwap: resourcePolicy.memLimitMb * 1024 * 1024, // pinned equal to Memory — no swap
+        PidsLimit: resourcePolicy.pidsLimit,
+        PublishAllPorts: false, // no public host port mappings — requirement #5
+        StorageOpt: resourcePolicy.storageLimitMb ? { size: `${resourcePolicy.storageLimitMb}m` } : undefined,
+      },
+      Env: [
+        `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+        `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+        `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`,
+      ],
+    });
+    await container.start();
+    const containerInfo = await container.inspect();
+    const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress;
+
+    await pool.query(
+      `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`,
+      [container.id, containerName, internalIp, env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'container_created', 'system', { container_name: containerName, internal_ip: internalIp });
+
+    // Readiness
+    const healthPath = health.path || '/api/health';
+    const timeoutMs = health.timeoutMs || 120000;
+    const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs);
+    if (!ready) {
+      throw new Error(`environment ${env.environment_id} did not become ready within ${timeoutMs}ms`);
+    }
+
+    const { rows } = await pool.query(
+      `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`,
+      [env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'ready', 'system', {});
+    return rows[0];
+  } catch (e) {
+    // Best-effort teardown of whatever got created before the failure — a provisioning
+    // failure must not leave an orphaned container/network/firewall chain running
+    // indefinitely. Found by testing: a readiness timeout left a real container and firewall
+    // chain in place with only the DB row marked 'error', which is exactly the kind of
+    // resource leak requirement #10 (idle reaper) exists to prevent — except this environment
+    // never reaches 'running', so it would never have had a last_active_at to age out on
+    // (see the reaper fix in reaper.js). Cleaning up here directly is more honest than relying
+    // on the reaper to catch a case it wasn't actually able to catch.
+    try {
+      const fresh = await getEnvironment(pool, env.environment_id);
+      const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [fresh.lab_host_id]);
+      const docker = clientFor(labHostRes.rows[0]);
+      if (fresh.container_id) {
+        await docker.getContainer(fresh.container_id).remove({ force: true }).catch(() => {});
+      }
+      if (fresh.network_subnet) {
+        await teardownEgressChain({ environmentId: fresh.environment_id, subnet: fresh.network_subnet }).catch(() => {});
+      }
+      if (fresh.network_id) {
+        await docker.getNetwork(fresh.network_id).remove().catch(() => {});
+      }
+    } catch (cleanupErr) {
+      // Cleanup itself failing is exactly what the reaper's COALESCE(last_active_at,
+      // created_at) fallback exists to catch eventually — log and move on, don't mask the
+      // original provisioning error with a cleanup error.
+      await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'provision_failure_cleanup', message: cleanupErr.message }).catch(() => {});
+    }
+
+    await pool.query(
+      `UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+      [e.message, env.environment_id]
+    );
+    await emitEvent(pool, env.environment_id, 'error', 'system', { message: e.message });
+    throw e;
+  }
+}
+
+/**
+ * destroy — stop+remove container, remove network+firewall chain, mark destroyed.
+ * Never touches academic data — that lives entirely outside the Lab (Platform SQLite today,
+ * see docs/LAB_ARCHITECTURE.md §3) — destroying an environment is purely infrastructure teardown.
+ */
+async function destroyEnvironment({ pool, environmentId, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+  if (env.status === 'destroyed') return env;
+
+  await pool.query(`UPDATE environments SET status = 'destroying' WHERE environment_id = $1`, [environmentId]);
+
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+
+  if (env.container_id) {
+    try {
+      const container = docker.getContainer(env.container_id);
+      await container.stop({ t: 10 }).catch(() => {}); // already stopped/gone is fine
+      await container.remove({ force: true }).catch(() => {});
+    } catch (e) {
+      // Container already gone — proceed with cleanup regardless (failure-safe, per requirement #10).
+    }
+  }
+
+  if (env.network_subnet) {
+    await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch((e) => {
+      // Log via event but don't block destroy on firewall cleanup failure — an orphaned
+      // per-env chain for a destroyed environment is a cleanup task, not a live security hole
+      // (its subnet is no longer bound to any running container).
+      return emitEvent(pool, environmentId, 'error', actor, { stage: 'firewall_teardown', message: e.message });
+    });
+  }
+
+  if (env.network_id) {
+    try {
+      await docker.getNetwork(env.network_id).remove();
+    } catch (e) { /* already gone is fine */ }
+  }
+
+  const { rows } = await pool.query(
+    `UPDATE environments SET status = 'destroyed' WHERE environment_id = $1 RETURNING *`,
+    [environmentId]
+  );
+  await emitEvent(pool, environmentId, 'destroyed', actor, {});
+  return rows[0];
+}
+
+/**
+ * reset — destroy + provision again, same environment_id/routing_token (same DB row).
+ */
+async function resetEnvironment({ pool, environmentId, resourcePolicy, egressConfig, health, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+
+  await pool.query(`UPDATE environments SET status = 'resetting' WHERE environment_id = $1`, [environmentId]);
+  await emitEvent(pool, environmentId, 'reset', actor, {});
+
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+
+  if (env.container_id) {
+    const container = docker.getContainer(env.container_id);
+    await container.stop({ t: 10 }).catch(() => {});
+    await container.remove({ force: true }).catch(() => {});
+  }
+  if (env.network_subnet) {
+    await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch(() => {});
+  }
+  if (env.network_id) {
+    await docker.getNetwork(env.network_id).remove().catch(() => {});
+  }
+
+  // Recreate against the SAME row: same environment_id/routing_token, fresh everything else.
+  const id = shortId(env.environment_id);
+  const networkName = `rootnode-lab-net-${id}`;
+  const containerName = `rootnode-lab-${id}`;
+
+  const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' });
+  const netInfo = await network.inspect();
+  const subnet = netInfo.IPAM.Config[0].Subnet;
+  const proxyHost = netInfo.IPAM.Config[0].Gateway; // see provisionEnvironment for why this is per-network, not static
+  await pool.query(
+    `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3, container_id = NULL, container_name = NULL, internal_ip = NULL, error_reason = NULL WHERE environment_id = $4`,
+    [network.id, networkName, subnet, environmentId]
+  );
+  await emitEvent(pool, environmentId, 'network_created', actor, { network_name: networkName, subnet });
+
+  await setupEgressChain({
+    environmentId, subnet,
+    egressProxyHost: proxyHost, egressProxyPort: egressConfig.proxyPort,
+    dnsResolver: egressConfig.dnsResolver, extraAllowedHosts: egressConfig.extraAllowedHosts || [],
+  });
+  await emitEvent(pool, environmentId, 'firewall_configured', actor, { subnet });
+
+  const container = await docker.createContainer({
+    name: containerName,
+    Image: env.image_ref,
+    HostConfig: {
+      Privileged: true,
+      NetworkMode: networkName,
+      NanoCpus: Math.round(Number(env.cpu_limit) * 1e9),
+      Memory: env.mem_limit_mb * 1024 * 1024,
+      MemorySwap: env.mem_limit_mb * 1024 * 1024,
+      PidsLimit: env.pids_limit,
+      PublishAllPorts: false,
+      StorageOpt: env.storage_limit_mb ? { size: `${env.storage_limit_mb}m` } : undefined,
+    },
+    Env: [
+      `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+      `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`,
+      `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`,
+    ],
+  });
+  await container.start();
+  const containerInfo = await container.inspect();
+  const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress;
+  await pool.query(
+    `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`,
+    [container.id, containerName, internalIp, environmentId]
+  );
+  await emitEvent(pool, environmentId, 'container_created', actor, { container_name: containerName, internal_ip: internalIp });
+
+  const healthPath = (health && health.path) || '/api/health';
+  const timeoutMs = (health && health.timeoutMs) || 120000;
+  const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs);
+  if (!ready) {
+    await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+      [`reset: did not become ready within ${timeoutMs}ms`, environmentId]);
+    await emitEvent(pool, environmentId, 'error', actor, { stage: 'reset_readiness' });
+    throw new Error(`environment ${environmentId} did not become ready after reset`);
+  }
+
+  const { rows } = await pool.query(
+    `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`,
+    [environmentId]
+  );
+  await emitEvent(pool, environmentId, 'ready', actor, {});
+  return rows[0];
+}
+
+// ── Suspend/resume — EXPERIMENTAL ONLY (decision #3 / correction A) ────────────────────────
+// Never called by the reaper or any guaranteed-path logic. Exists so the test plan
+// (docs/PHASE1_PLATFORM_FOUNDATION.md §8 test 7) can measure whether it's viable at all.
+
+async function suspendEnvironment({ pool, environmentId, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+  await docker.getContainer(env.container_id).stop({ t: 10 });
+  const { rows } = await pool.query(`UPDATE environments SET status = 'suspended' WHERE environment_id = $1 RETURNING *`, [environmentId]);
+  await emitEvent(pool, environmentId, 'suspend', actor, { experimental: true });
+  return rows[0];
+}
+
+async function resumeEnvironment({ pool, environmentId, health = {}, actor = 'system' }) {
+  const env = await getEnvironment(pool, environmentId);
+  if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' });
+  const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+  const docker = clientFor(labHostRes.rows[0]);
+  await docker.getContainer(env.container_id).start();
+
+  const healthPath = health.path || '/api/health';
+  const timeoutMs = health.timeoutMs || 120000;
+  const ready = await pollHealth(env.internal_ip, env.internal_port, healthPath, timeoutMs);
+  if (!ready) {
+    await pool.query(`UPDATE environments SET status = 'error', error_reason = 'resume: did not become ready' WHERE environment_id = $1`, [environmentId]);
+    await emitEvent(pool, environmentId, 'error', actor, { stage: 'resume_readiness', experimental: true });
+    throw new Error(`environment ${environmentId} did not become ready after resume (suspend/resume is experimental — see decision #3)`);
+  }
+  const { rows } = await pool.query(`UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, [environmentId]);
+  await emitEvent(pool, environmentId, 'resume', actor, { experimental: true });
+  return rows[0];
+}
+
+// ── Reconciliation on startup — docs/LAB_ARCHITECTURE.md §8 ────────────────────────────────
+// "On Provisioner restart, it must reconcile its registry against actual running containers
+// rather than trust stale DB state." Anything the registry claims is running/in-flight that
+// isn't actually running on its lab host gets marked error, not silently left wrong.
+async function reconcileOnStartup(pool) {
+  const { rows } = await pool.query(
+    `SELECT * FROM environments WHERE status IN ('provisioning', 'running', 'resetting', 'destroying', 'suspended')`
+  );
+  const results = { checked: rows.length, flagged: 0, ok: 0 };
+  for (const env of rows) {
+    try {
+      const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]);
+      if (!labHostRes.rows.length) {
+        await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+          ['reconcile: lab_host no longer configured', env.environment_id]);
+        await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'lab_host missing' });
+        results.flagged++;
+        continue;
+      }
+      const docker = clientFor(labHostRes.rows[0]);
+      let actuallyRunning = false;
+      if (env.container_id) {
+        try {
+          const info = await docker.getContainer(env.container_id).inspect();
+          actuallyRunning = info.State && info.State.Running;
+        } catch {
+          actuallyRunning = false; // container doesn't exist on this host at all
+        }
+      }
+      const expectedRunning = env.status === 'running' || env.status === 'suspended';
+      if (expectedRunning && !actuallyRunning) {
+        await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+          ['reconcile: DB claimed running/suspended but container is not actually running', env.environment_id]);
+        await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'container not running' });
+        results.flagged++;
+      } else if (['provisioning', 'resetting', 'destroying'].includes(env.status)) {
+        // Any environment caught mid-transition by a Provisioner crash/restart is not
+        // trustworthy either way — never silently assume it finished.
+        await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`,
+          [`reconcile: caught mid-transition (${env.status}) on restart`, env.environment_id]);
+        await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: `mid-transition:${env.status}` });
+        results.flagged++;
+      } else {
+        results.ok++;
+      }
+    } catch (e) {
+      await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', message: e.message }).catch(() => {});
+      results.flagged++;
+    }
+  }
+  return results;
+}
+
+module.exports = {
+  generateRoutingToken, emitEvent, getEnvironment, listEnvironments, getActiveEnvironmentForStudent,
+  provisionEnvironment, destroyEnvironment, resetEnvironment,
+  suspendEnvironment, resumeEnvironment, pollHealth, shortId, reconcileOnStartup,
+};