// Environment lifecycle orchestration — the state machine from // docs/PHASE1_PLATFORM_FOUNDATION.md §3, wired to docker.js (lab-host abstraction), egress.js // (firewall), and db.js (registry + audit). // // Guaranteed path: requested -> provisioning -> running -> resetting -> running // (provisioning|running|error) -> destroying -> destroyed // Suspend/resume is experimental only (decision #3 / correction A) — see suspendEnvironment/ // resumeEnvironment below, both explicitly opt-in and never on the guaranteed path. const crypto = require('crypto'); const http = require('http'); const { selectLabHost, clientFor } = require('./docker'); const { setupEgressChain, teardownEgressChain } = require('./egress'); function shortId(environmentId) { return environmentId.replace(/-/g, '').slice(0, 12); } function generateRoutingToken() { return crypto.randomBytes(32).toString('base64url'); // opaque, CSPRNG, URL-safe } // Never let a routing token reach the audit log in plaintext (requirement #9). function redact(detail) { if (!detail || typeof detail !== 'object') return detail; const copy = { ...detail }; if (copy.routing_token) copy.routing_token = '[redacted]'; return copy; } async function emitEvent(pool, environmentId, eventType, actor, detail = null) { await pool.query( `INSERT INTO environment_events (environment_id, event_type, actor, detail) VALUES ($1, $2, $3, $4)`, [environmentId, eventType, actor, detail ? JSON.stringify(redact(detail)) : null] ); } async function getEnvironment(pool, environmentId) { const { rows } = await pool.query('SELECT * FROM environments WHERE environment_id = $1', [environmentId]); return rows[0] || null; } // Phase 4.1 — the Gateway's read path (GET /environments/by-token/:token) needs to know HOW // to reach the Lab, not just its Docker-internal address: is this lab_host 'direct' (Gateway // reaches env.internal_ip itself) or 'router' (Gateway must go through that lab_host's Lab // Host Router instead)? Joining lab_hosts here keeps that decision entirely data-driven — // the Gateway never hardcodes or guesses a host's routing mode. async function getEnvironmentByTokenWithRouting(pool, token) { const { rows } = await pool.query( `SELECT e.*, h.routing_mode AS lab_host_routing_mode, h.router_endpoint AS lab_host_router_endpoint FROM environments e JOIN lab_hosts h ON h.lab_host_id = e.lab_host_id WHERE e.routing_token = $1`, [token] ); return rows[0] || null; } async function listEnvironments(pool, studentId) { const { rows } = await pool.query( 'SELECT * FROM environments WHERE student_id = $1 ORDER BY created_at DESC', [studentId] ); return rows; } async function getActiveEnvironmentForStudent(pool, studentId) { const { rows } = await pool.query( `SELECT * FROM environments WHERE student_id = $1 AND status NOT IN ('destroyed', 'error') ORDER BY created_at DESC LIMIT 1`, [studentId] ); return rows[0] || null; } async function pollHealth(ip, port, path, timeoutMs) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const ok = await new Promise((resolve) => { const req = http.get({ host: ip, port, path, timeout: 2000 }, (res) => { res.resume(); resolve(res.statusCode >= 200 && res.statusCode < 300); }); req.on('error', () => resolve(false)); req.on('timeout', () => { req.destroy(); resolve(false); }); }); if (ok) return true; await new Promise((r) => setTimeout(r, 2000)); } return false; } /** * provision — the full requested -> provisioning -> running (or error) path. * @param {object} opts * @param {import('pg').Pool} opts.pool * @param {string} opts.studentId - opaque, not validated against a roster (correction C) * @param {object} opts.resourcePolicy - { cpuLimit, memLimitMb, pidsLimit, storageLimitMb } * @param {string} opts.imageRef * @param {object} opts.egressConfig - { proxyHost, proxyPort, dnsResolver, extraAllowedHosts, noProxyCidrs } * @param {object} [opts.health] - { path='/api/health', timeoutMs=120000 } — overridable for testing */ async function provisionEnvironment({ pool, studentId, resourcePolicy, imageRef, egressConfig, health = {} }) { const existing = await getActiveEnvironmentForStudent(pool, studentId); if (existing) { throw Object.assign(new Error(`student '${studentId}' already has a non-terminal environment (${existing.environment_id})`), { code: 'ALREADY_EXISTS', existing }); } // Admission control (Phase 4.1 §26) — checked BEFORE touching Docker at all, so a full host // fails cleanly and fast (503, no partial network/container left behind to clean up) instead // of failing mid-provision with a generic 500 once Docker itself runs out of room. No-op // when maxConcurrentLabs isn't configured (see resourcePolicyFor's comment on why that's the // default). if (resourcePolicy.maxConcurrentLabs) { const { rows } = await pool.query( `SELECT count(*)::int AS n FROM environments WHERE status NOT IN ('destroyed', 'error')` ); if (rows[0].n >= resourcePolicy.maxConcurrentLabs) { throw Object.assign( new Error('Lab capacity is temporarily full. Try again shortly.'), { code: 'CAPACITY_EXCEEDED', current: rows[0].n, max: resourcePolicy.maxConcurrentLabs } ); } } const labHost = await selectLabHost(pool); const docker = clientFor(labHost); const routingToken = generateRoutingToken(); const insertResult = await pool.query( `INSERT INTO environments (student_id, lab_host_id, status, routing_token, image_ref, cpu_limit, mem_limit_mb, pids_limit, storage_limit_mb) VALUES ($1, $2, 'requested', $3, $4, $5, $6, $7, $8) RETURNING *`, [studentId, labHost.lab_host_id, routingToken, imageRef, resourcePolicy.cpuLimit, resourcePolicy.memLimitMb, resourcePolicy.pidsLimit, resourcePolicy.storageLimitMb] ); const env = insertResult.rows[0]; await emitEvent(pool, env.environment_id, 'provision_requested', 'system', { student_id: studentId }); await emitEvent(pool, env.environment_id, 'host_selected', 'system', { lab_host_id: labHost.lab_host_id }); try { await pool.query(`UPDATE environments SET status = 'provisioning' WHERE environment_id = $1`, [env.environment_id]); const id = shortId(env.environment_id); const networkName = `rootnode-lab-net-${id}`; const containerName = `rootnode-lab-${id}`; // Network const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' }); const netInfo = await network.inspect(); const subnet = netInfo.IPAM.Config[0].Subnet; // The egress proxy runs with host networking (see docker-compose.dev.yml), so it's present // on every bridge network's gateway interface — that gateway IP, not a fixed hostname, is // what's actually reachable from *this* environment's private network. Resolving it per // network (rather than one static proxyHost) is what makes the proxy reachable at all once // there's more than one isolated student network. const proxyHost = netInfo.IPAM.Config[0].Gateway; await pool.query( `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3 WHERE environment_id = $4`, [network.id, networkName, subnet, env.environment_id] ); await emitEvent(pool, env.environment_id, 'network_created', 'system', { network_name: networkName, subnet }); // Firewall — installed before the container starts, so there's never a window where the // container has network access but no restriction in place. await setupEgressChain({ environmentId: env.environment_id, subnet, egressProxyHost: proxyHost, egressProxyPort: egressConfig.proxyPort, dnsResolver: egressConfig.dnsResolver, extraAllowedHosts: egressConfig.extraAllowedHosts || [], }); await emitEvent(pool, env.environment_id, 'firewall_configured', 'system', { subnet }); // Container const container = await docker.createContainer({ name: containerName, Image: imageRef, // Phase 4.1: labeled by routing token (not environment_id alone) so the Lab Host Router // — which runs on this same Docker host and has no Postgres access — can resolve // "which local container does this opaque token address" purely from local `docker ps`, // with no cross-VM call back to the Provisioner needed at request time. See // docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1, platform/lab-router/src/router.js. Labels: { 'rootnode.routing-token': routingToken, 'rootnode.environment-id': env.environment_id, }, HostConfig: { Privileged: true, // required for k3s — see docs/LAB_ARCHITECTURE.md §7, not avoidable NetworkMode: networkName, NanoCpus: Math.round(resourcePolicy.cpuLimit * 1e9), Memory: resourcePolicy.memLimitMb * 1024 * 1024, MemorySwap: resourcePolicy.memLimitMb * 1024 * 1024, // pinned equal to Memory — no swap PidsLimit: resourcePolicy.pidsLimit, PublishAllPorts: false, // no public host port mappings — requirement #5 StorageOpt: resourcePolicy.storageLimitMb ? { size: `${resourcePolicy.storageLimitMb}m` } : undefined, }, Env: [ `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`, ], }); await container.start(); const containerInfo = await container.inspect(); const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress; await pool.query( `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`, [container.id, containerName, internalIp, env.environment_id] ); await emitEvent(pool, env.environment_id, 'container_created', 'system', { container_name: containerName, internal_ip: internalIp }); // Readiness const healthPath = health.path || '/api/health'; const timeoutMs = health.timeoutMs || 120000; const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs); if (!ready) { throw new Error(`environment ${env.environment_id} did not become ready within ${timeoutMs}ms`); } const { rows } = await pool.query( `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, [env.environment_id] ); await emitEvent(pool, env.environment_id, 'ready', 'system', {}); return rows[0]; } catch (e) { // Best-effort teardown of whatever got created before the failure — a provisioning // failure must not leave an orphaned container/network/firewall chain running // indefinitely. Found by testing: a readiness timeout left a real container and firewall // chain in place with only the DB row marked 'error', which is exactly the kind of // resource leak requirement #10 (idle reaper) exists to prevent — except this environment // never reaches 'running', so it would never have had a last_active_at to age out on // (see the reaper fix in reaper.js). Cleaning up here directly is more honest than relying // on the reaper to catch a case it wasn't actually able to catch. try { const fresh = await getEnvironment(pool, env.environment_id); const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [fresh.lab_host_id]); const docker = clientFor(labHostRes.rows[0]); if (fresh.container_id) { await docker.getContainer(fresh.container_id).remove({ force: true }).catch(() => {}); } if (fresh.network_subnet) { await teardownEgressChain({ environmentId: fresh.environment_id, subnet: fresh.network_subnet }).catch(() => {}); } if (fresh.network_id) { await docker.getNetwork(fresh.network_id).remove().catch(() => {}); } } catch (cleanupErr) { // Cleanup itself failing is exactly what the reaper's COALESCE(last_active_at, // created_at) fallback exists to catch eventually — log and move on, don't mask the // original provisioning error with a cleanup error. await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'provision_failure_cleanup', message: cleanupErr.message }).catch(() => {}); } await pool.query( `UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, [e.message, env.environment_id] ); await emitEvent(pool, env.environment_id, 'error', 'system', { message: e.message }); throw e; } } /** * destroy — stop+remove container, remove network+firewall chain, mark destroyed. * Never touches academic data — that lives entirely outside the Lab (Platform SQLite today, * see docs/LAB_ARCHITECTURE.md §3) — destroying an environment is purely infrastructure teardown. */ async function destroyEnvironment({ pool, environmentId, actor = 'system' }) { const env = await getEnvironment(pool, environmentId); if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); if (env.status === 'destroyed') return env; await pool.query(`UPDATE environments SET status = 'destroying' WHERE environment_id = $1`, [environmentId]); const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); const docker = clientFor(labHostRes.rows[0]); if (env.container_id) { try { const container = docker.getContainer(env.container_id); await container.stop({ t: 10 }).catch(() => {}); // already stopped/gone is fine await container.remove({ force: true }).catch(() => {}); } catch (e) { // Container already gone — proceed with cleanup regardless (failure-safe, per requirement #10). } } if (env.network_subnet) { await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch((e) => { // Log via event but don't block destroy on firewall cleanup failure — an orphaned // per-env chain for a destroyed environment is a cleanup task, not a live security hole // (its subnet is no longer bound to any running container). return emitEvent(pool, environmentId, 'error', actor, { stage: 'firewall_teardown', message: e.message }); }); } if (env.network_id) { try { await docker.getNetwork(env.network_id).remove(); } catch (e) { /* already gone is fine */ } } const { rows } = await pool.query( `UPDATE environments SET status = 'destroyed' WHERE environment_id = $1 RETURNING *`, [environmentId] ); await emitEvent(pool, environmentId, 'destroyed', actor, {}); return rows[0]; } /** * reset — destroy + provision again, same environment_id/routing_token (same DB row). */ async function resetEnvironment({ pool, environmentId, resourcePolicy, egressConfig, health, actor = 'system' }) { const env = await getEnvironment(pool, environmentId); if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); await pool.query(`UPDATE environments SET status = 'resetting' WHERE environment_id = $1`, [environmentId]); await emitEvent(pool, environmentId, 'reset', actor, {}); const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); const docker = clientFor(labHostRes.rows[0]); if (env.container_id) { const container = docker.getContainer(env.container_id); await container.stop({ t: 10 }).catch(() => {}); await container.remove({ force: true }).catch(() => {}); } if (env.network_subnet) { await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch(() => {}); } if (env.network_id) { await docker.getNetwork(env.network_id).remove().catch(() => {}); } // Recreate against the SAME row: same environment_id/routing_token, fresh everything else. const id = shortId(env.environment_id); const networkName = `rootnode-lab-net-${id}`; const containerName = `rootnode-lab-${id}`; const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' }); const netInfo = await network.inspect(); const subnet = netInfo.IPAM.Config[0].Subnet; const proxyHost = netInfo.IPAM.Config[0].Gateway; // see provisionEnvironment for why this is per-network, not static await pool.query( `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3, container_id = NULL, container_name = NULL, internal_ip = NULL, error_reason = NULL WHERE environment_id = $4`, [network.id, networkName, subnet, environmentId] ); await emitEvent(pool, environmentId, 'network_created', actor, { network_name: networkName, subnet }); await setupEgressChain({ environmentId, subnet, egressProxyHost: proxyHost, egressProxyPort: egressConfig.proxyPort, dnsResolver: egressConfig.dnsResolver, extraAllowedHosts: egressConfig.extraAllowedHosts || [], }); await emitEvent(pool, environmentId, 'firewall_configured', actor, { subnet }); const container = await docker.createContainer({ name: containerName, Image: env.image_ref, // Phase 4.1 — same routing-token label as provisionEnvironment; a reset keeps the same // environment_id/routing_token (same DB row) so this must relabel the RECREATED container // the same way, or the Lab Host Router would resolve stale/no container for this token. Labels: { 'rootnode.routing-token': env.routing_token, 'rootnode.environment-id': environmentId, }, HostConfig: { Privileged: true, NetworkMode: networkName, NanoCpus: Math.round(Number(env.cpu_limit) * 1e9), Memory: env.mem_limit_mb * 1024 * 1024, MemorySwap: env.mem_limit_mb * 1024 * 1024, PidsLimit: env.pids_limit, PublishAllPorts: false, StorageOpt: env.storage_limit_mb ? { size: `${env.storage_limit_mb}m` } : undefined, }, Env: [ `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`, ], }); await container.start(); const containerInfo = await container.inspect(); const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress; await pool.query( `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`, [container.id, containerName, internalIp, environmentId] ); await emitEvent(pool, environmentId, 'container_created', actor, { container_name: containerName, internal_ip: internalIp }); const healthPath = (health && health.path) || '/api/health'; const timeoutMs = (health && health.timeoutMs) || 120000; const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs); if (!ready) { await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, [`reset: did not become ready within ${timeoutMs}ms`, environmentId]); await emitEvent(pool, environmentId, 'error', actor, { stage: 'reset_readiness' }); throw new Error(`environment ${environmentId} did not become ready after reset`); } const { rows } = await pool.query( `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, [environmentId] ); await emitEvent(pool, environmentId, 'ready', actor, {}); return rows[0]; } // ── Suspend/resume — EXPERIMENTAL ONLY (decision #3 / correction A) ──────────────────────── // Never called by the reaper or any guaranteed-path logic. Exists so the test plan // (docs/PHASE1_PLATFORM_FOUNDATION.md §8 test 7) can measure whether it's viable at all. async function suspendEnvironment({ pool, environmentId, actor = 'system' }) { const env = await getEnvironment(pool, environmentId); if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); const docker = clientFor(labHostRes.rows[0]); await docker.getContainer(env.container_id).stop({ t: 10 }); const { rows } = await pool.query(`UPDATE environments SET status = 'suspended' WHERE environment_id = $1 RETURNING *`, [environmentId]); await emitEvent(pool, environmentId, 'suspend', actor, { experimental: true }); return rows[0]; } async function resumeEnvironment({ pool, environmentId, health = {}, actor = 'system' }) { const env = await getEnvironment(pool, environmentId); if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); const docker = clientFor(labHostRes.rows[0]); await docker.getContainer(env.container_id).start(); const healthPath = health.path || '/api/health'; const timeoutMs = health.timeoutMs || 120000; const ready = await pollHealth(env.internal_ip, env.internal_port, healthPath, timeoutMs); if (!ready) { await pool.query(`UPDATE environments SET status = 'error', error_reason = 'resume: did not become ready' WHERE environment_id = $1`, [environmentId]); await emitEvent(pool, environmentId, 'error', actor, { stage: 'resume_readiness', experimental: true }); throw new Error(`environment ${environmentId} did not become ready after resume (suspend/resume is experimental — see decision #3)`); } const { rows } = await pool.query(`UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, [environmentId]); await emitEvent(pool, environmentId, 'resume', actor, { experimental: true }); return rows[0]; } // ── Reconciliation on startup — docs/LAB_ARCHITECTURE.md §8 ──────────────────────────────── // "On Provisioner restart, it must reconcile its registry against actual running containers // rather than trust stale DB state." Anything the registry claims is running/in-flight that // isn't actually running on its lab host gets marked error, not silently left wrong. async function reconcileOnStartup(pool) { const { rows } = await pool.query( `SELECT * FROM environments WHERE status IN ('provisioning', 'running', 'resetting', 'destroying', 'suspended')` ); const results = { checked: rows.length, flagged: 0, ok: 0 }; for (const env of rows) { try { const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); if (!labHostRes.rows.length) { await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, ['reconcile: lab_host no longer configured', env.environment_id]); await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'lab_host missing' }); results.flagged++; continue; } const docker = clientFor(labHostRes.rows[0]); let actuallyRunning = false; if (env.container_id) { try { const info = await docker.getContainer(env.container_id).inspect(); actuallyRunning = info.State && info.State.Running; } catch { actuallyRunning = false; // container doesn't exist on this host at all } } const expectedRunning = env.status === 'running' || env.status === 'suspended'; if (expectedRunning && !actuallyRunning) { await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, ['reconcile: DB claimed running/suspended but container is not actually running', env.environment_id]); await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'container not running' }); results.flagged++; } else if (['provisioning', 'resetting', 'destroying'].includes(env.status)) { // Any environment caught mid-transition by a Provisioner crash/restart is not // trustworthy either way — never silently assume it finished. await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, [`reconcile: caught mid-transition (${env.status}) on restart`, env.environment_id]); await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: `mid-transition:${env.status}` }); results.flagged++; } else { results.ok++; } } catch (e) { await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', message: e.message }).catch(() => {}); results.flagged++; } } return results; } module.exports = { generateRoutingToken, emitEvent, getEnvironment, getEnvironmentByTokenWithRouting, listEnvironments, getActiveEnvironmentForStudent, provisionEnvironment, destroyEnvironment, resetEnvironment, suspendEnvironment, resumeEnvironment, pollHealth, shortId, reconcileOnStartup, };