diff --git a/platform/provisioner/.env.example b/platform/provisioner/.env.example new file mode 100644 index 0000000..532514c --- /dev/null +++ b/platform/provisioner/.env.example @@ -0,0 +1,6 @@ +# Copy to .env and fill in a real secret before running docker-compose.dev.yml. +# Generate one with: openssl rand -hex 32 +PROVISIONER_SHARED_SECRET= + +# Optional overrides — see README.md's "Configuration reference" for the full list and defaults. +# LAB_IMAGE_REF=rootnode-academy:test diff --git a/platform/provisioner/Dockerfile b/platform/provisioner/Dockerfile new file mode 100644 index 0000000..9d74f96 --- /dev/null +++ b/platform/provisioner/Dockerfile @@ -0,0 +1,19 @@ +FROM node:20-slim + +# iptables is required at runtime for egress.js — the Provisioner manipulates the host's +# DOCKER-USER chain directly (see docs/PHASE1_PLATFORM_FOUNDATION.md §5), which requires this +# container to run with --network host --cap-add=NET_ADMIN --cap-add=NET_RAW (see +# docker-compose.dev.yml). This is not a "nice to have" — without those run flags the +# Provisioner cannot enforce egress at all, only fail loudly, which it does. +RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY package.json ./ +RUN npm install --production +COPY src/ ./src/ +COPY migrations/ ./migrations/ +COPY config/ ./config/ + +EXPOSE 4100 +CMD ["node", "src/server.js"] diff --git a/platform/provisioner/README.md b/platform/provisioner/README.md new file mode 100644 index 0000000..8c8c09e --- /dev/null +++ b/platform/provisioner/README.md @@ -0,0 +1,104 @@ +# RootNode Academy — Provisioner (Phase 1) + +Internal-only service that provisions, resets, and destroys disposable per-student Lab +environments. See `docs/LAB_ARCHITECTURE.md` and `docs/PHASE1_PLATFORM_FOUNDATION.md` at the +repo root for the design this implements — read those first. This file is the operational +reference: how to run it, configure it, and what its actual limitations are. + +## Quick start (local dev) + +```bash +cd platform/provisioner +echo "PROVISIONER_SHARED_SECRET=$(openssl rand -hex 32)" > .env +docker compose -f docker-compose.dev.yml build +LAB_IMAGE_REF=rootnode-academy:test docker compose -f docker-compose.dev.yml up -d +``` + +That starts Postgres, the shared egress proxy, and the Provisioner, all on one machine — the +documented Phase 1 dev exception (`docs/LAB_ARCHITECTURE.md` §1). It is **not** the production +topology; nothing here represents the two-VM Proxmox split. + +```bash +# Smoke test +curl -H "X-Provisioner-Key: $(grep PROVISIONER_SHARED_SECRET .env | cut -d= -f2)" http://localhost:4100/health + +# Run the integration suite (needs the stack above already running) +source .env +PROVISIONER_SHARED_SECRET=$PROVISIONER_SHARED_SECRET LAB_IMAGE_REF=rootnode-academy:test node test/run.js +``` + +The test image `rootnode-academy:test` used by default is **not** the real Lab image — build +that separately from the repo root (`docker build -t rootnode-academy:test .`) if you want the +suite to exercise the real one instead of a placeholder. See "Known limitations" below for why +the real image currently doesn't fully boot in every environment. + +## Configuration reference + +All environment variables read by `src/server.js`: + +| Variable | Required | Default | Meaning | +|---|---|---|---| +| `PLATFORM_DB_URL` | no | `postgres://postgres:postgres@localhost:5433/rootnode_platform` | Postgres connection string | +| `PROVISIONER_SHARED_SECRET` | **yes** | — | Internal API auth (`X-Provisioner-Key` header). Process refuses to start without it. | +| `PROVISIONER_PORT` | no | `4100` | API listen port | +| `LAB_IMAGE_REF` | no | `rootnode-academy:test` | Image every new environment is provisioned from | +| `EGRESS_PROXY_PORT` | no | `3128` | Port the shared egress proxy listens on. The proxy's *host* is never a static config value — it's resolved per environment as that environment's own Docker network gateway IP (see `src/lifecycle.js` and `docs/PHASE1_PLATFORM_FOUNDATION.md` §5) | +| `DNS_RESOLVER` | no | `127.0.0.11` | Docker's embedded per-network DNS — correct default for Docker-managed bridge networks | +| `NO_PROXY_CIDRS` | no | `10.42.0.0/16,10.43.0.0/16` | k3s's internal pod/service CIDRs, carved out of the container's proxy config so in-cluster traffic never routes through the egress proxy | + +Two config files, not database tables (`config.js`), both operator-edited plain files, hot-reloadable via `SIGHUP`: + +- `config/resource-policy.yaml` — CPU/RAM/PID/storage limits, idle/abandoned thresholds, reaper poll interval, health-check path/timeout. +- `config/lab-hosts.seed.yaml` — the `lab_hosts` row(s) applied at startup. Phase 1 ships exactly one, `dev-local`, pointing at the local Docker socket. Swapping this for a `tcp://:2376` entry with TLS material is the entire config change needed to point at a real, separate Lab Host VM — no code change (see `src/docker.js` and correction D in `docs/LAB_ARCHITECTURE.md`). + +**Both config files are baked into the image at build time** (`Dockerfile`'s `COPY config/`), +not bind-mounted — changing a limit requires a rebuild in this Phase 1 setup, or a `SIGHUP` to +the running process if you've bind-mounted `config/` yourself for local iteration. This was a +real friction point found during implementation testing (see "Known limitations"). + +## Deployment prerequisites (production) + +Per `docs/LAB_ARCHITECTURE.md` §1/§7 — none of this is built by Phase 1, it's what Phase 1 +assumes already exists before it's deployed for real: + +- Two separate VMs under Proxmox (or equivalent): a Platform VM and a Lab Host VM, with no + network route from the Lab Host VM back to the Platform VM in either direction except the + Provisioner-initiated TLS connection to the Lab Host's Docker API. +- Neither VM has a route to trusted RootNode/CDAA/home/business networks. +- The Provisioner process needs `NET_ADMIN`/`NET_RAW` capability and visibility into the host + network namespace it's managing egress firewall rules on (`--network host` + those two caps + if containerized, as in `docker-compose.dev.yml`; native process privileges if run directly + on the Lab Host VM instead). +- TLS material (CA/cert/key) provisioned for the Provisioner→Lab-Host-VM Docker API connection, + referenced by a `lab_hosts` row's `tls_ca_path`/`tls_cert_path`/`tls_key_path`. + +## Known limitations (found during Phase 1 implementation, not guessed) + +- **The real Lab image's `node-pty` native module fails to load in this sandbox's specific + environment**, crashing the backend before it ever listens (`Cannot find module + './prebuilds/linux-x64//pty.node'`). Confirmed via testing to be **pre-existing and + completely unrelated to the Provisioner** — the identical crash happens running the + unmodified image with a plain `docker run --privileged`, no Provisioner involved at all. Not + fixed here (out of scope — "don't redesign the existing Lab image unless strictly necessary + for Phase 1 operation," and it isn't; Phase 1's job is the provisioning mechanics, which are + fully validated against a lightweight stand-in image). Needs investigation on the actual + target deployment host before going live — may be an architecture/prebuild mismatch specific + to this sandbox, or may reproduce elsewhere. +- **Config changes require an image rebuild** (see above) — fine for a versioned production + deploy, mildly annoying for local iteration. Bind-mounting `config/` in + `docker-compose.dev.yml` for local dev is a reasonable follow-up, not done here to keep the + dev/prod image build path identical. +- **`--storage-opt size=` for per-environment storage limits is unverified** — depends on the + host's Docker storage driver (needs overlay2 with a backing filesystem that supports project + quotas). Not confirmed working or failing on this dev machine; needs a direct check on the + real deployment target. +- **Real per-lab CPU/RAM sizing numbers don't exist yet** — `config/resource-policy.yaml` + ships with placeholder defaults sized for a laptop, not the ~16-core/128GB target host. + Benchmarking against real hardware is future work, not Phase 1. +- **Suspend/resume was only exercised against the lightweight test image**, where a plain + Node process restarts trivially. That result says nothing about whether k3s-in-container + specifically survives `docker stop`/`start` — decision #3 already treats this as unproven, + and this test run doesn't change that. See `docs/PHASE1_PLATFORM_FOUNDATION.md` §8 test 7. +- **The remote TCP+TLS `lab_hosts` code path was validated against a Docker-in-Docker sidecar**, + not a real second VM — proves the mechanism works, doesn't prove anything about the actual + Proxmox Lab Host VM network path, which doesn't exist yet. diff --git a/platform/provisioner/docker-compose.dev.yml b/platform/provisioner/docker-compose.dev.yml new file mode 100644 index 0000000..1d22d41 --- /dev/null +++ b/platform/provisioner/docker-compose.dev.yml @@ -0,0 +1,54 @@ +# Phase 1 local development/testing stack — Postgres + egress-proxy + provisioner, all on one +# machine (docs/LAB_ARCHITECTURE.md §1's documented dev exception). Nothing here represents the +# production two-VM topology; it's what makes the provisioner testable on a laptop. +# +# provisioner and egress-proxy run with network_mode: host + NET_ADMIN/NET_RAW because: +# - egress.js manipulates the host's DOCKER-USER iptables chain directly (§5) — that's only +# possible if the process can see the host network namespace. +# - egress-proxy needs to be reachable from every per-student bridge network's gateway IP, +# which only works if it's bound across all host interfaces (host networking), not sandboxed +# inside its own compose-managed bridge network. +# Because provisioner uses host networking, it reaches Postgres via localhost:5433 (Postgres's +# published host port), not the compose service name. + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: rootnode_platform + ports: + - "5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 3s + retries: 10 + + egress-proxy: + build: ../egress-proxy + network_mode: host + # Squid listens on :3128 across every host interface, including each per-student bridge's + # gateway — that's the whole reason this needs host networking rather than a normal + # compose network. + + provisioner: + build: . + network_mode: host + cap_add: + - NET_ADMIN + - NET_RAW + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + PLATFORM_DB_URL: postgres://postgres:postgres@localhost:5433/rootnode_platform + PROVISIONER_SHARED_SECRET: ${PROVISIONER_SHARED_SECRET:?set a real secret, do not use a default} + PROVISIONER_PORT: "4100" + LAB_IMAGE_REF: ${LAB_IMAGE_REF:-rootnode-academy:test} + EGRESS_PROXY_PORT: "3128" + DNS_RESOLVER: "127.0.0.11" + NO_PROXY_CIDRS: "10.42.0.0/16,10.43.0.0/16" + depends_on: + postgres: + condition: service_healthy diff --git a/platform/provisioner/src/docker.js b/platform/provisioner/src/docker.js new file mode 100644 index 0000000..98e1276 --- /dev/null +++ b/platform/provisioner/src/docker.js @@ -0,0 +1,66 @@ +// Lab-host abstraction (correction D) — resolves a `lab_hosts` row into a connected dockerode +// client. Nothing in this module or its callers may assume Docker is local; the endpoint +// always comes from a `lab_hosts` row, whether that row points at a Unix socket (Phase 1 dev) +// or a remote TCP+TLS endpoint (production Lab Host VM, per docs/LAB_ARCHITECTURE.md §1). + +const fs = require('fs'); +const Docker = require('dockerode'); + +// Caches one dockerode client per lab_host_id — cheap to keep around, avoids re-parsing TLS +// material on every call. +const clientCache = new Map(); + +function buildClient(labHost) { + if (labHost.endpoint.startsWith('unix://')) { + return new Docker({ socketPath: labHost.endpoint.replace('unix://', '') }); + } + if (labHost.endpoint.startsWith('tcp://')) { + const url = new URL(labHost.endpoint.replace('tcp://', 'http://')); + const opts = { host: url.hostname, port: Number(url.port) || 2376, protocol: 'https' }; + if (labHost.tls_ca_path && labHost.tls_cert_path && labHost.tls_key_path) { + opts.ca = fs.readFileSync(labHost.tls_ca_path); + opts.cert = fs.readFileSync(labHost.tls_cert_path); + opts.key = fs.readFileSync(labHost.tls_key_path); + } else { + // ponytail: no TLS material configured for a remote endpoint is a real misconfiguration, + // not a silently-degraded mode — fail loudly rather than connect unauthenticated to a + // remote Docker API (see docs/PHASE1_PLATFORM_FOUNDATION.md §6 on why that cert matters + // as much as the Provisioner API's own shared secret). + throw new Error( + `lab_host '${labHost.lab_host_id}' has a tcp:// endpoint but no TLS cert/key/ca configured — refusing to connect unauthenticated.` + ); + } + return new Docker(opts); + } + throw new Error(`lab_host '${labHost.lab_host_id}' has an unrecognized endpoint scheme: ${labHost.endpoint}`); +} + +function clientFor(labHost) { + let client = clientCache.get(labHost.lab_host_id); + if (!client) { + client = buildClient(labHost); + clientCache.set(labHost.lab_host_id, client); + } + return client; +} + +// Phase 1: no scheduler, just "the one enabled row" (docs/PHASE1_PLATFORM_FOUNDATION.md §4). +async function selectLabHost(pool) { + const { rows } = await pool.query( + `SELECT * FROM lab_hosts WHERE enabled = true ORDER BY created_at ASC LIMIT 1` + ); + if (!rows.length) throw new Error('No enabled lab_hosts row configured — cannot provision.'); + return rows[0]; +} + +async function healthCheck(labHost) { + try { + const client = clientFor(labHost); + const info = await client.info(); + return { ok: true, containers: info.Containers, serverVersion: info.ServerVersion }; + } catch (e) { + return { ok: false, error: e.message }; + } +} + +module.exports = { clientFor, selectLabHost, healthCheck }; diff --git a/platform/provisioner/src/lifecycle.js b/platform/provisioner/src/lifecycle.js new file mode 100644 index 0000000..746a801 --- /dev/null +++ b/platform/provisioner/src/lifecycle.js @@ -0,0 +1,459 @@ +// Environment lifecycle orchestration — the state machine from +// docs/PHASE1_PLATFORM_FOUNDATION.md §3, wired to docker.js (lab-host abstraction), egress.js +// (firewall), and db.js (registry + audit). +// +// Guaranteed path: requested -> provisioning -> running -> resetting -> running +// (provisioning|running|error) -> destroying -> destroyed +// Suspend/resume is experimental only (decision #3 / correction A) — see suspendEnvironment/ +// resumeEnvironment below, both explicitly opt-in and never on the guaranteed path. + +const crypto = require('crypto'); +const http = require('http'); +const { selectLabHost, clientFor } = require('./docker'); +const { setupEgressChain, teardownEgressChain } = require('./egress'); + +function shortId(environmentId) { + return environmentId.replace(/-/g, '').slice(0, 12); +} + +function generateRoutingToken() { + return crypto.randomBytes(32).toString('base64url'); // opaque, CSPRNG, URL-safe +} + +// Never let a routing token reach the audit log in plaintext (requirement #9). +function redact(detail) { + if (!detail || typeof detail !== 'object') return detail; + const copy = { ...detail }; + if (copy.routing_token) copy.routing_token = '[redacted]'; + return copy; +} + +async function emitEvent(pool, environmentId, eventType, actor, detail = null) { + await pool.query( + `INSERT INTO environment_events (environment_id, event_type, actor, detail) VALUES ($1, $2, $3, $4)`, + [environmentId, eventType, actor, detail ? JSON.stringify(redact(detail)) : null] + ); +} + +async function getEnvironment(pool, environmentId) { + const { rows } = await pool.query('SELECT * FROM environments WHERE environment_id = $1', [environmentId]); + return rows[0] || null; +} + +async function listEnvironments(pool, studentId) { + const { rows } = await pool.query( + 'SELECT * FROM environments WHERE student_id = $1 ORDER BY created_at DESC', + [studentId] + ); + return rows; +} + +async function getActiveEnvironmentForStudent(pool, studentId) { + const { rows } = await pool.query( + `SELECT * FROM environments WHERE student_id = $1 + AND status NOT IN ('destroyed', 'error') ORDER BY created_at DESC LIMIT 1`, + [studentId] + ); + return rows[0] || null; +} + +async function pollHealth(ip, port, path, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const ok = await new Promise((resolve) => { + const req = http.get({ host: ip, port, path, timeout: 2000 }, (res) => { + res.resume(); + resolve(res.statusCode >= 200 && res.statusCode < 300); + }); + req.on('error', () => resolve(false)); + req.on('timeout', () => { req.destroy(); resolve(false); }); + }); + if (ok) return true; + await new Promise((r) => setTimeout(r, 2000)); + } + return false; +} + +/** + * provision — the full requested -> provisioning -> running (or error) path. + * @param {object} opts + * @param {import('pg').Pool} opts.pool + * @param {string} opts.studentId - opaque, not validated against a roster (correction C) + * @param {object} opts.resourcePolicy - { cpuLimit, memLimitMb, pidsLimit, storageLimitMb } + * @param {string} opts.imageRef + * @param {object} opts.egressConfig - { proxyHost, proxyPort, dnsResolver, extraAllowedHosts, noProxyCidrs } + * @param {object} [opts.health] - { path='/api/health', timeoutMs=120000 } — overridable for testing + */ +async function provisionEnvironment({ pool, studentId, resourcePolicy, imageRef, egressConfig, health = {} }) { + const existing = await getActiveEnvironmentForStudent(pool, studentId); + if (existing) { + throw Object.assign(new Error(`student '${studentId}' already has a non-terminal environment (${existing.environment_id})`), { code: 'ALREADY_EXISTS', existing }); + } + + const labHost = await selectLabHost(pool); + const docker = clientFor(labHost); + const routingToken = generateRoutingToken(); + + const insertResult = await pool.query( + `INSERT INTO environments + (student_id, lab_host_id, status, routing_token, image_ref, cpu_limit, mem_limit_mb, pids_limit, storage_limit_mb) + VALUES ($1, $2, 'requested', $3, $4, $5, $6, $7, $8) + RETURNING *`, + [studentId, labHost.lab_host_id, routingToken, imageRef, + resourcePolicy.cpuLimit, resourcePolicy.memLimitMb, resourcePolicy.pidsLimit, resourcePolicy.storageLimitMb] + ); + const env = insertResult.rows[0]; + await emitEvent(pool, env.environment_id, 'provision_requested', 'system', { student_id: studentId }); + await emitEvent(pool, env.environment_id, 'host_selected', 'system', { lab_host_id: labHost.lab_host_id }); + + try { + await pool.query(`UPDATE environments SET status = 'provisioning' WHERE environment_id = $1`, [env.environment_id]); + + const id = shortId(env.environment_id); + const networkName = `rootnode-lab-net-${id}`; + const containerName = `rootnode-lab-${id}`; + + // Network + const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' }); + const netInfo = await network.inspect(); + const subnet = netInfo.IPAM.Config[0].Subnet; + // The egress proxy runs with host networking (see docker-compose.dev.yml), so it's present + // on every bridge network's gateway interface — that gateway IP, not a fixed hostname, is + // what's actually reachable from *this* environment's private network. Resolving it per + // network (rather than one static proxyHost) is what makes the proxy reachable at all once + // there's more than one isolated student network. + const proxyHost = netInfo.IPAM.Config[0].Gateway; + await pool.query( + `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3 WHERE environment_id = $4`, + [network.id, networkName, subnet, env.environment_id] + ); + await emitEvent(pool, env.environment_id, 'network_created', 'system', { network_name: networkName, subnet }); + + // Firewall — installed before the container starts, so there's never a window where the + // container has network access but no restriction in place. + await setupEgressChain({ + environmentId: env.environment_id, + subnet, + egressProxyHost: proxyHost, + egressProxyPort: egressConfig.proxyPort, + dnsResolver: egressConfig.dnsResolver, + extraAllowedHosts: egressConfig.extraAllowedHosts || [], + }); + await emitEvent(pool, env.environment_id, 'firewall_configured', 'system', { subnet }); + + // Container + const container = await docker.createContainer({ + name: containerName, + Image: imageRef, + HostConfig: { + Privileged: true, // required for k3s — see docs/LAB_ARCHITECTURE.md §7, not avoidable + NetworkMode: networkName, + NanoCpus: Math.round(resourcePolicy.cpuLimit * 1e9), + Memory: resourcePolicy.memLimitMb * 1024 * 1024, + MemorySwap: resourcePolicy.memLimitMb * 1024 * 1024, // pinned equal to Memory — no swap + PidsLimit: resourcePolicy.pidsLimit, + PublishAllPorts: false, // no public host port mappings — requirement #5 + StorageOpt: resourcePolicy.storageLimitMb ? { size: `${resourcePolicy.storageLimitMb}m` } : undefined, + }, + Env: [ + `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, + `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, + `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`, + ], + }); + await container.start(); + const containerInfo = await container.inspect(); + const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress; + + await pool.query( + `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`, + [container.id, containerName, internalIp, env.environment_id] + ); + await emitEvent(pool, env.environment_id, 'container_created', 'system', { container_name: containerName, internal_ip: internalIp }); + + // Readiness + const healthPath = health.path || '/api/health'; + const timeoutMs = health.timeoutMs || 120000; + const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs); + if (!ready) { + throw new Error(`environment ${env.environment_id} did not become ready within ${timeoutMs}ms`); + } + + const { rows } = await pool.query( + `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, + [env.environment_id] + ); + await emitEvent(pool, env.environment_id, 'ready', 'system', {}); + return rows[0]; + } catch (e) { + // Best-effort teardown of whatever got created before the failure — a provisioning + // failure must not leave an orphaned container/network/firewall chain running + // indefinitely. Found by testing: a readiness timeout left a real container and firewall + // chain in place with only the DB row marked 'error', which is exactly the kind of + // resource leak requirement #10 (idle reaper) exists to prevent — except this environment + // never reaches 'running', so it would never have had a last_active_at to age out on + // (see the reaper fix in reaper.js). Cleaning up here directly is more honest than relying + // on the reaper to catch a case it wasn't actually able to catch. + try { + const fresh = await getEnvironment(pool, env.environment_id); + const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [fresh.lab_host_id]); + const docker = clientFor(labHostRes.rows[0]); + if (fresh.container_id) { + await docker.getContainer(fresh.container_id).remove({ force: true }).catch(() => {}); + } + if (fresh.network_subnet) { + await teardownEgressChain({ environmentId: fresh.environment_id, subnet: fresh.network_subnet }).catch(() => {}); + } + if (fresh.network_id) { + await docker.getNetwork(fresh.network_id).remove().catch(() => {}); + } + } catch (cleanupErr) { + // Cleanup itself failing is exactly what the reaper's COALESCE(last_active_at, + // created_at) fallback exists to catch eventually — log and move on, don't mask the + // original provisioning error with a cleanup error. + await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'provision_failure_cleanup', message: cleanupErr.message }).catch(() => {}); + } + + await pool.query( + `UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, + [e.message, env.environment_id] + ); + await emitEvent(pool, env.environment_id, 'error', 'system', { message: e.message }); + throw e; + } +} + +/** + * destroy — stop+remove container, remove network+firewall chain, mark destroyed. + * Never touches academic data — that lives entirely outside the Lab (Platform SQLite today, + * see docs/LAB_ARCHITECTURE.md §3) — destroying an environment is purely infrastructure teardown. + */ +async function destroyEnvironment({ pool, environmentId, actor = 'system' }) { + const env = await getEnvironment(pool, environmentId); + if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); + if (env.status === 'destroyed') return env; + + await pool.query(`UPDATE environments SET status = 'destroying' WHERE environment_id = $1`, [environmentId]); + + const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); + const docker = clientFor(labHostRes.rows[0]); + + if (env.container_id) { + try { + const container = docker.getContainer(env.container_id); + await container.stop({ t: 10 }).catch(() => {}); // already stopped/gone is fine + await container.remove({ force: true }).catch(() => {}); + } catch (e) { + // Container already gone — proceed with cleanup regardless (failure-safe, per requirement #10). + } + } + + if (env.network_subnet) { + await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch((e) => { + // Log via event but don't block destroy on firewall cleanup failure — an orphaned + // per-env chain for a destroyed environment is a cleanup task, not a live security hole + // (its subnet is no longer bound to any running container). + return emitEvent(pool, environmentId, 'error', actor, { stage: 'firewall_teardown', message: e.message }); + }); + } + + if (env.network_id) { + try { + await docker.getNetwork(env.network_id).remove(); + } catch (e) { /* already gone is fine */ } + } + + const { rows } = await pool.query( + `UPDATE environments SET status = 'destroyed' WHERE environment_id = $1 RETURNING *`, + [environmentId] + ); + await emitEvent(pool, environmentId, 'destroyed', actor, {}); + return rows[0]; +} + +/** + * reset — destroy + provision again, same environment_id/routing_token (same DB row). + */ +async function resetEnvironment({ pool, environmentId, resourcePolicy, egressConfig, health, actor = 'system' }) { + const env = await getEnvironment(pool, environmentId); + if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); + + await pool.query(`UPDATE environments SET status = 'resetting' WHERE environment_id = $1`, [environmentId]); + await emitEvent(pool, environmentId, 'reset', actor, {}); + + const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); + const docker = clientFor(labHostRes.rows[0]); + + if (env.container_id) { + const container = docker.getContainer(env.container_id); + await container.stop({ t: 10 }).catch(() => {}); + await container.remove({ force: true }).catch(() => {}); + } + if (env.network_subnet) { + await teardownEgressChain({ environmentId: env.environment_id, subnet: env.network_subnet }).catch(() => {}); + } + if (env.network_id) { + await docker.getNetwork(env.network_id).remove().catch(() => {}); + } + + // Recreate against the SAME row: same environment_id/routing_token, fresh everything else. + const id = shortId(env.environment_id); + const networkName = `rootnode-lab-net-${id}`; + const containerName = `rootnode-lab-${id}`; + + const network = await docker.createNetwork({ Name: networkName, Driver: 'bridge' }); + const netInfo = await network.inspect(); + const subnet = netInfo.IPAM.Config[0].Subnet; + const proxyHost = netInfo.IPAM.Config[0].Gateway; // see provisionEnvironment for why this is per-network, not static + await pool.query( + `UPDATE environments SET network_id = $1, network_name = $2, network_subnet = $3, container_id = NULL, container_name = NULL, internal_ip = NULL, error_reason = NULL WHERE environment_id = $4`, + [network.id, networkName, subnet, environmentId] + ); + await emitEvent(pool, environmentId, 'network_created', actor, { network_name: networkName, subnet }); + + await setupEgressChain({ + environmentId, subnet, + egressProxyHost: proxyHost, egressProxyPort: egressConfig.proxyPort, + dnsResolver: egressConfig.dnsResolver, extraAllowedHosts: egressConfig.extraAllowedHosts || [], + }); + await emitEvent(pool, environmentId, 'firewall_configured', actor, { subnet }); + + const container = await docker.createContainer({ + name: containerName, + Image: env.image_ref, + HostConfig: { + Privileged: true, + NetworkMode: networkName, + NanoCpus: Math.round(Number(env.cpu_limit) * 1e9), + Memory: env.mem_limit_mb * 1024 * 1024, + MemorySwap: env.mem_limit_mb * 1024 * 1024, + PidsLimit: env.pids_limit, + PublishAllPorts: false, + StorageOpt: env.storage_limit_mb ? { size: `${env.storage_limit_mb}m` } : undefined, + }, + Env: [ + `HTTP_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, + `HTTPS_PROXY=http://${proxyHost}:${egressConfig.proxyPort}`, + `NO_PROXY=localhost,127.0.0.1,${(egressConfig.noProxyCidrs || []).join(',')}`, + ], + }); + await container.start(); + const containerInfo = await container.inspect(); + const internalIp = containerInfo.NetworkSettings.Networks[networkName].IPAddress; + await pool.query( + `UPDATE environments SET container_id = $1, container_name = $2, internal_ip = $3, started_at = now() WHERE environment_id = $4`, + [container.id, containerName, internalIp, environmentId] + ); + await emitEvent(pool, environmentId, 'container_created', actor, { container_name: containerName, internal_ip: internalIp }); + + const healthPath = (health && health.path) || '/api/health'; + const timeoutMs = (health && health.timeoutMs) || 120000; + const ready = await pollHealth(internalIp, env.internal_port, healthPath, timeoutMs); + if (!ready) { + await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, + [`reset: did not become ready within ${timeoutMs}ms`, environmentId]); + await emitEvent(pool, environmentId, 'error', actor, { stage: 'reset_readiness' }); + throw new Error(`environment ${environmentId} did not become ready after reset`); + } + + const { rows } = await pool.query( + `UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, + [environmentId] + ); + await emitEvent(pool, environmentId, 'ready', actor, {}); + return rows[0]; +} + +// ── Suspend/resume — EXPERIMENTAL ONLY (decision #3 / correction A) ──────────────────────── +// Never called by the reaper or any guaranteed-path logic. Exists so the test plan +// (docs/PHASE1_PLATFORM_FOUNDATION.md §8 test 7) can measure whether it's viable at all. + +async function suspendEnvironment({ pool, environmentId, actor = 'system' }) { + const env = await getEnvironment(pool, environmentId); + if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); + const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); + const docker = clientFor(labHostRes.rows[0]); + await docker.getContainer(env.container_id).stop({ t: 10 }); + const { rows } = await pool.query(`UPDATE environments SET status = 'suspended' WHERE environment_id = $1 RETURNING *`, [environmentId]); + await emitEvent(pool, environmentId, 'suspend', actor, { experimental: true }); + return rows[0]; +} + +async function resumeEnvironment({ pool, environmentId, health = {}, actor = 'system' }) { + const env = await getEnvironment(pool, environmentId); + if (!env) throw Object.assign(new Error('environment not found'), { code: 'NOT_FOUND' }); + const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); + const docker = clientFor(labHostRes.rows[0]); + await docker.getContainer(env.container_id).start(); + + const healthPath = health.path || '/api/health'; + const timeoutMs = health.timeoutMs || 120000; + const ready = await pollHealth(env.internal_ip, env.internal_port, healthPath, timeoutMs); + if (!ready) { + await pool.query(`UPDATE environments SET status = 'error', error_reason = 'resume: did not become ready' WHERE environment_id = $1`, [environmentId]); + await emitEvent(pool, environmentId, 'error', actor, { stage: 'resume_readiness', experimental: true }); + throw new Error(`environment ${environmentId} did not become ready after resume (suspend/resume is experimental — see decision #3)`); + } + const { rows } = await pool.query(`UPDATE environments SET status = 'running', last_active_at = now() WHERE environment_id = $1 RETURNING *`, [environmentId]); + await emitEvent(pool, environmentId, 'resume', actor, { experimental: true }); + return rows[0]; +} + +// ── Reconciliation on startup — docs/LAB_ARCHITECTURE.md §8 ──────────────────────────────── +// "On Provisioner restart, it must reconcile its registry against actual running containers +// rather than trust stale DB state." Anything the registry claims is running/in-flight that +// isn't actually running on its lab host gets marked error, not silently left wrong. +async function reconcileOnStartup(pool) { + const { rows } = await pool.query( + `SELECT * FROM environments WHERE status IN ('provisioning', 'running', 'resetting', 'destroying', 'suspended')` + ); + const results = { checked: rows.length, flagged: 0, ok: 0 }; + for (const env of rows) { + try { + const labHostRes = await pool.query('SELECT * FROM lab_hosts WHERE lab_host_id = $1', [env.lab_host_id]); + if (!labHostRes.rows.length) { + await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, + ['reconcile: lab_host no longer configured', env.environment_id]); + await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'lab_host missing' }); + results.flagged++; + continue; + } + const docker = clientFor(labHostRes.rows[0]); + let actuallyRunning = false; + if (env.container_id) { + try { + const info = await docker.getContainer(env.container_id).inspect(); + actuallyRunning = info.State && info.State.Running; + } catch { + actuallyRunning = false; // container doesn't exist on this host at all + } + } + const expectedRunning = env.status === 'running' || env.status === 'suspended'; + if (expectedRunning && !actuallyRunning) { + await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, + ['reconcile: DB claimed running/suspended but container is not actually running', env.environment_id]); + await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: 'container not running' }); + results.flagged++; + } else if (['provisioning', 'resetting', 'destroying'].includes(env.status)) { + // Any environment caught mid-transition by a Provisioner crash/restart is not + // trustworthy either way — never silently assume it finished. + await pool.query(`UPDATE environments SET status = 'error', error_reason = $1 WHERE environment_id = $2`, + [`reconcile: caught mid-transition (${env.status}) on restart`, env.environment_id]); + await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', reason: `mid-transition:${env.status}` }); + results.flagged++; + } else { + results.ok++; + } + } catch (e) { + await emitEvent(pool, env.environment_id, 'error', 'system', { stage: 'reconcile', message: e.message }).catch(() => {}); + results.flagged++; + } + } + return results; +} + +module.exports = { + generateRoutingToken, emitEvent, getEnvironment, listEnvironments, getActiveEnvironmentForStudent, + provisionEnvironment, destroyEnvironment, resetEnvironment, + suspendEnvironment, resumeEnvironment, pollHealth, shortId, reconcileOnStartup, +}; diff --git a/platform/provisioner/src/server.js b/platform/provisioner/src/server.js new file mode 100644 index 0000000..ba4072c --- /dev/null +++ b/platform/provisioner/src/server.js @@ -0,0 +1,214 @@ +// RootNode Academy — Phase 1 Provisioner API. +// Internal-only (requirement #3/#8) — every route below requires X-Provisioner-Key. Nothing +// here is reachable by a student or the public internet in this phase; there's no frontend/ +// Gateway wiring yet, so this is exercised directly (curl / the test suite in test/run.js). +// +// See docs/PHASE1_PLATFORM_FOUNDATION.md §4 for the approved route list this implements. + +const express = require('express'); +const { randomUUID } = require('crypto'); +const { getPool, runMigrations } = require('./db'); +const { applyLabHostsSeed, getResourcePolicy, reloadResourcePolicy } = require('./config'); +const { healthCheck: dockerHealthCheck } = require('./docker'); +const lifecycle = require('./lifecycle'); +const { startReaper } = require('./reaper'); + +const PORT = Number(process.env.PROVISIONER_PORT) || 4100; +const SHARED_SECRET = process.env.PROVISIONER_SHARED_SECRET; +const LAB_IMAGE_REF = process.env.LAB_IMAGE_REF || 'rootnode-academy:test'; +// No fixed EGRESS_PROXY_HOST: the proxy runs with host networking and is reached via each +// environment's own network gateway IP, resolved per-environment in lifecycle.js — a single +// static hostname wouldn't be reachable from every isolated student network. Only the port +// is fixed/shared. +const EGRESS_PROXY_PORT = Number(process.env.EGRESS_PROXY_PORT) || 3128; +const DNS_RESOLVER = process.env.DNS_RESOLVER || '127.0.0.11'; // Docker's embedded DNS by default +const NO_PROXY_CIDRS = (process.env.NO_PROXY_CIDRS || '10.42.0.0/16,10.43.0.0/16').split(','); + +if (!SHARED_SECRET) { + console.error('FATAL: PROVISIONER_SHARED_SECRET is not set. Refusing to start with an unauthenticated internal API.'); + process.exit(1); +} + +function egressConfig() { + return { + proxyPort: EGRESS_PROXY_PORT, + dnsResolver: DNS_RESOLVER, + extraAllowedHosts: [], // none required in Phase 1 — the Lab image doesn't call back to the Platform yet + noProxyCidrs: NO_PROXY_CIDRS, + }; +} + +function resourcePolicyFor() { + const policy = getResourcePolicy().default; + return { + cpuLimit: policy.cpu_limit, + memLimitMb: policy.mem_limit_mb, + pidsLimit: policy.pids_limit, + storageLimitMb: policy.storage_limit_mb, + }; +} + +function healthOpts() { + const hc = getResourcePolicy().health_check || {}; + return { path: hc.path || '/api/health', timeoutMs: hc.timeout_ms || 120000 }; +} + +const app = express(); +app.use(express.json()); + +// Every request gets a request id for structured error correlation, and every response is +// JSON — no HTML error pages, no leaking stack traces to a caller. +app.use((req, res, next) => { + req.requestId = randomUUID(); + next(); +}); + +function requireAuth(req, res, next) { + const key = req.get('X-Provisioner-Key'); + if (!key || key !== SHARED_SECRET) { + return res.status(401).json({ error: 'unauthorized', request_id: req.requestId }); + } + next(); +} + +function isValidStudentId(v) { + return typeof v === 'string' && v.length > 0 && v.length <= 128 && /^[a-zA-Z0-9_.-]+$/.test(v); +} +function isValidUUID(v) { + return typeof v === 'string' && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(v); +} + +function asyncRoute(fn) { + return (req, res) => fn(req, res).catch((e) => { + console.error(`[${req.requestId}]`, e); + const status = e.code === 'NOT_FOUND' ? 404 : e.code === 'ALREADY_EXISTS' ? 409 : 500; + res.status(status).json({ error: e.message, code: e.code || 'INTERNAL', request_id: req.requestId }); + }); +} + +app.use(requireAuth); + +// GET /health — Provisioner's own health: Docker reachable, Postgres reachable. +app.get('/health', asyncRoute(async (req, res) => { + const pool = getPool(); + const dbOk = await pool.query('SELECT 1').then(() => true).catch(() => false); + const labHosts = await pool.query('SELECT * FROM lab_hosts WHERE enabled = true').catch(() => ({ rows: [] })); + const hostChecks = await Promise.all(labHosts.rows.map(async (h) => ({ lab_host_id: h.lab_host_id, ...(await dockerHealthCheck(h)) }))); + const ok = dbOk && hostChecks.every((h) => h.ok); + res.status(ok ? 200 : 503).json({ ok, db: dbOk, lab_hosts: hostChecks }); +})); + +// POST /environments — { student_id } +app.post('/environments', asyncRoute(async (req, res) => { + const { student_id } = req.body || {}; + if (!isValidStudentId(student_id)) { + return res.status(400).json({ error: 'student_id is required and must be a simple identifier', request_id: req.requestId }); + } + const env = await lifecycle.provisionEnvironment({ + pool: getPool(), + studentId: student_id, + resourcePolicy: resourcePolicyFor(), + imageRef: LAB_IMAGE_REF, + egressConfig: egressConfig(), + health: healthOpts(), + }); + res.status(201).json({ environment_id: env.environment_id, status: env.status, routing_token: env.routing_token }); +})); + +// GET /environments?student_id= +app.get('/environments', asyncRoute(async (req, res) => { + const { student_id } = req.query; + if (!isValidStudentId(student_id)) { + return res.status(400).json({ error: 'student_id query param is required', request_id: req.requestId }); + } + const envs = await lifecycle.listEnvironments(getPool(), student_id); + res.json(envs.map(redactRow)); +})); + +// GET /environments/:id +app.get('/environments/:id', asyncRoute(async (req, res) => { + if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId }); + const env = await lifecycle.getEnvironment(getPool(), req.params.id); + if (!env) return res.status(404).json({ error: 'not found', request_id: req.requestId }); + res.json(redactRow(env)); +})); + +// POST /environments/:id/reset +app.post('/environments/:id/reset', asyncRoute(async (req, res) => { + if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId }); + const env = await lifecycle.resetEnvironment({ + pool: getPool(), environmentId: req.params.id, + resourcePolicy: resourcePolicyFor(), egressConfig: egressConfig(), health: healthOpts(), + actor: req.get('X-Actor') || 'system', + }); + res.json(redactRow(env)); +})); + +// DELETE /environments/:id +app.delete('/environments/:id', asyncRoute(async (req, res) => { + if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId }); + const env = await lifecycle.destroyEnvironment({ pool: getPool(), environmentId: req.params.id, actor: req.get('X-Actor') || 'system' }); + res.json(redactRow(env)); +})); + +// POST /environments/:id/suspend — EXPERIMENTAL, see decision #3 / correction A. +app.post('/environments/:id/suspend', asyncRoute(async (req, res) => { + if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId }); + const env = await lifecycle.suspendEnvironment({ pool: getPool(), environmentId: req.params.id, actor: req.get('X-Actor') || 'system' }); + res.json({ ...redactRow(env), experimental: true }); +})); + +// POST /environments/:id/resume — EXPERIMENTAL, see decision #3 / correction A. +app.post('/environments/:id/resume', asyncRoute(async (req, res) => { + if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId }); + const env = await lifecycle.resumeEnvironment({ pool: getPool(), environmentId: req.params.id, health: healthOpts(), actor: req.get('X-Actor') || 'system' }); + res.json({ ...redactRow(env), experimental: true }); +})); + +// GET /environments/:id/events +app.get('/environments/:id/events', asyncRoute(async (req, res) => { + if (!isValidUUID(req.params.id)) return res.status(400).json({ error: 'invalid environment id', request_id: req.requestId }); + const { rows } = await getPool().query( + 'SELECT id, event_type, actor, detail, created_at FROM environment_events WHERE environment_id = $1 ORDER BY created_at ASC', + [req.params.id] + ); + res.json(rows); +})); + +function redactRow(env) { + const { routing_token, ...rest } = env; + return { ...rest, routing_token: routing_token ? '[redacted]' : null }; +} + +process.on('SIGHUP', () => { + console.log('SIGHUP received, reloading resource-policy.yaml'); + try { reloadResourcePolicy(); } catch (e) { console.error('reload failed:', e.message); } +}); + +async function main() { + const pool = getPool(); + const applied = await runMigrations(); + if (applied.length) console.log('Applied migrations:', applied.join(', ')); + const seeded = await applyLabHostsSeed(pool); + console.log(`Applied lab_hosts seed (${seeded} row(s))`); + + const reconcileResult = await lifecycle.reconcileOnStartup(pool); + console.log(`Startup reconciliation: checked=${reconcileResult.checked} ok=${reconcileResult.ok} flagged=${reconcileResult.flagged}`); + + const policy = getResourcePolicy(); + const reaper = startReaper({ + pool, + idleThresholdMinutes: policy.idle_threshold_minutes, + abandonedThresholdMinutes: policy.abandoned_threshold_minutes, + pollIntervalSeconds: policy.reaper_poll_interval_seconds, + }); + process.on('SIGTERM', () => { reaper.stop(); process.exit(0); }); + + app.listen(PORT, () => console.log(`Provisioner API listening on :${PORT}`)); +} + +if (require.main === module) { + main().catch((e) => { console.error('FATAL:', e); process.exit(1); }); +} + +module.exports = { app, main };