feat(routing): Lab Host Router closes the Gateway-to-Lab cross-VM gap

Adds platform/lab-router/ — a narrow service that runs on the Lab Host,
authenticates the Gateway via mTLS, and resolves an opaque routing token to
a local container purely from Docker label state (no Postgres access, no
callback to the Provisioner). Replaces the Gateway's env.internal_ip
assumption (only valid when Gateway and Lab share a Docker daemon) with a
data-driven direct/router decision per lab_host
(lab_hosts.routing_mode/router_endpoint, migration 004), resolved in one
place (platform/gateway/src/lab-endpoint.js's labTarget()) that every
proxy/fetch call site in server.js now goes through.

Extends the existing Docker TLS CA (infra/scripts/docker-tls/generate-docker-tls.sh)
with a second mTLS leaf pair (router-server/gateway-client) for this new
data-plane connection, deployed by the docker-tls Ansible role into a
separate directory from the control-plane pair. Firewall
(rn-lab-host-01.rules) replaces the old speculative wide port-range
allowance with one narrow rule: platform_ip -> the Router's port only.

dev-local stays routing_mode: direct — zero behavior change for local dev,
confirmed by the full existing regression suite passing unmodified.

platform/provisioner/src/config.js

@@ -24,14 +24,19 @@ function loadLabHostsSeed() {
 async function applyLabHostsSeed(pool) {
   const hosts = loadLabHostsSeed();
   for (const h of hosts) {
+    const routingMode = h.routing_mode || 'direct'; // Phase 4.1 — see migration 004's comment
+    if (routingMode === 'router' && !h.router_endpoint) {
+      throw new Error(`lab_host '${h.lab_host_id}' has routing_mode: router but no router_endpoint configured`);
+    }
     await pool.query(
-      `INSERT INTO lab_hosts (lab_host_id, display_name, endpoint, tls_ca_path, tls_cert_path, tls_key_path, enabled)
-       VALUES ($1, $2, $3, $4, $5, $6, $7)
+      `INSERT INTO lab_hosts (lab_host_id, display_name, endpoint, tls_ca_path, tls_cert_path, tls_key_path, enabled, routing_mode, router_endpoint)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
        ON CONFLICT (lab_host_id) DO UPDATE SET
          display_name = excluded.display_name, endpoint = excluded.endpoint,
          tls_ca_path = excluded.tls_ca_path, tls_cert_path = excluded.tls_cert_path,
-         tls_key_path = excluded.tls_key_path, enabled = excluded.enabled`,
-      [h.lab_host_id, h.display_name, h.endpoint, h.tls_ca_path || null, h.tls_cert_path || null, h.tls_key_path || null, h.enabled !== false]
+         tls_key_path = excluded.tls_key_path, enabled = excluded.enabled,
+         routing_mode = excluded.routing_mode, router_endpoint = excluded.router_endpoint`,
+      [h.lab_host_id, h.display_name, h.endpoint, h.tls_ca_path || null, h.tls_cert_path || null, h.tls_key_path || null, h.enabled !== false, routingMode, h.router_endpoint || null]
     );
   }
   return hosts.length;

platform/provisioner/src/lifecycle.js

@@ -40,6 +40,21 @@ async function getEnvironment(pool, environmentId) {
   return rows[0] || null;
 }
 
+// Phase 4.1 — the Gateway's read path (GET /environments/by-token/:token) needs to know HOW
+// to reach the Lab, not just its Docker-internal address: is this lab_host 'direct' (Gateway
+// reaches env.internal_ip itself) or 'router' (Gateway must go through that lab_host's Lab
+// Host Router instead)? Joining lab_hosts here keeps that decision entirely data-driven —
+// the Gateway never hardcodes or guesses a host's routing mode.
+async function getEnvironmentByTokenWithRouting(pool, token) {
+  const { rows } = await pool.query(
+    `SELECT e.*, h.routing_mode AS lab_host_routing_mode, h.router_endpoint AS lab_host_router_endpoint
+     FROM environments e JOIN lab_hosts h ON h.lab_host_id = e.lab_host_id
+     WHERE e.routing_token = $1`,
+    [token]
+  );
+  return rows[0] || null;
+}
+
 async function listEnvironments(pool, studentId) {
   const { rows } = await pool.query(
     'SELECT * FROM environments WHERE student_id = $1 ORDER BY created_at DESC',
@@ -90,6 +105,23 @@ async function provisionEnvironment({ pool, studentId, resourcePolicy, imageRef,
     throw Object.assign(new Error(`student '${studentId}' already has a non-terminal environment (${existing.environment_id})`), { code: 'ALREADY_EXISTS', existing });
   }
 
+  // Admission control (Phase 4.1 §26) — checked BEFORE touching Docker at all, so a full host
+  // fails cleanly and fast (503, no partial network/container left behind to clean up) instead
+  // of failing mid-provision with a generic 500 once Docker itself runs out of room. No-op
+  // when maxConcurrentLabs isn't configured (see resourcePolicyFor's comment on why that's the
+  // default).
+  if (resourcePolicy.maxConcurrentLabs) {
+    const { rows } = await pool.query(
+      `SELECT count(*)::int AS n FROM environments WHERE status NOT IN ('destroyed', 'error')`
+    );
+    if (rows[0].n >= resourcePolicy.maxConcurrentLabs) {
+      throw Object.assign(
+        new Error('Lab capacity is temporarily full. Try again shortly.'),
+        { code: 'CAPACITY_EXCEEDED', current: rows[0].n, max: resourcePolicy.maxConcurrentLabs }
+      );
+    }
+  }
+
   const labHost = await selectLabHost(pool);
   const docker = clientFor(labHost);
   const routingToken = generateRoutingToken();
@@ -145,6 +177,15 @@ async function provisionEnvironment({ pool, studentId, resourcePolicy, imageRef,
     const container = await docker.createContainer({
       name: containerName,
       Image: imageRef,
+      // Phase 4.1: labeled by routing token (not environment_id alone) so the Lab Host Router
+      // — which runs on this same Docker host and has no Postgres access — can resolve
+      // "which local container does this opaque token address" purely from local `docker ps`,
+      // with no cross-VM call back to the Provisioner needed at request time. See
+      // docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1, platform/lab-router/src/router.js.
+      Labels: {
+        'rootnode.routing-token': routingToken,
+        'rootnode.environment-id': env.environment_id,
+      },
       HostConfig: {
         Privileged: true, // required for k3s — see docs/LAB_ARCHITECTURE.md §7, not avoidable
         NetworkMode: networkName,
@@ -321,6 +362,13 @@ async function resetEnvironment({ pool, environmentId, resourcePolicy, egressCon
   const container = await docker.createContainer({
     name: containerName,
     Image: env.image_ref,
+    // Phase 4.1 — same routing-token label as provisionEnvironment; a reset keeps the same
+    // environment_id/routing_token (same DB row) so this must relabel the RECREATED container
+    // the same way, or the Lab Host Router would resolve stale/no container for this token.
+    Labels: {
+      'rootnode.routing-token': env.routing_token,
+      'rootnode.environment-id': environmentId,
+    },
     HostConfig: {
       Privileged: true,
       NetworkMode: networkName,
@@ -453,7 +501,7 @@ async function reconcileOnStartup(pool) {
 }
 
 module.exports = {
-  generateRoutingToken, emitEvent, getEnvironment, listEnvironments, getActiveEnvironmentForStudent,
+  generateRoutingToken, emitEvent, getEnvironment, getEnvironmentByTokenWithRouting, listEnvironments, getActiveEnvironmentForStudent,
   provisionEnvironment, destroyEnvironment, resetEnvironment,
   suspendEnvironment, resumeEnvironment, pollHealth, shortId, reconcileOnStartup,
 };

platform/provisioner/src/server.js

@@ -40,11 +40,18 @@ function egressConfig() {
 
 function resourcePolicyFor() {
   const policy = getResourcePolicy().default;
+  // Phase 4.1 §26 — optional; null/unset (the default) means no cap is enforced, matching the
+  // Phase 4 rule against fabricating a concurrency number before it's measured. Set this from
+  // infra/scripts/capacity-model.js's real output once the host exists (see
+  // infra/environments/production/capacity.yml.example's hard-admission-control field) — see
+  // docs/PHASE4_1_PREPRODUCTION_HARDENING.md §26.
+  const admission = getResourcePolicy().admission_control || {};
   return {
     cpuLimit: policy.cpu_limit,
     memLimitMb: policy.mem_limit_mb,
     pidsLimit: policy.pids_limit,
     storageLimitMb: policy.storage_limit_mb,
+    maxConcurrentLabs: admission.max_concurrent_labs || null,
   };
 }
 
@@ -81,7 +88,10 @@ function isValidUUID(v) {
 function asyncRoute(fn) {
   return (req, res) => fn(req, res).catch((e) => {
     console.error(`[${req.requestId}]`, e);
-    const status = e.code === 'NOT_FOUND' ? 404 : e.code === 'ALREADY_EXISTS' ? 409 : 500;
+    // CAPACITY_EXCEEDED (Phase 4.1 §26) is 503, not 500 — the host being full is an expected,
+    // recoverable operational state, not an application error. See lifecycle.js's admission
+    // check, which runs BEFORE any Docker/network resources are touched.
+    const status = e.code === 'NOT_FOUND' ? 404 : e.code === 'ALREADY_EXISTS' ? 409 : e.code === 'CAPACITY_EXCEEDED' ? 503 : 500;
     res.status(status).json({ error: e.message, code: e.code || 'INTERNAL', request_id: req.requestId });
   });
 }
@@ -98,6 +108,14 @@ app.get('/health', asyncRoute(async (req, res) => {
   res.status(ok ? 200 : 503).json({ ok, db: dbOk, lab_hosts: hostChecks });
 }));
 
+// GET /capacity — Phase 4.1 §26. Internal-only, same shared-secret auth as everything else
+// here — lets the Platform API's instructor dashboard show real capacity state instead of
+// instructors only discovering a full host when a student's launch fails.
+app.get('/capacity', asyncRoute(async (req, res) => {
+  const { rows } = await getPool().query(`SELECT count(*)::int AS n FROM environments WHERE status NOT IN ('destroyed', 'error')`);
+  res.json({ current: rows[0].n, max: resourcePolicyFor().maxConcurrentLabs });
+}));
+
 // POST /environments — { student_id }
 app.post('/environments', asyncRoute(async (req, res) => {
   const { student_id } = req.body || {};
@@ -130,9 +148,12 @@ app.get('/environments', asyncRoute(async (req, res) => {
 // owning student_id) before proxying. Internal-only, same shared-secret auth as every other
 // route here — the Gateway is the only caller, browsers never reach this directly.
 app.get('/environments/by-token/:token', asyncRoute(async (req, res) => {
-  const { rows } = await getPool().query('SELECT * FROM environments WHERE routing_token = $1', [req.params.token]);
-  if (!rows.length) return res.status(404).json({ error: 'not found', request_id: req.requestId });
-  res.json(redactRow(rows[0]));
+  // Phase 4.1: also joins lab_host_routing_mode/lab_host_router_endpoint so the Gateway can
+  // decide direct-vs-router without knowing anything about Docker topology itself — see
+  // lifecycle.getEnvironmentByTokenWithRouting and docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1.
+  const env = await lifecycle.getEnvironmentByTokenWithRouting(getPool(), req.params.token);
+  if (!env) return res.status(404).json({ error: 'not found', request_id: req.requestId });
+  res.json(redactRow(env));
 }));
 
 // GET /environments/:id/token — Phase 2A addition. Every other read path deliberately redacts