feat(routing): Lab Host Router closes the Gateway-to-Lab cross-VM gap

Adds platform/lab-router/ — a narrow service that runs on the Lab Host,
authenticates the Gateway via mTLS, and resolves an opaque routing token to
a local container purely from Docker label state (no Postgres access, no
callback to the Provisioner). Replaces the Gateway's env.internal_ip
assumption (only valid when Gateway and Lab share a Docker daemon) with a
data-driven direct/router decision per lab_host
(lab_hosts.routing_mode/router_endpoint, migration 004), resolved in one
place (platform/gateway/src/lab-endpoint.js's labTarget()) that every
proxy/fetch call site in server.js now goes through.

Extends the existing Docker TLS CA (infra/scripts/docker-tls/generate-docker-tls.sh)
with a second mTLS leaf pair (router-server/gateway-client) for this new
data-plane connection, deployed by the docker-tls Ansible role into a
separate directory from the control-plane pair. Firewall
(rn-lab-host-01.rules) replaces the old speculative wide port-range
allowance with one narrow rule: platform_ip -> the Router's port only.

dev-local stays routing_mode: direct — zero behavior change for local dev,
confirmed by the full existing regression suite passing unmodified.
This commit is contained in:
2026-08-24 20:14:39 -05:00
parent bf1e1b5c76
commit eae8e48d43
27 changed files with 1431 additions and 68 deletions
+1 -1
View File
@@ -13,7 +13,7 @@ lab_host_ip: __LAB_HOST_VM_IP__
management_cidr: __YOUR_ADMIN_ACCESS_CIDR__ # e.g. a VPN/WireGuard subnet, never 0.0.0.0/0
proxmox_mgmt_cidr: __PROXMOX_HOST_MGMT_CIDR__
lab_subnet_range: "10.42.0.0/16" # matches NO_PROXY_CIDRS already used by the app
lab_gateway_port_range: "30000:32767" # see docs/PRODUCTION_NETWORK_SECURITY.md — Gateway-to-Lab reachability follow-up
lab_router_port: "8443" # Gateway -> Lab Host Router (data plane) — see docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1-3
# ── Storage (Phase 4 §4 — a decision framework, not a final layout; see docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md §4)
rootnode_data_root: /srv/rootnode-data # expected to be a ZFS dataset mountpoint on the real host
@@ -4,3 +4,19 @@
name: docker
state: restarted
daemon_reload: true
# Phase 4.1 — these restart the whole per-host compose wrapper unit (no per-container systemd
# units exist in this project's convention, see the `platform`/`lab-host` roles), which is
# heavier than a surgical single-container restart but consistent with how every other
# config-changed-so-restart-the-stack case is already handled here.
- name: restart lab-router
ansible.builtin.systemd:
name: rootnode-lab-host.service
state: restarted
when: inventory_hostname in groups['lab_host']
- name: restart gateway
ansible.builtin.systemd:
name: rootnode-platform.service
state: restarted
when: inventory_hostname in groups['platform']
@@ -87,3 +87,50 @@
mode: "0644"
when: inventory_hostname in groups['lab_host']
notify: restart docker with tls
# ── Phase 4.1: the DATA-plane mTLS pair (Gateway <-> Lab Host Router). Same CA as the
# CONTROL-plane pair above, separate directory (/etc/rootnode/lab-router-tls) so the two
# concerns stay bind-mount-separable — the `lab-router`/`gateway` containers only ever need
# their own pair mounted in, never the Docker TLS one. See
# docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1-3.
- name: Create /etc/rootnode/lab-router-tls
ansible.builtin.file:
path: /etc/rootnode/lab-router-tls
state: directory
owner: root
group: root
mode: "0700"
- name: Deploy CA cert for the data-plane pair (public, both hosts)
ansible.builtin.copy:
src: "{{ docker_tls_local_bundle_dir }}/ca.pem"
dest: /etc/rootnode/lab-router-tls/ca.pem
owner: root
group: root
mode: "0444"
- name: Deploy Lab Host Router server cert/key (rn-lab-host-01 only)
ansible.builtin.copy:
src: "{{ docker_tls_local_bundle_dir }}/{{ item.src }}"
dest: "/etc/rootnode/lab-router-tls/{{ item.dest }}"
owner: root
group: root
mode: "{{ item.mode }}"
loop:
- { src: router-server-cert.pem, dest: router-server-cert.pem, mode: "0444" }
- { src: router-server-key.pem, dest: router-server-key.pem, mode: "0400" }
when: inventory_hostname in groups['lab_host']
notify: restart lab-router
- name: Deploy Gateway client cert/key (rn-platform-01 only)
ansible.builtin.copy:
src: "{{ docker_tls_local_bundle_dir }}/{{ item.src }}"
dest: "/etc/rootnode/lab-router-tls/{{ item.dest }}"
owner: root
group: root
mode: "{{ item.mode }}"
loop:
- { src: gateway-client-cert.pem, dest: gateway-client-cert.pem, mode: "0444" }
- { src: gateway-client-key.pem, dest: gateway-client-key.pem, mode: "0400" }
when: inventory_hostname in groups['platform']
notify: restart gateway
@@ -3,3 +3,11 @@ PLATFORM_API_URL=http://api:4300
PROVISIONER_URL=http://provisioner:4100
PROVISIONER_SHARED_SECRET={{ vault_provisioner_shared_secret }}
GATEWAY_PORT=4200
# Phase 4.1 — mTLS client identity for the Lab Host Router (data plane). Only used when a
# lab_host's routing_mode is 'router' (see platform/gateway/src/lab-endpoint.js); harmless if
# every configured lab_host is still 'direct'. Paths are inside the container — mounted from
# /etc/rootnode/lab-router-tls on the host by the `docker-tls` role, same pattern as the
# Provisioner's Docker TLS client cert.
LAB_ROUTER_TLS_CA=/tls/ca.pem
LAB_ROUTER_TLS_CERT=/tls/gateway-client-cert.pem
LAB_ROUTER_TLS_KEY=/tls/gateway-client-key.pem
+1
View File
@@ -8,6 +8,7 @@
| `rootnode/gateway` | `platform/gateway/Dockerfile` | `{{ registry }}/rootnode/gateway` |
| `rootnode/provisioner` | `platform/provisioner/Dockerfile` | `{{ registry }}/rootnode/provisioner` |
| `rootnode/egress-proxy` | `platform/egress-proxy/Dockerfile` | `{{ registry }}/rootnode/egress-proxy` |
| `rootnode/lab-router` | `platform/lab-router/Dockerfile` (Phase 4.1) | `{{ registry }}/rootnode/lab-router` |
| `rootnode/lab` | root `Dockerfile` (the student Lab image) | `{{ registry }}/rootnode/lab` |
## Tagging
@@ -1,10 +1,13 @@
# Production compose for rn-lab-host-01. Deployed by infra/ansible/roles/lab-host/ to
# /opt/rootnode/lab-host/docker-compose.yml, run as a systemd unit
# (rootnode-lab-host.service.j2). Runs exactly one long-lived service: the shared egress proxy
# every student Lab's outbound traffic passes through (platform/egress-proxy/, unchanged since
# Phase 1). Student Lab containers themselves are NOT part of this compose file — they're
# created/destroyed individually by the Provisioner over the remote Docker TLS endpoint
# (dockerd itself, not compose), same mechanism as dev, just remote.
# (rootnode-lab-host.service.j2). Runs the shared egress proxy every student Lab's outbound
# traffic passes through (platform/egress-proxy/, unchanged since Phase 1) and, as of Phase
# 4.1, the Lab Host Router (platform/lab-router/) that gives the Gateway on rn-platform-01 a
# way to reach a running Lab without ever touching a Docker-internal address — see
# docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1-3. Student Lab containers themselves are NOT
# part of this compose file — they're created/destroyed individually by the Provisioner over
# the remote Docker TLS endpoint (dockerd itself, not compose), same mechanism as dev, just
# remote; the Router finds them by label, purely from local Docker state (see its own README).
networks:
rn-egress:
@@ -22,6 +25,26 @@ services:
# is the actual control, see platform/egress-proxy/allowlist.txt)
networks: [rn-egress]
lab-router:
image: "{{ registry }}/rootnode/lab-router:{{ image_tag }}"
restart: unless-stopped
# Needs the host's own routing table to reach whichever per-Lab bridge network
# (rootnode-lab-net-<id>) the Provisioner created for a given container — those networks
# aren't known ahead of time and aren't attachable one-by-one, the same reason egress-proxy
# would need this too if it initiated connections outward instead of just accepting them.
# Restricted to platform_ip on port {{ lab_router_port | default('8443') }} at the host
# firewall (infra/firewall/rn-lab-host-01.rules) — this is the ONLY thing that makes host
# networking here safe, same defense-in-depth pattern as the Docker TLS port.
network_mode: host
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- /etc/rootnode/lab-router-tls:/tls:ro
environment:
LAB_ROUTER_PORT: "{{ lab_router_port | default('8443') }}"
LAB_ROUTER_TLS_CA: /tls/ca.pem
LAB_ROUTER_TLS_CERT: /tls/router-server-cert.pem
LAB_ROUTER_TLS_KEY: /tls/router-server-key.pem
node-exporter:
image: prom/node-exporter:latest
restart: unless-stopped
@@ -88,6 +88,11 @@ services:
image: "{{ registry }}/rootnode/platform-api:{{ image_tag }}"
restart: unless-stopped
env_file: /etc/rootnode/platform.env
environment:
# Phase 4.1 §25 — not a secret, just the deployed tag; set directly here rather than in
# the env template since it's the same value already used for the image tag above, not a
# separately-managed setting.
GIT_SHA: "{{ image_tag }}"
depends_on:
postgres:
condition: service_healthy
@@ -99,6 +104,10 @@ services:
image: "{{ registry }}/rootnode/gateway:{{ image_tag }}"
restart: unless-stopped
env_file: /etc/rootnode/gateway.env
volumes:
# mTLS client identity for the Lab Host Router (data plane) — see gateway.env.j2's
# LAB_ROUTER_TLS_* vars. Read-only; the private key never leaves this one bind mount.
- /etc/rootnode/lab-router-tls:/tls:ro
depends_on: [provisioner, api]
networks: [rn-platform-internal]
@@ -1,7 +1,12 @@
# Production lab_hosts seed — points at the real rn-lab-host-01 VM instead of Phase 1's dev
# Unix socket. Same mechanism the Provisioner has had since Phase 1
# (platform/provisioner/config/lab-hosts.seed.yaml), just pointed at a real remote endpoint —
# no application code changes required for this, see docs/PRODUCTION_NETWORK_SECURITY.md §7.
# (platform/provisioner/config/lab-hosts.seed.yaml), just pointed at real remote endpoints.
#
# Phase 4.1 correction: routing_mode/router_endpoint is REQUIRED here — Phase 4's original
# claim that "no application code changes required" was true for the Provisioner's CONTROL
# plane (Docker mTLS) but false for the Gateway's DATA plane, which used to assume
# env.internal_ip was directly reachable. It is not, across two real VMs. See
# docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1 and docs/PRODUCTION_NETWORK_SECURITY.md.
#
# {{ lab_host_ip }} is an Ansible template variable — see infra/ansible/group_vars/production.yml.
# Rendered by the `platform` role; the raw file here keeps the placeholder for readability.
@@ -9,8 +14,14 @@
lab_hosts:
- lab_host_id: labhost-01
display_name: "RootNode Lab Host VM 01 (production)"
# CONTROL plane: Provisioner -> dockerd, mTLS, port 2376.
endpoint: "tcp://{{ lab_host_ip }}:2376"
tls_ca_path: /etc/rootnode/docker-tls/ca.pem
tls_cert_path: /etc/rootnode/docker-tls/provisioner-client-cert.pem
tls_key_path: /etc/rootnode/docker-tls/provisioner-client-key.pem
enabled: true
# DATA plane: Gateway -> Lab Host Router, mTLS, router's own port (8443 by default —
# infra/docker/lab-host/docker-compose.production.yml). Gateway never touches a
# Docker-internal address once routing_mode is 'router'.
routing_mode: router
router_endpoint: "https://{{ lab_host_ip }}:8443"
@@ -1,16 +1,17 @@
# Production lab_hosts seed — points at the real rn-lab-host-01 VM instead of Phase 1's dev
# Unix socket. Same mechanism the Provisioner has had since Phase 1
# (platform/provisioner/config/lab-hosts.seed.yaml), just pointed at a real remote endpoint —
# no application code changes required for this, see docs/PRODUCTION_NETWORK_SECURITY.md §7.
#
# {{ lab_host_ip }} is an Ansible template variable — see infra/ansible/group_vars/production.yml.
# Rendered by the `platform` role; the raw file here keeps the placeholder for readability.
# Staging lab_hosts seed — see infra/environments/production/provisioner-config/lab-hosts.yaml
# for the full explanation; this is the same shape, lower stakes. Staging's `lab_host_ip` is
# distinct from production's own (infra/ansible/group_vars/staging.yml), so it needs the same
# router routing_mode as production once staging genuinely runs on two hosts. If staging is
# ever collapsed onto a single host instead, change routing_mode to `direct` and drop
# router_endpoint — that's the only edit needed either way.
lab_hosts:
- lab_host_id: labhost-01
display_name: "RootNode Lab Host VM 01 (production)"
display_name: "RootNode Lab Host VM 01 (staging)"
endpoint: "tcp://{{ lab_host_ip }}:2376"
tls_ca_path: /etc/rootnode/docker-tls/ca.pem
tls_cert_path: /etc/rootnode/docker-tls/provisioner-client-cert.pem
tls_key_path: /etc/rootnode/docker-tls/provisioner-client-key.pem
enabled: true
routing_mode: router
router_endpoint: "https://{{ lab_host_ip }}:8443"
+8 -7
View File
@@ -38,13 +38,14 @@
# endpoint: ALLOW" example in the Phase 4 spec, encoded literally here.
-A INPUT -p tcp --dport 2376 -s {{ platform_ip }} -m conntrack --ctstate NEW -j ACCEPT
# Gateway -> per-Lab published port range, rn-platform-01 only. REQUIRED so the Gateway can
# reach each student's Lab UI/terminal over the network once Labs run on a genuinely separate
# host — see docs/PRODUCTION_NETWORK_SECURITY.md §"Gateway-to-Lab cross-VM reachability" for
# why this exists and the small Provisioner/Gateway code change it depends on (NOT YET BUILT —
# this firewall rule is the network-policy half; the port-publishing half is an explicit,
# reviewed application-code follow-up, tracked there rather than silently assumed here).
-A INPUT -p tcp --dport {{ lab_gateway_port_range | default('30000:32767') }} -s {{ platform_ip }} -m conntrack --ctstate NEW -j ACCEPT
# Lab Host Router (8443) — rn-platform-01 (the Gateway) only. This is the DATA-plane
# equivalent of the Docker TLS rule above: the Gateway never reaches a Lab's Docker-internal
# address directly (no per-Lab port publishing, no wide port range needed) — it reaches this
# one mTLS-authenticated Router instead, which resolves the request to a local container on
# ITS OWN host. See docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1-3 and
# docs/PRODUCTION_NETWORK_SECURITY.md — this closes the "Gateway-to-Lab cross-VM reachability"
# gap Phase 4 flagged as a required follow-up; it is no longer open.
-A INPUT -p tcp --dport {{ lab_router_port | default('8443') }} -s {{ platform_ip }} -m conntrack --ctstate NEW -j ACCEPT
# Prometheus (rn-platform-01) scraping this host's node-exporter/cadvisor — metrics only,
# read-only exposition endpoints, still restricted to the one legitimate scraper.
+1 -1
View File
@@ -21,7 +21,7 @@ render() {
-e 's/{{ lab_host_ip }}/10.10.10.20/g' \
-e 's/{{ lab_subnet_range }}/10.42.0.0\/16/g' \
-e 's/{{ proxmox_mgmt_cidr }}/203.0.113.0\/24/g' \
-e "s/{{ lab_gateway_port_range | default('30000:32767') }}/30000:32767/g" \
-e "s/{{ lab_router_port | default('8443') }}/8443/g" \
"$1" > "$2"
}
+1
View File
@@ -22,6 +22,7 @@ declare -A IMAGES=(
[rootnode/gateway]="platform/gateway/Dockerfile platform/gateway"
[rootnode/provisioner]="platform/provisioner/Dockerfile platform/provisioner"
[rootnode/egress-proxy]="platform/egress-proxy/Dockerfile platform/egress-proxy"
[rootnode/lab-router]="platform/lab-router/Dockerfile platform/lab-router"
[rootnode/lab]="Dockerfile ."
)
+43 -12
View File
@@ -1,14 +1,21 @@
#!/usr/bin/env bash
# Generates a private CA and a mutual-TLS cert/key pair for the Provisioner <-> rn-lab-host-01
# Docker Engine connection (docs/PRODUCTION_NETWORK_SECURITY.md §7). The application already
# supports this (platform/provisioner/src/docker.js buildClient() has required mTLS since
# Phase 1) — this script only ever needed to exist for the CERTIFICATES, not new app code.
# Generates a private CA and TWO mutual-TLS cert/key pairs, both under that one CA:
# 1. Provisioner <-> rn-lab-host-01 Docker Engine (CONTROL plane, port 2376). The application
# already supports this (platform/provisioner/src/docker.js buildClient() has required
# mTLS since Phase 1) — this script only ever needed to exist for the CERTIFICATES.
# 2. Gateway <-> rn-lab-host-01 Lab Host Router (DATA plane, port 8443) — Phase 4.1, see
# docs/PHASE4_1_PREPRODUCTION_HARDENING.md §1-3 and docs/PRODUCTION_NETWORK_SECURITY.md.
# One CA for both keeps the PKI surface small (one thing to protect, one rotation runbook) —
# see docs/RUNBOOK_DOCKER_TLS_ROTATION.md, which now covers all four leaf certs, not just two.
#
# Output goes to $OUT_DIR (default ./out, gitignored) — never into a tracked path. Deploy the
# resulting files with the `docker-tls` Ansible role, which places:
# rn-lab-host-01: ca.pem, server-cert.pem, server-key.pem -> dockerd's --tlsverify config
# rn-platform-01: ca.pem, provisioner-client-cert.pem, provisioner-client-key.pem
# rn-lab-host-01: ca.pem, server-cert.pem, server-key.pem -> dockerd --tlsverify
# ca.pem, router-server-cert.pem, router-server-key.pem -> lab-router (data plane)
# rn-platform-01: ca.pem, provisioner-client-cert.pem, provisioner-client-key.pem
# -> /etc/rootnode/docker-tls/ (referenced by lab-hosts.yaml)
# ca.pem, gateway-client-cert.pem, gateway-client-key.pem
# -> /etc/rootnode/lab-router-tls/ (referenced by gateway's LAB_ROUTER_TLS_*)
#
# Usage:
# ./generate-docker-tls.sh [--out-dir DIR] [--lab-host-ip IP] [--lab-host-dns NAME] [--days N]
@@ -61,13 +68,37 @@ EOF
openssl x509 -req -days "$DAYS" -sha256 -in client.csr -CA ca.pem -CAkey ca-key.pem \
-CAcreateserial -out provisioner-client-cert.pem -extfile client-extfile.cnf
rm -f server.csr client.csr server-extfile.cnf client-extfile.cnf
chmod 0400 ca-key.pem server-key.pem provisioner-client-key.pem
chmod 0444 ca.pem server-cert.pem provisioner-client-cert.pem
echo "==> Generating server cert for rn-lab-host-01 (Lab Host Router, data plane)"
openssl genrsa -out router-server-key.pem 4096
openssl req -subj "/CN=$LAB_HOST_DNS" -sha256 -new -key router-server-key.pem -out router-server.csr
cat > router-server-extfile.cnf <<EOF
subjectAltName = DNS:$LAB_HOST_DNS,IP:$LAB_HOST_IP
extendedKeyUsage = serverAuth
EOF
openssl x509 -req -days "$DAYS" -sha256 -in router-server.csr -CA ca.pem -CAkey ca-key.pem \
-CAcreateserial -out router-server-cert.pem -extfile router-server-extfile.cnf
echo "==> Generating client cert for the Gateway (data plane)"
openssl genrsa -out gateway-client-key.pem 4096
openssl req -subj "/CN=rootnode-gateway" -new -key gateway-client-key.pem -out gateway-client.csr
openssl x509 -req -days "$DAYS" -sha256 -in gateway-client.csr -CA ca.pem -CAkey ca-key.pem \
-CAcreateserial -out gateway-client-cert.pem -extfile client-extfile.cnf
rm -f server.csr client.csr router-server.csr gateway-client.csr server-extfile.cnf client-extfile.cnf router-server-extfile.cnf
chmod 0400 ca-key.pem server-key.pem provisioner-client-key.pem router-server-key.pem gateway-client-key.pem
chmod 0444 ca.pem server-cert.pem provisioner-client-cert.pem router-server-cert.pem gateway-client-cert.pem
echo "==> Done. Generated in $OUT_DIR:"
ls -la
echo
echo "Deploy server-{cert,key}.pem + ca.pem to rn-lab-host-01 (docker-tls Ansible role)."
echo "Deploy provisioner-client-{cert,key}.pem + ca.pem to rn-platform-01 (same role)."
echo "Verify with: ../verify-docker-tls.sh $OUT_DIR <lab-host-ip>"
echo "CONTROL plane (Provisioner -> dockerd, port 2376):"
echo " Deploy server-{cert,key}.pem + ca.pem to rn-lab-host-01 (docker-tls Ansible role)."
echo " Deploy provisioner-client-{cert,key}.pem + ca.pem to rn-platform-01 (same role)."
echo "DATA plane (Gateway -> Lab Host Router, port 8443):"
echo " Deploy router-server-{cert,key}.pem + ca.pem to rn-lab-host-01 (lab-router Ansible role)."
echo " Deploy gateway-client-{cert,key}.pem + ca.pem to rn-platform-01 (platform Ansible role)."
echo
echo "Verify Docker control-plane TLS with: ../verify-docker-tls.sh $OUT_DIR <lab-host-ip>"
+14
View File
@@ -36,3 +36,17 @@ sed -e 's/{{ academy_domain }}/academy.example.com/g' -e 's/{{ management_cidr }
chmod a+r "$TMPDIR/Caddyfile"
docker run --rm -v "$TMPDIR:/etc/caddy:ro" caddy:2-alpine caddy validate --config /etc/caddy/Caddyfile 2>&1 | grep -q "Valid configuration" \
&& echo "OK: Caddyfile is valid" || { echo "FAIL: Caddyfile did not validate"; exit 1; }
# Phase 4.1: rn-lab-host-01's compose file (egress-proxy, lab-router, observability agents)
# was never covered here before — added when the lab-router service landed.
echo
echo "--- lab-host production compose ---"
LABHOST_TMPDIR="$(mktemp -d)"
sed \
-e 's/{{ registry }}/registry.example.com/g' \
-e 's/{{ image_tag }}/test/g' \
-e "s/{{ lab_router_port | default('8443') }}/8443/g" \
../lab-host/docker-compose.production.yml > "$LABHOST_TMPDIR/docker-compose.yml"
docker compose -f "$LABHOST_TMPDIR/docker-compose.yml" config --quiet
echo "OK: lab-host production compose file is well-formed YAML and passes docker compose's own schema validation"
rm -rf "$LABHOST_TMPDIR"