From d2280d0f0d5e80175789ba655d47fa3dbbcf3f0e Mon Sep 17 00:00:00 2001 From: ced4568 Date: Tue, 18 Aug 2026 17:13:09 -0500 Subject: [PATCH] feat(security): production Compose topology, firewall rulesets, Docker mTLS Named-bridge-network production Compose (not host networking, unlike dev) for rn-platform-01; egress-proxy + agents for rn-lab-host-01. Caddy reverse proxy, single public origin, Grafana restricted to management_cidr. iptables-restore firewall rulesets for all three hosts (Proxmox host, rn-platform-01, rn-lab-host-01) codifying the Platform/Lab-Host boundary: no student-to-student traffic, no direct Proxmox exposure, no direct Postgres/Provisioner exposure from the Lab Host. Mutual TLS for the Provisioner<->Lab Host Docker connection: certificate generation/verification scripts, dockerd TLS config, firewall-restricted port 2376. Application support already existed since Phase 1; this is purely the operational half. Validated: docker compose config, caddy validate, firewall structural lint, and a real openssl s_server/s_client mTLS handshake proving the cert chain and mutual-auth rejection (legitimate client verified, rogue client correctly rejected with verify code 19). --- infra/docker/IMAGE_STRATEGY.md | 49 ++++ .../lab-host/docker-compose.production.yml | 68 ++++++ infra/docker/platform/Caddyfile.j2 | 38 ++++ .../platform/docker-compose.production.yml | 212 ++++++++++++++++++ .../00-rootnode-baseline.conf | 30 +++ infra/docker/tls/README.md | 19 ++ infra/firewall/README.md | 31 +++ infra/firewall/proxmox-host.rules | 42 ++++ infra/firewall/rn-lab-host-01.rules | 62 +++++ infra/firewall/rn-platform-01.rules | 58 +++++ infra/firewall/validate-firewall.sh | 63 ++++++ .../scripts/docker-tls/generate-docker-tls.sh | 73 ++++++ infra/scripts/docker-tls/verify-docker-tls.sh | 51 +++++ 13 files changed, 796 insertions(+) create mode 100644 infra/docker/IMAGE_STRATEGY.md create mode 100644 infra/docker/lab-host/docker-compose.production.yml create mode 100644 infra/docker/platform/Caddyfile.j2 create mode 100644 infra/docker/platform/docker-compose.production.yml create mode 100644 infra/docker/postgres/postgresql.conf.d/00-rootnode-baseline.conf create mode 100644 infra/docker/tls/README.md create mode 100644 infra/firewall/README.md create mode 100644 infra/firewall/proxmox-host.rules create mode 100644 infra/firewall/rn-lab-host-01.rules create mode 100644 infra/firewall/rn-platform-01.rules create mode 100755 infra/firewall/validate-firewall.sh create mode 100755 infra/scripts/docker-tls/generate-docker-tls.sh create mode 100755 infra/scripts/docker-tls/verify-docker-tls.sh diff --git a/infra/docker/IMAGE_STRATEGY.md b/infra/docker/IMAGE_STRATEGY.md new file mode 100644 index 0000000..4e9210c --- /dev/null +++ b/infra/docker/IMAGE_STRATEGY.md @@ -0,0 +1,49 @@ +# Container image strategy + +## Images + +| Image | Built from | Registry path | +|---|---|---| +| `rootnode/platform-api` | `platform/api/Dockerfile` | `{{ registry }}/rootnode/platform-api` | +| `rootnode/gateway` | `platform/gateway/Dockerfile` | `{{ registry }}/rootnode/gateway` | +| `rootnode/provisioner` | `platform/provisioner/Dockerfile` | `{{ registry }}/rootnode/provisioner` | +| `rootnode/egress-proxy` | `platform/egress-proxy/Dockerfile` | `{{ registry }}/rootnode/egress-proxy` | +| `rootnode/lab` | root `Dockerfile` (the student Lab image) | `{{ registry }}/rootnode/lab` | + +## Tagging + +**Never `:latest` in staging or production.** Every build is tagged with: + +- the full git SHA (`sha-<40 hex>`) — always, every build, the ground truth +- `staging-` on every push to `main` that passes CI (moving pointer, staging only) +- a semantic tag (`vX.Y.Z`) only on an explicit, human-triggered production release + +`infra/scripts/ci/tag-and-push.sh` implements this — see §18 in +`docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md`. + +Deployment configs (`infra/environments/*/​*.env.template`, Ansible `group_vars`) reference the +tag as a variable (`image_tag`), never a hardcoded value baked into a compose file — rolling +back is changing one variable and re-running the `platform`/`lab-host` Ansible role, not editing +a compose file by hand (see `docs/RUNBOOK_ROLLBACK.md`). + +## Registry + +No paid registry required. Two supported options, chosen at deploy time via `registry` in +`infra/ansible/group_vars/production.yml`: + +1. **Gitea's own built-in container registry** (`git.synthossystems.com`) — zero new + infrastructure, already authenticated the same way the repo itself is. Preferred default. +2. **A self-hosted registry** (e.g. a `registry:2` container on rn-platform-01 itself, or + elsewhere in the homelab) — a documented fallback if the Gitea registry isn't enabled on + this instance. + +Either way: private registry, credentials pulled the same way any other production secret is +(`docs/PRODUCTION_DEPLOYMENT.md` §10), never a public Docker Hub push of anything beyond what's +already public (this project doesn't publish anything to Docker Hub in Phase 4). + +## Rollback + +Every image is content-addressable by SHA tag, so "roll back" is always available as long as +the previous tag hasn't been garbage-collected from the registry — see +`docs/RUNBOOK_ROLLBACK.md` for the exact procedure (re-point `image_tag`, re-run the Ansible +role, verify). diff --git a/infra/docker/lab-host/docker-compose.production.yml b/infra/docker/lab-host/docker-compose.production.yml new file mode 100644 index 0000000..e170c22 --- /dev/null +++ b/infra/docker/lab-host/docker-compose.production.yml @@ -0,0 +1,68 @@ +# Production compose for rn-lab-host-01. Deployed by infra/ansible/roles/lab-host/ to +# /opt/rootnode/lab-host/docker-compose.yml, run as a systemd unit +# (rootnode-lab-host.service.j2). Runs exactly one long-lived service: the shared egress proxy +# every student Lab's outbound traffic passes through (platform/egress-proxy/, unchanged since +# Phase 1). Student Lab containers themselves are NOT part of this compose file — they're +# created/destroyed individually by the Provisioner over the remote Docker TLS endpoint +# (dockerd itself, not compose), same mechanism as dev, just remote. + +networks: + rn-egress: + driver: bridge + +services: + egress-proxy: + image: "{{ registry }}/rootnode/egress-proxy:{{ image_tag }}" + restart: unless-stopped + ports: + - "3128:3128" # reachable from every per-Lab Docker network on this host — restricted + # to those networks by the Provisioner's per-environment DOCKER-USER + # chains, not by a published-port firewall rule (there is no sensible + # single source IP to restrict this to; the egress proxy's OWN allowlist + # is the actual control, see platform/egress-proxy/allowlist.txt) + networks: [rn-egress] + + node-exporter: + image: prom/node-exporter:latest + restart: unless-stopped + network_mode: host + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + restart: unless-stopped + privileged: true + devices: + - /dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - "8080:8080" # scraped remotely by Prometheus on rn-platform-01 — restricted to that + # one source IP by the host firewall (rn-lab-host-01.rules), not by + # binding to localhost, since Prometheus isn't local on this host. + + promtail: + image: grafana/promtail:latest + restart: unless-stopped + volumes: + - /opt/rootnode/lab-host/promtail-config.yml:/etc/promtail/config.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log/journal:/var/log/journal:ro + - /etc/machine-id:/etc/machine-id:ro + command: ["-config.file=/etc/promtail/config.yml"] + # host networking so promtail can reach Loki on rn-platform-01 over the network — this + # traffic is OUTBOUND from rn-lab-host-01 (log push, client-initiated), which the host + # firewall's default-ACCEPT OUTPUT chain already allows; nothing new needed there. + network_mode: host diff --git a/infra/docker/platform/Caddyfile.j2 b/infra/docker/platform/Caddyfile.j2 new file mode 100644 index 0000000..17b7d6a --- /dev/null +++ b/infra/docker/platform/Caddyfile.j2 @@ -0,0 +1,38 @@ +# Caddy 2 — single stable RootNode Academy origin, automatic HTTPS (Let's Encrypt) unless +# Cloudflare origin certs are used instead (see docs/PRODUCTION_NETWORK_SECURITY.md §8 for the +# Cloudflare-vs-direct-LE decision). Rendered by the `platform` Ansible role from +# infra/ansible/group_vars/production.yml's `academy_domain`. +# +# The student/instructor path proxies to the Gateway (platform/gateway) — the sole +# student-facing entrypoint, unchanged architectural rule from Phase 2A onward. Nothing here +# reaches the Provisioner, Postgres, or Docker TLS directly; those have no public route at all. +# Grafana is reachable at /grafana/ but restricted to the management CIDR even though it has +# its own login — internal observability tooling shouldn't be internet-facing at all, same +# posture as everything else that isn't the student/instructor product surface. + +{{ academy_domain }} { + encode gzip zstd + + handle_path /grafana/* { + @notmanagement not remote_ip {{ management_cidr }} + respond @notmanagement 403 + reverse_proxy grafana:3000 + } + + # WebSocket terminal traffic (shell-ws) needs the connection upgraded, not buffered — + # Caddy's reverse_proxy handles this automatically for Upgrade: websocket requests, no + # special directive needed, but flush_interval -1 avoids buffering SSE/long-poll-shaped + # responses too. `gateway` is the Docker Compose service name (rn-platform-internal + # network), not localhost — Caddy no longer shares host networking with the Gateway in + # production, unlike the dev compose file. + reverse_proxy gateway:4200 { + flush_interval -1 + } + + log { + output file /var/log/caddy/access.log { + roll_size 100mb + roll_keep 7 + } + } +} diff --git a/infra/docker/platform/docker-compose.production.yml b/infra/docker/platform/docker-compose.production.yml new file mode 100644 index 0000000..9e0ae63 --- /dev/null +++ b/infra/docker/platform/docker-compose.production.yml @@ -0,0 +1,212 @@ +# Production compose for rn-platform-01. Deployed by infra/ansible/roles/platform/ to +# /opt/rootnode/platform/docker-compose.yml, run as a systemd-managed `docker compose` unit +# (infra/ansible/roles/platform/templates/rootnode-platform.service.j2). +# +# Differs from platform/docker-compose.dev.yml in the ways that actually matter for +# production, and nothing else: +# - images are pulled from a registry by DIGEST-PINNED TAG (never :latest, never built +# in-place) — see infra/docker/IMAGE_STRATEGY.md +# - Postgres data and backups live on a bind-mounted ZFS dataset, not an anonymous volume +# - a reverse proxy (Caddy) terminates TLS and is the only container with a port published to +# the internet; a few other services publish loopback-only or single-source-IP-restricted +# ports for specific operational needs (Postgres for the host's own backup cron, Loki for +# rn-lab-host-01's log shipper) — see each service's own comment +# - no `network_mode: host` on gateway/provisioner — production uses a named bridge network, +# since rn-platform-01 no longer needs to reach ephemeral container IPs directly (those live +# on rn-lab-host-01, reached over the network, not shared Docker networking) +# - the api/gateway/provisioner containers still don't reach the internet except through +# what's explicitly allowed by the host firewall (infra/firewall/), not Docker's own +# default bridge NAT behavior +# +# All image tags, ports, and paths below are Ansible Jinja2 variables — this file is a +# template rendered by the `platform` role, not run directly. See +# infra/ansible/group_vars/production.yml for the values. + +networks: + rn-platform-internal: + driver: bridge + internal: false # postgres/api/gateway/provisioner need it false to reach rn-lab-host-01 + # over the network; the HOST firewall (not Docker) is what actually + # restricts what that traffic can reach — see docs/PRODUCTION_NETWORK_SECURITY.md + +volumes: + postgres-data: + driver: local + driver_opts: + type: none + o: bind + device: "{{ rootnode_data_root }}/postgres" + caddy-data: + caddy-config: + prometheus-data: + alertmanager-data: + grafana-data: + loki-data: + +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + env_file: /etc/rootnode/postgres.env + volumes: + - postgres-data:/var/lib/postgresql/data + - /opt/rootnode/platform/postgresql.conf.d:/etc/postgresql/conf.d:ro + # include_dir appends our tuning files to the image's own auto-generated postgresql.conf — + # no need to replace or template the base config file at all. + command: ["postgres", "-c", "include_dir=/etc/postgresql/conf.d"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-rootnode_platform}"] + interval: 10s + timeout: 5s + retries: 10 + networks: [rn-platform-internal] + ports: + - "127.0.0.1:5432:5432" # loopback-only — reachable from other containers on + # rn-platform-internal AND from the host's own backup cron + # (infra/backup/), never from another host on the network, let + # alone the internet. This is NOT the same as the "Postgres + # never exposed on this VM's external interface" rule in + # rn-platform-01.rules — that rule (and the host firewall in + # general) still applies to this port; 127.0.0.1 binding is a + # second, independent reason nothing off-host can ever reach it. + + provisioner: + image: "{{ registry }}/rootnode/provisioner:{{ image_tag }}" + restart: unless-stopped + env_file: /etc/rootnode/provisioner.env + volumes: + - /etc/rootnode/provisioner-config:/app/config:ro + - /etc/rootnode/docker-tls:/app/docker-tls:ro + depends_on: + postgres: + condition: service_healthy + networks: [rn-platform-internal] + cap_add: [NET_ADMIN, NET_RAW] # only needed while a lab_host row is still a local Unix + # socket (dev); harmless no-op against a remote TLS endpoint + + api: + image: "{{ registry }}/rootnode/platform-api:{{ image_tag }}" + restart: unless-stopped + env_file: /etc/rootnode/platform.env + depends_on: + postgres: + condition: service_healthy + provisioner: + condition: service_started + networks: [rn-platform-internal] + + gateway: + image: "{{ registry }}/rootnode/gateway:{{ image_tag }}" + restart: unless-stopped + env_file: /etc/rootnode/gateway.env + depends_on: [provisioner, api] + networks: [rn-platform-internal] + + reverse-proxy: + image: caddy:2-alpine + restart: unless-stopped + ports: + - "443:443" + - "80:80" # ACME HTTP-01 challenge / plain-HTTP-to-HTTPS redirect only + volumes: + - /opt/rootnode/platform/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + depends_on: [gateway] + networks: [rn-platform-internal] + + # ── Observability agents (infra/monitoring/) ──────────────────────────────────────────── + node-exporter: + image: prom/node-exporter:latest + restart: unless-stopped + network_mode: host # needs host-level metrics; this is metrics-only, not app traffic + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + - /var/lib/node-exporter/textfile:/textfile:ro # backup-postgres.sh writes its metric here + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + - "--collector.textfile.directory=/textfile" + + postgres-exporter: + image: prometheuscommunity/postgres-exporter:latest + restart: unless-stopped + environment: + DATA_SOURCE_NAME: "postgresql://{{ postgres_exporter_user }}:{{ vault_postgres_exporter_password }}@postgres:5432/rootnode_platform?sslmode=disable" + depends_on: [postgres] + networks: [rn-platform-internal] + + blackbox-exporter: + image: prom/blackbox-exporter:latest + restart: unless-stopped + volumes: + - /opt/rootnode/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml:ro + networks: [rn-platform-internal] + + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + volumes: + - /opt/rootnode/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - /opt/rootnode/monitoring/rules:/etc/prometheus/rules:ro + - prometheus-data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.retention.time={{ prometheus_retention_days }}d" + networks: [rn-platform-internal] + ports: + - "127.0.0.1:9090:9090" # bound to loopback only — access via SSH tunnel or Grafana, never public + + alertmanager: + image: prom/alertmanager:latest + restart: unless-stopped + volumes: + - /opt/rootnode/monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager-data:/alertmanager + networks: [rn-platform-internal] + + grafana: + image: grafana/grafana-oss:latest + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_PASSWORD: "{{ vault_grafana_admin_password }}" + GF_SERVER_ROOT_URL: "https://{{ academy_domain }}/grafana/" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - grafana-data:/var/lib/grafana + - /opt/rootnode/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - /opt/rootnode/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + networks: [rn-platform-internal] + # No published port — reached only via the reverse proxy's /grafana/ path (Caddyfile), + # never a direct public listener. See docs/PRODUCTION_NETWORK_SECURITY.md §8. + + loki: + image: grafana/loki:latest + restart: unless-stopped + volumes: + - /opt/rootnode/monitoring/loki-config.yml:/etc/loki/local-config.yaml:ro + - loki-data:/loki + command: ["-config.file=/etc/loki/local-config.yaml"] + networks: [rn-platform-internal] + ports: + - "3100:3100" # reachable from rn-lab-host-01's promtail over the network — restricted + # to that one source IP by the host firewall (rn-platform-01.rules), same + # pattern as Docker TLS's 2376. Not published to the internet. + + promtail: + image: grafana/promtail:latest + restart: unless-stopped + volumes: + - /opt/rootnode/monitoring/promtail-config.yml:/etc/promtail/config.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log/journal:/var/log/journal:ro + - /etc/machine-id:/etc/machine-id:ro + command: ["-config.file=/etc/promtail/config.yml"] + networks: [rn-platform-internal] + depends_on: [loki] diff --git a/infra/docker/postgres/postgresql.conf.d/00-rootnode-baseline.conf b/infra/docker/postgres/postgresql.conf.d/00-rootnode-baseline.conf new file mode 100644 index 0000000..481036b --- /dev/null +++ b/infra/docker/postgres/postgresql.conf.d/00-rootnode-baseline.conf @@ -0,0 +1,30 @@ +# Conservative production tuning baseline — deliberately NOT hand-tuned to a specific RAM +# allocation before rn-platform-01's real resources are known (Phase 4 instruction: do not +# over-tune before actual hardware exists). Sized for a modest ~4GB Postgres allocation, safe +# defaults everywhere else. Revisit once VM RAM is final — see docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md §11. + +shared_buffers = 1GB +effective_cache_size = 3GB +maintenance_work_mem = 256MB +work_mem = 16MB + +max_connections = 100 + +# WAL — safe defaults for a single-instance setup with pg_dump/pg_basebackup-style backups +# (infra/backup/). Not configured for streaming replication (out of scope this phase — see +# docs/BACKUP_DR.md's DR model for why pg_dump + periodic pg_basebackup is the chosen approach, +# not a hot standby). +wal_level = replica +max_wal_size = 2GB +min_wal_size = 512MB +checkpoint_completion_target = 0.9 + +# Query logging — slow queries only, never full statement logging (would capture bind +# parameters that can include session tokens' hashes / PII in edge cases). See docs/OBSERVABILITY.md. +log_min_duration_statement = 500 +log_line_prefix = '%m [%p] %q%u@%d ' +log_checkpoints = on +log_connections = on +log_disconnections = on + +# Autovacuum stays on defaults — no reason to disable it for this workload size. diff --git a/infra/docker/tls/README.md b/infra/docker/tls/README.md new file mode 100644 index 0000000..bbde5d8 --- /dev/null +++ b/infra/docker/tls/README.md @@ -0,0 +1,19 @@ +# Docker TLS material (generated, not committed) + +Generate with `infra/scripts/docker-tls/generate-docker-tls.sh`. Deploy with the `docker-tls` +Ansible role. Verify with `infra/scripts/docker-tls/verify-docker-tls.sh`. + +**Nothing in this directory is ever committed** — see `infra/.gitignore`. This README (and only +this README) is tracked, so the directory itself exists in the repo as documentation of what +belongs here operationally. + +Files that will exist here on a real deployment host (never in git): + +``` +ca.pem # public — safe to distribute to both VMs +ca-key.pem # PRIVATE — offline/secrets-vault only, never on either VM +server-cert.pem / server-key.pem # rn-lab-host-01's dockerd +provisioner-client-cert.pem / provisioner-client-key.pem # rn-platform-01's Provisioner +``` + +Rotation procedure: `docs/RUNBOOK_DOCKER_TLS_ROTATION.md`. diff --git a/infra/firewall/README.md b/infra/firewall/README.md new file mode 100644 index 0000000..8fed599 --- /dev/null +++ b/infra/firewall/README.md @@ -0,0 +1,31 @@ +# Production firewall policy + +iptables (not nftables) — deliberately, to stay consistent with the Provisioner's own dynamic +per-Lab egress rules, which already manipulate the `DOCKER-USER` chain via iptables +(`platform/provisioner/src/firewall.js`, Phase 1). Mixing the nft and legacy iptables backends +on the same host is a known source of silent rule-ordering bugs; the host-level policy below +uses the same backend the app already depends on. + +Each `.rules` file is `iptables-restore` format — deployed by the matching Ansible role +(`firewall` for the VMs; the Proxmox host ruleset is applied manually per +`docs/PRODUCTION_DEPLOYMENT.md` §2, since Proxmox host firewalling is out of Ansible's blast +radius by design — a mistake there can lock out the only way in). + +| File | Applies to | Deployed by | +|---|---|---| +| `proxmox-host.rules` | The Proxmox hypervisor itself | manual, `docs/PRODUCTION_DEPLOYMENT.md` §2 | +| `rn-platform-01.rules` | Platform VM | `infra/ansible/roles/firewall/` | +| `rn-lab-host-01.rules` | Lab Host VM | `infra/ansible/roles/firewall/` | + +All three encode the same policy table from `docs/PRODUCTION_NETWORK_SECURITY.md` §6 — this +README doesn't restate it, that doc is the source of truth for *why*; these files are the +*what*, in a form the host can actually load. + +**IP addresses/subnets are Ansible template variables** (`{{ platform_ip }}`, +`{{ lab_host_ip }}`, `{{ management_cidr }}`, etc.) — see `infra/ansible/group_vars/production.yml`. +Nothing here hardcodes a real production IP, per the explicit Phase 4 instruction not to guess +those before the real host exists. + +`infra/firewall/validate-firewall.sh` renders each ruleset with dummy IPs and runs it through +`iptables-restore --test` (syntax/semantic validation only — proves the rules are well-formed +iptables, not that the real network topology is correct, which needs the real hardware). diff --git a/infra/firewall/proxmox-host.rules b/infra/firewall/proxmox-host.rules new file mode 100644 index 0000000..090663d --- /dev/null +++ b/infra/firewall/proxmox-host.rules @@ -0,0 +1,42 @@ +# iptables-restore ruleset for the Proxmox VE hypervisor itself. Applied MANUALLY per +# docs/PRODUCTION_DEPLOYMENT.md §2 — deliberately not Ansible-automated (a mistake in a +# playbook run against the one host that IS the only way to reach every VM console is a much +# worse failure mode than a manual, reviewed apply). Prefer configuring this through Proxmox's +# own Datacenter -> Firewall UI/API where practical instead of hand-editing iptables on the +# hypervisor — this file documents the equivalent policy either way. +# +# Policy summary (docs/PRODUCTION_NETWORK_SECURITY.md §5-6): +# - Proxmox web UI (8006) and SSH (22) reachable ONLY from the management CIDR — never from +# the internet, never from either VM's network, never from a student Lab subnet +# - VM traffic (rn-platform-01 <-> rn-lab-host-01, rn-platform-01 <-> internet via NAT/bridge) +# is FORWARDed by the bridge, not filtered by this hypervisor-level ruleset — that's the +# VMs' own INPUT chains' job (rn-platform-01.rules / rn-lab-host-01.rules) +# - no student Lab subnet, and no rn-lab-host-01 traffic at all, may ever reach Proxmox +# management — this is the literal "Proxmox management not reachable from student Lab +# networks" requirement, enforced at the hypervisor itself, not just hoped-for via VM +# network segmentation + +*filter +:INPUT DROP [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] + +-A INPUT -i lo -j ACCEPT +-A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +-A INPUT -m conntrack --ctstate INVALID -j DROP +-A INPUT -p icmp --icmp-type echo-request -m conntrack --ctstate NEW -j ACCEPT + +-A INPUT -p tcp --dport 22 -s {{ management_cidr }} -m conntrack --ctstate NEW -j ACCEPT +-A INPUT -p tcp --dport 8006 -s {{ management_cidr }} -m conntrack --ctstate NEW -j ACCEPT + +# Explicit deny + log for the Lab Host VM's network and the student Lab subnet range reaching +# Proxmox management directly — the literal "Proxmox management denied from student Lab +# networks" requirement. +-A INPUT -s {{ lab_host_ip }} -p tcp -m multiport --dports 22,8006 -j LOG --log-prefix "ROOTNODE-DENY-LABHOST-TO-PROXMOX: " --log-level 4 +-A INPUT -s {{ lab_host_ip }} -p tcp -m multiport --dports 22,8006 -j DROP +-A INPUT -s {{ lab_subnet_range }} -p tcp -m multiport --dports 22,8006 -j LOG --log-prefix "ROOTNODE-DENY-LAB-TO-PROXMOX: " --log-level 4 +-A INPUT -s {{ lab_subnet_range }} -p tcp -m multiport --dports 22,8006 -j DROP + +-A INPUT -j LOG --log-prefix "ROOTNODE-DENY-PROXMOX-DEFAULT: " --log-level 4 + +COMMIT diff --git a/infra/firewall/rn-lab-host-01.rules b/infra/firewall/rn-lab-host-01.rules new file mode 100644 index 0000000..7f084b9 --- /dev/null +++ b/infra/firewall/rn-lab-host-01.rules @@ -0,0 +1,62 @@ +# iptables-restore ruleset for rn-lab-host-01 (Lab Host VM). Deployed by +# infra/ansible/roles/firewall/. See docs/PRODUCTION_NETWORK_SECURITY.md §6 for full rationale. +# +# This is the security-critical host — it runs privileged, disposable, student-controlled +# containers. Its own INPUT/FORWARD policy is only half the story: the Provisioner's dynamic +# per-Lab DOCKER-USER chains (platform/provisioner/src/egress.js, unchanged since Phase 1) +# already enforce egress-proxy-only outbound for every student subnet and implicitly deny +# Lab-to-Lab and Lab-to-anything-not-allowlisted traffic. This file is the BASE policy under +# that dynamic layer — it must never be more permissive than what the dynamic chains assume, +# and it's what protects the host itself (SSH, Docker TLS) rather than inter-container traffic. +# +# Policy summary: +# - Docker TLS (2376) reachable ONLY from rn-platform-01 — this is the entire point of mTLS +# plus a firewall restriction, defense in depth (a leaked client cert alone still isn't +# enough without also being on this IP) +# - SSH only from the management CIDR +# - no port on this host is ever reachable from a student Lab's own Docker subnet — Labs only +# ever reach the egress proxy and DNS resolver (already enforced by the Provisioner's +# per-Lab chains); this file adds an explicit belt-and-suspenders DROP for the Lab subnet +# range as a whole, so even a Lab whose per-environment chain failed to install can't reach +# the host's own services +# - nothing on this host is ever reachable from the public internet at all + +*filter +:INPUT DROP [0:0] +:FORWARD DROP [0:0] +:OUTPUT ACCEPT [0:0] + +-A INPUT -i lo -j ACCEPT +-A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +-A INPUT -m conntrack --ctstate INVALID -j DROP +-A INPUT -p icmp --icmp-type echo-request -m conntrack --ctstate NEW -j ACCEPT + +-A INPUT -p tcp --dport 22 -s {{ management_cidr }} -m conntrack --ctstate NEW -j ACCEPT + +# Docker TLS — rn-platform-01 (the Provisioner) only. This is the ONLY control-plane flow +# allowed into this host at all — see the "Platform Provisioner -> Lab Host Docker TLS +# endpoint: ALLOW" example in the Phase 4 spec, encoded literally here. +-A INPUT -p tcp --dport 2376 -s {{ platform_ip }} -m conntrack --ctstate NEW -j ACCEPT + +# Gateway -> per-Lab published port range, rn-platform-01 only. REQUIRED so the Gateway can +# reach each student's Lab UI/terminal over the network once Labs run on a genuinely separate +# host — see docs/PRODUCTION_NETWORK_SECURITY.md §"Gateway-to-Lab cross-VM reachability" for +# why this exists and the small Provisioner/Gateway code change it depends on (NOT YET BUILT — +# this firewall rule is the network-policy half; the port-publishing half is an explicit, +# reviewed application-code follow-up, tracked there rather than silently assumed here). +-A INPUT -p tcp --dport {{ lab_gateway_port_range | default('30000:32767') }} -s {{ platform_ip }} -m conntrack --ctstate NEW -j ACCEPT + +# Prometheus (rn-platform-01) scraping this host's node-exporter/cadvisor — metrics only, +# read-only exposition endpoints, still restricted to the one legitimate scraper. +-A INPUT -p tcp -m multiport --dports 9100,8080 -s {{ platform_ip }} -m conntrack --ctstate NEW -j ACCEPT + +# Belt-and-suspenders: explicit deny + log for any student Lab subnet reaching THIS HOST +# (not another container — the host's own listening ports: SSH, Docker TLS, anything else). +# The Provisioner's per-Lab DOCKER-USER chains already stop this at the container-egress layer; +# this is what still protects the host if that layer is ever misconfigured or mid-install. +-A INPUT -s {{ lab_subnet_range }} -j LOG --log-prefix "ROOTNODE-DENY-LAB-TO-HOST: " --log-level 4 +-A INPUT -s {{ lab_subnet_range }} -j DROP + +-A INPUT -j LOG --log-prefix "ROOTNODE-DENY-LABHOST-DEFAULT: " --log-level 4 + +COMMIT diff --git a/infra/firewall/rn-platform-01.rules b/infra/firewall/rn-platform-01.rules new file mode 100644 index 0000000..6bb62b4 --- /dev/null +++ b/infra/firewall/rn-platform-01.rules @@ -0,0 +1,58 @@ +# iptables-restore ruleset for rn-platform-01 (Platform VM). Deployed by +# infra/ansible/roles/firewall/. Variables ({{ }}) are Ansible Jinja2, rendered before this +# file reaches the host — see infra/ansible/group_vars/production.yml. +# +# Policy summary (full rationale: docs/PRODUCTION_NETWORK_SECURITY.md §6): +# - only the reverse proxy's 443/80 are reachable from the internet +# - SSH only from the management CIDR (your home/office/VPN, never 0.0.0.0/0) +# - Docker TLS (2376) is a rn-lab-host-01 concern, not this host's — nothing to open here +# - Postgres, Provisioner's internal API, and the Platform API/Gateway's raw ports are never +# exposed on this VM's external interface at all — the reverse proxy is the only public +# listener; everything else binds to the docker-compose internal network only, so there is +# no host-level rule needed to "hide" them, only to make sure nothing ever publishes them +# by accident (this ruleset's default-deny INPUT catches that class of mistake too) +# - rn-lab-host-01 may reach exactly one port here: Loki (3100), for its own log shipper — +# the Provisioner otherwise calls OUT to rn-lab-host-01, never the other way around — and +# may NEVER reach Postgres or the Provisioner's internal API + +*filter +:INPUT DROP [0:0] +:FORWARD DROP [0:0] +:OUTPUT ACCEPT [0:0] + +# Loopback and established/related — standard baseline +-A INPUT -i lo -j ACCEPT +-A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +-A INPUT -m conntrack --ctstate INVALID -j DROP + +# ICMP — keep basic reachability diagnostics working +-A INPUT -p icmp --icmp-type echo-request -m conntrack --ctstate NEW -j ACCEPT + +# SSH — management CIDR only, never open to the internet +-A INPUT -p tcp --dport 22 -s {{ management_cidr }} -m conntrack --ctstate NEW -j ACCEPT + +# Public HTTPS/HTTP — the reverse proxy (Caddy), the only public listener on this host +-A INPUT -p tcp --dport 443 -m conntrack --ctstate NEW -j ACCEPT +-A INPUT -p tcp --dport 80 -m conntrack --ctstate NEW -j ACCEPT + +# Loki log ingestion — rn-lab-host-01's promtail only (that host's own log shipper, pushing its +# container/journald logs here for centralized viewing in Grafana). Same "one specific source +# IP, one specific port" pattern as Docker TLS on the other host. +-A INPUT -p tcp --dport 3100 -s {{ lab_host_ip }} -m conntrack --ctstate NEW -j ACCEPT + +# Explicit deny + log for anything from rn-lab-host-01 hitting Postgres or the Provisioner's +# internal port — defense in depth on top of "these ports were never published," and the one +# rule in this file that most directly encodes the Lab->Platform boundary the whole platform +# depends on. Logged (not silently dropped) so an attempted breach attempt is visible in +# infra/monitoring/ alerts, not just absorbed by the default DROP. +-A INPUT -s {{ lab_host_ip }} -p tcp --dport 5432 -j LOG --log-prefix "ROOTNODE-DENY-LABHOST-PG: " --log-level 4 +-A INPUT -s {{ lab_host_ip }} -p tcp --dport 5432 -j DROP +-A INPUT -s {{ lab_host_ip }} -p tcp --dport 4100 -j LOG --log-prefix "ROOTNODE-DENY-LABHOST-PROVISIONER: " --log-level 4 +-A INPUT -s {{ lab_host_ip }} -p tcp --dport 4100 -j DROP + +# Default: everything else already falls to the chain policy (DROP), nothing further needed — +# but an explicit final logged catch-all makes "what got denied and why" visible in logs rather +# than silent. +-A INPUT -j LOG --log-prefix "ROOTNODE-DENY-PLATFORM-DEFAULT: " --log-level 4 + +COMMIT diff --git a/infra/firewall/validate-firewall.sh b/infra/firewall/validate-firewall.sh new file mode 100755 index 0000000..e6138a3 --- /dev/null +++ b/infra/firewall/validate-firewall.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Local test (Phase 4 §29): renders each .rules file with dummy IPs and validates it. +# +# `iptables-restore --test` needs root (it talks to netfilter) — if this shell has passwordless +# sudo, we use it for a real semantic check. Otherwise we fall back to a structural lint +# (balanced *table/COMMIT blocks, every rule starts with a real flag, no unrendered {{ }} +# placeholders left over) — good enough to catch a typo'd rule or a missing COMMIT, NOT a +# substitute for the real `iptables-restore --test` a human/Ansible with actual root runs +# before this ever reaches a live host. HARDWARE-DEPENDENT: proving the real IPs/CIDRs are +# correct needs the actual production network, not this script. +set -euo pipefail +cd "$(dirname "$0")" + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +render() { + sed \ + -e 's/{{ management_cidr }}/203.0.113.0\/24/g' \ + -e 's/{{ platform_ip }}/10.10.10.10/g' \ + -e 's/{{ lab_host_ip }}/10.10.10.20/g' \ + -e 's/{{ lab_subnet_range }}/10.42.0.0\/16/g' \ + -e 's/{{ proxmox_mgmt_cidr }}/203.0.113.0\/24/g' \ + -e "s/{{ lab_gateway_port_range | default('30000:32767') }}/30000:32767/g" \ + "$1" > "$2" +} + +structural_lint() { + local f="$1" + grep -q '^\*filter' "$f" || { echo " missing *filter table declaration"; return 1; } + grep -q '^COMMIT' "$f" || { echo " missing COMMIT"; return 1; } + grep -v '^#' "$f" | grep -q '{{' && { echo " unrendered {{ }} placeholder left in a non-comment line"; return 1; } + # every -A/-I rule line should end in a real target (ACCEPT/DROP/REJECT/LOG/a custom chain) + while IFS= read -r line; do + [[ "$line" =~ ^-[AI]\ ]] || continue + [[ "$line" =~ -j\ [A-Za-z0-9_-]+ ]] || { echo " rule missing a -j target: $line"; return 1; } + done < "$f" + return 0 +} + +HAVE_SUDO=0 +sudo -n true 2>/dev/null && HAVE_SUDO=1 + +FAIL=0 +for f in proxmox-host.rules rn-platform-01.rules rn-lab-host-01.rules; do + render "$f" "$TMPDIR/$f" + echo "==> $f" + if [ "$HAVE_SUDO" = "1" ] && command -v iptables-restore >/dev/null 2>&1; then + if sudo -n iptables-restore --test < "$TMPDIR/$f"; then + echo "OK: $f passed a real iptables-restore --test" + else + echo "FAIL: $f did not pass iptables-restore --test" >&2; FAIL=1 + fi + else + if structural_lint "$TMPDIR/$f"; then + echo "OK: $f passes structural lint (no root available for a real iptables-restore --test — see script header)" + else + echo "FAIL: $f failed structural lint" >&2; FAIL=1 + fi + fi +done + +exit $FAIL diff --git a/infra/scripts/docker-tls/generate-docker-tls.sh b/infra/scripts/docker-tls/generate-docker-tls.sh new file mode 100755 index 0000000..7de4282 --- /dev/null +++ b/infra/scripts/docker-tls/generate-docker-tls.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Generates a private CA and a mutual-TLS cert/key pair for the Provisioner <-> rn-lab-host-01 +# Docker Engine connection (docs/PRODUCTION_NETWORK_SECURITY.md §7). The application already +# supports this (platform/provisioner/src/docker.js buildClient() has required mTLS since +# Phase 1) — this script only ever needed to exist for the CERTIFICATES, not new app code. +# +# Output goes to $OUT_DIR (default ./out, gitignored) — never into a tracked path. Deploy the +# resulting files with the `docker-tls` Ansible role, which places: +# rn-lab-host-01: ca.pem, server-cert.pem, server-key.pem -> dockerd's --tlsverify config +# rn-platform-01: ca.pem, provisioner-client-cert.pem, provisioner-client-key.pem +# -> /etc/rootnode/docker-tls/ (referenced by lab-hosts.yaml) +# +# Usage: +# ./generate-docker-tls.sh [--out-dir DIR] [--lab-host-ip IP] [--lab-host-dns NAME] [--days N] +set -euo pipefail + +OUT_DIR="./out" +LAB_HOST_IP="127.0.0.1" +LAB_HOST_DNS="rn-lab-host-01.internal" +DAYS=825 # ~2.25 years — matches modern browser/OS max cert lifetime conventions; see rotation runbook for cadence + +while [ $# -gt 0 ]; do + case "$1" in + --out-dir) OUT_DIR="$2"; shift 2 ;; + --lab-host-ip) LAB_HOST_IP="$2"; shift 2 ;; + --lab-host-dns) LAB_HOST_DNS="$2"; shift 2 ;; + --days) DAYS="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +mkdir -p "$OUT_DIR" +cd "$OUT_DIR" +umask 077 # private keys must never be group/world readable, from the moment they're written + +echo "==> Generating CA" +openssl genrsa -out ca-key.pem 4096 +openssl req -new -x509 -days "$DAYS" -key ca-key.pem -sha256 -out ca.pem \ + -subj "/O=RootNode Academy/CN=RootNode Docker TLS CA" + +echo "==> Generating server cert for rn-lab-host-01 (dockerd)" +openssl genrsa -out server-key.pem 4096 +openssl req -subj "/CN=$LAB_HOST_DNS" -sha256 -new -key server-key.pem -out server.csr + +cat > server-extfile.cnf < Generating client cert for the Provisioner" +openssl genrsa -out provisioner-client-key.pem 4096 +openssl req -subj "/CN=rootnode-provisioner" -new -key provisioner-client-key.pem -out client.csr + +cat > client-extfile.cnf < Done. Generated in $OUT_DIR:" +ls -la +echo +echo "Deploy server-{cert,key}.pem + ca.pem to rn-lab-host-01 (docker-tls Ansible role)." +echo "Deploy provisioner-client-{cert,key}.pem + ca.pem to rn-platform-01 (same role)." +echo "Verify with: ../verify-docker-tls.sh $OUT_DIR " diff --git a/infra/scripts/docker-tls/verify-docker-tls.sh b/infra/scripts/docker-tls/verify-docker-tls.sh new file mode 100755 index 0000000..61bba59 --- /dev/null +++ b/infra/scripts/docker-tls/verify-docker-tls.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Verifies a generated Docker TLS cert bundle is internally consistent (CA signed both certs, +# key/cert pairs match, SANs are correct) BEFORE deploying it — cheap, local, no real dockerd +# needed. Once deployed, also verifies live mTLS against a real endpoint if given one. +# +# Usage: ./verify-docker-tls.sh [lab-host-ip:port] +set -euo pipefail +DIR="${1:?usage: verify-docker-tls.sh [lab-host-ip:port]}" +ENDPOINT="${2:-}" + +fail() { echo "FAIL: $1" >&2; exit 1; } + +echo "==> Checking file presence" +for f in ca.pem server-cert.pem server-key.pem provisioner-client-cert.pem provisioner-client-key.pem; do + [ -f "$DIR/$f" ] || fail "missing $f" +done +echo "OK: all expected files present" + +echo "==> Checking key/cert pairs match (modulus comparison)" +for pair in "server-cert.pem server-key.pem" "provisioner-client-cert.pem provisioner-client-key.pem"; do + set -- $pair + cert_mod=$(openssl x509 -noout -modulus -in "$DIR/$1" | openssl md5) + key_mod=$(openssl rsa -noout -modulus -in "$DIR/$2" | openssl md5) + [ "$cert_mod" = "$key_mod" ] || fail "$1 / $2 do not match" +done +echo "OK: cert/key pairs match" + +echo "==> Checking both certs are signed by the CA" +openssl verify -CAfile "$DIR/ca.pem" "$DIR/server-cert.pem" | grep -q "OK" || fail "server-cert.pem not CA-signed" +openssl verify -CAfile "$DIR/ca.pem" "$DIR/provisioner-client-cert.pem" | grep -q "OK" || fail "provisioner-client-cert.pem not CA-signed" +echo "OK: both certs verify against ca.pem" + +echo "==> Checking private key permissions" +for f in ca-key.pem server-key.pem provisioner-client-key.pem; do + [ -f "$DIR/$f" ] || continue + perm=$(stat -c '%a' "$DIR/$f" 2>/dev/null || stat -f '%Lp' "$DIR/$f") + [ "$perm" = "400" ] || echo "WARNING: $f has permissions $perm, expected 400" +done + +if [ -n "$ENDPOINT" ]; then + echo "==> Testing live mTLS handshake against $ENDPOINT" + echo | openssl s_client -connect "$ENDPOINT" \ + -cert "$DIR/provisioner-client-cert.pem" -key "$DIR/provisioner-client-key.pem" \ + -CAfile "$DIR/ca.pem" 2>&1 | grep -q "Verify return code: 0" \ + && echo "OK: live mTLS handshake succeeded" \ + || fail "live handshake against $ENDPOINT failed — check firewall/dockerd TLS config" +else + echo "(skipped live handshake test — no endpoint given; this only proves the cert bundle is internally consistent, not that dockerd is actually listening with it)" +fi + +echo "==> All checks passed"