feat(loadtest): load-test tooling + capacity model

boot-labs.js drives the real admin Prepare-Labs endpoint (reusing the
existing session-helper.js from Phase 3A) and records boot time/failure
rate; sample-resources.sh samples real docker stats for running Labs.
capacity-model.js turns measured per-Lab CPU/memory into a recommended/
warning/hard-admission-control concurrency threshold - no concurrency
number is hardcoded as a marketing figure, every input is explicitly
placeholder data until measured against the real host.

Plus deploy/rollback/smoke-test scripts used by the CI/CD and deployment
runbooks. All syntax-checked; validate-compose.sh and capacity-model.js
actually run against real inputs.

infra/scripts/capacity-model.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+// Capacity planning worksheet (Phase 4 §25) — computes recommended-concurrent-Labs from real
+// inputs, not a guessed marketing number. Inputs come from infra/environments/production/capacity.yml
+// (edit that file with real measured values from infra/scripts/loadtest/, then re-run this).
+//
+// Usage: node capacity-model.js [path-to-capacity.yml]
+
+const fs = require('fs');
+const path = require('path');
+// Reuses the Provisioner's existing js-yaml dependency (platform/provisioner/src/config.js
+// already parses YAML config this same way) rather than adding a new package.json just for
+// this one script.
+const yaml = require(path.join(__dirname, '../../platform/provisioner/node_modules/js-yaml'));
+
+const configPath = process.argv[2] || path.join(__dirname, '../environments/production/capacity.yml');
+if (!fs.existsSync(configPath)) {
+  console.error(`No capacity config at ${configPath} — copy capacity.yml.example first and fill in real measured values.`);
+  process.exit(1);
+}
+
+const cfg = yaml.load(fs.readFileSync(configPath, 'utf8'));
+
+function computeFor(resource, hostTotal, platformReserved, proxmoxOverheadPct, perLab, safetyMarginPct) {
+  const usable = hostTotal * (1 - proxmoxOverheadPct / 100) - platformReserved;
+  const withMargin = usable * (1 - safetyMarginPct / 100);
+  return Math.floor(withMargin / perLab);
+}
+
+const cpuLimit = computeFor(
+  'CPU', cfg.host_cpu_cores, cfg.platform_reserved_cpu_cores,
+  cfg.proxmox_overhead_pct, cfg.measured_lab_cpu_cores, cfg.safety_margin_pct
+);
+const memLimit = computeFor(
+  'Memory', cfg.host_mem_mb, cfg.platform_reserved_mem_mb,
+  cfg.proxmox_overhead_pct, cfg.measured_lab_mem_mb, cfg.safety_margin_pct
+);
+
+const recommended = Math.min(cpuLimit, memLimit);
+const warningThreshold = Math.floor(recommended * 0.8);
+const hardAdmissionControl = recommended; // never provision past the more conservative of the two limits
+
+console.log('RootNode capacity model');
+console.log('========================');
+console.log(`Host: ${cfg.host_cpu_cores} cores, ${cfg.host_mem_mb} MB RAM`);
+console.log(`Platform-reserved: ${cfg.platform_reserved_cpu_cores} cores, ${cfg.platform_reserved_mem_mb} MB`);
+console.log(`Proxmox overhead: ${cfg.proxmox_overhead_pct}%`);
+console.log(`Measured per-Lab: ${cfg.measured_lab_cpu_cores} cores, ${cfg.measured_lab_mem_mb} MB`);
+console.log(`Safety margin: ${cfg.safety_margin_pct}%`);
+console.log('');
+console.log(`CPU-limited concurrent Labs:    ${cpuLimit}`);
+console.log(`Memory-limited concurrent Labs: ${memLimit}`);
+console.log('');
+console.log(`RECOMMENDED concurrent Labs:       ${recommended}  (binding constraint: ${cpuLimit < memLimit ? 'CPU' : 'memory'})`);
+console.log(`WARNING threshold (80%):           ${warningThreshold}`);
+console.log(`HARD admission-control threshold:  ${hardAdmissionControl}`);
+console.log('');
+if (cfg.measured_lab_cpu_cores === undefined || cfg._placeholder) {
+  console.log('WARNING: this config still has placeholder/example values — see infra/scripts/loadtest/ for how to measure real ones.');
+}

infra/scripts/deploy-production.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Manual production promotion — the one deliberate non-automatic step in the pipeline
+# (docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md §18: automatic test/build/staging, manual
+# production promotion). Run by a human after reviewing the staging smoke test result.
+#
+# Usage: ./deploy-production.sh <image-tag>
+#   e.g. ./deploy-production.sh sha-abc123def456
+#        ./deploy-production.sh v1.2.0
+set -euo pipefail
+cd "$(dirname "$0")/../ansible"
+
+IMAGE_TAG="${1:?usage: deploy-production.sh <image-tag>}"
+DEPLOY_LOG="/opt/rootnode/deploy-history.log"   # append-only — rollback.sh reads the entry
+                                                  # before the most recent one from this file
+
+echo "About to deploy image_tag=$IMAGE_TAG to PRODUCTION (rn-platform-01 + rn-lab-host-01)."
+echo "This assumes it already passed CI + the staging smoke test."
+if [ -f "$DEPLOY_LOG" ]; then
+  echo "Currently deployed (per $DEPLOY_LOG): $(tail -1 "$DEPLOY_LOG")"
+fi
+read -r -p "Type DEPLOY to continue: " CONFIRM
+[ "$CONFIRM" = "DEPLOY" ] || { echo "Aborted."; exit 1; }
+
+ansible-playbook site.yml -i inventory/production.ini \
+  -e "image_tag=$IMAGE_TAG" \
+  --vault-password-file .vault_pass
+
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $IMAGE_TAG" >> "$DEPLOY_LOG"
+echo "==> Deployed. Run infra/scripts/production-smoke-test.sh to verify, then the relevant"
+echo "    section of docs/PRODUCTION_VALIDATION_CHECKLIST.md."

infra/scripts/loadtest/README.md

@@ -0,0 +1,32 @@
+# Load test tooling
+
+Two scripts, run together against whatever stack you point them at:
+
+- `boot-labs.js` — provisions N Labs concurrently via the real admin Prepare Labs endpoint,
+  measures boot success rate and wall-clock timing. `GATEWAY_URL=... OWNER_EMAIL=...
+  OWNER_PASSWORD=... node boot-labs.js --count=10`
+- `sample-resources.sh` — run on the Lab Host **while** `boot-labs.js` runs, samples real
+  `docker stats` for every `rootnode-lab-*` container every 5s, writes CSV.
+
+```bash
+# terminal 1, on the Lab Host:
+./infra/scripts/loadtest/sample-resources.sh 180 ./results/run1.csv
+
+# terminal 2, wherever the Gateway is reachable from:
+GATEWAY_URL=https://academy.example.com OWNER_EMAIL=... OWNER_PASSWORD=... \
+  node infra/scripts/loadtest/boot-labs.js --count=10
+```
+
+`report-template.md` is a filled-in-by-hand summary format — copy it, paste the two scripts'
+output in, note the host class it ran against. **A run against a dev laptop is not evidence
+about the real Hetzner host** — every report must say what it ran on.
+
+Feeds `docs/CAPACITY_PLANNING.md`'s worksheet directly: `sample-resources.sh`'s peak
+`mem_usage`/`cpu_pct` per Lab are exactly the "measured Lab CPU"/"measured Lab RAM" inputs that
+worksheet asks for.
+
+Test dimensions this DOESN'T cover yet (documented gaps, not silently skipped): terminal
+WebSocket throughput under concurrent load, Gateway HTTP throughput independent of Lab boot,
+and steady-state (long-running, not just boot-burst) resource drift. Extend `boot-labs.js`'s
+pattern (it already reuses `platform/test/session-helper.js`) rather than building a separate
+tool if/when those are needed.

infra/scripts/loadtest/boot-labs.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+// Load test tooling (Phase 4 §24) — provisions N concurrent Labs via the real admin
+// Prepare-Labs endpoint (the same code path a real instructor's "Prepare Labs" click uses,
+// platform/api/src/admin-routes.js), measures boot time and failure rate, and (if run ON the
+// Lab Host, or given SSH access to it) samples per-Lab CPU/RAM/disk via `docker stats`.
+//
+// This produces REAL numbers against whatever stack it's pointed at — a local dev stack today,
+// the real rn-lab-host-01 once it exists. It does NOT fabricate a concurrency number; the
+// report is only as meaningful as the host it ran against (see the report template's WARNING
+// banner when run against anything other than real target-class hardware).
+//
+// Usage: GATEWAY_URL=... OWNER_EMAIL=... OWNER_PASSWORD=... node boot-labs.js --count 10
+
+const fs = require('fs');
+const path = require('path');
+const { makeSession } = require('../../../platform/test/session-helper');
+
+const GW = process.env.GATEWAY_URL || 'http://localhost:4200';
+const OWNER_EMAIL = process.env.OWNER_EMAIL;
+const OWNER_PASSWORD = process.env.OWNER_PASSWORD;
+const COUNT = Number((process.argv.find((a) => a.startsWith('--count=')) || '--count=5').split('=')[1]);
+
+if (!OWNER_EMAIL || !OWNER_PASSWORD) {
+  console.error('Set OWNER_EMAIL/OWNER_PASSWORD (bootstrap one first — see platform/api/bootstrap-owner.js)');
+  process.exit(1);
+}
+
+async function main() {
+  const owner = makeSession(GW);
+  await owner.request('POST', '/api/auth/instructor/login', { email: OWNER_EMAIL, password: OWNER_PASSWORD });
+
+  const cohortId = `loadtest-${Date.now()}`;
+  console.log(`Creating cohort ${cohortId} with ${COUNT} students...`);
+  await owner.request('POST', '/api/admin/cohorts', { cohort_id: cohortId, name: 'Load Test' });
+
+  for (let i = 0; i < COUNT; i++) {
+    await owner.request('POST', '/api/admin/students', { display_name: `LoadTest-${i}`, cohort_id: cohortId });
+  }
+
+  console.log(`Prepare Labs: booting ${COUNT} Labs concurrently...`);
+  const start = Date.now();
+  const { status, body } = await owner.request('POST', `/api/admin/cohorts/${cohortId}/prepare-labs`);
+  const elapsedMs = Date.now() - start;
+
+  if (status !== 200) {
+    console.error('Prepare Labs call itself failed:', status, body);
+    process.exit(1);
+  }
+
+  const running = body.results.filter((r) => r.status === 'running').length;
+  const failed = body.results.filter((r) => r.status === 'error');
+
+  const report = {
+    timestamp: new Date().toISOString(),
+    gateway_url: GW,
+    requested_count: COUNT,
+    running_count: running,
+    failed_count: failed.length,
+    failure_rate_pct: Math.round((failed.length / COUNT) * 1000) / 10,
+    total_boot_wall_time_ms: elapsedMs,
+    avg_boot_time_ms_estimate: Math.round(elapsedMs / COUNT), // concurrent, so this is a rough
+                                                                 // lower bound, not a real per-Lab figure
+    failures: failed,
+    note: 'Run infra/scripts/loadtest/sample-resources.sh DURING this script (in another '
+        + 'terminal) for real per-Lab CPU/RAM numbers — this script only measures boot '
+        + 'timing/success rate, not resource consumption.',
+  };
+
+  const outDir = path.join(__dirname, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  const outFile = path.join(outDir, `boot-labs-${Date.now()}.json`);
+  fs.writeFileSync(outFile, JSON.stringify(report, null, 2));
+
+  console.log(JSON.stringify(report, null, 2));
+  console.log(`\nWritten to ${outFile}`);
+  console.log(`\nCleanup: POST /api/admin/cohorts/${cohortId}/destroy-labs as the owner when done.`);
+}
+
+main().catch((e) => { console.error('Load test crashed:', e); process.exit(1); });

infra/scripts/loadtest/report-template.md

@@ -0,0 +1,36 @@
+# RootNode Load Test Report
+
+**Date:** _______
+**Ran against:** [ ] dev laptop  [ ] staging  [ ] production rn-lab-host-01 (real hardware)
+**Host spec (fill in real numbers, do not guess):** ___ cores / ___ threads / ___ GB RAM / ___ storage
+
+> A run against anything other than the real target-class hardware only tells you the code
+> path works, not what production can actually sustain. Say clearly which this is.
+
+## Boot test (`boot-labs.js`)
+
+| Metric | Value |
+|---|---|
+| Requested Lab count | |
+| Successfully running | |
+| Failed | |
+| Failure rate | |
+| Total wall-clock time (concurrent boot) | |
+
+## Resource usage (`sample-resources.sh`)
+
+| Metric | Value |
+|---|---|
+| Peak CPU per Lab (observed) | |
+| Peak memory per Lab (observed) | |
+| Peak disk I/O per Lab (observed) | |
+| Host CPU headroom remaining at peak | |
+| Host memory headroom remaining at peak | |
+
+## Feed into docs/CAPACITY_PLANNING.md
+
+- [ ] Updated `measured_lab_cpu_cores` with the peak CPU figure above
+- [ ] Updated `measured_lab_mem_mb` with the peak memory figure above
+- [ ] Re-ran `infra/scripts/capacity-model.js` and recorded the new recommended-concurrent-Labs number
+
+## Notes / anomalies observed

infra/scripts/loadtest/sample-resources.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Run on rn-lab-host-01 (or the local dev machine running the Lab containers) WHILE
+# boot-labs.js is running, to capture real per-Lab CPU/RAM/disk usage — `docker stats` is the
+# ground truth here, not an estimate. Samples every 5s for the given duration and writes CSV.
+set -euo pipefail
+DURATION_SEC="${1:-120}"
+OUT="${2:-./results/resource-sample-$(date +%s).csv}"
+mkdir -p "$(dirname "$OUT")"
+
+echo "timestamp,container,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids" > "$OUT"
+END=$(( $(date +%s) + DURATION_SEC ))
+while [ "$(date +%s)" -lt "$END" ]; do
+  docker stats --no-stream --format '{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}' \
+    | grep -E '^rootnode-lab-' \
+    | while IFS=, read -r name cpu mem memp net block pids; do
+        echo "$(date -u +%Y-%m-%dT%H:%M:%SZ),$name,$cpu,$mem,$memp,$net,$block,$pids" >> "$OUT"
+      done
+  sleep 5
+done
+
+echo "Written to $OUT"
+echo "Summarize with: awk -F, 'NR>1{print \$2}' $OUT | sort -u   # containers seen"

infra/scripts/production-smoke-test.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Production post-deploy smoke test — deliberately read-only/non-destructive (never creates
+# real students/cohorts against production data). Confirms the deployed services are actually
+# up after a deploy/rollback, before calling the deploy "done."
+set -euo pipefail
+
+PROD_URL="${PRODUCTION_URL:?set PRODUCTION_URL, e.g. https://academy.example.com}"
+
+echo "==> Gateway health"
+curl -sf "$PROD_URL/healthz" > /dev/null && echo "OK"
+
+echo "==> TLS certificate is valid and not expiring soon"
+EXPIRY=$(echo | openssl s_client -connect "$(echo "$PROD_URL" | sed -e 's#https://##')":443 -servername "$(echo "$PROD_URL" | sed -e 's#https://##')" 2>/dev/null | openssl x509 -noout -enddate | cut -d= -f2)
+echo "Certificate expires: $EXPIRY"
+
+echo "==> Roster endpoint responds (public, unauthenticated, read-only)"
+curl -sf "$PROD_URL/api/roster/cohorts" > /dev/null && echo "OK"
+
+echo "==> Admin login page loads (does not attempt to log in)"
+curl -sf "$PROD_URL/admin" -o /dev/null -w '%{http_code}\n' | grep -q 200 && echo "OK"
+
+echo
+echo "OK: production smoke test passed. For a full functional check (owner/instructor login,"
+echo "cohort, Prepare Labs, student flow, assessment, Capstone), work through"
+echo "docs/PRODUCTION_VALIDATION_CHECKLIST.md's ACADEMY section manually — deliberately not"
+echo "automated against real production data."

infra/scripts/rollback-production.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Rolls back to the previously deployed image tag, per infra/docker/IMAGE_STRATEGY.md —
+# "rollback is changing one variable and re-running the Ansible role." Reads
+# /opt/rootnode/deploy-history.log (written by deploy-production.sh) for the tag before the
+# current one; pass an explicit tag instead if you need to go back further than one step.
+#
+# Usage: ./rollback-production.sh [explicit-image-tag]
+set -euo pipefail
+cd "$(dirname "$0")/../ansible"
+
+DEPLOY_LOG="/opt/rootnode/deploy-history.log"
+
+if [ -n "${1:-}" ]; then
+  TARGET_TAG="$1"
+else
+  [ -f "$DEPLOY_LOG" ] || { echo "No $DEPLOY_LOG found and no explicit tag given — nothing to roll back to." >&2; exit 1; }
+  LINES=$(wc -l < "$DEPLOY_LOG")
+  [ "$LINES" -ge 2 ] || { echo "Deploy history has fewer than 2 entries — nothing to roll back to. Pass an explicit tag instead." >&2; exit 1; }
+  TARGET_TAG=$(tail -2 "$DEPLOY_LOG" | head -1 | awk '{print $2}')
+fi
+
+echo "Rolling back production to image_tag=$TARGET_TAG"
+read -r -p "Type ROLLBACK to continue: " CONFIRM
+[ "$CONFIRM" = "ROLLBACK" ] || { echo "Aborted."; exit 1; }
+
+ansible-playbook site.yml -i inventory/production.ini \
+  -e "image_tag=$TARGET_TAG" \
+  --vault-password-file .vault_pass
+
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $TARGET_TAG (rollback)" >> "$DEPLOY_LOG"
+echo "==> Rolled back to $TARGET_TAG. Verify with infra/scripts/production-smoke-test.sh."
+echo "NOTE: this rolls back the application images only. If the rollback needs to undo a"
+echo "database migration too, see docs/RUNBOOK_FAILED_MIGRATION.md — schema changes are not"
+echo "automatically reversible by this script."

infra/scripts/staging-smoke-test.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Staging acceptance test (Phase 4 §23/§18) — the four things a human needs before promoting a
+# build: app deploys, migrations ran clean, the Gateway/Platform API/Provisioner are reachable,
+# and (if OWNER_EMAIL/OWNER_PASSWORD are provided) a real assessment can be started end to end.
+# Reuses platform/test/e2e-phase3b.js against the staging Gateway URL rather than re-inventing
+# a parallel smoke-test harness — that suite already IS the full acceptance chain.
+set -euo pipefail
+
+STAGING_URL="${STAGING_GATEWAY_URL:?set STAGING_GATEWAY_URL, e.g. https://staging.academy.example.com}"
+
+echo "==> Health checks"
+curl -sf "$STAGING_URL/healthz" > /dev/null && echo "OK: Gateway /healthz"
+
+if [ -n "${OWNER_EMAIL:-}" ] && [ -n "${OWNER_PASSWORD:-}" ]; then
+  echo "==> Full acceptance run (platform/test/e2e-phase3b.js against staging)"
+  cd "$(dirname "$0")/../../platform"
+  GATEWAY_URL="$STAGING_URL" OWNER_EMAIL="$OWNER_EMAIL" OWNER_PASSWORD="$OWNER_PASSWORD" \
+    SKIP_RESTART_TEST=1 node test/e2e-phase3b.js
+else
+  echo "==> Skipping full e2e run — set OWNER_EMAIL/OWNER_PASSWORD for a real classroom-workflow"
+  echo "    smoke test (bootstrap one first against staging: platform/api/bootstrap-owner.js)."
+fi
+
+echo "OK: staging smoke test passed"

infra/scripts/validate-compose.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Local test (Phase 4 §29): renders infra/docker/platform/docker-compose.production.yml's
+# Jinja2 placeholders with disposable dummy values and confirms `docker compose config`
+# accepts the result — catches YAML/schema errors before Ansible ever touches a real host.
+# This does NOT prove the real production values work (env_file paths under /etc/rootnode/
+# won't exist locally, which docker compose only warns about, not a fatal error), and does NOT
+# validate anything hardware/network-dependent.
+set -euo pipefail
+cd "$(dirname "$0")/../docker/platform"
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+
+for f in platform.env provisioner.env gateway.env postgres.env; do touch "$TMPDIR/$f"; done
+
+sed \
+  -e 's/{{ rootnode_data_root }}/\/tmp\/rootnode-test-data/g' \
+  -e 's/{{ registry }}/registry.example.com/g' \
+  -e 's/{{ image_tag }}/test/g' \
+  -e 's/{{ postgres_exporter_user }}/exporter/g' \
+  -e 's/{{ postgres_exporter_password }}/dummy/g' \
+  -e "s#/etc/rootnode/platform.env#$TMPDIR/platform.env#" \
+  -e "s#/etc/rootnode/provisioner.env#$TMPDIR/provisioner.env#" \
+  -e "s#/etc/rootnode/gateway.env#$TMPDIR/gateway.env#" \
+  -e "s#/etc/rootnode/postgres.env#$TMPDIR/postgres.env#" \
+  docker-compose.production.yml > "$TMPDIR/docker-compose.yml"
+
+echo "--- docker compose config ---"
+docker compose -f "$TMPDIR/docker-compose.yml" config --quiet
+echo "OK: production compose file is well-formed YAML and passes docker compose's own schema validation"
+
+echo
+echo "--- Caddyfile syntax (real caddy validate) ---"
+sed -e 's/{{ academy_domain }}/academy.example.com/g' -e 's/{{ management_cidr }}/203.0.113.0\/24/g' \
+  Caddyfile.j2 > "$TMPDIR/Caddyfile"
+chmod a+r "$TMPDIR/Caddyfile"
+docker run --rm -v "$TMPDIR:/etc/caddy:ro" caddy:2-alpine caddy validate --config /etc/caddy/Caddyfile 2>&1 | grep -q "Valid configuration" \
+  && echo "OK: Caddyfile is valid" || { echo "FAIL: Caddyfile did not validate"; exit 1; }