diff --git a/infra/scripts/capacity-model.js b/infra/scripts/capacity-model.js new file mode 100755 index 0000000..c7bf611 --- /dev/null +++ b/infra/scripts/capacity-model.js @@ -0,0 +1,59 @@ +#!/usr/bin/env node +// Capacity planning worksheet (Phase 4 §25) — computes recommended-concurrent-Labs from real +// inputs, not a guessed marketing number. Inputs come from infra/environments/production/capacity.yml +// (edit that file with real measured values from infra/scripts/loadtest/, then re-run this). +// +// Usage: node capacity-model.js [path-to-capacity.yml] + +const fs = require('fs'); +const path = require('path'); +// Reuses the Provisioner's existing js-yaml dependency (platform/provisioner/src/config.js +// already parses YAML config this same way) rather than adding a new package.json just for +// this one script. +const yaml = require(path.join(__dirname, '../../platform/provisioner/node_modules/js-yaml')); + +const configPath = process.argv[2] || path.join(__dirname, '../environments/production/capacity.yml'); +if (!fs.existsSync(configPath)) { + console.error(`No capacity config at ${configPath} — copy capacity.yml.example first and fill in real measured values.`); + process.exit(1); +} + +const cfg = yaml.load(fs.readFileSync(configPath, 'utf8')); + +function computeFor(resource, hostTotal, platformReserved, proxmoxOverheadPct, perLab, safetyMarginPct) { + const usable = hostTotal * (1 - proxmoxOverheadPct / 100) - platformReserved; + const withMargin = usable * (1 - safetyMarginPct / 100); + return Math.floor(withMargin / perLab); +} + +const cpuLimit = computeFor( + 'CPU', cfg.host_cpu_cores, cfg.platform_reserved_cpu_cores, + cfg.proxmox_overhead_pct, cfg.measured_lab_cpu_cores, cfg.safety_margin_pct +); +const memLimit = computeFor( + 'Memory', cfg.host_mem_mb, cfg.platform_reserved_mem_mb, + cfg.proxmox_overhead_pct, cfg.measured_lab_mem_mb, cfg.safety_margin_pct +); + +const recommended = Math.min(cpuLimit, memLimit); +const warningThreshold = Math.floor(recommended * 0.8); +const hardAdmissionControl = recommended; // never provision past the more conservative of the two limits + +console.log('RootNode capacity model'); +console.log('========================'); +console.log(`Host: ${cfg.host_cpu_cores} cores, ${cfg.host_mem_mb} MB RAM`); +console.log(`Platform-reserved: ${cfg.platform_reserved_cpu_cores} cores, ${cfg.platform_reserved_mem_mb} MB`); +console.log(`Proxmox overhead: ${cfg.proxmox_overhead_pct}%`); +console.log(`Measured per-Lab: ${cfg.measured_lab_cpu_cores} cores, ${cfg.measured_lab_mem_mb} MB`); +console.log(`Safety margin: ${cfg.safety_margin_pct}%`); +console.log(''); +console.log(`CPU-limited concurrent Labs: ${cpuLimit}`); +console.log(`Memory-limited concurrent Labs: ${memLimit}`); +console.log(''); +console.log(`RECOMMENDED concurrent Labs: ${recommended} (binding constraint: ${cpuLimit < memLimit ? 'CPU' : 'memory'})`); +console.log(`WARNING threshold (80%): ${warningThreshold}`); +console.log(`HARD admission-control threshold: ${hardAdmissionControl}`); +console.log(''); +if (cfg.measured_lab_cpu_cores === undefined || cfg._placeholder) { + console.log('WARNING: this config still has placeholder/example values — see infra/scripts/loadtest/ for how to measure real ones.'); +} diff --git a/infra/scripts/deploy-production.sh b/infra/scripts/deploy-production.sh new file mode 100755 index 0000000..5417611 --- /dev/null +++ b/infra/scripts/deploy-production.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Manual production promotion — the one deliberate non-automatic step in the pipeline +# (docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md §18: automatic test/build/staging, manual +# production promotion). Run by a human after reviewing the staging smoke test result. +# +# Usage: ./deploy-production.sh +# e.g. ./deploy-production.sh sha-abc123def456 +# ./deploy-production.sh v1.2.0 +set -euo pipefail +cd "$(dirname "$0")/../ansible" + +IMAGE_TAG="${1:?usage: deploy-production.sh }" +DEPLOY_LOG="/opt/rootnode/deploy-history.log" # append-only — rollback.sh reads the entry + # before the most recent one from this file + +echo "About to deploy image_tag=$IMAGE_TAG to PRODUCTION (rn-platform-01 + rn-lab-host-01)." +echo "This assumes it already passed CI + the staging smoke test." +if [ -f "$DEPLOY_LOG" ]; then + echo "Currently deployed (per $DEPLOY_LOG): $(tail -1 "$DEPLOY_LOG")" +fi +read -r -p "Type DEPLOY to continue: " CONFIRM +[ "$CONFIRM" = "DEPLOY" ] || { echo "Aborted."; exit 1; } + +ansible-playbook site.yml -i inventory/production.ini \ + -e "image_tag=$IMAGE_TAG" \ + --vault-password-file .vault_pass + +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $IMAGE_TAG" >> "$DEPLOY_LOG" +echo "==> Deployed. Run infra/scripts/production-smoke-test.sh to verify, then the relevant" +echo " section of docs/PRODUCTION_VALIDATION_CHECKLIST.md." diff --git a/infra/scripts/loadtest/README.md b/infra/scripts/loadtest/README.md new file mode 100644 index 0000000..eeb7228 --- /dev/null +++ b/infra/scripts/loadtest/README.md @@ -0,0 +1,32 @@ +# Load test tooling + +Two scripts, run together against whatever stack you point them at: + +- `boot-labs.js` — provisions N Labs concurrently via the real admin Prepare Labs endpoint, + measures boot success rate and wall-clock timing. `GATEWAY_URL=... OWNER_EMAIL=... + OWNER_PASSWORD=... node boot-labs.js --count=10` +- `sample-resources.sh` — run on the Lab Host **while** `boot-labs.js` runs, samples real + `docker stats` for every `rootnode-lab-*` container every 5s, writes CSV. + +```bash +# terminal 1, on the Lab Host: +./infra/scripts/loadtest/sample-resources.sh 180 ./results/run1.csv + +# terminal 2, wherever the Gateway is reachable from: +GATEWAY_URL=https://academy.example.com OWNER_EMAIL=... OWNER_PASSWORD=... \ + node infra/scripts/loadtest/boot-labs.js --count=10 +``` + +`report-template.md` is a filled-in-by-hand summary format — copy it, paste the two scripts' +output in, note the host class it ran against. **A run against a dev laptop is not evidence +about the real Hetzner host** — every report must say what it ran on. + +Feeds `docs/CAPACITY_PLANNING.md`'s worksheet directly: `sample-resources.sh`'s peak +`mem_usage`/`cpu_pct` per Lab are exactly the "measured Lab CPU"/"measured Lab RAM" inputs that +worksheet asks for. + +Test dimensions this DOESN'T cover yet (documented gaps, not silently skipped): terminal +WebSocket throughput under concurrent load, Gateway HTTP throughput independent of Lab boot, +and steady-state (long-running, not just boot-burst) resource drift. Extend `boot-labs.js`'s +pattern (it already reuses `platform/test/session-helper.js`) rather than building a separate +tool if/when those are needed. diff --git a/infra/scripts/loadtest/boot-labs.js b/infra/scripts/loadtest/boot-labs.js new file mode 100644 index 0000000..5ece16f --- /dev/null +++ b/infra/scripts/loadtest/boot-labs.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// Load test tooling (Phase 4 §24) — provisions N concurrent Labs via the real admin +// Prepare-Labs endpoint (the same code path a real instructor's "Prepare Labs" click uses, +// platform/api/src/admin-routes.js), measures boot time and failure rate, and (if run ON the +// Lab Host, or given SSH access to it) samples per-Lab CPU/RAM/disk via `docker stats`. +// +// This produces REAL numbers against whatever stack it's pointed at — a local dev stack today, +// the real rn-lab-host-01 once it exists. It does NOT fabricate a concurrency number; the +// report is only as meaningful as the host it ran against (see the report template's WARNING +// banner when run against anything other than real target-class hardware). +// +// Usage: GATEWAY_URL=... OWNER_EMAIL=... OWNER_PASSWORD=... node boot-labs.js --count 10 + +const fs = require('fs'); +const path = require('path'); +const { makeSession } = require('../../../platform/test/session-helper'); + +const GW = process.env.GATEWAY_URL || 'http://localhost:4200'; +const OWNER_EMAIL = process.env.OWNER_EMAIL; +const OWNER_PASSWORD = process.env.OWNER_PASSWORD; +const COUNT = Number((process.argv.find((a) => a.startsWith('--count=')) || '--count=5').split('=')[1]); + +if (!OWNER_EMAIL || !OWNER_PASSWORD) { + console.error('Set OWNER_EMAIL/OWNER_PASSWORD (bootstrap one first — see platform/api/bootstrap-owner.js)'); + process.exit(1); +} + +async function main() { + const owner = makeSession(GW); + await owner.request('POST', '/api/auth/instructor/login', { email: OWNER_EMAIL, password: OWNER_PASSWORD }); + + const cohortId = `loadtest-${Date.now()}`; + console.log(`Creating cohort ${cohortId} with ${COUNT} students...`); + await owner.request('POST', '/api/admin/cohorts', { cohort_id: cohortId, name: 'Load Test' }); + + for (let i = 0; i < COUNT; i++) { + await owner.request('POST', '/api/admin/students', { display_name: `LoadTest-${i}`, cohort_id: cohortId }); + } + + console.log(`Prepare Labs: booting ${COUNT} Labs concurrently...`); + const start = Date.now(); + const { status, body } = await owner.request('POST', `/api/admin/cohorts/${cohortId}/prepare-labs`); + const elapsedMs = Date.now() - start; + + if (status !== 200) { + console.error('Prepare Labs call itself failed:', status, body); + process.exit(1); + } + + const running = body.results.filter((r) => r.status === 'running').length; + const failed = body.results.filter((r) => r.status === 'error'); + + const report = { + timestamp: new Date().toISOString(), + gateway_url: GW, + requested_count: COUNT, + running_count: running, + failed_count: failed.length, + failure_rate_pct: Math.round((failed.length / COUNT) * 1000) / 10, + total_boot_wall_time_ms: elapsedMs, + avg_boot_time_ms_estimate: Math.round(elapsedMs / COUNT), // concurrent, so this is a rough + // lower bound, not a real per-Lab figure + failures: failed, + note: 'Run infra/scripts/loadtest/sample-resources.sh DURING this script (in another ' + + 'terminal) for real per-Lab CPU/RAM numbers — this script only measures boot ' + + 'timing/success rate, not resource consumption.', + }; + + const outDir = path.join(__dirname, 'results'); + fs.mkdirSync(outDir, { recursive: true }); + const outFile = path.join(outDir, `boot-labs-${Date.now()}.json`); + fs.writeFileSync(outFile, JSON.stringify(report, null, 2)); + + console.log(JSON.stringify(report, null, 2)); + console.log(`\nWritten to ${outFile}`); + console.log(`\nCleanup: POST /api/admin/cohorts/${cohortId}/destroy-labs as the owner when done.`); +} + +main().catch((e) => { console.error('Load test crashed:', e); process.exit(1); }); diff --git a/infra/scripts/loadtest/report-template.md b/infra/scripts/loadtest/report-template.md new file mode 100644 index 0000000..437e571 --- /dev/null +++ b/infra/scripts/loadtest/report-template.md @@ -0,0 +1,36 @@ +# RootNode Load Test Report + +**Date:** _______ +**Ran against:** [ ] dev laptop [ ] staging [ ] production rn-lab-host-01 (real hardware) +**Host spec (fill in real numbers, do not guess):** ___ cores / ___ threads / ___ GB RAM / ___ storage + +> A run against anything other than the real target-class hardware only tells you the code +> path works, not what production can actually sustain. Say clearly which this is. + +## Boot test (`boot-labs.js`) + +| Metric | Value | +|---|---| +| Requested Lab count | | +| Successfully running | | +| Failed | | +| Failure rate | | +| Total wall-clock time (concurrent boot) | | + +## Resource usage (`sample-resources.sh`) + +| Metric | Value | +|---|---| +| Peak CPU per Lab (observed) | | +| Peak memory per Lab (observed) | | +| Peak disk I/O per Lab (observed) | | +| Host CPU headroom remaining at peak | | +| Host memory headroom remaining at peak | | + +## Feed into docs/CAPACITY_PLANNING.md + +- [ ] Updated `measured_lab_cpu_cores` with the peak CPU figure above +- [ ] Updated `measured_lab_mem_mb` with the peak memory figure above +- [ ] Re-ran `infra/scripts/capacity-model.js` and recorded the new recommended-concurrent-Labs number + +## Notes / anomalies observed diff --git a/infra/scripts/loadtest/sample-resources.sh b/infra/scripts/loadtest/sample-resources.sh new file mode 100755 index 0000000..a83ef8c --- /dev/null +++ b/infra/scripts/loadtest/sample-resources.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run on rn-lab-host-01 (or the local dev machine running the Lab containers) WHILE +# boot-labs.js is running, to capture real per-Lab CPU/RAM/disk usage — `docker stats` is the +# ground truth here, not an estimate. Samples every 5s for the given duration and writes CSV. +set -euo pipefail +DURATION_SEC="${1:-120}" +OUT="${2:-./results/resource-sample-$(date +%s).csv}" +mkdir -p "$(dirname "$OUT")" + +echo "timestamp,container,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids" > "$OUT" +END=$(( $(date +%s) + DURATION_SEC )) +while [ "$(date +%s)" -lt "$END" ]; do + docker stats --no-stream --format '{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}' \ + | grep -E '^rootnode-lab-' \ + | while IFS=, read -r name cpu mem memp net block pids; do + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ),$name,$cpu,$mem,$memp,$net,$block,$pids" >> "$OUT" + done + sleep 5 +done + +echo "Written to $OUT" +echo "Summarize with: awk -F, 'NR>1{print \$2}' $OUT | sort -u # containers seen" diff --git a/infra/scripts/production-smoke-test.sh b/infra/scripts/production-smoke-test.sh new file mode 100755 index 0000000..382d84a --- /dev/null +++ b/infra/scripts/production-smoke-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Production post-deploy smoke test — deliberately read-only/non-destructive (never creates +# real students/cohorts against production data). Confirms the deployed services are actually +# up after a deploy/rollback, before calling the deploy "done." +set -euo pipefail + +PROD_URL="${PRODUCTION_URL:?set PRODUCTION_URL, e.g. https://academy.example.com}" + +echo "==> Gateway health" +curl -sf "$PROD_URL/healthz" > /dev/null && echo "OK" + +echo "==> TLS certificate is valid and not expiring soon" +EXPIRY=$(echo | openssl s_client -connect "$(echo "$PROD_URL" | sed -e 's#https://##')":443 -servername "$(echo "$PROD_URL" | sed -e 's#https://##')" 2>/dev/null | openssl x509 -noout -enddate | cut -d= -f2) +echo "Certificate expires: $EXPIRY" + +echo "==> Roster endpoint responds (public, unauthenticated, read-only)" +curl -sf "$PROD_URL/api/roster/cohorts" > /dev/null && echo "OK" + +echo "==> Admin login page loads (does not attempt to log in)" +curl -sf "$PROD_URL/admin" -o /dev/null -w '%{http_code}\n' | grep -q 200 && echo "OK" + +echo +echo "OK: production smoke test passed. For a full functional check (owner/instructor login," +echo "cohort, Prepare Labs, student flow, assessment, Capstone), work through" +echo "docs/PRODUCTION_VALIDATION_CHECKLIST.md's ACADEMY section manually — deliberately not" +echo "automated against real production data." diff --git a/infra/scripts/rollback-production.sh b/infra/scripts/rollback-production.sh new file mode 100755 index 0000000..08646a9 --- /dev/null +++ b/infra/scripts/rollback-production.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Rolls back to the previously deployed image tag, per infra/docker/IMAGE_STRATEGY.md — +# "rollback is changing one variable and re-running the Ansible role." Reads +# /opt/rootnode/deploy-history.log (written by deploy-production.sh) for the tag before the +# current one; pass an explicit tag instead if you need to go back further than one step. +# +# Usage: ./rollback-production.sh [explicit-image-tag] +set -euo pipefail +cd "$(dirname "$0")/../ansible" + +DEPLOY_LOG="/opt/rootnode/deploy-history.log" + +if [ -n "${1:-}" ]; then + TARGET_TAG="$1" +else + [ -f "$DEPLOY_LOG" ] || { echo "No $DEPLOY_LOG found and no explicit tag given — nothing to roll back to." >&2; exit 1; } + LINES=$(wc -l < "$DEPLOY_LOG") + [ "$LINES" -ge 2 ] || { echo "Deploy history has fewer than 2 entries — nothing to roll back to. Pass an explicit tag instead." >&2; exit 1; } + TARGET_TAG=$(tail -2 "$DEPLOY_LOG" | head -1 | awk '{print $2}') +fi + +echo "Rolling back production to image_tag=$TARGET_TAG" +read -r -p "Type ROLLBACK to continue: " CONFIRM +[ "$CONFIRM" = "ROLLBACK" ] || { echo "Aborted."; exit 1; } + +ansible-playbook site.yml -i inventory/production.ini \ + -e "image_tag=$TARGET_TAG" \ + --vault-password-file .vault_pass + +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $TARGET_TAG (rollback)" >> "$DEPLOY_LOG" +echo "==> Rolled back to $TARGET_TAG. Verify with infra/scripts/production-smoke-test.sh." +echo "NOTE: this rolls back the application images only. If the rollback needs to undo a" +echo "database migration too, see docs/RUNBOOK_FAILED_MIGRATION.md — schema changes are not" +echo "automatically reversible by this script." diff --git a/infra/scripts/staging-smoke-test.sh b/infra/scripts/staging-smoke-test.sh new file mode 100755 index 0000000..9ab240f --- /dev/null +++ b/infra/scripts/staging-smoke-test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Staging acceptance test (Phase 4 §23/§18) — the four things a human needs before promoting a +# build: app deploys, migrations ran clean, the Gateway/Platform API/Provisioner are reachable, +# and (if OWNER_EMAIL/OWNER_PASSWORD are provided) a real assessment can be started end to end. +# Reuses platform/test/e2e-phase3b.js against the staging Gateway URL rather than re-inventing +# a parallel smoke-test harness — that suite already IS the full acceptance chain. +set -euo pipefail + +STAGING_URL="${STAGING_GATEWAY_URL:?set STAGING_GATEWAY_URL, e.g. https://staging.academy.example.com}" + +echo "==> Health checks" +curl -sf "$STAGING_URL/healthz" > /dev/null && echo "OK: Gateway /healthz" + +if [ -n "${OWNER_EMAIL:-}" ] && [ -n "${OWNER_PASSWORD:-}" ]; then + echo "==> Full acceptance run (platform/test/e2e-phase3b.js against staging)" + cd "$(dirname "$0")/../../platform" + GATEWAY_URL="$STAGING_URL" OWNER_EMAIL="$OWNER_EMAIL" OWNER_PASSWORD="$OWNER_PASSWORD" \ + SKIP_RESTART_TEST=1 node test/e2e-phase3b.js +else + echo "==> Skipping full e2e run — set OWNER_EMAIL/OWNER_PASSWORD for a real classroom-workflow" + echo " smoke test (bootstrap one first against staging: platform/api/bootstrap-owner.js)." +fi + +echo "OK: staging smoke test passed" diff --git a/infra/scripts/validate-compose.sh b/infra/scripts/validate-compose.sh new file mode 100755 index 0000000..5cd1b43 --- /dev/null +++ b/infra/scripts/validate-compose.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Local test (Phase 4 §29): renders infra/docker/platform/docker-compose.production.yml's +# Jinja2 placeholders with disposable dummy values and confirms `docker compose config` +# accepts the result — catches YAML/schema errors before Ansible ever touches a real host. +# This does NOT prove the real production values work (env_file paths under /etc/rootnode/ +# won't exist locally, which docker compose only warns about, not a fatal error), and does NOT +# validate anything hardware/network-dependent. +set -euo pipefail +cd "$(dirname "$0")/../docker/platform" + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +for f in platform.env provisioner.env gateway.env postgres.env; do touch "$TMPDIR/$f"; done + +sed \ + -e 's/{{ rootnode_data_root }}/\/tmp\/rootnode-test-data/g' \ + -e 's/{{ registry }}/registry.example.com/g' \ + -e 's/{{ image_tag }}/test/g' \ + -e 's/{{ postgres_exporter_user }}/exporter/g' \ + -e 's/{{ postgres_exporter_password }}/dummy/g' \ + -e "s#/etc/rootnode/platform.env#$TMPDIR/platform.env#" \ + -e "s#/etc/rootnode/provisioner.env#$TMPDIR/provisioner.env#" \ + -e "s#/etc/rootnode/gateway.env#$TMPDIR/gateway.env#" \ + -e "s#/etc/rootnode/postgres.env#$TMPDIR/postgres.env#" \ + docker-compose.production.yml > "$TMPDIR/docker-compose.yml" + +echo "--- docker compose config ---" +docker compose -f "$TMPDIR/docker-compose.yml" config --quiet +echo "OK: production compose file is well-formed YAML and passes docker compose's own schema validation" + +echo +echo "--- Caddyfile syntax (real caddy validate) ---" +sed -e 's/{{ academy_domain }}/academy.example.com/g' -e 's/{{ management_cidr }}/203.0.113.0\/24/g' \ + Caddyfile.j2 > "$TMPDIR/Caddyfile" +chmod a+r "$TMPDIR/Caddyfile" +docker run --rm -v "$TMPDIR:/etc/caddy:ro" caddy:2-alpine caddy validate --config /etc/caddy/Caddyfile 2>&1 | grep -q "Valid configuration" \ + && echo "OK: Caddyfile is valid" || { echo "FAIL: Caddyfile did not validate"; exit 1; }