ced4568
f3c356951c
- infra/scripts/simulate-two-host/: real two-Docker-network simulation (Gateway genuinely isolated from any Lab bridge network) proving the Lab Host Router fix - 13/13 real assertions (HTTP+WebSocket through the Router, Alice/Bob isolation, garbage/destroyed tokens, mTLS-only Router access). - infra/scripts/drills/: Lab Host loss/rebuild (9/9), deployment rollback, CI-fails-closed (6 required failure modes), fake cohort dress rehearsal (14/14, one continuous narrative through the real app). - infra/ci/gitea-workflows/build-and-test.yml: new security-scan job (npm audit, gitleaks, hadolint, shellcheck, trivy - all free/open-source), severity policy in docs/SECURITY_SCANNING.md (CRITICAL/HIGH blocks, MEDIUM/LOW reported only). Gates build-images alongside regression. Found and fixed two real pre-existing bugs via the CI-fails-closed drill: staging-smoke-test.sh and production-smoke-test.sh both had a classic set -e gotcha (curl ... && echo "OK" as one statement - a failing command that isn't last in an && chain doesn't trigger -e) that let an unreachable/broken endpoint fall through silently instead of failing the smoke test. platform/test/e2e-phase3a.js updated for the new paginated /api/admin/audit-events response shape (Phase 4.1 pagination work).
75 lines
4.4 KiB
Bash
Executable File
75 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Phase 4.1 §22 — deployment rollback drill. Builds two real gateway image versions (a known
|
|
# good build, and a deliberately broken one), "deploys" each in turn to a local container,
|
|
# proves the smoke test genuinely catches the regression, then rolls back using the EXACT same
|
|
# decision logic rollback-production.sh uses against deploy-history.log (tail -2 | head -1 |
|
|
# awk '{print $2}' — the tag before the current one), and proves the rollback recovers.
|
|
#
|
|
# What this proves: the deterministic-tag + deploy-history.log rollback MECHANISM is real and
|
|
# correct. What it does NOT prove: the actual `ansible-playbook site.yml -e image_tag=...` step
|
|
# rollback-production.sh runs against a real host — that needs the real hardware
|
|
# (docs/PRODUCTION_VALIDATION_CHECKLIST.md). This drill swaps a local Docker container's image
|
|
# tag directly instead, as the closest local equivalent.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")/../../../platform/gateway"
|
|
|
|
CONTAINER=rootnode-rollback-drill
|
|
PORT=4295
|
|
DEPLOY_LOG="$(mktemp)"
|
|
trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1 || true; rm -f "$DEPLOY_LOG"; rm -rf /tmp/rootnode-rollback-drill-broken' EXIT
|
|
|
|
smoke_test() {
|
|
curl -sf "http://localhost:$PORT/healthz" >/dev/null 2>&1
|
|
}
|
|
|
|
echo "==> [1/6] Building the known-good gateway image (current source)"
|
|
docker build -q -t rootnode-gateway-drill:known-good -f Dockerfile . >/dev/null
|
|
|
|
echo "==> [2/6] Building a deliberately broken version (healthz returns 500)"
|
|
rm -rf /tmp/rootnode-rollback-drill-broken
|
|
cp -r . /tmp/rootnode-rollback-drill-broken
|
|
sed -i "s#app.get('/healthz', (req, res) => res.json({ ok: true }));#app.get('/healthz', (req, res) => res.status(500).json({ ok: false, broken: 'DELIBERATE_DRILL_REGRESSION' }));#" \
|
|
/tmp/rootnode-rollback-drill-broken/src/server.js
|
|
grep -q DELIBERATE_DRILL_REGRESSION /tmp/rootnode-rollback-drill-broken/src/server.js || { echo "FAIL: sed didn't apply — healthz line text must have changed upstream"; exit 1; }
|
|
docker build -q -t rootnode-gateway-drill:broken -f /tmp/rootnode-rollback-drill-broken/Dockerfile /tmp/rootnode-rollback-drill-broken >/dev/null
|
|
|
|
deploy() {
|
|
local tag="$1"
|
|
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
|
docker run -d --name "$CONTAINER" -p "$PORT:4200" -e GATEWAY_PORT=4200 -e PROVISIONER_URL=http://127.0.0.1:1 \
|
|
-e PROVISIONER_SHARED_SECRET=drill -e PLATFORM_API_URL=http://127.0.0.1:1 \
|
|
"rootnode-gateway-drill:$tag" >/dev/null
|
|
sleep 2
|
|
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $tag" >> "$DEPLOY_LOG"
|
|
}
|
|
|
|
echo "==> [3/6] Deploying known-good (deploy #1)"
|
|
deploy known-good
|
|
smoke_test && echo " smoke test: PASS (as expected)" || { echo "FAIL: known-good should have passed"; exit 1; }
|
|
|
|
echo "==> [4/6] Deploying the broken version (deploy #2 — the 'bad release' this drill catches)"
|
|
deploy broken
|
|
if smoke_test; then echo "FAIL: the broken deploy's regression was NOT caught by the smoke test — that defeats the whole point"; exit 1; fi
|
|
echo " smoke test: FAIL (correctly caught — this is exactly what should stop a real promotion, see docs/RUNBOOK_ROLLBACK.md)"
|
|
|
|
echo "==> [5/6] Computing the rollback target — same logic as infra/scripts/rollback-production.sh"
|
|
TARGET_TAG=$(tail -2 "$DEPLOY_LOG" | head -1 | awk '{print $2}')
|
|
[ "$TARGET_TAG" = "known-good" ] || { echo "FAIL: expected rollback target 'known-good', computed '$TARGET_TAG'"; exit 1; }
|
|
echo " rollback target: $TARGET_TAG (correctly identified as the tag before the broken one)"
|
|
|
|
echo "==> [6/6] Rolling back"
|
|
deploy "$TARGET_TAG"
|
|
smoke_test && echo " smoke test: PASS — rollback recovered service" || { echo "FAIL: rollback did not recover"; exit 1; }
|
|
|
|
echo
|
|
echo "OK: deployment rollback drill passed. Deterministic image tags + deploy-history.log"
|
|
echo "correctly identify and recover from a bad release."
|
|
echo
|
|
echo "On migration compatibility (docs/RUNBOOK_FAILED_MIGRATION.md, docs/RUNBOOK_ROLLBACK.md):"
|
|
echo "this script rolls back APPLICATION IMAGES ONLY. There is no automatic down-migration —"
|
|
echo "if the bad release also ran a schema-changing migration, rolling the image back does NOT"
|
|
echo "undo that migration, and old code may not run correctly against the new schema. That"
|
|
echo "case requires restoring from the pre-migration backup (docs/RUNBOOK_POSTGRES_RESTORE.md),"
|
|
echo "not a rollback, which is exactly why migrations in this project are check-before-alter"
|
|
echo "and additive-only (docs/ENGINEERING_SOP.md §6) — to make image-only rollback usually safe."
|