#!/usr/bin/env bash # Phase 4.1 §22 — deployment rollback drill. Builds two real gateway image versions (a known # good build, and a deliberately broken one), "deploys" each in turn to a local container, # proves the smoke test genuinely catches the regression, then rolls back using the EXACT same # decision logic rollback-production.sh uses against deploy-history.log (tail -2 | head -1 | # awk '{print $2}' — the tag before the current one), and proves the rollback recovers. # # What this proves: the deterministic-tag + deploy-history.log rollback MECHANISM is real and # correct. What it does NOT prove: the actual `ansible-playbook site.yml -e image_tag=...` step # rollback-production.sh runs against a real host — that needs the real hardware # (docs/PRODUCTION_VALIDATION_CHECKLIST.md). This drill swaps a local Docker container's image # tag directly instead, as the closest local equivalent. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../../../platform/gateway" CONTAINER=rootnode-rollback-drill PORT=4295 DEPLOY_LOG="$(mktemp)" trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1 || true; rm -f "$DEPLOY_LOG"; rm -rf /tmp/rootnode-rollback-drill-broken' EXIT smoke_test() { curl -sf "http://localhost:$PORT/healthz" >/dev/null 2>&1 } echo "==> [1/6] Building the known-good gateway image (current source)" docker build -q -t rootnode-gateway-drill:known-good -f Dockerfile . >/dev/null echo "==> [2/6] Building a deliberately broken version (healthz returns 500)" rm -rf /tmp/rootnode-rollback-drill-broken cp -r . /tmp/rootnode-rollback-drill-broken sed -i "s#app.get('/healthz', (req, res) => res.json({ ok: true }));#app.get('/healthz', (req, res) => res.status(500).json({ ok: false, broken: 'DELIBERATE_DRILL_REGRESSION' }));#" \ /tmp/rootnode-rollback-drill-broken/src/server.js grep -q DELIBERATE_DRILL_REGRESSION /tmp/rootnode-rollback-drill-broken/src/server.js || { echo "FAIL: sed didn't apply — healthz line text must have changed upstream"; exit 1; } docker build -q -t rootnode-gateway-drill:broken -f /tmp/rootnode-rollback-drill-broken/Dockerfile /tmp/rootnode-rollback-drill-broken >/dev/null deploy() { local tag="$1" docker rm -f "$CONTAINER" >/dev/null 2>&1 || true docker run -d --name "$CONTAINER" -p "$PORT:4200" -e GATEWAY_PORT=4200 -e PROVISIONER_URL=http://127.0.0.1:1 \ -e PROVISIONER_SHARED_SECRET=drill -e PLATFORM_API_URL=http://127.0.0.1:1 \ "rootnode-gateway-drill:$tag" >/dev/null sleep 2 echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $tag" >> "$DEPLOY_LOG" } echo "==> [3/6] Deploying known-good (deploy #1)" deploy known-good smoke_test && echo " smoke test: PASS (as expected)" || { echo "FAIL: known-good should have passed"; exit 1; } echo "==> [4/6] Deploying the broken version (deploy #2 — the 'bad release' this drill catches)" deploy broken if smoke_test; then echo "FAIL: the broken deploy's regression was NOT caught by the smoke test — that defeats the whole point"; exit 1; fi echo " smoke test: FAIL (correctly caught — this is exactly what should stop a real promotion, see docs/RUNBOOK_ROLLBACK.md)" echo "==> [5/6] Computing the rollback target — same logic as infra/scripts/rollback-production.sh" TARGET_TAG=$(tail -2 "$DEPLOY_LOG" | head -1 | awk '{print $2}') [ "$TARGET_TAG" = "known-good" ] || { echo "FAIL: expected rollback target 'known-good', computed '$TARGET_TAG'"; exit 1; } echo " rollback target: $TARGET_TAG (correctly identified as the tag before the broken one)" echo "==> [6/6] Rolling back" deploy "$TARGET_TAG" smoke_test && echo " smoke test: PASS — rollback recovered service" || { echo "FAIL: rollback did not recover"; exit 1; } echo echo "OK: deployment rollback drill passed. Deterministic image tags + deploy-history.log" echo "correctly identify and recover from a bad release." echo echo "On migration compatibility (docs/RUNBOOK_FAILED_MIGRATION.md, docs/RUNBOOK_ROLLBACK.md):" echo "this script rolls back APPLICATION IMAGES ONLY. There is no automatic down-migration —" echo "if the bad release also ran a schema-changing migration, rolling the image back does NOT" echo "undo that migration, and old code may not run correctly against the new schema. That" echo "case requires restoring from the pre-migration backup (docs/RUNBOOK_POSTGRES_RESTORE.md)," echo "not a rollback, which is exactly why migrations in this project are check-before-alter" echo "and additive-only (docs/ENGINEERING_SOP.md §6) — to make image-only rollback usually safe."