feat(observability): Prometheus/Alertmanager/Grafana/Loki stack

Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
2026-08-18 17:13:14 -05:00
parent d2280d0f0d
commit 4e8fa1d413
11 changed files with 461 additions and 0 deletions
@@ -0,0 +1,97 @@
+# Alert rules (Phase 4 §16). CRITICAL pages immediately; WARNING is for daytime attention.
+# Deliberately does NOT alert on individual student scenario failures or validation
+# failures — those are instructional signal (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md's Needs
+# Attention view), not operational incidents.
+
+groups:
+  - name: rootnode-critical
+    rules:
+      - alert: PlatformAPIDown
+        expr: probe_success{job="blackbox-platform-api"} == 0
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "Platform API is unreachable"
+          description: "/healthz on rn-platform-01's Platform API has failed for 2+ minutes."
+
+      - alert: GatewayDown
+        expr: probe_success{job="blackbox-gateway"} == 0
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "Gateway is unreachable"
+          description: "The Gateway's /healthz has failed for 2+ minutes — students cannot reach any Lab."
+
+      - alert: PostgresDown
+        expr: pg_up == 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "Postgres is down"
+          description: "postgres-exporter cannot reach rootnode_platform Postgres."
+
+      - alert: LabHostDown
+        expr: up{job="lab-host-node"} == 0
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "rn-lab-host-01 is unreachable"
+          description: "node-exporter on the Lab Host VM has stopped responding — no student can launch or use a Lab."
+
+      - alert: BackupFailed
+        expr: rootnode_backup_last_success_timestamp_seconds < (time() - 90000)
+        labels: { severity: critical }
+        annotations:
+          summary: "Postgres backup hasn't succeeded in 25+ hours"
+          description: "infra/backup/backup-postgres.sh's last successful run is stale — see docs/RUNBOOK_BACKUP_FAILURE.md."
+
+      - alert: DiskCriticallyFull
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
+        for: 5m
+        labels: { severity: critical }
+        annotations:
+          summary: "Root filesystem below 10% free on {{ $labels.host }}"
+
+  - name: rootnode-warning
+    rules:
+      - alert: LabHostHighMemory
+        expr: (1 - (node_memory_MemAvailable_bytes{job="lab-host-node"} / node_memory_MemTotal_bytes{job="lab-host-node"})) > 0.85
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "rn-lab-host-01 memory usage above 85% for 10+ minutes"
+          description: "Check active Lab count against docs/CAPACITY_PLANNING.md's admission thresholds."
+
+      - alert: HighDiskUsage
+        expr: (1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) > 0.80
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "Disk usage above 80% on {{ $labels.host }}"
+
+      - alert: RepeatedProvisioningFailures
+        expr: increase(rootnode_provisioning_failures_total[15m]) >= 3
+        labels: { severity: warning }
+        annotations:
+          summary: "3+ Lab provisioning failures in 15 minutes"
+          description: >-
+            Depends on the Provisioner exposing this counter — see
+            docs/OBSERVABILITY.md's "application metrics — not yet built" note. Placeholder rule,
+            inert until that counter exists.
+
+      - alert: BackupAging
+        expr: rootnode_backup_last_success_timestamp_seconds < (time() - 43200)
+        labels: { severity: warning }
+        annotations:
+          summary: "Postgres backup hasn't succeeded in 12+ hours"
+
+      - alert: ExcessiveFailedLogins
+        expr: increase(rootnode_audit_login_failures_total[5m]) > 20
+        labels: { severity: warning }
+        annotations:
+          summary: "20+ failed instructor logins in 5 minutes"
+          description: >-
+            Depends on exporting audit_events counts — see docs/OBSERVABILITY.md's
+            "application metrics — not yet built" note. Placeholder rule, inert until built;
+            the audit_events table itself already exists (Phase 3A) and can be queried
+            manually via /api/admin/audit-events in the meantime.