feat(observability): Prometheus/Alertmanager/Grafana/Loki stack

Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
2026-08-18 17:13:14 -05:00
parent d2280d0f0d
commit 4e8fa1d413
11 changed files with 461 additions and 0 deletions
@@ -0,0 +1,93 @@
+# Prometheus config — deployed to rn-platform-01 (runs alongside the app stack; see
+# infra/ansible/roles/monitoring-agent/). Scrapes both hosts remotely — node-exporter/
+# postgres-exporter here on rn-platform-01 (same-host, local network), node-exporter/cadvisor
+# on rn-lab-host-01 over the network (allowed by the firewall rule added for exactly this,
+# infra/firewall/rn-lab-host-01.rules).
+#
+# Deliberately does NOT scrape or duplicate the application's own instructor learning
+# analytics (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §12) — operational observability
+# (uptime/CPU/errors) and learning analytics (completion rates/hint usage) are separate
+# concerns with separate audiences, per the explicit Phase 4 instruction.
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["localhost:9093"]
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: platform-node
+    static_configs:
+      - targets: ["localhost:9100"]
+        labels: { host: rn-platform-01 }
+
+  - job_name: platform-postgres
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+        labels: { host: rn-platform-01 }
+
+  - job_name: lab-host-node
+    static_configs:
+      - targets: ["{{ lab_host_ip }}:9100"]
+        labels: { host: rn-lab-host-01 }
+
+  - job_name: lab-host-cadvisor
+    static_configs:
+      - targets: ["{{ lab_host_ip }}:8080"]
+        labels: { host: rn-lab-host-01 }
+
+  # Uptime checks (Phase 4 §17) — blackbox_exporter probing the healthz endpoints added this
+  # phase (Platform API, Gateway) plus the Provisioner's existing /health (Phase 1).
+  - job_name: blackbox-gateway
+    metrics_path: /probe
+    params: { module: [http_2xx] }
+    static_configs:
+      - targets: ["https://{{ academy_domain }}/healthz"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9115
+
+  - job_name: blackbox-platform-api
+    metrics_path: /probe
+    params: { module: [http_2xx] }
+    static_configs:
+      - targets: ["http://api:4300/healthz"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9115
+
+  - job_name: blackbox-provisioner
+    metrics_path: /probe
+    params: { module: [http_2xx] }
+    static_configs:
+      - targets: ["http://provisioner:4100/health"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9115
+
+  # RootNode application metrics — see docs/OBSERVABILITY.md §"exposing /metrics" for the
+  # documented, NOT-YET-BUILT follow-up (Platform API/Gateway/Provisioner don't currently
+  # expose a Prometheus /metrics endpoint; this job is ready for when that lands, disabled
+  # until then rather than scraping a 404 forever).
+  # - job_name: rootnode-platform-api
+  #   static_configs:
+  #     - targets: ["api:4300"]
+  #   metrics_path: /metrics