feat(observability): Prometheus/Alertmanager/Grafana/Loki stack

Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
2026-08-18 17:13:14 -05:00
parent d2280d0f0d
commit 4e8fa1d413
11 changed files with 461 additions and 0 deletions
@@ -0,0 +1,54 @@
+{
+  "title": "RootNode Academy — Infrastructure Overview",
+  "uid": "rootnode-overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "timezone": "utc",
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "id": 1, "title": "Platform VM CPU", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [{ "expr": "100 - (avg(rate(node_cpu_seconds_total{job=\"platform-node\",mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }]
+    },
+    {
+      "id": 2, "title": "Lab Host VM CPU", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [{ "expr": "100 - (avg(rate(node_cpu_seconds_total{job=\"lab-host-node\",mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }]
+    },
+    {
+      "id": 3, "title": "Platform VM Memory Available", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [{ "expr": "node_memory_MemAvailable_bytes{job=\"platform-node\"}", "legendFormat": "Available" }]
+    },
+    {
+      "id": 4, "title": "Lab Host VM Memory Available", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [{ "expr": "node_memory_MemAvailable_bytes{job=\"lab-host-node\"}", "legendFormat": "Available" }]
+    },
+    {
+      "id": 5, "title": "Disk Usage (root filesystem, both hosts)", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "targets": [{ "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", "legendFormat": "{{host}}" }]
+    },
+    {
+      "id": 6, "title": "Postgres — Up / Connections", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "targets": [
+        { "expr": "pg_up", "legendFormat": "up" },
+        { "expr": "pg_stat_database_numbackends{datname=\"rootnode_platform\"}", "legendFormat": "connections" }
+      ]
+    },
+    {
+      "id": 7, "title": "Running Docker Containers — Lab Host", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "targets": [{ "expr": "count(container_last_seen{name=~\"rootnode-lab-.*\"})", "legendFormat": "active labs (best-effort via cadvisor label match)" }]
+    },
+    {
+      "id": 8, "title": "Uptime Probes (Gateway / Platform API / Provisioner)", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "targets": [{ "expr": "probe_success", "legendFormat": "{{job}}" }]
+    }
+  ]
+}
@@ -0,0 +1,7 @@
+apiVersion: 1
+providers:
+  - name: RootNode
+    folder: RootNode
+    type: file
+    options:
+      path: /var/lib/grafana/dashboards
@@ -0,0 +1,11 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100