feat(observability): Prometheus/Alertmanager/Grafana/Loki stack

Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes
against the new /healthz endpoints, alert rules split critical (pages) vs
warning (informational), one starter Grafana dashboard. Deliberately
separate from the application's instructor learning analytics (Phase 3B) -
different audience, different data.

Validated: promtool check config (valid), promtool check rules (11 rules,
valid), amtool check-config (1 inhibit rule, 2 receivers, valid).

infra/monitoring/alertmanager/alertmanager.yml.j2

@@ -0,0 +1,39 @@
+# Alertmanager config — routes CRITICAL alerts and WARNING alerts to the same webhook by
+# default (Uptime Kuma's push-monitor webhook, or ntfy, or another already-available target —
+# not a new paid service; the user's stated preference for Uptime Kuma is honored here without
+# forcing it if a cleaner existing target turns out to be available at deploy time, per the
+# Phase 4 instruction). Swap the receiver's url for whatever's actually wired up.
+
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: rootnode-default
+  group_by: ["alertname", "host"]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  routes:
+    - match:
+        severity: critical
+      receiver: rootnode-critical
+      repeat_interval: 30m
+
+receivers:
+  - name: rootnode-default
+    webhook_configs:
+      - url: "{{ vault_alertmanager_webhook_url | default('http://example.invalid/webhook') }}"
+        send_resolved: true
+
+  - name: rootnode-critical
+    webhook_configs:
+      - url: "{{ vault_alertmanager_webhook_url | default('http://example.invalid/webhook') }}"
+        send_resolved: true
+
+inhibit_rules:
+  # If the whole Lab Host is down, don't also page separately for every dependent warning.
+  - source_match:
+      alertname: LabHostDown
+    target_match:
+      severity: warning
+    equal: ["host"]

infra/monitoring/grafana/dashboards/rootnode-overview.json

@@ -0,0 +1,54 @@
+{
+  "title": "RootNode Academy — Infrastructure Overview",
+  "uid": "rootnode-overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "timezone": "utc",
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "id": 1, "title": "Platform VM CPU", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [{ "expr": "100 - (avg(rate(node_cpu_seconds_total{job=\"platform-node\",mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }]
+    },
+    {
+      "id": 2, "title": "Lab Host VM CPU", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [{ "expr": "100 - (avg(rate(node_cpu_seconds_total{job=\"lab-host-node\",mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }]
+    },
+    {
+      "id": 3, "title": "Platform VM Memory Available", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [{ "expr": "node_memory_MemAvailable_bytes{job=\"platform-node\"}", "legendFormat": "Available" }]
+    },
+    {
+      "id": 4, "title": "Lab Host VM Memory Available", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [{ "expr": "node_memory_MemAvailable_bytes{job=\"lab-host-node\"}", "legendFormat": "Available" }]
+    },
+    {
+      "id": 5, "title": "Disk Usage (root filesystem, both hosts)", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "targets": [{ "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", "legendFormat": "{{host}}" }]
+    },
+    {
+      "id": 6, "title": "Postgres — Up / Connections", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "targets": [
+        { "expr": "pg_up", "legendFormat": "up" },
+        { "expr": "pg_stat_database_numbackends{datname=\"rootnode_platform\"}", "legendFormat": "connections" }
+      ]
+    },
+    {
+      "id": 7, "title": "Running Docker Containers — Lab Host", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "targets": [{ "expr": "count(container_last_seen{name=~\"rootnode-lab-.*\"})", "legendFormat": "active labs (best-effort via cadvisor label match)" }]
+    },
+    {
+      "id": 8, "title": "Uptime Probes (Gateway / Platform API / Provisioner)", "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "targets": [{ "expr": "probe_success", "legendFormat": "{{job}}" }]
+    }
+  ]
+}

infra/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+providers:
+  - name: RootNode
+    folder: RootNode
+    type: file
+    options:
+      path: /var/lib/grafana/dashboards

infra/monitoring/grafana/provisioning/datasources/datasources.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100

infra/monitoring/loki/loki-config.yml

@@ -0,0 +1,36 @@
+# Minimal single-binary Loki config — local filesystem storage, no external dependency, sized
+# for this deployment (a handful of services' logs, not internet-scale ingestion).
+# Retention is set here, not left to grow forever — see docs/OBSERVABILITY.md §"retention."
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: 720h   # 30 days — matches Prometheus's default retention (production.yml var)
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true

infra/monitoring/prometheus/blackbox.yml

@@ -0,0 +1,13 @@
+# blackbox_exporter config — HTTP probes for uptime monitoring (Phase 4 §17). Probes the
+# /healthz endpoints added to Platform API and Gateway this phase, plus the Provisioner's
+# existing /health (Phase 1). Prometheus scrapes blackbox_exporter with these probe targets
+# (see prometheus.yml.j2's blackbox-* jobs — add them there once blackbox_exporter is deployed;
+# left as a documented follow-up wiring step rather than duplicated here).
+
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [200]
+      method: GET

infra/monitoring/prometheus/prometheus.yml.j2

@@ -0,0 +1,93 @@
+# Prometheus config — deployed to rn-platform-01 (runs alongside the app stack; see
+# infra/ansible/roles/monitoring-agent/). Scrapes both hosts remotely — node-exporter/
+# postgres-exporter here on rn-platform-01 (same-host, local network), node-exporter/cadvisor
+# on rn-lab-host-01 over the network (allowed by the firewall rule added for exactly this,
+# infra/firewall/rn-lab-host-01.rules).
+#
+# Deliberately does NOT scrape or duplicate the application's own instructor learning
+# analytics (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §12) — operational observability
+# (uptime/CPU/errors) and learning analytics (completion rates/hint usage) are separate
+# concerns with separate audiences, per the explicit Phase 4 instruction.
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["localhost:9093"]
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: platform-node
+    static_configs:
+      - targets: ["localhost:9100"]
+        labels: { host: rn-platform-01 }
+
+  - job_name: platform-postgres
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+        labels: { host: rn-platform-01 }
+
+  - job_name: lab-host-node
+    static_configs:
+      - targets: ["{{ lab_host_ip }}:9100"]
+        labels: { host: rn-lab-host-01 }
+
+  - job_name: lab-host-cadvisor
+    static_configs:
+      - targets: ["{{ lab_host_ip }}:8080"]
+        labels: { host: rn-lab-host-01 }
+
+  # Uptime checks (Phase 4 §17) — blackbox_exporter probing the healthz endpoints added this
+  # phase (Platform API, Gateway) plus the Provisioner's existing /health (Phase 1).
+  - job_name: blackbox-gateway
+    metrics_path: /probe
+    params: { module: [http_2xx] }
+    static_configs:
+      - targets: ["https://{{ academy_domain }}/healthz"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9115
+
+  - job_name: blackbox-platform-api
+    metrics_path: /probe
+    params: { module: [http_2xx] }
+    static_configs:
+      - targets: ["http://api:4300/healthz"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9115
+
+  - job_name: blackbox-provisioner
+    metrics_path: /probe
+    params: { module: [http_2xx] }
+    static_configs:
+      - targets: ["http://provisioner:4100/health"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9115
+
+  # RootNode application metrics — see docs/OBSERVABILITY.md §"exposing /metrics" for the
+  # documented, NOT-YET-BUILT follow-up (Platform API/Gateway/Provisioner don't currently
+  # expose a Prometheus /metrics endpoint; this job is ready for when that lands, disabled
+  # until then rather than scraping a 404 forever).
+  # - job_name: rootnode-platform-api
+  #   static_configs:
+  #     - targets: ["api:4300"]
+  #   metrics_path: /metrics

infra/monitoring/prometheus/rules/rootnode.yml

@@ -0,0 +1,97 @@
+# Alert rules (Phase 4 §16). CRITICAL pages immediately; WARNING is for daytime attention.
+# Deliberately does NOT alert on individual student scenario failures or validation
+# failures — those are instructional signal (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md's Needs
+# Attention view), not operational incidents.
+
+groups:
+  - name: rootnode-critical
+    rules:
+      - alert: PlatformAPIDown
+        expr: probe_success{job="blackbox-platform-api"} == 0
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "Platform API is unreachable"
+          description: "/healthz on rn-platform-01's Platform API has failed for 2+ minutes."
+
+      - alert: GatewayDown
+        expr: probe_success{job="blackbox-gateway"} == 0
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "Gateway is unreachable"
+          description: "The Gateway's /healthz has failed for 2+ minutes — students cannot reach any Lab."
+
+      - alert: PostgresDown
+        expr: pg_up == 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "Postgres is down"
+          description: "postgres-exporter cannot reach rootnode_platform Postgres."
+
+      - alert: LabHostDown
+        expr: up{job="lab-host-node"} == 0
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "rn-lab-host-01 is unreachable"
+          description: "node-exporter on the Lab Host VM has stopped responding — no student can launch or use a Lab."
+
+      - alert: BackupFailed
+        expr: rootnode_backup_last_success_timestamp_seconds < (time() - 90000)
+        labels: { severity: critical }
+        annotations:
+          summary: "Postgres backup hasn't succeeded in 25+ hours"
+          description: "infra/backup/backup-postgres.sh's last successful run is stale — see docs/RUNBOOK_BACKUP_FAILURE.md."
+
+      - alert: DiskCriticallyFull
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
+        for: 5m
+        labels: { severity: critical }
+        annotations:
+          summary: "Root filesystem below 10% free on {{ $labels.host }}"
+
+  - name: rootnode-warning
+    rules:
+      - alert: LabHostHighMemory
+        expr: (1 - (node_memory_MemAvailable_bytes{job="lab-host-node"} / node_memory_MemTotal_bytes{job="lab-host-node"})) > 0.85
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "rn-lab-host-01 memory usage above 85% for 10+ minutes"
+          description: "Check active Lab count against docs/CAPACITY_PLANNING.md's admission thresholds."
+
+      - alert: HighDiskUsage
+        expr: (1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) > 0.80
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "Disk usage above 80% on {{ $labels.host }}"
+
+      - alert: RepeatedProvisioningFailures
+        expr: increase(rootnode_provisioning_failures_total[15m]) >= 3
+        labels: { severity: warning }
+        annotations:
+          summary: "3+ Lab provisioning failures in 15 minutes"
+          description: >-
+            Depends on the Provisioner exposing this counter — see
+            docs/OBSERVABILITY.md's "application metrics — not yet built" note. Placeholder rule,
+            inert until that counter exists.
+
+      - alert: BackupAging
+        expr: rootnode_backup_last_success_timestamp_seconds < (time() - 43200)
+        labels: { severity: warning }
+        annotations:
+          summary: "Postgres backup hasn't succeeded in 12+ hours"
+
+      - alert: ExcessiveFailedLogins
+        expr: increase(rootnode_audit_login_failures_total[5m]) > 20
+        labels: { severity: warning }
+        annotations:
+          summary: "20+ failed instructor logins in 5 minutes"
+          description: >-
+            Depends on exporting audit_events counts — see docs/OBSERVABILITY.md's
+            "application metrics — not yet built" note. Placeholder rule, inert until built;
+            the audit_events table itself already exists (Phase 3A) and can be queried
+            manually via /api/admin/audit-events in the meantime.

infra/monitoring/promtail/promtail-config.lab-host.yml.j2

@@ -0,0 +1,38 @@
+# Promtail on rn-lab-host-01 — same shape as promtail-config.platform.yml, but pushes to Loki
+# over the network (Loki lives on rn-platform-01, not locally) since these two hosts don't
+# share a Docker network. {{ platform_ip }} is an Ansible variable, rendered by the lab-host
+# role — see infra/firewall/rn-platform-01.rules for the matching inbound rule (3100, this
+# host's IP only).
+
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: "http://{{ platform_ip }}:3100/loki/api/v1/push"
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: [__meta_docker_container_name]
+        target_label: container
+      - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+        target_label: service
+    pipeline_stages:
+      - replace:
+          expression: '(?i)(password|token|secret)=[^&\s]+'
+          replace: '$1=***REDACTED***'
+
+  - job_name: journal
+    journal:
+      max_age: 24h
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit

infra/monitoring/promtail/promtail-config.platform.yml

@@ -0,0 +1,42 @@
+# Promtail — ships container logs (via the Docker socket, all containers on this host) and
+# journald (systemd units: rootnode-platform.service, rootnode-lab-host.service, docker,
+# sshd, fail2ban) to Loki. Deliberately does NOT centralize raw student terminal/command
+# output — the Lab image's own stdout is Lab-lifecycle noise (k3s boot logs, entrypoint
+# output), not the actual PTY session content, which never leaves the container's terminal
+# WebSocket in the first place. See docs/OBSERVABILITY.md §"what's excluded and why."
+
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: [__meta_docker_container_name]
+        target_label: container
+      - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+        target_label: service
+    pipeline_stages:
+      # Never ship secrets — a defense-in-depth scrub in case a service ever accidentally logs
+      # one (the app itself is expected to never log tokens/passwords per its own conventions,
+      # e.g. docs/PHASE3A_IDENTITY_AND_INSTRUCTOR_ACCESS.md's "never log raw session tokens").
+      - replace:
+          expression: '(?i)(password|token|secret)=[^&\s]+'
+          replace: '$1=***REDACTED***'
+
+  - job_name: journal
+    journal:
+      max_age: 24h
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit

infra/monitoring/validate-monitoring.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Local test (Phase 4 §29): renders prometheus.yml.j2 with dummy values and validates it plus
+# the alert rules with the REAL promtool (via the official Prometheus Docker image, no local
+# install needed). Also validates alertmanager.yml.j2 with amtool if the image supports it.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+mkdir -p "$TMPDIR/rules"
+
+sed -e 's/{{ lab_host_ip }}/10.10.10.20/g' prometheus/prometheus.yml.j2 > "$TMPDIR/prometheus.yml"
+cp prometheus/rules/*.yml "$TMPDIR/rules/"
+chmod -R a+rX "$TMPDIR"
+
+echo "==> promtool check config"
+docker run --rm -v "$TMPDIR:/etc/prometheus:ro" --entrypoint promtool prom/prometheus:latest \
+  check config /etc/prometheus/prometheus.yml
+
+echo
+echo "==> amtool check-config (Alertmanager)"
+TMPDIR2="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR" "$TMPDIR2"' EXIT
+sed -e "s/{{ vault_alertmanager_webhook_url | default('http:\/\/example.invalid\/webhook') }}/http:\/\/example.invalid\/webhook/g" \
+  alertmanager/alertmanager.yml.j2 > "$TMPDIR2/alertmanager.yml"
+chmod -R a+rX "$TMPDIR2"
+docker run --rm -v "$TMPDIR2:/etc/alertmanager:ro" --entrypoint amtool prom/alertmanager:latest \
+  check-config /etc/alertmanager/alertmanager.yml
+
+echo
+echo "OK: monitoring configs are valid"