feat(observability): Prometheus/Alertmanager/Grafana/Loki stack

Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
2026-08-18 17:13:14 -05:00
parent d2280d0f0d
commit 4e8fa1d413
11 changed files with 461 additions and 0 deletions
@@ -0,0 +1,38 @@
+# Promtail on rn-lab-host-01 — same shape as promtail-config.platform.yml, but pushes to Loki
+# over the network (Loki lives on rn-platform-01, not locally) since these two hosts don't
+# share a Docker network. {{ platform_ip }} is an Ansible variable, rendered by the lab-host
+# role — see infra/firewall/rn-platform-01.rules for the matching inbound rule (3100, this
+# host's IP only).
+
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: "http://{{ platform_ip }}:3100/loki/api/v1/push"
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: [__meta_docker_container_name]
+        target_label: container
+      - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+        target_label: service
+    pipeline_stages:
+      - replace:
+          expression: '(?i)(password|token|secret)=[^&\s]+'
+          replace: '$1=***REDACTED***'
+
+  - job_name: journal
+    journal:
+      max_age: 24h
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit
@@ -0,0 +1,42 @@
+# Promtail — ships container logs (via the Docker socket, all containers on this host) and
+# journald (systemd units: rootnode-platform.service, rootnode-lab-host.service, docker,
+# sshd, fail2ban) to Loki. Deliberately does NOT centralize raw student terminal/command
+# output — the Lab image's own stdout is Lab-lifecycle noise (k3s boot logs, entrypoint
+# output), not the actual PTY session content, which never leaves the container's terminal
+# WebSocket in the first place. See docs/OBSERVABILITY.md §"what's excluded and why."
+
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: [__meta_docker_container_name]
+        target_label: container
+      - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+        target_label: service
+    pipeline_stages:
+      # Never ship secrets — a defense-in-depth scrub in case a service ever accidentally logs
+      # one (the app itself is expected to never log tokens/passwords per its own conventions,
+      # e.g. docs/PHASE3A_IDENTITY_AND_INSTRUCTOR_ACCESS.md's "never log raw session tokens").
+      - replace:
+          expression: '(?i)(password|token|secret)=[^&\s]+'
+          replace: '$1=***REDACTED***'
+
+  - job_name: journal
+    journal:
+      max_age: 24h
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit