diff --git a/infra/monitoring/alertmanager/alertmanager.yml.j2 b/infra/monitoring/alertmanager/alertmanager.yml.j2 new file mode 100644 index 0000000..191a685 --- /dev/null +++ b/infra/monitoring/alertmanager/alertmanager.yml.j2 @@ -0,0 +1,39 @@ +# Alertmanager config — routes CRITICAL alerts and WARNING alerts to the same webhook by +# default (Uptime Kuma's push-monitor webhook, or ntfy, or another already-available target — +# not a new paid service; the user's stated preference for Uptime Kuma is honored here without +# forcing it if a cleaner existing target turns out to be available at deploy time, per the +# Phase 4 instruction). Swap the receiver's url for whatever's actually wired up. + +global: + resolve_timeout: 5m + +route: + receiver: rootnode-default + group_by: ["alertname", "host"] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - match: + severity: critical + receiver: rootnode-critical + repeat_interval: 30m + +receivers: + - name: rootnode-default + webhook_configs: + - url: "{{ vault_alertmanager_webhook_url | default('http://example.invalid/webhook') }}" + send_resolved: true + + - name: rootnode-critical + webhook_configs: + - url: "{{ vault_alertmanager_webhook_url | default('http://example.invalid/webhook') }}" + send_resolved: true + +inhibit_rules: + # If the whole Lab Host is down, don't also page separately for every dependent warning. + - source_match: + alertname: LabHostDown + target_match: + severity: warning + equal: ["host"] diff --git a/infra/monitoring/grafana/dashboards/rootnode-overview.json b/infra/monitoring/grafana/dashboards/rootnode-overview.json new file mode 100644 index 0000000..9b6e780 --- /dev/null +++ b/infra/monitoring/grafana/dashboards/rootnode-overview.json @@ -0,0 +1,54 @@ +{ + "title": "RootNode Academy — Infrastructure Overview", + "uid": "rootnode-overview", + "schemaVersion": 39, + "version": 1, + "timezone": "utc", + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "panels": [ + { + "id": 1, "title": "Platform VM CPU", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [{ "expr": "100 - (avg(rate(node_cpu_seconds_total{job=\"platform-node\",mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }] + }, + { + "id": 2, "title": "Lab Host VM CPU", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [{ "expr": "100 - (avg(rate(node_cpu_seconds_total{job=\"lab-host-node\",mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %" }] + }, + { + "id": 3, "title": "Platform VM Memory Available", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [{ "expr": "node_memory_MemAvailable_bytes{job=\"platform-node\"}", "legendFormat": "Available" }] + }, + { + "id": 4, "title": "Lab Host VM Memory Available", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [{ "expr": "node_memory_MemAvailable_bytes{job=\"lab-host-node\"}", "legendFormat": "Available" }] + }, + { + "id": 5, "title": "Disk Usage (root filesystem, both hosts)", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [{ "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", "legendFormat": "{{host}}" }] + }, + { + "id": 6, "title": "Postgres — Up / Connections", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "targets": [ + { "expr": "pg_up", "legendFormat": "up" }, + { "expr": "pg_stat_database_numbackends{datname=\"rootnode_platform\"}", "legendFormat": "connections" } + ] + }, + { + "id": 7, "title": "Running Docker Containers — Lab Host", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "targets": [{ "expr": "count(container_last_seen{name=~\"rootnode-lab-.*\"})", "legendFormat": "active labs (best-effort via cadvisor label match)" }] + }, + { + "id": 8, "title": "Uptime Probes (Gateway / Platform API / Provisioner)", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "targets": [{ "expr": "probe_success", "legendFormat": "{{job}}" }] + } + ] +} diff --git a/infra/monitoring/grafana/provisioning/dashboards/dashboards.yml b/infra/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..c9aed37 --- /dev/null +++ b/infra/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: RootNode + folder: RootNode + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/infra/monitoring/grafana/provisioning/datasources/datasources.yml b/infra/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..8904117 --- /dev/null +++ b/infra/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,11 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + type: loki + access: proxy + url: http://loki:3100 diff --git a/infra/monitoring/loki/loki-config.yml b/infra/monitoring/loki/loki-config.yml new file mode 100644 index 0000000..d96224e --- /dev/null +++ b/infra/monitoring/loki/loki-config.yml @@ -0,0 +1,36 @@ +# Minimal single-binary Loki config — local filesystem storage, no external dependency, sized +# for this deployment (a handful of services' logs, not internet-scale ingestion). +# Retention is set here, not left to grow forever — see docs/OBSERVABILITY.md §"retention." + +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 720h # 30 days — matches Prometheus's default retention (production.yml var) + +compactor: + working_directory: /loki/compactor + retention_enabled: true diff --git a/infra/monitoring/prometheus/blackbox.yml b/infra/monitoring/prometheus/blackbox.yml new file mode 100644 index 0000000..2f4cd16 --- /dev/null +++ b/infra/monitoring/prometheus/blackbox.yml @@ -0,0 +1,13 @@ +# blackbox_exporter config — HTTP probes for uptime monitoring (Phase 4 §17). Probes the +# /healthz endpoints added to Platform API and Gateway this phase, plus the Provisioner's +# existing /health (Phase 1). Prometheus scrapes blackbox_exporter with these probe targets +# (see prometheus.yml.j2's blackbox-* jobs — add them there once blackbox_exporter is deployed; +# left as a documented follow-up wiring step rather than duplicated here). + +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_status_codes: [200] + method: GET diff --git a/infra/monitoring/prometheus/prometheus.yml.j2 b/infra/monitoring/prometheus/prometheus.yml.j2 new file mode 100644 index 0000000..1184307 --- /dev/null +++ b/infra/monitoring/prometheus/prometheus.yml.j2 @@ -0,0 +1,93 @@ +# Prometheus config — deployed to rn-platform-01 (runs alongside the app stack; see +# infra/ansible/roles/monitoring-agent/). Scrapes both hosts remotely — node-exporter/ +# postgres-exporter here on rn-platform-01 (same-host, local network), node-exporter/cadvisor +# on rn-lab-host-01 over the network (allowed by the firewall rule added for exactly this, +# infra/firewall/rn-lab-host-01.rules). +# +# Deliberately does NOT scrape or duplicate the application's own instructor learning +# analytics (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §12) — operational observability +# (uptime/CPU/errors) and learning analytics (completion rates/hint usage) are separate +# concerns with separate audiences, per the explicit Phase 4 instruction. + +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: ["localhost:9093"] + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: platform-node + static_configs: + - targets: ["localhost:9100"] + labels: { host: rn-platform-01 } + + - job_name: platform-postgres + static_configs: + - targets: ["postgres-exporter:9187"] + labels: { host: rn-platform-01 } + + - job_name: lab-host-node + static_configs: + - targets: ["{{ lab_host_ip }}:9100"] + labels: { host: rn-lab-host-01 } + + - job_name: lab-host-cadvisor + static_configs: + - targets: ["{{ lab_host_ip }}:8080"] + labels: { host: rn-lab-host-01 } + + # Uptime checks (Phase 4 §17) — blackbox_exporter probing the healthz endpoints added this + # phase (Platform API, Gateway) plus the Provisioner's existing /health (Phase 1). + - job_name: blackbox-gateway + metrics_path: /probe + params: { module: [http_2xx] } + static_configs: + - targets: ["https://{{ academy_domain }}/healthz"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9115 + + - job_name: blackbox-platform-api + metrics_path: /probe + params: { module: [http_2xx] } + static_configs: + - targets: ["http://api:4300/healthz"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9115 + + - job_name: blackbox-provisioner + metrics_path: /probe + params: { module: [http_2xx] } + static_configs: + - targets: ["http://provisioner:4100/health"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9115 + + # RootNode application metrics — see docs/OBSERVABILITY.md §"exposing /metrics" for the + # documented, NOT-YET-BUILT follow-up (Platform API/Gateway/Provisioner don't currently + # expose a Prometheus /metrics endpoint; this job is ready for when that lands, disabled + # until then rather than scraping a 404 forever). + # - job_name: rootnode-platform-api + # static_configs: + # - targets: ["api:4300"] + # metrics_path: /metrics diff --git a/infra/monitoring/prometheus/rules/rootnode.yml b/infra/monitoring/prometheus/rules/rootnode.yml new file mode 100644 index 0000000..4eb43ef --- /dev/null +++ b/infra/monitoring/prometheus/rules/rootnode.yml @@ -0,0 +1,97 @@ +# Alert rules (Phase 4 §16). CRITICAL pages immediately; WARNING is for daytime attention. +# Deliberately does NOT alert on individual student scenario failures or validation +# failures — those are instructional signal (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md's Needs +# Attention view), not operational incidents. + +groups: + - name: rootnode-critical + rules: + - alert: PlatformAPIDown + expr: probe_success{job="blackbox-platform-api"} == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: "Platform API is unreachable" + description: "/healthz on rn-platform-01's Platform API has failed for 2+ minutes." + + - alert: GatewayDown + expr: probe_success{job="blackbox-gateway"} == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: "Gateway is unreachable" + description: "The Gateway's /healthz has failed for 2+ minutes — students cannot reach any Lab." + + - alert: PostgresDown + expr: pg_up == 0 + for: 1m + labels: { severity: critical } + annotations: + summary: "Postgres is down" + description: "postgres-exporter cannot reach rootnode_platform Postgres." + + - alert: LabHostDown + expr: up{job="lab-host-node"} == 0 + for: 2m + labels: { severity: critical } + annotations: + summary: "rn-lab-host-01 is unreachable" + description: "node-exporter on the Lab Host VM has stopped responding — no student can launch or use a Lab." + + - alert: BackupFailed + expr: rootnode_backup_last_success_timestamp_seconds < (time() - 90000) + labels: { severity: critical } + annotations: + summary: "Postgres backup hasn't succeeded in 25+ hours" + description: "infra/backup/backup-postgres.sh's last successful run is stale — see docs/RUNBOOK_BACKUP_FAILURE.md." + + - alert: DiskCriticallyFull + expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10 + for: 5m + labels: { severity: critical } + annotations: + summary: "Root filesystem below 10% free on {{ $labels.host }}" + + - name: rootnode-warning + rules: + - alert: LabHostHighMemory + expr: (1 - (node_memory_MemAvailable_bytes{job="lab-host-node"} / node_memory_MemTotal_bytes{job="lab-host-node"})) > 0.85 + for: 10m + labels: { severity: warning } + annotations: + summary: "rn-lab-host-01 memory usage above 85% for 10+ minutes" + description: "Check active Lab count against docs/CAPACITY_PLANNING.md's admission thresholds." + + - alert: HighDiskUsage + expr: (1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) > 0.80 + for: 15m + labels: { severity: warning } + annotations: + summary: "Disk usage above 80% on {{ $labels.host }}" + + - alert: RepeatedProvisioningFailures + expr: increase(rootnode_provisioning_failures_total[15m]) >= 3 + labels: { severity: warning } + annotations: + summary: "3+ Lab provisioning failures in 15 minutes" + description: >- + Depends on the Provisioner exposing this counter — see + docs/OBSERVABILITY.md's "application metrics — not yet built" note. Placeholder rule, + inert until that counter exists. + + - alert: BackupAging + expr: rootnode_backup_last_success_timestamp_seconds < (time() - 43200) + labels: { severity: warning } + annotations: + summary: "Postgres backup hasn't succeeded in 12+ hours" + + - alert: ExcessiveFailedLogins + expr: increase(rootnode_audit_login_failures_total[5m]) > 20 + labels: { severity: warning } + annotations: + summary: "20+ failed instructor logins in 5 minutes" + description: >- + Depends on exporting audit_events counts — see docs/OBSERVABILITY.md's + "application metrics — not yet built" note. Placeholder rule, inert until built; + the audit_events table itself already exists (Phase 3A) and can be queried + manually via /api/admin/audit-events in the meantime. diff --git a/infra/monitoring/promtail/promtail-config.lab-host.yml.j2 b/infra/monitoring/promtail/promtail-config.lab-host.yml.j2 new file mode 100644 index 0000000..2015bf6 --- /dev/null +++ b/infra/monitoring/promtail/promtail-config.lab-host.yml.j2 @@ -0,0 +1,38 @@ +# Promtail on rn-lab-host-01 — same shape as promtail-config.platform.yml, but pushes to Loki +# over the network (Loki lives on rn-platform-01, not locally) since these two hosts don't +# share a Docker network. {{ platform_ip }} is an Ansible variable, rendered by the lab-host +# role — see infra/firewall/rn-platform-01.rules for the matching inbound rule (3100, this +# host's IP only). + +server: + http_listen_port: 9080 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: "http://{{ platform_ip }}:3100/loki/api/v1/push" + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + relabel_configs: + - source_labels: [__meta_docker_container_name] + target_label: container + - source_labels: [__meta_docker_container_label_com_docker_compose_service] + target_label: service + pipeline_stages: + - replace: + expression: '(?i)(password|token|secret)=[^&\s]+' + replace: '$1=***REDACTED***' + + - job_name: journal + journal: + max_age: 24h + labels: + job: systemd-journal + relabel_configs: + - source_labels: [__journal__systemd_unit] + target_label: unit diff --git a/infra/monitoring/promtail/promtail-config.platform.yml b/infra/monitoring/promtail/promtail-config.platform.yml new file mode 100644 index 0000000..452010a --- /dev/null +++ b/infra/monitoring/promtail/promtail-config.platform.yml @@ -0,0 +1,42 @@ +# Promtail — ships container logs (via the Docker socket, all containers on this host) and +# journald (systemd units: rootnode-platform.service, rootnode-lab-host.service, docker, +# sshd, fail2ban) to Loki. Deliberately does NOT centralize raw student terminal/command +# output — the Lab image's own stdout is Lab-lifecycle noise (k3s boot logs, entrypoint +# output), not the actual PTY session content, which never leaves the container's terminal +# WebSocket in the first place. See docs/OBSERVABILITY.md §"what's excluded and why." + +server: + http_listen_port: 9080 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + relabel_configs: + - source_labels: [__meta_docker_container_name] + target_label: container + - source_labels: [__meta_docker_container_label_com_docker_compose_service] + target_label: service + pipeline_stages: + # Never ship secrets — a defense-in-depth scrub in case a service ever accidentally logs + # one (the app itself is expected to never log tokens/passwords per its own conventions, + # e.g. docs/PHASE3A_IDENTITY_AND_INSTRUCTOR_ACCESS.md's "never log raw session tokens"). + - replace: + expression: '(?i)(password|token|secret)=[^&\s]+' + replace: '$1=***REDACTED***' + + - job_name: journal + journal: + max_age: 24h + labels: + job: systemd-journal + relabel_configs: + - source_labels: [__journal__systemd_unit] + target_label: unit diff --git a/infra/monitoring/validate-monitoring.sh b/infra/monitoring/validate-monitoring.sh new file mode 100755 index 0000000..391db46 --- /dev/null +++ b/infra/monitoring/validate-monitoring.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Local test (Phase 4 §29): renders prometheus.yml.j2 with dummy values and validates it plus +# the alert rules with the REAL promtool (via the official Prometheus Docker image, no local +# install needed). Also validates alertmanager.yml.j2 with amtool if the image supports it. +set -euo pipefail +cd "$(dirname "$0")" + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT +mkdir -p "$TMPDIR/rules" + +sed -e 's/{{ lab_host_ip }}/10.10.10.20/g' prometheus/prometheus.yml.j2 > "$TMPDIR/prometheus.yml" +cp prometheus/rules/*.yml "$TMPDIR/rules/" +chmod -R a+rX "$TMPDIR" + +echo "==> promtool check config" +docker run --rm -v "$TMPDIR:/etc/prometheus:ro" --entrypoint promtool prom/prometheus:latest \ + check config /etc/prometheus/prometheus.yml + +echo +echo "==> amtool check-config (Alertmanager)" +TMPDIR2="$(mktemp -d)" +trap 'rm -rf "$TMPDIR" "$TMPDIR2"' EXIT +sed -e "s/{{ vault_alertmanager_webhook_url | default('http:\/\/example.invalid\/webhook') }}/http:\/\/example.invalid\/webhook/g" \ + alertmanager/alertmanager.yml.j2 > "$TMPDIR2/alertmanager.yml" +chmod -R a+rX "$TMPDIR2" +docker run --rm -v "$TMPDIR2:/etc/alertmanager:ro" --entrypoint amtool prom/alertmanager:latest \ + check-config /etc/alertmanager/alertmanager.yml + +echo +echo "OK: monitoring configs are valid"