feat(observability): Prometheus/Alertmanager/Grafana/Loki stack
Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
# blackbox_exporter config — HTTP probes for uptime monitoring (Phase 4 §17). Probes the
|
||||
# /healthz endpoints added to Platform API and Gateway this phase, plus the Provisioner's
|
||||
# existing /health (Phase 1). Prometheus scrapes blackbox_exporter with these probe targets
|
||||
# (see prometheus.yml.j2's blackbox-* jobs — add them there once blackbox_exporter is deployed;
|
||||
# left as a documented follow-up wiring step rather than duplicated here).
|
||||
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
valid_status_codes: [200]
|
||||
method: GET
|
||||
@@ -0,0 +1,93 @@
|
||||
# Prometheus config — deployed to rn-platform-01 (runs alongside the app stack; see
|
||||
# infra/ansible/roles/monitoring-agent/). Scrapes both hosts remotely — node-exporter/
|
||||
# postgres-exporter here on rn-platform-01 (same-host, local network), node-exporter/cadvisor
|
||||
# on rn-lab-host-01 over the network (allowed by the firewall rule added for exactly this,
|
||||
# infra/firewall/rn-lab-host-01.rules).
|
||||
#
|
||||
# Deliberately does NOT scrape or duplicate the application's own instructor learning
|
||||
# analytics (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md §12) — operational observability
|
||||
# (uptime/CPU/errors) and learning analytics (completion rates/hint usage) are separate
|
||||
# concerns with separate audiences, per the explicit Phase 4 instruction.
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: ["localhost:9093"]
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: platform-node
|
||||
static_configs:
|
||||
- targets: ["localhost:9100"]
|
||||
labels: { host: rn-platform-01 }
|
||||
|
||||
- job_name: platform-postgres
|
||||
static_configs:
|
||||
- targets: ["postgres-exporter:9187"]
|
||||
labels: { host: rn-platform-01 }
|
||||
|
||||
- job_name: lab-host-node
|
||||
static_configs:
|
||||
- targets: ["{{ lab_host_ip }}:9100"]
|
||||
labels: { host: rn-lab-host-01 }
|
||||
|
||||
- job_name: lab-host-cadvisor
|
||||
static_configs:
|
||||
- targets: ["{{ lab_host_ip }}:8080"]
|
||||
labels: { host: rn-lab-host-01 }
|
||||
|
||||
# Uptime checks (Phase 4 §17) — blackbox_exporter probing the healthz endpoints added this
|
||||
# phase (Platform API, Gateway) plus the Provisioner's existing /health (Phase 1).
|
||||
- job_name: blackbox-gateway
|
||||
metrics_path: /probe
|
||||
params: { module: [http_2xx] }
|
||||
static_configs:
|
||||
- targets: ["https://{{ academy_domain }}/healthz"]
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: localhost:9115
|
||||
|
||||
- job_name: blackbox-platform-api
|
||||
metrics_path: /probe
|
||||
params: { module: [http_2xx] }
|
||||
static_configs:
|
||||
- targets: ["http://api:4300/healthz"]
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: localhost:9115
|
||||
|
||||
- job_name: blackbox-provisioner
|
||||
metrics_path: /probe
|
||||
params: { module: [http_2xx] }
|
||||
static_configs:
|
||||
- targets: ["http://provisioner:4100/health"]
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: localhost:9115
|
||||
|
||||
# RootNode application metrics — see docs/OBSERVABILITY.md §"exposing /metrics" for the
|
||||
# documented, NOT-YET-BUILT follow-up (Platform API/Gateway/Provisioner don't currently
|
||||
# expose a Prometheus /metrics endpoint; this job is ready for when that lands, disabled
|
||||
# until then rather than scraping a 404 forever).
|
||||
# - job_name: rootnode-platform-api
|
||||
# static_configs:
|
||||
# - targets: ["api:4300"]
|
||||
# metrics_path: /metrics
|
||||
@@ -0,0 +1,97 @@
|
||||
# Alert rules (Phase 4 §16). CRITICAL pages immediately; WARNING is for daytime attention.
|
||||
# Deliberately does NOT alert on individual student scenario failures or validation
|
||||
# failures — those are instructional signal (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md's Needs
|
||||
# Attention view), not operational incidents.
|
||||
|
||||
groups:
|
||||
- name: rootnode-critical
|
||||
rules:
|
||||
- alert: PlatformAPIDown
|
||||
expr: probe_success{job="blackbox-platform-api"} == 0
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "Platform API is unreachable"
|
||||
description: "/healthz on rn-platform-01's Platform API has failed for 2+ minutes."
|
||||
|
||||
- alert: GatewayDown
|
||||
expr: probe_success{job="blackbox-gateway"} == 0
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "Gateway is unreachable"
|
||||
description: "The Gateway's /healthz has failed for 2+ minutes — students cannot reach any Lab."
|
||||
|
||||
- alert: PostgresDown
|
||||
expr: pg_up == 0
|
||||
for: 1m
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "Postgres is down"
|
||||
description: "postgres-exporter cannot reach rootnode_platform Postgres."
|
||||
|
||||
- alert: LabHostDown
|
||||
expr: up{job="lab-host-node"} == 0
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "rn-lab-host-01 is unreachable"
|
||||
description: "node-exporter on the Lab Host VM has stopped responding — no student can launch or use a Lab."
|
||||
|
||||
- alert: BackupFailed
|
||||
expr: rootnode_backup_last_success_timestamp_seconds < (time() - 90000)
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "Postgres backup hasn't succeeded in 25+ hours"
|
||||
description: "infra/backup/backup-postgres.sh's last successful run is stale — see docs/RUNBOOK_BACKUP_FAILURE.md."
|
||||
|
||||
- alert: DiskCriticallyFull
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
|
||||
for: 5m
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "Root filesystem below 10% free on {{ $labels.host }}"
|
||||
|
||||
- name: rootnode-warning
|
||||
rules:
|
||||
- alert: LabHostHighMemory
|
||||
expr: (1 - (node_memory_MemAvailable_bytes{job="lab-host-node"} / node_memory_MemTotal_bytes{job="lab-host-node"})) > 0.85
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "rn-lab-host-01 memory usage above 85% for 10+ minutes"
|
||||
description: "Check active Lab count against docs/CAPACITY_PLANNING.md's admission thresholds."
|
||||
|
||||
- alert: HighDiskUsage
|
||||
expr: (1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) > 0.80
|
||||
for: 15m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "Disk usage above 80% on {{ $labels.host }}"
|
||||
|
||||
- alert: RepeatedProvisioningFailures
|
||||
expr: increase(rootnode_provisioning_failures_total[15m]) >= 3
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "3+ Lab provisioning failures in 15 minutes"
|
||||
description: >-
|
||||
Depends on the Provisioner exposing this counter — see
|
||||
docs/OBSERVABILITY.md's "application metrics — not yet built" note. Placeholder rule,
|
||||
inert until that counter exists.
|
||||
|
||||
- alert: BackupAging
|
||||
expr: rootnode_backup_last_success_timestamp_seconds < (time() - 43200)
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "Postgres backup hasn't succeeded in 12+ hours"
|
||||
|
||||
- alert: ExcessiveFailedLogins
|
||||
expr: increase(rootnode_audit_login_failures_total[5m]) > 20
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "20+ failed instructor logins in 5 minutes"
|
||||
description: >-
|
||||
Depends on exporting audit_events counts — see docs/OBSERVABILITY.md's
|
||||
"application metrics — not yet built" note. Placeholder rule, inert until built;
|
||||
the audit_events table itself already exists (Phase 3A) and can be queried
|
||||
manually via /api/admin/audit-events in the meantime.
|
||||
Reference in New Issue
Block a user