ced4568
4e8fa1d413
Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
98 lines
4.1 KiB
YAML
98 lines
4.1 KiB
YAML
# Alert rules (Phase 4 §16). CRITICAL pages immediately; WARNING is for daytime attention.
|
|
# Deliberately does NOT alert on individual student scenario failures or validation
|
|
# failures — those are instructional signal (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md's Needs
|
|
# Attention view), not operational incidents.
|
|
|
|
groups:
|
|
- name: rootnode-critical
|
|
rules:
|
|
- alert: PlatformAPIDown
|
|
expr: probe_success{job="blackbox-platform-api"} == 0
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: "Platform API is unreachable"
|
|
description: "/healthz on rn-platform-01's Platform API has failed for 2+ minutes."
|
|
|
|
- alert: GatewayDown
|
|
expr: probe_success{job="blackbox-gateway"} == 0
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: "Gateway is unreachable"
|
|
description: "The Gateway's /healthz has failed for 2+ minutes — students cannot reach any Lab."
|
|
|
|
- alert: PostgresDown
|
|
expr: pg_up == 0
|
|
for: 1m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: "Postgres is down"
|
|
description: "postgres-exporter cannot reach rootnode_platform Postgres."
|
|
|
|
- alert: LabHostDown
|
|
expr: up{job="lab-host-node"} == 0
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: "rn-lab-host-01 is unreachable"
|
|
description: "node-exporter on the Lab Host VM has stopped responding — no student can launch or use a Lab."
|
|
|
|
- alert: BackupFailed
|
|
expr: rootnode_backup_last_success_timestamp_seconds < (time() - 90000)
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: "Postgres backup hasn't succeeded in 25+ hours"
|
|
description: "infra/backup/backup-postgres.sh's last successful run is stale — see docs/RUNBOOK_BACKUP_FAILURE.md."
|
|
|
|
- alert: DiskCriticallyFull
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
|
|
for: 5m
|
|
labels: { severity: critical }
|
|
annotations:
|
|
summary: "Root filesystem below 10% free on {{ $labels.host }}"
|
|
|
|
- name: rootnode-warning
|
|
rules:
|
|
- alert: LabHostHighMemory
|
|
expr: (1 - (node_memory_MemAvailable_bytes{job="lab-host-node"} / node_memory_MemTotal_bytes{job="lab-host-node"})) > 0.85
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: "rn-lab-host-01 memory usage above 85% for 10+ minutes"
|
|
description: "Check active Lab count against docs/CAPACITY_PLANNING.md's admission thresholds."
|
|
|
|
- alert: HighDiskUsage
|
|
expr: (1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) > 0.80
|
|
for: 15m
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: "Disk usage above 80% on {{ $labels.host }}"
|
|
|
|
- alert: RepeatedProvisioningFailures
|
|
expr: increase(rootnode_provisioning_failures_total[15m]) >= 3
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: "3+ Lab provisioning failures in 15 minutes"
|
|
description: >-
|
|
Depends on the Provisioner exposing this counter — see
|
|
docs/OBSERVABILITY.md's "application metrics — not yet built" note. Placeholder rule,
|
|
inert until that counter exists.
|
|
|
|
- alert: BackupAging
|
|
expr: rootnode_backup_last_success_timestamp_seconds < (time() - 43200)
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: "Postgres backup hasn't succeeded in 12+ hours"
|
|
|
|
- alert: ExcessiveFailedLogins
|
|
expr: increase(rootnode_audit_login_failures_total[5m]) > 20
|
|
labels: { severity: warning }
|
|
annotations:
|
|
summary: "20+ failed instructor logins in 5 minutes"
|
|
description: >-
|
|
Depends on exporting audit_events counts — see docs/OBSERVABILITY.md's
|
|
"application metrics — not yet built" note. Placeholder rule, inert until built;
|
|
the audit_events table itself already exists (Phase 3A) and can be queried
|
|
manually via /api/admin/audit-events in the meantime.
|