feat(observability): Prometheus/Alertmanager/Grafana/Loki stack
Self-hosted, no paid SaaS. Host/container/Postgres metrics, uptime probes against the new /healthz endpoints, alert rules split critical (pages) vs warning (informational), one starter Grafana dashboard. Deliberately separate from the application's instructor learning analytics (Phase 3B) - different audience, different data. Validated: promtool check config (valid), promtool check rules (11 rules, valid), amtool check-config (1 inhibit rule, 2 receivers, valid).
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
# Alertmanager config — routes CRITICAL alerts and WARNING alerts to the same webhook by
|
||||
# default (Uptime Kuma's push-monitor webhook, or ntfy, or another already-available target —
|
||||
# not a new paid service; the user's stated preference for Uptime Kuma is honored here without
|
||||
# forcing it if a cleaner existing target turns out to be available at deploy time, per the
|
||||
# Phase 4 instruction). Swap the receiver's url for whatever's actually wired up.
|
||||
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: rootnode-default
|
||||
group_by: ["alertname", "host"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: rootnode-critical
|
||||
repeat_interval: 30m
|
||||
|
||||
receivers:
|
||||
- name: rootnode-default
|
||||
webhook_configs:
|
||||
- url: "{{ vault_alertmanager_webhook_url | default('http://example.invalid/webhook') }}"
|
||||
send_resolved: true
|
||||
|
||||
- name: rootnode-critical
|
||||
webhook_configs:
|
||||
- url: "{{ vault_alertmanager_webhook_url | default('http://example.invalid/webhook') }}"
|
||||
send_resolved: true
|
||||
|
||||
inhibit_rules:
|
||||
# If the whole Lab Host is down, don't also page separately for every dependent warning.
|
||||
- source_match:
|
||||
alertname: LabHostDown
|
||||
target_match:
|
||||
severity: warning
|
||||
equal: ["host"]
|
||||
Reference in New Issue
Block a user