# Alert rules (Phase 4 §16). CRITICAL pages immediately; WARNING is for daytime attention.
# Deliberately does NOT alert on individual student scenario failures or validation
# failures — those are instructional signal (docs/PHASE3B_LEARNING_AND_ASSESSMENT.md's Needs
# Attention view), not operational incidents.

groups:
  - name: rootnode-critical
    rules:
      - alert: PlatformAPIDown
        expr: probe_success{job="blackbox-platform-api"} == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "Platform API is unreachable"
          description: "/healthz on rn-platform-01's Platform API has failed for 2+ minutes."

      - alert: GatewayDown
        expr: probe_success{job="blackbox-gateway"} == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "Gateway is unreachable"
          description: "The Gateway's /healthz has failed for 2+ minutes — students cannot reach any Lab."

      - alert: PostgresDown
        expr: pg_up == 0
        for: 1m
        labels: { severity: critical }
        annotations:
          summary: "Postgres is down"
          description: "postgres-exporter cannot reach rootnode_platform Postgres."

      - alert: LabHostDown
        expr: up{job="lab-host-node"} == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "rn-lab-host-01 is unreachable"
          description: "node-exporter on the Lab Host VM has stopped responding — no student can launch or use a Lab."

      - alert: BackupFailed
        expr: rootnode_backup_last_success_timestamp_seconds < (time() - 90000)
        labels: { severity: critical }
        annotations:
          summary: "Postgres backup hasn't succeeded in 25+ hours"
          description: "infra/backup/backup-postgres.sh's last successful run is stale — see docs/RUNBOOK_BACKUP_FAILURE.md."

      - alert: DiskCriticallyFull
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
        for: 5m
        labels: { severity: critical }
        annotations:
          summary: "Root filesystem below 10% free on {{ $labels.host }}"

  - name: rootnode-warning
    rules:
      - alert: LabHostHighMemory
        expr: (1 - (node_memory_MemAvailable_bytes{job="lab-host-node"} / node_memory_MemTotal_bytes{job="lab-host-node"})) > 0.85
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "rn-lab-host-01 memory usage above 85% for 10+ minutes"
          description: "Check active Lab count against docs/CAPACITY_PLANNING.md's admission thresholds."

      - alert: HighDiskUsage
        expr: (1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) > 0.80
        for: 15m
        labels: { severity: warning }
        annotations:
          summary: "Disk usage above 80% on {{ $labels.host }}"

      - alert: RepeatedProvisioningFailures
        expr: increase(rootnode_provisioning_failures_total[15m]) >= 3
        labels: { severity: warning }
        annotations:
          summary: "3+ Lab provisioning failures in 15 minutes"
          description: >-
            Depends on the Provisioner exposing this counter — see
            docs/OBSERVABILITY.md's "application metrics — not yet built" note. Placeholder rule,
            inert until that counter exists.

      - alert: BackupAging
        expr: rootnode_backup_last_success_timestamp_seconds < (time() - 43200)
        labels: { severity: warning }
        annotations:
          summary: "Postgres backup hasn't succeeded in 12+ hours"

      - alert: ExcessiveFailedLogins
        expr: increase(rootnode_audit_login_failures_total[5m]) > 20
        labels: { severity: warning }
        annotations:
          summary: "20+ failed instructor logins in 5 minutes"
          description: >-
            Depends on exporting audit_events counts — see docs/OBSERVABILITY.md's
            "application metrics — not yet built" note. Placeholder rule, inert until built;
            the audit_events table itself already exists (Phase 3A) and can be queried
            manually via /api/admin/audit-events in the meantime.