feat(observability): notification delivery metrics, no PII/high-cardinality labels

- rootnode_notifications_total{notification_type,status} (computed fresh from the DB, like
  the existing enrollment_breakdown gauge — never drifts from what's actually stored),
  rootnode_notification_queue_depth, rootnode_notification_oldest_queued_seconds,
  rootnode_notification_delivery_attempts_total{provider,result} (a real in-process counter —
  per-attempt outcomes aren't reconstructable from the DB after the fact),
  rootnode_student_activations_{success,invalid}_total. Every label is a bounded enum —
  never student_id/email/token/notification_id/external_event_id.
This commit is contained in:
2026-08-25 14:43:48 -05:00
parent f25535027c
commit be92be81cd
+52 -1
View File
@@ -25,6 +25,22 @@ function recordEnrollmentEvent(result) {
if (result in enrollmentEventTotals) enrollmentEventTotals[result]++;
}
// Phase 5C §18 — per-attempt delivery outcomes (a real counter, since "how many send attempts
// happened" is an ephemeral event count, not something a DB GROUP BY can reconstruct after the
// fact the way notifications_total below can). `provider` is 'dev'|'gws' (bounded), `result` is
// 'sent'|'retry'|'failed' (bounded) — never a notification_id/student_id/provider message id.
const notificationAttemptTotals = {};
function recordNotificationAttempt(provider, result) {
const key = `${provider}|${result}`;
notificationAttemptTotals[key] = (notificationAttemptTotals[key] || 0) + 1;
}
// student.activate outcomes — 'success' | 'invalid_or_expired_token'. Bounded, no student id.
const studentActivationTotals = {};
function recordStudentActivation(result) {
studentActivationTotals[result] = (studentActivationTotals[result] || 0) + 1;
}
// Gauges are computed fresh at scrape time from Postgres/the Provisioner — cheap enough at a
// normal 15-30s Prometheus scrape interval, and always exactly correct (no separate value to
// keep in sync with reality).
@@ -58,6 +74,15 @@ async function collectGauges(pool, provisioner) {
pool.query(`SELECT program_id, count(*)::int AS n FROM enrollments WHERE status = 'completed' GROUP BY program_id`),
]);
// Phase 5C — notification lifecycle, computed fresh from the DB (like enrollment_breakdown
// above) rather than an in-process counter, so "how many notifications are in each
// type/status right now" can never drift from what's actually stored.
const [{ rows: notificationBreakdown }, { rows: queueStats }] = await Promise.all([
pool.query(`SELECT notification_type, status, count(*)::int AS n FROM notification_events GROUP BY notification_type, status`),
pool.query(`SELECT count(*)::int AS depth, COALESCE(extract(epoch FROM now() - min(created_at)), 0)::int AS oldest_seconds
FROM notification_events WHERE status = 'queued'`),
]);
return {
classroom_sessions_live: liveClassrooms.length,
classroom_participants_online: onlineParticipants,
@@ -68,6 +93,9 @@ async function collectGauges(pool, provisioner) {
enrollment_breakdown: enrollmentBreakdown,
selfpaced_active_enrollments: selfPacedActive[0].n,
completions_by_program: completionsByProgram,
notification_breakdown: notificationBreakdown,
notification_queue_depth: queueStats[0].depth,
notification_oldest_queued_seconds: queueStats[0].oldest_seconds,
};
}
@@ -85,6 +113,20 @@ function labeledGauge(name, help, rows, labelCols, valueCol) {
return header + body + '\n';
}
// Same idea as labeledGauge, for the flat "label-key -> count" dicts the in-process counters
// above use (e.g. notificationAttemptTotals's `${provider}|${result}` keys).
function labeledCounter(name, help, totals, labelNames) {
const header = `# HELP ${name} ${help}\n# TYPE ${name} counter\n`;
const keys = Object.keys(totals);
if (!keys.length) return header;
const body = keys.map((key) => {
const values = key.split('|');
const labels = labelNames.map((n, i) => `${n}="${values[i].replace(/"/g, '\\"')}"`).join(',');
return `${name}{${labels}} ${totals[key]}`;
}).join('\n');
return header + body + '\n';
}
async function renderMetrics(pool, provisioner) {
const gauges = await collectGauges(pool, provisioner);
return [
@@ -102,7 +144,16 @@ async function renderMetrics(pool, provisioner) {
line('rootnode_enrollment_events_created_total', 'Integration enrollment events that created a new enrollment.', 'counter', enrollmentEventTotals.created),
line('rootnode_enrollment_events_duplicate_total', 'Integration enrollment events that were idempotent replays of an already-processed event.', 'counter', enrollmentEventTotals.duplicate),
line('rootnode_enrollment_events_rejected_total', 'Integration enrollment events rejected (bad auth, unknown program, invalid payload).', 'counter', enrollmentEventTotals.rejected),
labeledGauge('rootnode_notifications_total', 'Number of notification rows by type and lifecycle status.', gauges.notification_breakdown, ['notification_type', 'status'], 'n'),
line('rootnode_notification_queue_depth', 'Number of notifications currently queued for delivery.', 'gauge', gauges.notification_queue_depth),
line('rootnode_notification_oldest_queued_seconds', 'Age in seconds of the oldest still-queued notification (0 if none queued).', 'gauge', gauges.notification_oldest_queued_seconds),
labeledCounter('rootnode_notification_delivery_attempts_total', 'Notification delivery attempts by provider and outcome.', notificationAttemptTotals, ['provider', 'result']),
line('rootnode_student_activations_success_total', 'Student accounts successfully activated.', 'counter', studentActivationTotals.success || 0),
line('rootnode_student_activations_invalid_total', 'Activation attempts with an invalid or expired token.', 'counter', studentActivationTotals.invalid_or_expired_token || 0),
].join('\n');
}
module.exports = { recordStudentLoginSuccess, recordStudentLoginFailure, recordEnrollmentEvent, renderMetrics };
module.exports = {
recordStudentLoginSuccess, recordStudentLoginFailure, recordEnrollmentEvent,
recordNotificationAttempt, recordStudentActivation, renderMetrics,
};