feat(observability): notification delivery metrics, no PII/high-cardinality labels
- rootnode_notifications_total{notification_type,status} (computed fresh from the DB, like
the existing enrollment_breakdown gauge — never drifts from what's actually stored),
rootnode_notification_queue_depth, rootnode_notification_oldest_queued_seconds,
rootnode_notification_delivery_attempts_total{provider,result} (a real in-process counter —
per-attempt outcomes aren't reconstructable from the DB after the fact),
rootnode_student_activations_{success,invalid}_total. Every label is a bounded enum —
never student_id/email/token/notification_id/external_event_id.
This commit is contained in:
@@ -25,6 +25,22 @@ function recordEnrollmentEvent(result) {
|
||||
if (result in enrollmentEventTotals) enrollmentEventTotals[result]++;
|
||||
}
|
||||
|
||||
// Phase 5C §18 — per-attempt delivery outcomes (a real counter, since "how many send attempts
|
||||
// happened" is an ephemeral event count, not something a DB GROUP BY can reconstruct after the
|
||||
// fact the way notifications_total below can). `provider` is 'dev'|'gws' (bounded), `result` is
|
||||
// 'sent'|'retry'|'failed' (bounded) — never a notification_id/student_id/provider message id.
|
||||
const notificationAttemptTotals = {};
|
||||
function recordNotificationAttempt(provider, result) {
|
||||
const key = `${provider}|${result}`;
|
||||
notificationAttemptTotals[key] = (notificationAttemptTotals[key] || 0) + 1;
|
||||
}
|
||||
|
||||
// student.activate outcomes — 'success' | 'invalid_or_expired_token'. Bounded, no student id.
|
||||
const studentActivationTotals = {};
|
||||
function recordStudentActivation(result) {
|
||||
studentActivationTotals[result] = (studentActivationTotals[result] || 0) + 1;
|
||||
}
|
||||
|
||||
// Gauges are computed fresh at scrape time from Postgres/the Provisioner — cheap enough at a
|
||||
// normal 15-30s Prometheus scrape interval, and always exactly correct (no separate value to
|
||||
// keep in sync with reality).
|
||||
@@ -58,6 +74,15 @@ async function collectGauges(pool, provisioner) {
|
||||
pool.query(`SELECT program_id, count(*)::int AS n FROM enrollments WHERE status = 'completed' GROUP BY program_id`),
|
||||
]);
|
||||
|
||||
// Phase 5C — notification lifecycle, computed fresh from the DB (like enrollment_breakdown
|
||||
// above) rather than an in-process counter, so "how many notifications are in each
|
||||
// type/status right now" can never drift from what's actually stored.
|
||||
const [{ rows: notificationBreakdown }, { rows: queueStats }] = await Promise.all([
|
||||
pool.query(`SELECT notification_type, status, count(*)::int AS n FROM notification_events GROUP BY notification_type, status`),
|
||||
pool.query(`SELECT count(*)::int AS depth, COALESCE(extract(epoch FROM now() - min(created_at)), 0)::int AS oldest_seconds
|
||||
FROM notification_events WHERE status = 'queued'`),
|
||||
]);
|
||||
|
||||
return {
|
||||
classroom_sessions_live: liveClassrooms.length,
|
||||
classroom_participants_online: onlineParticipants,
|
||||
@@ -68,6 +93,9 @@ async function collectGauges(pool, provisioner) {
|
||||
enrollment_breakdown: enrollmentBreakdown,
|
||||
selfpaced_active_enrollments: selfPacedActive[0].n,
|
||||
completions_by_program: completionsByProgram,
|
||||
notification_breakdown: notificationBreakdown,
|
||||
notification_queue_depth: queueStats[0].depth,
|
||||
notification_oldest_queued_seconds: queueStats[0].oldest_seconds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -85,6 +113,20 @@ function labeledGauge(name, help, rows, labelCols, valueCol) {
|
||||
return header + body + '\n';
|
||||
}
|
||||
|
||||
// Same idea as labeledGauge, for the flat "label-key -> count" dicts the in-process counters
|
||||
// above use (e.g. notificationAttemptTotals's `${provider}|${result}` keys).
|
||||
function labeledCounter(name, help, totals, labelNames) {
|
||||
const header = `# HELP ${name} ${help}\n# TYPE ${name} counter\n`;
|
||||
const keys = Object.keys(totals);
|
||||
if (!keys.length) return header;
|
||||
const body = keys.map((key) => {
|
||||
const values = key.split('|');
|
||||
const labels = labelNames.map((n, i) => `${n}="${values[i].replace(/"/g, '\\"')}"`).join(',');
|
||||
return `${name}{${labels}} ${totals[key]}`;
|
||||
}).join('\n');
|
||||
return header + body + '\n';
|
||||
}
|
||||
|
||||
async function renderMetrics(pool, provisioner) {
|
||||
const gauges = await collectGauges(pool, provisioner);
|
||||
return [
|
||||
@@ -102,7 +144,16 @@ async function renderMetrics(pool, provisioner) {
|
||||
line('rootnode_enrollment_events_created_total', 'Integration enrollment events that created a new enrollment.', 'counter', enrollmentEventTotals.created),
|
||||
line('rootnode_enrollment_events_duplicate_total', 'Integration enrollment events that were idempotent replays of an already-processed event.', 'counter', enrollmentEventTotals.duplicate),
|
||||
line('rootnode_enrollment_events_rejected_total', 'Integration enrollment events rejected (bad auth, unknown program, invalid payload).', 'counter', enrollmentEventTotals.rejected),
|
||||
labeledGauge('rootnode_notifications_total', 'Number of notification rows by type and lifecycle status.', gauges.notification_breakdown, ['notification_type', 'status'], 'n'),
|
||||
line('rootnode_notification_queue_depth', 'Number of notifications currently queued for delivery.', 'gauge', gauges.notification_queue_depth),
|
||||
line('rootnode_notification_oldest_queued_seconds', 'Age in seconds of the oldest still-queued notification (0 if none queued).', 'gauge', gauges.notification_oldest_queued_seconds),
|
||||
labeledCounter('rootnode_notification_delivery_attempts_total', 'Notification delivery attempts by provider and outcome.', notificationAttemptTotals, ['provider', 'result']),
|
||||
line('rootnode_student_activations_success_total', 'Student accounts successfully activated.', 'counter', studentActivationTotals.success || 0),
|
||||
line('rootnode_student_activations_invalid_total', 'Activation attempts with an invalid or expired token.', 'counter', studentActivationTotals.invalid_or_expired_token || 0),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
module.exports = { recordStudentLoginSuccess, recordStudentLoginFailure, recordEnrollmentEvent, renderMetrics };
|
||||
module.exports = {
|
||||
recordStudentLoginSuccess, recordStudentLoginFailure, recordEnrollmentEvent,
|
||||
recordNotificationAttempt, recordStudentActivation, renderMetrics,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user