feat(reaper): add idle environment cleanup

Background loop (src/reaper.js), config-driven idle/abandoned thresholds and
poll interval (no hard-coded numbers) — without this, "ephemeral by default"
is just a description: a closed browser tab does not stop a running k3s
cluster on its own.

Past the idle threshold: logs a warning event only. Past the abandoned
threshold: destroys via the same path DELETE /environments/:id uses, so
cleanup is identical either way, and logs event_type='reaped_idle'.
Failure-safe — one environment's teardown throwing never stops the reaper
from checking the rest, and a Postgres/Docker hiccup on one tick just means
"try again next tick," never a crashed process.

Uses COALESCE(last_active_at, created_at) rather than bare last_active_at.
Found during testing: an environment that fails during provisioning (before
ever reaching 'running') has a NULL last_active_at, and NULL < anything is
never true in SQL — without the fallback, a failed-and-never-cleaned-up
environment would never have aged out here regardless of how long it sat
there. This is the second, independent safety net on top of
provisionEnvironment's own best-effort cleanup on failure (see the
platform-provisioner commit).

Verified end-to-end: backdated an environment's last_active_at past the
abandoned threshold, waited for a real reaper tick, confirmed full
deterministic cleanup (container, network, firewall chain all gone) with the
correct actor='reaper' attribution in environment_events.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
2026-08-18 01:22:19 -05:00
co-authored by Claude Sonnet 5
parent 0116ee1522
commit 9359abc9cb
+71
View File
@@ -0,0 +1,71 @@
// Idle reaper — requirement #10. Without this, "ephemeral by default" is just a description:
// a closed browser tab does not stop a running k3s cluster on its own.
//
// Two thresholds, both config-driven (never hard-coded):
// - idleThresholdMinutes — past this, log a warning event only (future: notify student).
// - abandonedThresholdMinutes — past this, destroy for real via the same path DELETE
// /environments/:id uses, so cleanup is identical either way.
//
// Uses COALESCE(last_active_at, created_at): an environment that never reached 'running' has
// a NULL last_active_at, and `NULL < ...` is never true in SQL — without this fallback, an
// environment stuck in 'error' from a failed provision would never age out here at all. This
// is a second, independent safety net on top of provisionEnvironment's own best-effort cleanup
// on failure (lifecycle.js) — found by testing, not designed in from the start.
//
// Failure-safe: one environment's teardown throwing never stops the reaper from checking the
// rest, and a Postgres/Docker hiccup on one tick just means "try again next tick" rather than
// crashing the process.
const { destroyEnvironment, emitEvent } = require('./lifecycle');
function startReaper({ pool, idleThresholdMinutes, abandonedThresholdMinutes, pollIntervalSeconds }) {
let stopped = false;
let timer = null;
async function tick() {
if (stopped) return;
try {
const idleRows = await pool.query(
`SELECT environment_id FROM environments
WHERE status = 'running'
AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval
AND COALESCE(last_active_at, created_at) >= now() - ($2 || ' minutes')::interval`,
[String(idleThresholdMinutes), String(abandonedThresholdMinutes)]
);
for (const row of idleRows.rows) {
await emitEvent(pool, row.environment_id, 'idle_warning', 'reaper', { idleThresholdMinutes }).catch(() => {});
}
const abandonedRows = await pool.query(
`SELECT environment_id FROM environments
WHERE status IN ('running', 'error')
AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval`,
[String(abandonedThresholdMinutes)]
);
for (const row of abandonedRows.rows) {
try {
await destroyEnvironment({ pool, environmentId: row.environment_id, actor: 'reaper' });
await emitEvent(pool, row.environment_id, 'reaped_idle', 'reaper', { abandonedThresholdMinutes });
} catch (e) {
// One bad environment must not stop the reaper from handling the rest.
await emitEvent(pool, row.environment_id, 'error', 'reaper', { stage: 'reap', message: e.message }).catch(() => {});
}
}
} catch (e) {
// Postgres/Docker unreachable this tick — try again next tick, don't crash the process.
console.error('[reaper] tick failed:', e.message);
} finally {
if (!stopped) timer = setTimeout(tick, pollIntervalSeconds * 1000);
}
}
timer = setTimeout(tick, 0);
return {
stop() {
stopped = true;
if (timer) clearTimeout(timer);
},
};
}
module.exports = { startReaper };