feat(reaper): add idle environment cleanup
Background loop (src/reaper.js), config-driven idle/abandoned thresholds and poll interval (no hard-coded numbers) — without this, "ephemeral by default" is just a description: a closed browser tab does not stop a running k3s cluster on its own. Past the idle threshold: logs a warning event only. Past the abandoned threshold: destroys via the same path DELETE /environments/:id uses, so cleanup is identical either way, and logs event_type='reaped_idle'. Failure-safe — one environment's teardown throwing never stops the reaper from checking the rest, and a Postgres/Docker hiccup on one tick just means "try again next tick," never a crashed process. Uses COALESCE(last_active_at, created_at) rather than bare last_active_at. Found during testing: an environment that fails during provisioning (before ever reaching 'running') has a NULL last_active_at, and NULL < anything is never true in SQL — without the fallback, a failed-and-never-cleaned-up environment would never have aged out here regardless of how long it sat there. This is the second, independent safety net on top of provisionEnvironment's own best-effort cleanup on failure (see the platform-provisioner commit). Verified end-to-end: backdated an environment's last_active_at past the abandoned threshold, waited for a real reaper tick, confirmed full deterministic cleanup (container, network, firewall chain all gone) with the correct actor='reaper' attribution in environment_events. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
// Idle reaper — requirement #10. Without this, "ephemeral by default" is just a description:
|
||||
// a closed browser tab does not stop a running k3s cluster on its own.
|
||||
//
|
||||
// Two thresholds, both config-driven (never hard-coded):
|
||||
// - idleThresholdMinutes — past this, log a warning event only (future: notify student).
|
||||
// - abandonedThresholdMinutes — past this, destroy for real via the same path DELETE
|
||||
// /environments/:id uses, so cleanup is identical either way.
|
||||
//
|
||||
// Uses COALESCE(last_active_at, created_at): an environment that never reached 'running' has
|
||||
// a NULL last_active_at, and `NULL < ...` is never true in SQL — without this fallback, an
|
||||
// environment stuck in 'error' from a failed provision would never age out here at all. This
|
||||
// is a second, independent safety net on top of provisionEnvironment's own best-effort cleanup
|
||||
// on failure (lifecycle.js) — found by testing, not designed in from the start.
|
||||
//
|
||||
// Failure-safe: one environment's teardown throwing never stops the reaper from checking the
|
||||
// rest, and a Postgres/Docker hiccup on one tick just means "try again next tick" rather than
|
||||
// crashing the process.
|
||||
|
||||
const { destroyEnvironment, emitEvent } = require('./lifecycle');
|
||||
|
||||
function startReaper({ pool, idleThresholdMinutes, abandonedThresholdMinutes, pollIntervalSeconds }) {
|
||||
let stopped = false;
|
||||
let timer = null;
|
||||
|
||||
async function tick() {
|
||||
if (stopped) return;
|
||||
try {
|
||||
const idleRows = await pool.query(
|
||||
`SELECT environment_id FROM environments
|
||||
WHERE status = 'running'
|
||||
AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval
|
||||
AND COALESCE(last_active_at, created_at) >= now() - ($2 || ' minutes')::interval`,
|
||||
[String(idleThresholdMinutes), String(abandonedThresholdMinutes)]
|
||||
);
|
||||
for (const row of idleRows.rows) {
|
||||
await emitEvent(pool, row.environment_id, 'idle_warning', 'reaper', { idleThresholdMinutes }).catch(() => {});
|
||||
}
|
||||
|
||||
const abandonedRows = await pool.query(
|
||||
`SELECT environment_id FROM environments
|
||||
WHERE status IN ('running', 'error')
|
||||
AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval`,
|
||||
[String(abandonedThresholdMinutes)]
|
||||
);
|
||||
for (const row of abandonedRows.rows) {
|
||||
try {
|
||||
await destroyEnvironment({ pool, environmentId: row.environment_id, actor: 'reaper' });
|
||||
await emitEvent(pool, row.environment_id, 'reaped_idle', 'reaper', { abandonedThresholdMinutes });
|
||||
} catch (e) {
|
||||
// One bad environment must not stop the reaper from handling the rest.
|
||||
await emitEvent(pool, row.environment_id, 'error', 'reaper', { stage: 'reap', message: e.message }).catch(() => {});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Postgres/Docker unreachable this tick — try again next tick, don't crash the process.
|
||||
console.error('[reaper] tick failed:', e.message);
|
||||
} finally {
|
||||
if (!stopped) timer = setTimeout(tick, pollIntervalSeconds * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
timer = setTimeout(tick, 0);
|
||||
return {
|
||||
stop() {
|
||||
stopped = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { startReaper };
|
||||
Reference in New Issue
Block a user