diff --git a/platform/provisioner/src/reaper.js b/platform/provisioner/src/reaper.js new file mode 100644 index 0000000..c3fcf76 --- /dev/null +++ b/platform/provisioner/src/reaper.js @@ -0,0 +1,71 @@ +// Idle reaper — requirement #10. Without this, "ephemeral by default" is just a description: +// a closed browser tab does not stop a running k3s cluster on its own. +// +// Two thresholds, both config-driven (never hard-coded): +// - idleThresholdMinutes — past this, log a warning event only (future: notify student). +// - abandonedThresholdMinutes — past this, destroy for real via the same path DELETE +// /environments/:id uses, so cleanup is identical either way. +// +// Uses COALESCE(last_active_at, created_at): an environment that never reached 'running' has +// a NULL last_active_at, and `NULL < ...` is never true in SQL — without this fallback, an +// environment stuck in 'error' from a failed provision would never age out here at all. This +// is a second, independent safety net on top of provisionEnvironment's own best-effort cleanup +// on failure (lifecycle.js) — found by testing, not designed in from the start. +// +// Failure-safe: one environment's teardown throwing never stops the reaper from checking the +// rest, and a Postgres/Docker hiccup on one tick just means "try again next tick" rather than +// crashing the process. + +const { destroyEnvironment, emitEvent } = require('./lifecycle'); + +function startReaper({ pool, idleThresholdMinutes, abandonedThresholdMinutes, pollIntervalSeconds }) { + let stopped = false; + let timer = null; + + async function tick() { + if (stopped) return; + try { + const idleRows = await pool.query( + `SELECT environment_id FROM environments + WHERE status = 'running' + AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval + AND COALESCE(last_active_at, created_at) >= now() - ($2 || ' minutes')::interval`, + [String(idleThresholdMinutes), String(abandonedThresholdMinutes)] + ); + for (const row of idleRows.rows) { + await emitEvent(pool, row.environment_id, 'idle_warning', 'reaper', { idleThresholdMinutes }).catch(() => {}); + } + + const abandonedRows = await pool.query( + `SELECT environment_id FROM environments + WHERE status IN ('running', 'error') + AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval`, + [String(abandonedThresholdMinutes)] + ); + for (const row of abandonedRows.rows) { + try { + await destroyEnvironment({ pool, environmentId: row.environment_id, actor: 'reaper' }); + await emitEvent(pool, row.environment_id, 'reaped_idle', 'reaper', { abandonedThresholdMinutes }); + } catch (e) { + // One bad environment must not stop the reaper from handling the rest. + await emitEvent(pool, row.environment_id, 'error', 'reaper', { stage: 'reap', message: e.message }).catch(() => {}); + } + } + } catch (e) { + // Postgres/Docker unreachable this tick — try again next tick, don't crash the process. + console.error('[reaper] tick failed:', e.message); + } finally { + if (!stopped) timer = setTimeout(tick, pollIntervalSeconds * 1000); + } + } + + timer = setTimeout(tick, 0); + return { + stop() { + stopped = true; + if (timer) clearTimeout(timer); + }, + }; +} + +module.exports = { startReaper };