From 9359abc9cbc1d7e3192f57c5a94d67ec2b8ddf15 Mon Sep 17 00:00:00 2001
From: ced4568 <chasedumphord@gmail.com>
Date: Tue, 18 Aug 2026 01:22:19 -0500
Subject: [PATCH] feat(reaper): add idle environment cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background loop (src/reaper.js), config-driven idle/abandoned thresholds and
poll interval (no hard-coded numbers) — without this, "ephemeral by default"
is just a description: a closed browser tab does not stop a running k3s
cluster on its own.

Past the idle threshold: logs a warning event only. Past the abandoned
threshold: destroys via the same path DELETE /environments/:id uses, so
cleanup is identical either way, and logs event_type='reaped_idle'.
Failure-safe — one environment's teardown throwing never stops the reaper
from checking the rest, and a Postgres/Docker hiccup on one tick just means
"try again next tick," never a crashed process.

Uses COALESCE(last_active_at, created_at) rather than bare last_active_at.
Found during testing: an environment that fails during provisioning (before
ever reaching 'running') has a NULL last_active_at, and NULL < anything is
never true in SQL — without the fallback, a failed-and-never-cleaned-up
environment would never have aged out here regardless of how long it sat
there. This is the second, independent safety net on top of
provisionEnvironment's own best-effort cleanup on failure (see the
platform-provisioner commit).

Verified end-to-end: backdated an environment's last_active_at past the
abandoned threshold, waited for a real reaper tick, confirmed full
deterministic cleanup (container, network, firewall chain all gone) with the
correct actor='reaper' attribution in environment_events.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 platform/provisioner/src/reaper.js | 71 ++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 platform/provisioner/src/reaper.js

diff --git a/platform/provisioner/src/reaper.js b/platform/provisioner/src/reaper.js
new file mode 100644
index 0000000..c3fcf76
--- /dev/null
+++ b/platform/provisioner/src/reaper.js
@@ -0,0 +1,71 @@
+// Idle reaper — requirement #10. Without this, "ephemeral by default" is just a description:
+// a closed browser tab does not stop a running k3s cluster on its own.
+//
+// Two thresholds, both config-driven (never hard-coded):
+//  - idleThresholdMinutes  — past this, log a warning event only (future: notify student).
+//  - abandonedThresholdMinutes — past this, destroy for real via the same path DELETE
+//    /environments/:id uses, so cleanup is identical either way.
+//
+// Uses COALESCE(last_active_at, created_at): an environment that never reached 'running' has
+// a NULL last_active_at, and `NULL < ...` is never true in SQL — without this fallback, an
+// environment stuck in 'error' from a failed provision would never age out here at all. This
+// is a second, independent safety net on top of provisionEnvironment's own best-effort cleanup
+// on failure (lifecycle.js) — found by testing, not designed in from the start.
+//
+// Failure-safe: one environment's teardown throwing never stops the reaper from checking the
+// rest, and a Postgres/Docker hiccup on one tick just means "try again next tick" rather than
+// crashing the process.
+
+const { destroyEnvironment, emitEvent } = require('./lifecycle');
+
+function startReaper({ pool, idleThresholdMinutes, abandonedThresholdMinutes, pollIntervalSeconds }) {
+  let stopped = false;
+  let timer = null;
+
+  async function tick() {
+    if (stopped) return;
+    try {
+      const idleRows = await pool.query(
+        `SELECT environment_id FROM environments
+         WHERE status = 'running'
+           AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval
+           AND COALESCE(last_active_at, created_at) >= now() - ($2 || ' minutes')::interval`,
+        [String(idleThresholdMinutes), String(abandonedThresholdMinutes)]
+      );
+      for (const row of idleRows.rows) {
+        await emitEvent(pool, row.environment_id, 'idle_warning', 'reaper', { idleThresholdMinutes }).catch(() => {});
+      }
+
+      const abandonedRows = await pool.query(
+        `SELECT environment_id FROM environments
+         WHERE status IN ('running', 'error')
+           AND COALESCE(last_active_at, created_at) < now() - ($1 || ' minutes')::interval`,
+        [String(abandonedThresholdMinutes)]
+      );
+      for (const row of abandonedRows.rows) {
+        try {
+          await destroyEnvironment({ pool, environmentId: row.environment_id, actor: 'reaper' });
+          await emitEvent(pool, row.environment_id, 'reaped_idle', 'reaper', { abandonedThresholdMinutes });
+        } catch (e) {
+          // One bad environment must not stop the reaper from handling the rest.
+          await emitEvent(pool, row.environment_id, 'error', 'reaper', { stage: 'reap', message: e.message }).catch(() => {});
+        }
+      }
+    } catch (e) {
+      // Postgres/Docker unreachable this tick — try again next tick, don't crash the process.
+      console.error('[reaper] tick failed:', e.message);
+    } finally {
+      if (!stopped) timer = setTimeout(tick, pollIntervalSeconds * 1000);
+    }
+  }
+
+  timer = setTimeout(tick, 0);
+  return {
+    stop() {
+      stopped = true;
+      if (timer) clearTimeout(timer);
+    },
+  };
+}
+
+module.exports = { startReaper };