feat(engagement): retention — three sweeps and one recorded refusal

ENGAGEMENT.md Phase 14, the last phase of the workstream. Four engagement
tables grew on every fire and nothing had ever deleted from any of them.

Three of them now have a horizon, swept nightly by one worker
(utils/engagementRetentionPrune.js — setInterval + unref + stop(), batched
1000 x 50, each table's failure caught on its own so a lock timeout on one
does not leave the other two unbounded):

  engagement_sends      180 days   engagement_sends_retain_days      (7-3650)
  engagement_cooldowns   30 days   engagement_cooldowns_retain_days  (2-3650)
  engagement_outbox      30 days   engagement_outbox_retain_days     (2-3650)

The fourth, engagement_suppressions, does not expire, and that is the
recorded decision rather than an omission: a suppression is a standing
decision, and ageing out a hard bounce re-mails an address that already
bounced. The way out stays deliberate, and is now reachable per row.

Six decisions were settled by the org lead before any code. Two of them
widened the phase past what was offered:

  * the send-log horizon is admin-configurable, so retention got a SCREEN
    (Admin -> Engagement -> Retention) where team_activity and
    user_notifications keep theirs in invisible settings rows. The send-log
    horizon changes what an operator-facing page is able to show, so it has
    to be visible; the other two came with it, because "what does this
    deployment keep" is one question.
  * the suppression purge, which cost a Phase 9 decision. The list
    deliberately stripped address_hash from every row, so the only way out
    was a window.prompt asking the operator to retype an address the screen
    has never shown them. The row had no handle at all. The hash is now
    returned: this route is admin-only and an admin can already suppress and
    unsuppress any address they can name, so it grants no capability they
    lack. GET /sends still strips its own.

The outbox sweep is TERMINAL-ONLY and that is a correctness rule: a
scheduled row is a send this deployment still intends to make (delay_seconds
can put one a day out) and a sending row may be mid-flight.

One shipped defect had to be fixed for the sweep to be a bound at all.
reclaimStale returned every stale sending row to scheduled, and MAX_ATTEMPTS
is consulted only on a graceful retry outcome — so a send that killed the
process mid-flight cycled sending -> scheduled -> sending forever, never
terminal, therefore never eligible for any sweep. It now fails an exhausted
row BEFORE reclaiming the rest; the order is the fix.

Two indexes (idx_engo_sweep, idx_engs_sweep): every existing index on those
tables has created_at in second position, which serves a per-rule window and
is useless to a whole-table horizon.

Proved twice: engagementRetentionSql.test.js against a real MariaDB (7
tests, incl. the acceptance case and the wrong reclaim order run
deliberately), and the live stack, where a 90-day-old cancelled row was
swept and a 90-day-old scheduled row survived.

Co-Authored-By: Claude <noreply@anthropic.com>

server/src/utils/engagementRetentionPrune.js

@@ -0,0 +1,134 @@
+// ── Engagement retention worker ────────────────────────────────────────────
+//
+// ENGAGEMENT.md Phase 14. Three of the four engagement tables grow on every fire
+// and nothing has ever deleted from any of them: `engagement_cooldowns` (one row
+// per rule × user × subject × channel per fire), `engagement_outbox` (one row per
+// enqueued delivery, terminal rows included) and `engagement_sends` (one row per
+// attempt). `engagement_suppressions` is the fourth and does not expire — see
+// `engagementRetention.model.js` for why that is a decision rather than an
+// omission.
+//
+// **One worker, three sweeps, not three workers.** They share a timer, a batch
+// discipline and one settings-backed policy object; splitting them would give an
+// operator three independent nightly table-wide DELETEs to reason about and
+// three places for a horizon to be read differently.
+//
+// Same in-process shape as `teamActivityPrune` and `userNotificationsPrune` —
+// setInterval + unref + stop(), wired into server.js start/shutdown beside them,
+// with the first run delayed so a table-wide DELETE never lands in front of the
+// first request on a crash-looping deployment.
+
+const cooldownsDb = require('../model/engagement/engagementCooldowns.db')
+const outboxDb = require('../model/engagement/engagementOutbox.db')
+const sendsDb = require('../model/engagement/engagementSends.db')
+const retention = require('../model/engagement/engagementRetention.model')
+const log = require('./logger')('engagement')
+
+const INTERVAL_MS = Number(process.env.ENGAGEMENT_PRUNE_MS) || 24 * 60 * 60 * 1000
+const FIRST_RUN_MS = Number(process.env.ENGAGEMENT_PRUNE_DELAY_MS) || 10 * 60 * 1000
+
+// A bound per statement, so one run after a long outage is a series of bounded
+// DELETEs rather than one holding locks over a million rows. Each sweep repeats
+// until it clears and stops early rather than looping forever; a run that hits
+// the ceiling simply resumes tomorrow, which is what a horizon means anyway.
+const BATCH = 1000
+const MAX_BATCHES = 50
+
+const daysAgo = (days, now) => new Date(now.getTime() - days * 24 * 60 * 60 * 1000)
+
+/** Repeat a bounded delete until it stops filling its batch. Never throws. */
+async function sweep(name, del) {
+  let removed = 0
+  for (let i = 0; i < MAX_BATCHES; i += 1) {
+    const n = await del()
+    removed += n
+    if (n < BATCH) break
+  }
+  if (removed) log.info('engagement retention swept', { table: name, removed })
+  return removed
+}
+
+/**
+ * One pass over all three tables.
+ *
+ * Each sweep is caught on its own: a failure in one (a lock timeout on a huge
+ * outbox, say) must not stop the other two from being bounded. Never throws — it
+ * runs on a timer with nobody to catch it.
+ */
+async function tick(now = new Date()) {
+  const result = { cooldowns: 0, outbox: 0, sends: 0, warnings: [] }
+  let policy
+  try {
+    policy = await retention.get()
+  } catch (err) {
+    log.error('engagement retention policy unreadable; skipping this run', { message: err.message })
+    return result
+  }
+
+  // Phase 14's acceptance line: checked, not picked. A horizon shorter than a
+  // live cooldown is logged and swept anyway — see the model for that trade.
+  try {
+    const check = await retention.checkCooldownHorizon(policy.cooldowns)
+    if (!check.ok) {
+      result.warnings.push(check.message)
+      log.warn('cooldown retention is shorter than a live cooldown', {
+        retainDays: policy.cooldowns,
+        longestCooldownSeconds: check.longestCooldownSeconds,
+      })
+    }
+  } catch (err) {
+    log.debug('cooldown horizon check failed', { message: err.message })
+  }
+
+  try {
+    result.cooldowns = await sweep('engagement_cooldowns', () =>
+      cooldownsDb.prune(daysAgo(policy.cooldowns, now), BATCH))
+  } catch (err) {
+    log.error('cooldown prune failed', { message: err.message })
+  }
+
+  try {
+    result.outbox = await sweep('engagement_outbox', () =>
+      outboxDb.pruneTerminal(daysAgo(policy.outbox, now), BATCH))
+  } catch (err) {
+    log.error('outbox prune failed', { message: err.message })
+  }
+
+  try {
+    result.sends = await sweep('engagement_sends', () =>
+      sendsDb.prune(daysAgo(policy.sends, now), BATCH))
+  } catch (err) {
+    log.error('send-log prune failed', { message: err.message })
+  }
+
+  return result
+}
+
+let timer = null
+let firstRun = null
+
+function start() {
+  if (timer || firstRun) return timer
+  firstRun = setTimeout(() => {
+    firstRun = null
+    tick()
+    timer = setInterval(() => { tick() }, INTERVAL_MS)
+    if (timer.unref) timer.unref()
+  }, FIRST_RUN_MS)
+  if (firstRun.unref) firstRun.unref()
+  log.info('engagement retention started', { intervalMs: INTERVAL_MS, firstRunMs: FIRST_RUN_MS })
+  return timer
+}
+
+function stop() {
+  if (firstRun) {
+    clearTimeout(firstRun)
+    firstRun = null
+  }
+  if (timer) {
+    clearInterval(timer)
+    timer = null
+  }
+}
+
+module.exports = { start, stop, tick, BATCH, MAX_BATCHES, INTERVAL_MS, FIRST_RUN_MS }

server/src/utils/engagementWorker.js

@@ -153,7 +153,7 @@ async function processRow(row, now = new Date(), deliverFn = deliver) {
 
 async function tick(now = new Date()) {
   try {
-    await outboxDb.reclaimStale(new Date(now.getTime() - STALE_MS))
+    await outboxDb.reclaimStale(new Date(now.getTime() - STALE_MS), MAX_ATTEMPTS)
   } catch (err) {
     log.error('failed to reclaim stale rows', { message: err.message })
   }