feat(events): the runner (Phase 2)

`utils/eventRunner.js`, the eighth poller, wired into server.js beside
engagementWorker. Its tick reclaims stale leases, sweeps occurrences past their
grace window into `missed`, advances each due run through its phases, and drains
that phase's steps in `seq` order. The three core actions from Phase 1 get real
bodies, so a published event started from the existing run route now announces,
waits and completes on its own.

No routes are added: a runner has no surface, and the live controls stay Phase
3's.

Four things the org lead settled (2026-09-02): a parked step is `running` with a
NULL lease; `await: 'human'` and `holdFor` are ordinary success-envelope members
rather than special cases keyed on an action id; a run whose concurrency key is
held stays `scheduled` and lets its grace window decide; and `n` in §L's
`retry(n)` is a runner constant.

Co-Authored-By: Claude <noreply@anthropic.com>

server/src/model/events/eventRunLog.db.js

@@ -23,6 +23,16 @@ const KINDS = [
   'phase.entered',    // a phase's steps were materialised
   'step.status',      // a step transition, with the module's answer
   'note',             // a human action taken from the admin surface
+  // Phase 2's, all five of them answers to a question an operator asks out
+  // loud. `run.blocked` in particular is the whole reason this table exists
+  // rather than a server log line: "it did not start because run 37 holds
+  // invasion:Yew" is a fact with two run ids in it, and it has to be
+  // queryable from the run that did NOT start.
+  'run.blocked',      // an occurrence held off: another run has its concurrency key
+  'run.health',       // a health change, which is not a status change
+  'step.retry',       // a step failed transiently and will be attempted again
+  'step.parked',      // a step is waiting on a human and nothing is holding it
+  'phase.completed',  // every step of a phase reached a terminal status
 ]
 
 const hydrate = (row) => row && { ...row, detail: parseJson(row.detail, null) }
@@ -64,4 +74,33 @@ async function write({ runId, stepId = null, kind, phase = null, detail = null }
   }
 }
 
-module.exports = { KINDS, listForRun, write }
+/**
+ * Delete log lines belonging to runs that are both TERMINAL and older than
+ * `before`, a bounded number at a time.
+ *
+ * The schema comment beside `idx_evlog_at` parked this sweep here, and it is the
+ * rule Engagement Phase 14 arrived at applied to a second high-cardinality table:
+ * **only terminal rows are eligible.** A run still in flight keeps every line it
+ * has, however old — the log's whole job is answering "why didn't phase 3 start?"
+ * about a run that is, right now, not starting phase 3, and a horizon that could
+ * reach a live run would delete the answer while the question was still open.
+ *
+ * `LIMIT` makes one call a bounded amount of work rather than a table-sized
+ * transaction; the timer runs again and takes the next slice. The join is on the
+ * run's terminal status rather than on a precomputed id list so that a run which
+ * reached a terminal state between the two would not be missed.
+ */
+const pruneTerminal = async (before, limit = 5000) => {
+  const n = Math.min(Math.max(Number(limit) || 5000, 1), 50_000)
+  const result = await query(
+    `DELETE l FROM event_run_log l
+       JOIN event_runs r ON r.id = l.run_id
+      WHERE r.status IN ('completed','cancelled','failed','missed')
+        AND COALESCE(r.ended_at, r.updated_at) < ?
+      LIMIT ${n}`,
+    [before],
+  )
+  return Number(result?.affectedRows || 0)
+}
+
+module.exports = { KINDS, listForRun, write, pruneTerminal }

server/src/model/events/eventRunSteps.db.js

@@ -92,4 +92,201 @@ const statusCounts = async (runId) => {
   return Object.fromEntries(rows.map((r) => [r.status, Number(r.n)]))
 }
 
-module.exports = { listForRun, getById, materialisePhase, statusCounts, idempotencyKey }
+// ── Phase 2: draining a step ───────────────────────────────────────────────
+//
+// The CAS claim, the lease, the attempt counter and the terminal writes. Phase 1
+// left all of it out rather than stubbing it, and this is where it lands.
+//
+// **Two rules govern everything below, and both were paid for once already.**
+//
+// 1. `attempts` is incremented by the CLAIM and by nothing else, and no recovery
+//    path ever resets it. Engagement Phase 14's defect was a stale-row sweep that
+//    returned rows to their start state: the attempt ceiling became unreachable,
+//    so the row cycled forever, never reached a terminal status, and was
+//    therefore never eligible for any retention sweep.
+// 2. A PARKED step is `running` with a NULL lease, and the reclaim only ever
+//    touches a lease that is non-NULL and expired (the org lead's answer,
+//    2026-09-02). That is what lets a GM cue wait for a human overnight without a
+//    sweep re-dispatching the instruction every fifteen minutes.
+
+// A run's steps in authored order, for the phase the run is currently in.
+const listForPhase = async (runId, phase) =>
+  (
+    await query(
+      'SELECT * FROM event_run_steps WHERE run_id = ? AND phase = ? ORDER BY seq, id',
+      [runId, phase],
+    )
+  ).map(hydrate)
+
+/**
+ * The next step of a phase that the runner may work on, or null.
+ *
+ * **Steps within a phase are strictly serial.** This returns the lowest-`seq`
+ * step that is not terminal, and the runner does nothing with step N+1 until N
+ * has finished — which is the only reading under which `core.wait` means anything
+ * at all, and the only one under which a cue can gate what follows it.
+ *
+ * A parked or running step is returned too, so the caller can see that the phase
+ * is occupied rather than concluding it is finished.
+ */
+const nextOpenStep = async (runId, phase) => {
+  const [row] = await query(
+    `SELECT * FROM event_run_steps
+      WHERE run_id = ? AND phase = ?
+        AND status IN ('pending','running')
+      ORDER BY seq, id LIMIT 1`,
+    [runId, phase],
+  )
+  return hydrate(row) || null
+}
+
+/**
+ * Take ownership of one pending step: the CAS `pending -> running`, plus a lease.
+ *
+ * `due_at` is honoured here rather than in the caller's filter so that the whole
+ * decision — is it mine, is it due — is one statement the database arbitrates. A
+ * NULL `due_at` is due now, which is what materialisation writes for every step
+ * that is not sitting behind a `core.wait`.
+ */
+async function claim(id, owner, leaseUntil, now) {
+  const result = await query(
+    `UPDATE event_run_steps
+        SET status = 'running', attempts = attempts + 1, claimed_by = ?, claim_expires_at = ?,
+            started_at = COALESCE(started_at, NOW())
+      WHERE id = ? AND status = 'pending' AND (due_at IS NULL OR due_at <= ?)`,
+    [owner, leaseUntil, id, now],
+  )
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * Park a claimed step: it stays `running`, and its lease goes NULL.
+ *
+ * This is the whole mechanism behind `core.cue`. The step is genuinely in flight
+ * — an instruction has been posted and nothing else in the phase may proceed —
+ * but no process is holding it, so the reclaim must not take it back. A NULL
+ * lease says exactly that, and `reclaimStale` below is written to agree.
+ */
+const park = (id, note) =>
+  query(
+    `UPDATE event_run_steps SET claim_expires_at = NULL, last_error = ?
+      WHERE id = ? AND status = 'running'`,
+    [note ? String(note).slice(0, 500) : null, id],
+  )
+
+/**
+ * Release a claimed step back to `pending` for a later attempt.
+ *
+ * `attempts` is untouched — it was already incremented by the claim, which is the
+ * only place that may. Backoff is flat rather than exponential for the reason the
+ * outbox's is: `due_at` is also the event's own clock, and a doubling backoff
+ * pushes a step arbitrarily far past the moment the event was about.
+ */
+const reschedule = (id, dueAt, error) =>
+  query(
+    `UPDATE event_run_steps
+        SET status = 'pending', due_at = ?, claimed_by = NULL, claim_expires_at = NULL, last_error = ?
+      WHERE id = ? AND status = 'running'`,
+    [dueAt, error ? String(error).slice(0, 500) : null, id],
+  )
+
+/** A terminal outcome for one step: done, failed, skipped, refused or cancelled. */
+const finish = (id, status, error) =>
+  query(
+    `UPDATE event_run_steps
+        SET status = ?, last_error = ?, finished_at = NOW(),
+            claimed_by = NULL, claim_expires_at = NULL
+      WHERE id = ? AND status = 'running'`,
+    [status, error ? String(error).slice(0, 500) : null, id],
+  )
+
+/**
+ * Delay the next not-yet-started step of a phase — what `core.wait` actually does.
+ *
+ * The wait step itself completes normally; the pause is the NEXT step's `due_at`,
+ * owned by the runner. A `perform()` that slept would hold its claim for the
+ * duration and turn a five-minute pause into a five-minute lease, which is the
+ * one shape this must not have.
+ *
+ * Guarded on `status = 'pending'` and on the current `due_at` being sooner, so a
+ * re-dispatch of a wait whose ack was lost cannot push the following step further
+ * out a second time.
+ */
+const holdNext = async (runId, phase, afterSeq, dueAt) => {
+  const result = await query(
+    `UPDATE event_run_steps
+        SET due_at = ?
+      WHERE run_id = ? AND phase = ? AND seq > ? AND status = 'pending'
+        AND (due_at IS NULL OR due_at < ?)
+      ORDER BY seq LIMIT 1`,
+    [dueAt, runId, phase, afterSeq, dueAt],
+  )
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * Recover steps whose claim outlived the process that took it.
+ *
+ * **`attempts` is not reset and the lease being NULL is not staleness.** The
+ * first is Engagement Phase 14's rule; the second is what makes a parked cue
+ * survive. A step that has already burned its attempts leaves `running` as
+ * `failed` rather than being handed back, and in that order — a reclaim that ran
+ * first would return it to `pending` and it would be retried forever.
+ */
+const reclaimStale = async (now, maxAttempts = 0) => {
+  let failed = 0
+  if (Number(maxAttempts) > 0) {
+    const gaveUp = await query(
+      `UPDATE event_run_steps
+          SET status = 'failed', last_error = 'gave up after repeated interruptions',
+              finished_at = NOW(), claimed_by = NULL, claim_expires_at = NULL
+        WHERE status = 'running'
+          AND claim_expires_at IS NOT NULL AND claim_expires_at < ?
+          AND attempts >= ?`,
+      [now, Math.floor(maxAttempts)],
+    )
+    failed = Number(gaveUp?.affectedRows || 0)
+  }
+  const reclaimed = await query(
+    `UPDATE event_run_steps
+        SET status = 'pending', claimed_by = NULL, claim_expires_at = NULL
+      WHERE status = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at < ?`,
+    [now],
+  )
+  return { failed, reclaimed: Number(reclaimed?.affectedRows || 0) }
+}
+
+/**
+ * Cancel every step of a run that has not started (Phase 3's cancel, and the
+ * abort_run disposition).
+ *
+ * A `running` step is deliberately left alone, parked or not: nothing can recall
+ * a command already sent, and a second writer on that row would race the process
+ * that owns it (§L).
+ */
+const cancelPending = async (runId) => {
+  const result = await query(
+    `UPDATE event_run_steps
+        SET status = 'cancelled', finished_at = NOW()
+      WHERE run_id = ? AND status = 'pending'`,
+    [runId],
+  )
+  return Number(result?.affectedRows || 0)
+}
+
+module.exports = {
+  listForRun,
+  listForPhase,
+  getById,
+  materialisePhase,
+  statusCounts,
+  idempotencyKey,
+  nextOpenStep,
+  claim,
+  park,
+  reschedule,
+  finish,
+  holdNext,
+  reclaimStale,
+  cancelPending,
+}

server/src/model/events/eventRuns.db.js

@@ -17,6 +17,12 @@ const { parseJson } = require('./eventJson')
 
 const hydrate = (row) => row && { ...row, params: parseJson(row.params, null), rehearsal: Boolean(row.rehearsal) }
 
+// The statuses a run never leaves. A transition INTO one of these stamps
+// `ended_at` and drops the claim, and only rows in one of them are eligible for
+// the log retention sweep -- Engagement Phase 14's rule, which is a bound at all
+// only if every path a run can take reaches one of them.
+const TERMINAL = ['completed', 'cancelled', 'failed', 'missed']
+
 const SELECT_LIST = `
   SELECT r.*, d.title AS definition_title, d.slug AS definition_slug, v.version AS version_number
     FROM event_runs r
@@ -102,4 +108,256 @@ const countActiveForDefinition = async (definitionId) => {
   return Number(row?.n || 0)
 }
 
-module.exports = { list, getById, materialise, findOccurrence, countActiveForDefinition }
+// ── Phase 2: the claim, the transitions and the reclaim ────────────────────
+//
+// Everything below is the runner's, and none of it existed in Phase 1 for a
+// stated reason: a half-written claim is worse than no claim, because it reads
+// as protection. It is written here now, in full.
+//
+// **The division of labour with the unique index has not changed.** The index one
+// section up is what makes "one run per occurrence per scope" TRUE; the CAS below
+// decides only WHO advances an occurrence that already exists. Neither substitutes
+// for the other, and this deployment being single-instance (§N4) changes the test
+// rather than the design — the same two protections are what keep a tick that
+// overran into the next one from advancing a run twice.
+
+/**
+ * Runs the runner should look at this tick: due, and not yet terminal.
+ *
+ * It selects rather than claims — `claimStart` and `claimTick` below are one row
+ * at a time — so two sweepers see the same candidates and then disagree,
+ * harmlessly, about which of them owns each. `idx_evrun_due (status,
+ * scheduled_for)` is this query.
+ *
+ * `paused` is absent from the status list on purpose. A paused run is waiting on
+ * a human and must not be advanced by a tick; the only thing that moves it is
+ * Phase 3's resume control.
+ */
+const findDue = async (now, limit = 50) => {
+  const n = Math.min(Math.max(Number(limit) || 50, 1), 500)
+  return (
+    await query(
+      `SELECT * FROM event_runs
+        WHERE status IN ('scheduled','starting','running','ending')
+          AND scheduled_for <= ?
+        ORDER BY scheduled_for, id
+        LIMIT ${n}`,
+      [now],
+    )
+  ).map(hydrate)
+}
+
+/**
+ * Take ownership of a run that has not started: the CAS `scheduled -> starting`.
+ *
+ * Verbatim the outbox claim the org lead settled over `SELECT ... FOR UPDATE
+ * SKIP LOCKED` — the instance the server reports `affectedRows = 1` to owns the
+ * row, every other sweeper gets 0 and moves on. No transaction to hold open and
+ * no MariaDB version floor.
+ */
+async function claimStart(id, owner, leaseUntil) {
+  const result = await query(
+    `UPDATE event_runs
+        SET status = 'starting', claimed_by = ?, claim_expires_at = ?,
+            started_at = COALESCE(started_at, NOW())
+      WHERE id = ? AND status = 'scheduled'`,
+    [owner, leaseUntil, id],
+  )
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * Take a lease on a run already in flight, so one tick works on it at a time.
+ *
+ * Unlike `claimStart` this does not change `status` — the run is already
+ * `starting`, `running` or `ending`, and what is being claimed is the right to
+ * advance it.
+ *
+ * **A live lease is not re-enterable, not even by the process that took it**, and
+ * that is the whole point rather than an oversight. `setInterval` fires the next
+ * tick whether or not the last one has returned, so an owner-matches escape
+ * clause here would let one process advance one run twice at once — which is
+ * precisely the overrun the plan says the CAS is meant to protect against. A run
+ * this process still holds is a run this process is still working on; the tick
+ * skips it, and `releaseClaim` below is what ends that in the ordinary case.
+ */
+async function claimTick(id, owner, leaseUntil, now) {
+  const result = await query(
+    `UPDATE event_runs
+        SET claimed_by = ?, claim_expires_at = ?
+      WHERE id = ?
+        AND status IN ('starting','running','ending')
+        AND (claim_expires_at IS NULL OR claim_expires_at < ?)`,
+    [owner, leaseUntil, id, now],
+  )
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * Give a still-in-flight run back, so the next tick can pick it up at once.
+ *
+ * A run left parked on a GM cue, or waiting out a `core.wait`, is not finished
+ * and must not carry a lease: without this the run would be unadvanceable until
+ * the lease expired, which would turn every wait into `max(wait, leaseMs)`.
+ * Scoped to `claimed_by = ?` so a process can only release its own claim.
+ */
+async function releaseClaim(id, owner) {
+  const result = await query(
+    'UPDATE event_runs SET claimed_by = NULL, claim_expires_at = NULL WHERE id = ? AND claimed_by = ?',
+    [id, owner],
+  )
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * A guarded status transition: `from -> to`, and only from `from`.
+ *
+ * Every move the runner makes goes through here rather than through a bare
+ * UPDATE, so "did this transition actually happen" is answerable at each call
+ * site. A `false` is not an error — it is another worker, or this run having been
+ * cancelled from the admin surface between the read and the write, which is a
+ * race Phase 3's live controls make ordinary.
+ */
+async function transition(id, from, to, { phase, error, clearClaim = false } = {}) {
+  const sets = ['status = ?']
+  const args = [to]
+  if (phase !== undefined) {
+    sets.push('current_phase = ?')
+    args.push(phase)
+  }
+  if (error !== undefined) {
+    sets.push('last_error = ?')
+    args.push(error === null ? null : String(error).slice(0, 500))
+  }
+  if (TERMINAL.includes(to)) sets.push('ended_at = COALESCE(ended_at, NOW())')
+  if (clearClaim || TERMINAL.includes(to)) sets.push('claimed_by = NULL', 'claim_expires_at = NULL')
+
+  const froms = Array.isArray(from) ? from : [from]
+  const result = await query(
+    `UPDATE event_runs SET ${sets.join(', ')}
+      WHERE id = ? AND status IN (${froms.map(() => '?').join(',')})`,
+    [...args, id, ...froms],
+  )
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * Set health without touching status (§E).
+ *
+ * The two columns are separate because a run can be genuinely running and
+ * degraded at once — announcements landing, world writes parked — and one column
+ * cannot say both. Guarded on the current value so a tick that re-observes the
+ * same degradation does not restamp `updated_at`.
+ */
+async function setHealth(id, health) {
+  const result = await query('UPDATE event_runs SET health = ? WHERE id = ? AND health <> ?', [
+    health,
+    id,
+    health,
+  ])
+  return Number(result?.affectedRows || 0) === 1
+}
+
+/**
+ * Runs whose start instant passed more than their own grace window ago (§E, §L).
+ *
+ * The window is per definition, so the comparison is against `grace_seconds` on
+ * the joined row rather than against a constant here: an event whose announcement
+ * gives a fifteen-minute window and one that must start on the second are the
+ * same query with different data.
+ *
+ * Only `scheduled` runs qualify. A run that reached `starting` has begun, and
+ * "began and then stalled" is a different fact from "never began" — conflating
+ * them would let `missed` describe a run that had already announced itself.
+ */
+const findMissed = async (now, limit = 100) => {
+  const n = Math.min(Math.max(Number(limit) || 100, 1), 500)
+  return (
+    await query(
+      `SELECT r.* FROM event_runs r
+         JOIN event_definitions d ON d.id = r.definition_id
+        WHERE r.status = 'scheduled'
+          AND r.scheduled_for + INTERVAL d.grace_seconds SECOND < ?
+        ORDER BY r.scheduled_for
+        LIMIT ${n}`,
+      [now],
+    )
+  ).map(hydrate)
+}
+
+/**
+ * Is another run holding this concurrency key?
+ *
+ * The org lead's answer for a held key (2026-09-02) is to leave the run
+ * `scheduled` and let the grace window decide, so this is a READ rather than a
+ * claim: the caller holds off, logs which run holds the key, and tries again next
+ * tick. `idx_evrun_concurrency (concurrency_key, status)` is this query, and a
+ * NULL key is skipped by that index — which is right, because a definition with
+ * no key never contends.
+ */
+const concurrencyHolder = async (key, exceptRunId) => {
+  if (!key) return null
+  const [row] = await query(
+    `SELECT id, status, definition_id FROM event_runs
+      WHERE concurrency_key = ?
+        AND id <> ?
+        AND status IN ('starting','running','paused','ending')
+      ORDER BY id LIMIT 1`,
+    [key, exceptRunId],
+  )
+  return row || null
+}
+
+/**
+ * Recover runs whose claim outlived the process that took it.
+ *
+ * **It does not change status and it touches no counter.** All it releases is the
+ * lease; the run stays exactly where it was and the next tick picks it up through
+ * `findDue`. This is Engagement Phase 14's lesson applied one table over: a sweep
+ * that returned a stale row to its start state made the attempt ceiling
+ * unreachable, so the row cycled forever, never terminal, and therefore never
+ * eligible for any retention sweep.
+ */
+const reclaimStale = async (now) => {
+  const result = await query(
+    `UPDATE event_runs SET claimed_by = NULL, claim_expires_at = NULL
+      WHERE status IN ('starting','running','ending')
+        AND claim_expires_at IS NOT NULL
+        AND claim_expires_at < ?`,
+    [now],
+  )
+  return Number(result?.affectedRows || 0)
+}
+
+/** Terminal runs that ended before `before` — what the log retention sweep walks. */
+const terminalBefore = async (before, limit = 500) => {
+  const n = Math.min(Math.max(Number(limit) || 500, 1), 5000)
+  return (
+    await query(
+      `SELECT id FROM event_runs
+        WHERE status IN (${TERMINAL.map(() => '?').join(',')})
+          AND COALESCE(ended_at, updated_at) < ?
+        ORDER BY id LIMIT ${n}`,
+      [...TERMINAL, before],
+    )
+  ).map((r) => Number(r.id))
+}
+
+module.exports = {
+  list,
+  getById,
+  materialise,
+  findOccurrence,
+  countActiveForDefinition,
+  findDue,
+  findMissed,
+  claimStart,
+  claimTick,
+  releaseClaim,
+  transition,
+  setHealth,
+  concurrencyHolder,
+  reclaimStale,
+  terminalBefore,
+  TERMINAL,
+}