website/server/src/utils/eventRunner.js

// ── The event runner ───────────────────────────────────────────────────────
//
// EVENTS.md §E, and Phase 2 of EVENTS_PLAN.md. The eighth poller: same
// `setInterval` + `unref()` + `stop()` shape as `announceWorker`, the three Team
// sweepers and `engagementWorker`, wired into `server.js` beside them. In the
// **website** process rather than the bot, which cannot load module code.
//
// Its tick does four things, in this order:
//
//   1. **reclaim** — release leases whose holder died, never touching `attempts`
//   2. **materialise** — sweep occurrences past their grace window into `missed`
//   3. **advance** — claim each due run and move it through its phases
//   4. **prune** — the `event_run_log` retention sweep, on its own long clock
//
// **What a phase advances on, as of Phase 5.** Every step terminal, and — if the
// phase authored one — its GATE open as well. The gate is an ADDITIONAL
// condition and never a replacement: a phase whose steps are still running is
// not advanced by a boss that spawned early. `{ after: '30m' }` is closed by
// this file when its deadline passes; `{ on: '<trigger>', count: n }` is closed
// by the EMIT PATH, because a firing between two ticks is not observable from
// either of them. See `events/gates.js` for that division.
//
// **What "materialise" means, and why it is two halves.** Phase 4 completed it.
// The first half EXPANDS: every `ready` definition's recurrence is computed in
// its own IANA zone and every occurrence inside a fourteen-day horizon becomes a
// real `scheduled` row (`INSERT IGNORE` against the occurrence key, so the tick
// that already made one makes nothing). The second half SWEEPS: a run whose
// instant passed while the process was down does not start late and silently —
// it becomes `missed`, which is terminal and which a human can see (§L).
//
// **The two halves need each other, and the horizon is why.** Expansion looks
// forward from `now - grace` only, so an occurrence nobody ever materialised is
// never invented retroactively — waking up after three days down must not
// manufacture three days of history that no operator could have seen or
// cancelled. It does not have to: because rows exist a fortnight ahead of their
// instant, an outage that spans an occurrence finds the row already there, and
// the sweep marks it `missed` honestly. The horizon is what makes the missed
// sweep mean anything for a recurring event.
//
// **Two properties this file must not lose**, both already paid for once on this
// codebase:
//
// - **A reclaim never resets `attempts`.** Engagement Phase 14's defect: a sweep
//   that returned every stale row to its start state made the attempt ceiling
//   unreachable, so the row cycled forever, never terminal, and therefore never
//   eligible for any retention sweep.
// - **The unique index, not the claim, is what prevents a double run.** The claim
//   decides *who* advances an occurrence; `uq_evrun_occurrence` is what stops two
//   of them existing. Neither substitutes for the other.
//
// §N4 settled this deployment as single-instance, so the `--scale app=2` rig the
// engagement workstream used is deliberately not built. Every claim path is here
// exactly as §E specifies anyway, because the multi-instance case is not what
// they are for on a single container: the CAS is what protects a tick that
// overran into the next one, and the lease and its reclaim are what recover a
// step whose process died mid-dispatch. Both happen with one app.

const os = require('os')

const runsDb = require('../model/events/eventRuns.db')
const stepsDb = require('../model/events/eventRunSteps.db')
const logDb = require('../model/events/eventRunLog.db')
const versionsDb = require('../model/events/eventVersions.db')
const definitionsDb = require('../model/events/eventDefinitions.db')
const runsModel = require('../model/events/eventRuns.model')
const gatesDb = require('../model/events/eventPhaseGates.db')
const recurrence = require('../events/recurrence')
const gates = require('../events/gates')
const spec = require('../events/spec')
const registries = require('../modules/registries')
const { dispatchStep } = require('../events/dispatch')
const authorize = require('../events/authorize')
const log = require('./logger')('event-runner')

const POLL_MS = Number(process.env.EVENT_POLL_MS) || 15_000

// How many runs one sweep looks at, and how many steps it will drain from one
// run. Bounds rather than targets: the tick runs again in POLL_MS, and an
// unbounded batch is how a backlog turns one tick into a stall. The step bound
// also caps how long one run can hold the tick, which is what keeps a
// forty-step phase from starving every other run on the deployment.
const RUN_BATCH = Number(process.env.EVENT_RUN_BATCH) || 50
const STEPS_PER_TICK = Number(process.env.EVENT_STEPS_PER_TICK) || 25

// How far ahead a recurrence is turned into real rows (org lead, 2026-09-02).
// Fourteen days is a fortnight of occurrences an operator can see, cancel and
// reschedule ONE AT A TIME, which a projection is not — and it is short enough
// that a definition edited today affects almost everything still ahead of it.
// Beyond it the calendar projects rather than materialises, so a monthly event
// is still visible three weeks out without a row nobody will honour.
const HORIZON_DAYS = Number(process.env.EVENT_MATERIALISE_AHEAD_DAYS) || 14

// A bound on one definition's expansion in one tick, not a target. A daily
// schedule over a fortnight is fourteen; this is what stops a hand-written spec
// turning one tick into a thousand inserts.
const MAX_OCCURRENCES_PER_DEFINITION = 100

// A step's retries (org lead, 2026-09-02). §L specifies `retry(n) -> skip |
// pause | abort_run` and names no `n`; it lives here, as one number an operator
// can change, rather than in a column no authoring surface would ever show.
//
// Flat backoff rather than exponential, for the reason the outbox's is flat:
// `due_at` is also the event's own clock, and a doubling backoff pushes a step
// arbitrarily far past the moment the event was about.
const MAX_ATTEMPTS = Number(process.env.EVENT_STEP_MAX_ATTEMPTS) || 3
const RETRY_MS = Number(process.env.EVENT_STEP_RETRY_MS) || 60_000

// How long a claim may look alive before the reclaim takes it back. The run
// lease has to outlast a whole tick's work on one run; a step's is computed from
// its own action's `budgetMs` (see `leaseFor`), because a registry that lets an
// action declare an hour would otherwise have its steps reclaimed and
// re-dispatched fifty-nine minutes before they answered.
const RUN_LEASE_MS = Number(process.env.EVENT_RUN_LEASE_MS) || 15 * 60 * 1000
const STEP_LEASE_MARGIN_MS = 60_000

// The log retention horizon, and how often the sweep runs. It is folded into
// this tick rather than given a ninth timer because it shares the tick's only
// real dependency — a database — and a sweep that runs four times a day does not
// need an interval of its own. Only TERMINAL runs are ever eligible, which is the
// rule Engagement Phase 14 arrived at.
const LOG_RETENTION_DAYS = Number(process.env.EVENT_LOG_RETENTION_DAYS) || 90
const PRUNE_EVERY_MS = 6 * 60 * 60 * 1000

// Who this process is, for `claimed_by`. Host and pid, so a stranded claim in the
// table names the thing that stranded it.
const OWNER = `${os.hostname()}:${process.pid}`.slice(0, 64)

const RUN_TERMINAL = runsDb.TERMINAL

/** The lease a step's dispatch gets: its action's own budget, plus a margin. */
function leaseFor(step, now) {
  const action = registries.eventAction(step.action_id)
  const budget = action?.budgetMs || 10_000
  return new Date(now.getTime() + budget + STEP_LEASE_MARGIN_MS)
}

// ── The disposition of a step that has run out of road ─────────────────────
//
// **All three dispositions write the step `failed`.** `on_failure` says what
// happens to the RUN, not what happened to the step, and a step that was
// attempted three times and never worked is `failed` under every one of them.
// `skipped` is reserved for a step a human skipped from the run console (Phase
// 3) — a status that meant both "nobody ran this" and "this failed and we moved
// on" would make the run console's summary line unreadable.
async function applyFailure(run, step, error, { status = 'failed', kind = 'step.status', extra = null } = {}) {
  // **`refused` shares this whole function with `failed`, and that is decision 3
  // of Phase 6** (org lead, 2026-09-03): a cap breach or a disabled action takes
  // the same disposition a terminal failure takes, so a `change` step's default
  // `pause` stops the run where it stands and an operator raises the cap, edits,
  // and resumes. What differs is the two words on the record — the STATUS the
  // step ends in and the KIND the log line carries — because "nothing here is
  // broken, this deployment does not permit that" is a different sentence from
  // "the shard did not answer", and an operator reading a stopped run at 2am
  // needs to tell them apart at a glance.
  await stepsDb.finish(step.id, status, error)
  await logDb.write({
    runId: run.id,
    stepId: step.id,
    kind,
    phase: step.phase,
    detail: {
      to: status,
      action: step.action_id,
      attempts: step.attempts + 1,
      onFailure: step.on_failure,
      error,
      ...(extra || {}),
    },
  })

  // A run that lost a step is degraded whatever happens next. Health is not
  // status (§E): a run can be genuinely running and degraded at once, and the
  // admin surface needs to say so without claiming the run stopped.
  if (await runsDb.setHealth(run.id, 'degraded')) {
    await logDb.write({ runId: run.id, kind: 'run.health', detail: { to: 'degraded', because: step.action_id } })
  }

  if (step.on_failure === 'abort_run') {
    // §L: the disposition for `irreversible`. Nothing further is dispatched, and
    // the pending steps are cancelled rather than left looking due forever.
    const cancelled = await stepsDb.cancelPending(run.id)
    await runsDb.transition(run.id, ['starting', 'running', 'ending'], 'failed', { error })
    await logDb.write({
      runId: run.id,
      kind: 'run.status',
      phase: step.phase,
      detail: { to: 'failed', because: step.action_id, cancelledSteps: cancelled },
    })
    return 'stop'
  }

  if (step.on_failure === 'pause') {
    // The default for `change`, and the right one when the world is half-altered:
    // stop advancing and wait for a human. `paused` is excluded from `findDue`,
    // so nothing here picks it up again — Phase 3's resume control is the only
    // thing that moves it.
    await runsDb.transition(run.id, ['starting', 'running'], 'paused', { error })
    await logDb.write({
      runId: run.id,
      kind: 'run.status',
      phase: step.phase,
      detail: { to: 'paused', because: step.action_id },
    })
    return 'stop'
  }

  // 'skip': the run carries on, degraded, and the failure is on the record.
  return 'continue'
}

/**
 * Claim one step, dispatch it, and record what came back.
 *
 * Answers `'continue'` (the phase may proceed), `'stop'` (it may not, for now or
 * ever) or `'taken'` (somebody else claimed it first).
 */
async function drainStep(run, step, now, carry = {}) {
  if (!(await stepsDb.claim(step.id, OWNER, leaseFor(step, now), now))) return 'taken'

  // ── The permission check, and it is the LAST thing before the dispatch ──
  //
  // §K's four layers behind one function (Phase 6). It sits after the claim, not
  // before it: the cap is held by a conditional UPDATE and two ticks that both
  // priced a step before either claimed it would both spend. It sits before the
  // dispatch because a refusal means the action does not happen — nothing is
  // sent, nothing is created, and the step never reaches the module at all.
  //
  // **`user` is null here, and that is the design rather than an omission.** The
  // role was checked when a human published the version and again when a human
  // or the scheduler started the run; a run in flight is deliberately not
  // re-gated against its starter's current role, because demoting an admin at
  // midnight should not silently strand every event they started. Cancel is the
  // control for a run that should stop.
  //
  // **A retry does not pay twice.** The spend happens on the first attempt only.
  // A retry re-dispatches the same idempotent operation against the same key, and
  // charging a cap for a flaky socket would exhaust a deployment's allowance
  // through unreliability rather than through effect. The corollary is that a
  // step which spent and then failed for good keeps its spend: the attempt may
  // have half-run, and a refund would be core asserting that it did not.
  const action = registries.eventAction(step.action_id)
  if (action) {
    const verdict = await authorize.mayInvoke({
      action,
      params: step.params || {},
      run,
      spend: step.attempts === 0,
    })
    if (!verdict.ok) {
      return applyFailure(run, step, verdict.reason, {
        status: 'refused',
        kind: 'step.refused',
        extra: {
          code: verdict.code,
          dimension: verdict.dimension,
          requested: verdict.requested,
          cap: verdict.cap,
          consumed: verdict.consumed,
        },
      })
    }
  }

  const result = await dispatchStep(step, { run })

  if (result.actionVersionDrift) {
    await logDb.write({
      runId: run.id,
      stepId: step.id,
      kind: 'step.status',
      phase: step.phase,
      detail: { action: step.action_id, versionDrift: result.actionVersionDrift },
    })
  }

  if (result.outcome === 'parked') {
    // The GM cue. The step stays `running` with a NULL lease: genuinely in
    // flight, nothing holding it, so the stale reclaim passes it by and a cue
    // posted on Friday is still waiting on Monday. Phase 3's confirm control is
    // what ends it.
    await stepsDb.park(step.id, null)
    await logDb.write({
      runId: run.id,
      stepId: step.id,
      kind: 'step.parked',
      phase: step.phase,
      detail: { action: step.action_id, params: step.params },
    })
    return 'stop'
  }

  if (result.outcome === 'done') {
    await stepsDb.finish(step.id, 'done', null)
    if (result.holdSeconds > 0) {
      // `core.wait`, and any module action that answers `holdFor`. The pause is
      // the NEXT step's `due_at` and it is set here, by the runner, because a
      // `perform()` that slept would hold its claim for the duration.
      const until = new Date(now.getTime() + result.holdSeconds * 1000)
      if (!(await stepsDb.holdNext(run.id, step.phase, step.seq, until))) {
        // **Nothing after it in this phase**, which is the case a wait written as
        // the last step of a phase produces. Dropping the hold here would make
        // "announce, wait five minutes, then the next phase" start the next phase
        // at once — a wait that silently meant nothing. The later phase's steps do
        // not exist yet, so the instant is carried out to `advanceRun` and applied
        // when they are materialised.
        carry.holdUntil = until
      }
    }
    await logDb.write({
      runId: run.id,
      stepId: step.id,
      kind: 'step.status',
      phase: step.phase,
      detail: { to: 'done', action: step.action_id, holdSeconds: result.holdSeconds || 0 },
    })
    return 'continue'
  }

  if (result.outcome === 'retry' && step.attempts + 1 < MAX_ATTEMPTS) {
    await stepsDb.reschedule(step.id, new Date(now.getTime() + RETRY_MS), result.error)
    await logDb.write({
      runId: run.id,
      stepId: step.id,
      kind: 'step.retry',
      phase: step.phase,
      detail: { action: step.action_id, attempt: step.attempts + 1, of: MAX_ATTEMPTS, error: result.error },
    })
    // Degraded from the FIRST retry, not from the eventual failure. An event
    // whose announcements are landing on the second attempt is having trouble
    // now, and that is when an operator wants to know.
    if (await runsDb.setHealth(run.id, 'degraded')) {
      await logDb.write({ runId: run.id, kind: 'run.health', detail: { to: 'degraded', because: step.action_id } })
    }
    return 'stop'
  }

  // Terminal, or transient with the attempts spent. Same disposition either way:
  // §L's `retry(n) -> ...` has arrived at the arrow.
  return applyFailure(run, step, result.error)
}

/**
 * Open a phase's advance gate, if it authored one.
 *
 * Called at phase entry, immediately after `materialisePhase` and BEFORE the
 * transition that makes the phase current — which is the safe order rather than
 * the tidy one. A process that died between the transition and this call would
 * leave a phase whose gate does not exist, and a missing gate does not hold a
 * phase: it advances on its steps alone, silently ignoring the condition its
 * author wrote. Opening first risks only a row for a phase this tick did not win,
 * which the winner's INSERT IGNORE then finds already correct.
 */
async function openGate(runId, phase, now) {
  const advance = phase?.advance
  if (!advance) return false

  const created =
    advance.after !== undefined
      ? await gatesDb.open({
          runId,
          phase: phase.key,
          kind: 'after',
          // Re-derived from the authored string rather than stored beside it:
          // `events/spec.js` owns what `'2h'` means, and this is the one caller
          // that needs the number.
          afterSeconds: spec.parseAfter(advance.after),
          now,
        })
      : await gatesDb.open({
          runId,
          phase: phase.key,
          kind: 'on',
          triggerId: advance.on,
          conditions: advance.where ?? null,
          needed: advance.count || 1,
          now,
        })

  if (created) {
    await logDb.write({
      runId,
      kind: 'phase.gate',
      phase: phase.key,
      detail:
        advance.after !== undefined
          ? { kind: 'after', after: advance.after }
          : { kind: 'on', trigger: advance.on, needed: advance.count || 1, where: gates.phrase(advance.where ?? null) },
    })
  }
  return created
}

/**
 * A run whose phase has waited past `EVENT_PHASE_STALL_MS` is `stalled`.
 *
 * §E's third health value, and the first thing in this system ever to write it.
 * It is VISIBILITY and not a timeout: nothing advances, nothing fails, and a
 * human decides — which is the org lead's answer of 2026-09-02 and the reason
 * there is no authored deadline in the spec. What it must not be is quiet,
 * because a held run also holds its concurrency key, so every later occurrence
 * of the same definition goes `missed` behind it.
 *
 * `setHealth` only ever escalates, so this cannot undo a `degraded` a retry
 * earned, and the `run.health` line is written once because `setHealth` answers
 * whether it changed anything.
 */
async function noteStall(run, gate, now) {
  const described = gates.describe(gate, now)
  if (!described?.stalled) return false
  if (!(await runsDb.setHealth(run.id, 'stalled'))) return false
  await logDb.write({
    runId: run.id,
    kind: 'run.health',
    phase: gate.phase,
    detail: {
      to: 'stalled',
      because: `waiting ${described.elapsedSeconds}s on ${gate.trigger_id}`,
      seen: gate.tally,
      needed: gate.needed,
    },
  })
  return true
}

/**
 * Advance one claimed run as far as it will go this tick.
 *
 * The loop is bounded by `STEPS_PER_TICK` and exits on the first thing it cannot
 * get past — a parked step, a step whose `due_at` is in the future, a step
 * somebody else holds, or a phase that is not finished.
 */
async function advanceRun(run, now) {
  const version = await versionsDb.getById(run.version_id)
  const phases = version?.spec?.phases
  if (!Array.isArray(phases) || !phases.length) {
    // The pinned version is unreadable. `version_id`'s foreign key RESTRICTs
    // precisely so this cannot be a deleted row, so it is corruption rather than
    // an ordinary race — terminal, named, and not retried.
    await runsDb.transition(run.id, ['scheduled', 'starting', 'running', 'ending'], 'failed', {
      error: 'the pinned version has no phases',
    })
    await logDb.write({ runId: run.id, kind: 'run.status', detail: { to: 'failed', because: 'pinned version has no phases' } })
    return 'failed'
  }

  let phaseKey = run.current_phase

  if (run.status === 'starting') {
    // Entering the first phase. `materialisePhase` is INSERT IGNORE against
    // `uq_evstep_slot`, so doing it again over the rows Phase 1's `create()`
    // already wrote is a no-op — which is what makes recovery from a process that
    // died between the claim and here uneventful.
    const first = phases[0]
    await stepsDb.materialisePhase(run.id, first.key, first.steps || [])
    await openGate(run.id, first, now)
    if (!(await runsDb.transition(run.id, 'starting', 'running', { phase: first.key }))) return 'taken'
    phaseKey = first.key
    await logDb.write({ runId: run.id, kind: 'run.status', phase: first.key, detail: { from: 'starting', to: 'running' } })
  }

  if (run.status === 'ending') {
    // A run that reached the wind-down and then lost its process. Phase 8 puts
    // cleanup here; until then `ending` is a state a run passes through rather
    // than one it does work in, and completing it is the whole recovery.
    await runsDb.transition(run.id, 'ending', 'completed')
    await logDb.write({ runId: run.id, kind: 'run.status', detail: { from: 'ending', to: 'completed' } })
    return 'completed'
  }

  // A hold a `core.wait` could not place because nothing followed it in its own
  // phase. It crosses the phase boundary with the run rather than being dropped.
  const carry = {}

  for (let n = 0; n < STEPS_PER_TICK; n++) {
    // Re-read the run's status between steps, not just at the top of the tick.
    // This loop drains up to STEPS_PER_TICK steps from one run, and Phase 3 put
    // a pause and a cancel in a human's hand: without this, pausing a run in the
    // middle of a batch would answer by dispatching another two dozen steps,
    // which is not a pause. One indexed column read per step, against a control
    // whose entire value is that it takes effect at once.
    if (n > 0 && (await runsDb.statusOf(run.id)) !== 'running') return 'stopped'

    const phaseIndex = phases.findIndex((p) => p.key === phaseKey)
    if (phaseIndex < 0) {
      await runsDb.transition(run.id, ['running'], 'failed', { error: `phase "${phaseKey}" is not in the pinned version` })
      await logDb.write({ runId: run.id, kind: 'run.status', detail: { to: 'failed', because: `unknown phase "${phaseKey}"` } })
      return 'failed'
    }

    const step = await stepsDb.nextOpenStep(run.id, phaseKey)

    if (step && step.status === 'running') return 'in-flight' // parked, or somebody's dispatch
    if (step && step.due_at && new Date(step.due_at) > now) return 'waiting' // behind a core.wait

    if (step) {
      const outcome = await drainStep({ ...run, current_phase: phaseKey }, step, now, carry)
      if (outcome === 'continue') continue
      return outcome === 'taken' ? 'taken' : 'stopped'
    }

    // Every step of this phase is terminal — which is the whole of the advance
    // test for a phase with no gate, and half of it for a phase with one.
    const { open, gate } = await gates.check(run.id, phaseKey, now)
    if (!open) {
      await noteStall(run, gate, now)
      // **Nothing is logged per tick here, deliberately.** The gate row IS the
      // state — the tally, the deadline, the last related event — and the run
      // console reads it directly. A `phase.waiting` line every fifteen seconds
      // would bury `condition.evaluated`, which is the line that actually says
      // something happened.
      return 'waiting'
    }
    // **`phase.advanced` is written by whoever made the DECISION, and a forced
    // gate's decision was not this tick's.** The emit path closes an `on` gate
    // and logs only `condition.evaluated`, so the line for that one is the
    // runner's; `gates.check` closes an `after` gate here, so that one is too.
    // A human closing it through the advance control already wrote the line,
    // with the actor and the reason — things this tick does not have — and a
    // second line from here made the console show the phase advancing twice,
    // the less informative one last. Found in the live walk.
    if (gate && gate.satisfied_by !== 'forced') {
      await logDb.write({
        runId: run.id,
        kind: 'phase.advanced',
        phase: phaseKey,
        detail: {
          because: gate.satisfied_by,
          waitedSeconds: Math.max(0, Math.round((new Date(gate.satisfied_at) - new Date(gate.entered_at)) / 1000)),
          ...(gate.kind === 'on' ? { trigger: gate.trigger_id, seen: gate.tally, needed: gate.needed } : { after: gate.after_seconds }),
        },
      })
    }

    await logDb.write({ runId: run.id, kind: 'phase.completed', phase: phaseKey, detail: { index: phaseIndex } })

    const next = phases[phaseIndex + 1]
    if (!next) {
      // §E's `ending` exists for the reason `sending` does in the outbox — it is
      // what a claim sets — so the run passes through it even though Phase 2 has
      // no cleanup to do there. Phase 8 is what gives it work.
      if (!(await runsDb.transition(run.id, 'running', 'ending'))) return 'taken'
      await logDb.write({ runId: run.id, kind: 'run.status', phase: phaseKey, detail: { from: 'running', to: 'ending' } })
      await runsDb.transition(run.id, 'ending', 'completed')
      await logDb.write({ runId: run.id, kind: 'run.status', detail: { from: 'ending', to: 'completed' } })
      return 'completed'
    }

    await stepsDb.materialisePhase(run.id, next.key, next.steps || [])
    await openGate(run.id, next, now)
    if (carry.holdUntil) {
      // `seq > -1` is the first step of the phase just created. Applied after
      // materialisation because that is the first moment there is a row to hold.
      await stepsDb.holdNext(run.id, next.key, -1, carry.holdUntil)
      carry.holdUntil = null
    }
    // `running -> running` is not a no-op: it is a guarded write of
    // `current_phase` that fails if the run stopped being `running` underneath
    // this tick, which is what a cancel from the admin surface looks like.
    if (!(await runsDb.transition(run.id, 'running', 'running', { phase: next.key }))) return 'taken'
    phaseKey = next.key
    await logDb.write({ runId: run.id, kind: 'phase.entered', phase: next.key, detail: { steps: (next.steps || []).length } })
  }

  return 'bounded' // more to do; the next tick picks it up
}

/** Claim one due run, work it, and hand the lease back if it is still in flight. */
async function processRun(run, now = new Date()) {
  if (run.status === 'scheduled') {
    const holder = await runsDb.concurrencyHolder(run.concurrency_key, run.id)
    if (holder) {
      // The org lead's answer for a held key (2026-09-02): hold at `scheduled`
      // and let the grace window decide. Nothing is destroyed, nothing starts
      // silently late, and if the holder outlasts the window the missed sweep
      // makes this run terminal and visible.
      //
      // Logged only when the reason CHANGES. A blocked run is re-examined every
      // tick, and a line per tick for the length of a grace window would bury the
      // one line that matters under a thousand identical ones.
      const message = `held: run ${holder.id} has concurrency key "${run.concurrency_key}"`
      if (run.last_error !== message) {
        await runsDb.transition(run.id, 'scheduled', 'scheduled', { error: message })
        await logDb.write({
          runId: run.id,
          kind: 'run.blocked',
          detail: { concurrencyKey: run.concurrency_key, heldBy: holder.id, holderStatus: holder.status },
        })
      }
      return 'blocked'
    }

    if (!(await runsDb.claimStart(run.id, OWNER, new Date(now.getTime() + RUN_LEASE_MS)))) return 'taken'
    await logDb.write({ runId: run.id, kind: 'run.status', detail: { from: 'scheduled', to: 'starting' } })
    run = { ...run, status: 'starting' }
  } else if (!(await runsDb.claimTick(run.id, OWNER, new Date(now.getTime() + RUN_LEASE_MS), now))) {
    return 'taken'
  }

  try {
    const outcome = await advanceRun(run, now)
    // A run that is not finished must not keep its lease: it would be
    // unadvanceable until the lease expired, which would turn every `core.wait`
    // into `max(wait, RUN_LEASE_MS)`. A terminal transition already cleared it.
    if (!['completed', 'failed'].includes(outcome)) await runsDb.releaseClaim(run.id, OWNER)
    return outcome
  } catch (err) {
    await runsDb.releaseClaim(run.id, OWNER)
    throw err
  }
}

/** Occurrences that passed their own grace window while nothing was running (§L). */
/**
 * Turn every `ready` definition's recurrence into rows inside the horizon.
 *
 * The first half of the materialise leg (§E). Answers how many occurrences were
 * newly created, which is zero on almost every tick — the horizon moves fifteen
 * seconds at a time, so a weekly event creates one row a week and answers
 * `created: false` for the same fourteen occurrences in between.
 *
 * **The window starts at `now - grace`, not at `now`.** An occurrence whose
 * instant just passed is still startable inside the definition's own grace
 * window, and that is exactly the case of a definition published four minutes
 * before its first occurrence. An occurrence older than that is not materialised
 * at all rather than materialised-then-swept: a row nobody could ever have seen
 * is not history, and writing one would put a `missed` event on the calendar for
 * a date on which this deployment had no such event.
 *
 * **Expansion is per definition and one failure does not stop the sweep.** A
 * spec written directly into the database with a shape the validator would have
 * refused is a bad row, not a bad tick.
 */
async function expandSchedules(now) {
  const definitions = await definitionsDb.findSchedulable()
  let created = 0
  const heldUnverified = new Set()

  for (const definition of definitions) {
    const schedule = definition.version_spec?.schedule
    if (!schedule || schedule.kind === 'manual') continue

    const from = now.getTime() - Number(definition.grace_seconds || 0) * 1000
    const to = now.getTime() + HORIZON_DAYS * recurrence.DAY_MS

    let occurrences
    try {
      occurrences = recurrence.occurrencesBetween(schedule, definition.timezone || 'UTC', from, to, {
        limit: MAX_OCCURRENCES_PER_DEFINITION,
      })
    } catch (err) {
      log.error('could not expand a schedule', {
        definition: definition.id,
        timezone: definition.timezone,
        message: err.message,
      })
      continue
    }

    for (const occurrence of occurrences) {
      try {
        // `runsModel.create` rather than a second insert path: it re-checks that
        // the definition is still `ready` and the version still has phases, and
        // it materialises the first phase's steps with their idempotency keys.
        // Nobody is watching a scheduled occurrence, so it needs those checks
        // more than a hand-started one does.
        const result = await runsModel.create(
          definition.id,
          // Scope is empty, deliberately (org lead, 2026-09-02). A fan-out across
          // named scopes needs a registry of what a scope IS, which no phase owns
          // yet; inventing one here would be a contract the modules were never
          // asked about. An admin's own start route still takes any scope.
          { scope: '', scheduledFor: occurrence.at, source: 'schedule' },
          null,
        )
        if (!result.ok) {
          if (result.code === 'unverified') {
            // §K's gate, and it must not be silent. There is no run row to hang
            // a diagnostic line on — that is the point, nothing was created — so
            // it is said once per definition per tick rather than once per
            // occurrence, and the admin surface says it where the author is
            // looking: a `ready` definition carries `versionVerified: false` and
            // the editor shows the one button that clears it.
            if (!heldUnverified.has(definition.id)) {
              heldUnverified.add(definition.id)
              log.warn('scheduled occurrences held: the published version has never been verified', {
                definition: definition.id,
                title: definition.title,
                version: definition.current_version_id,
              })
            }
            break
          }
          continue
        }
        if (!result.created) continue
        created += 1
        if (occurrence.adjusted) {
          // Why the clock reads oddly, recorded where an operator will look for
          // it rather than left to be rediscovered at 3am on the last Sunday in
          // October.
          await logDb.write({
            runId: result.run.id,
            kind: 'run.created',
            detail: {
              dstAdjusted: occurrence.adjusted,
              shiftMinutes: occurrence.shiftMinutes,
              timezone: definition.timezone,
              scheduledFor: occurrence.at,
            },
          })
        }
      } catch (err) {
        log.error('could not materialise an occurrence', {
          definition: definition.id,
          at: occurrence.at,
          message: err.message,
        })
      }
    }
  }

  if (created) log.info('occurrences materialised', { created, horizonDays: HORIZON_DAYS })
  return created
}

async function sweepMissed(now) {
  const missed = await runsDb.findMissed(now)
  let n = 0
  for (const run of missed) {
    if (await runsDb.transition(run.id, 'scheduled', 'missed', { error: 'the grace window passed' })) {
      await stepsDb.cancelPending(run.id)
      await logDb.write({
        runId: run.id,
        kind: 'run.status',
        detail: { to: 'missed', scheduledFor: run.scheduled_for },
      })
      n += 1
    }
  }
  return n
}

let lastPruneAt = 0

async function prune(now) {
  if (now.getTime() - lastPruneAt < PRUNE_EVERY_MS) return 0
  lastPruneAt = now.getTime()
  const before = new Date(now.getTime() - LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000)
  const deleted = await logDb.pruneTerminal(before)
  if (deleted) log.info('event run log pruned', { deleted, before, retentionDays: LOG_RETENTION_DAYS })
  return deleted
}

async function tick(now = new Date()) {
  try {
    await runsDb.reclaimStale(now)
    await stepsDb.reclaimStale(now, MAX_ATTEMPTS)
  } catch (err) {
    log.error('failed to reclaim stale claims', { message: err.message })
  }

  try {
    await expandSchedules(now)
  } catch (err) {
    log.error('schedule expansion failed', { message: err.message })
  }

  try {
    await sweepMissed(now)
  } catch (err) {
    log.error('missed sweep failed', { message: err.message })
  }

  let due
  try {
    due = await runsDb.findDue(now, RUN_BATCH)
  } catch (err) {
    log.error('failed to load due runs', { message: err.message })
    return
  }

  const counts = {}
  for (const run of due || []) {
    try {
      const outcome = await processRun(run, now)
      counts[outcome] = (counts[outcome] || 0) + 1
    } catch (err) {
      log.error('run failed', { run: run.id, message: err.message })
    }
  }
  if (due && due.length) log.info('event runs swept', { due: due.length, ...counts })

  try {
    await prune(now)
  } catch (err) {
    log.error('log prune failed', { message: err.message })
  }
}

let timer = null
// Guards against this process running two ticks over the same runs at once.
// `setInterval` fires whether or not the last callback returned, and the CAS
// alone does not cover it now that a live lease is not re-enterable by its own
// owner — a second tick would simply find every run claimed and do nothing
// useful, one query at a time, for as long as the first one ran.
let ticking = false

function start() {
  if (timer) return timer
  timer = setInterval(() => {
    if (ticking) {
      log.warn('event tick still running; skipping this interval')
      return
    }
    ticking = true
    tick()
      .catch((err) => log.error('event tick failed', { message: err.message }))
      .finally(() => {
        ticking = false
      })
  }, POLL_MS)
  if (timer.unref) timer.unref() // don't keep the event loop alive (tests, shutdown)
  log.info('event runner started', { pollMs: POLL_MS, owner: OWNER, maxAttempts: MAX_ATTEMPTS })
  return timer
}

function stop() {
  if (timer) {
    clearInterval(timer)
    timer = null
  }
}

module.exports = {
  start,
  stop,
  tick,
  processRun,
  advanceRun,
  drainStep,
  sweepMissed,
  expandSchedules,
  prune,
  OWNER,
  POLL_MS,
  MAX_ATTEMPTS,
  RETRY_MS,
  HORIZON_DAYS,
  RUN_LEASE_MS,
  LOG_RETENTION_DAYS,
  RUN_TERMINAL,
}