Three screens, an Events nav group and the six live run controls Phase 1 left
absent on purpose because nothing was in flight. An admin can now author,
publish, start and watch an event that announces things and cues a human; a
moderator can stop one that is going wrong.
Six controls, not eight. `advance` is absent because a phase today advances when
its steps go terminal — the per-step skip already does that — and Phase 5 is what
gives a phase an advance condition. Cancel takes `{ reason }`, not `{ cleanup }`,
until Phase 8's ledger exists. Every control is a compare-and-set on the status it
may act from, so a console rendered thirty seconds ago cannot act on a run that
has moved.
Fixes a defect in the Phase 2 runner: `advanceRun` drained up to
EVENT_STEPS_PER_TICK steps while only checking the run's status at the top of the
tick, so a pause pressed mid-batch did nothing for up to 24 more steps.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T6t8mrAWhZU5vnyYgZTMtL
388 lines
15 KiB
JavaScript
388 lines
15 KiB
JavaScript
// ── event_runs — SQL only ──────────────────────────────────────────────────
|
|
//
|
|
// EVENTS.md §D and §E. Phase 1 writes exactly one kind of row — a `scheduled`
|
|
// occurrence — and reads them back for the admin surface. **The claim, the CAS
|
|
// transitions and the lease reclaim are Phase 2's** and are deliberately not
|
|
// stubbed here: a half-written claim is worse than no claim, because it reads as
|
|
// protection.
|
|
//
|
|
// What Phase 1 does own is the INSERT, and it owns the important half of it:
|
|
// materialisation is `INSERT IGNORE` against `UNIQUE (definition_id, scope,
|
|
// scheduled_for)`, so a second attempt at one occurrence writes nothing and
|
|
// answers honestly rather than raising a duplicate-key error a caller has to
|
|
// interpret.
|
|
|
|
const { query } = require('../../utils/db')
|
|
const { parseJson } = require('./eventJson')
|
|
|
|
const hydrate = (row) => row && { ...row, params: parseJson(row.params, null), rehearsal: Boolean(row.rehearsal) }
|
|
|
|
// The statuses a run never leaves. A transition INTO one of these stamps
|
|
// `ended_at` and drops the claim, and only rows in one of them are eligible for
|
|
// the log retention sweep -- Engagement Phase 14's rule, which is a bound at all
|
|
// only if every path a run can take reaches one of them.
|
|
const TERMINAL = ['completed', 'cancelled', 'failed', 'missed']
|
|
|
|
// `waiting_steps` is the count of PARKED steps: `running` with a NULL lease, the
|
|
// pair `park()` alone produces, which means a cue waiting on a human. It is a
|
|
// correlated subquery on an admin list bounded at 500 rows rather than a column,
|
|
// because it is derived from the steps and a column would be a second writer's
|
|
// opinion of them. It earns its cost on the list screen: a cue nobody notices is
|
|
// a run that never advances, and the run itself looks perfectly healthy until
|
|
// somebody opens it.
|
|
const SELECT_LIST = `
|
|
SELECT r.*, d.title AS definition_title, d.slug AS definition_slug, v.version AS version_number,
|
|
(SELECT COUNT(*) FROM event_run_steps s
|
|
WHERE s.run_id = r.id AND s.status = 'running' AND s.claim_expires_at IS NULL) AS waiting_steps
|
|
FROM event_runs r
|
|
JOIN event_definitions d ON d.id = r.definition_id
|
|
JOIN event_versions v ON v.id = r.version_id
|
|
`
|
|
|
|
/**
|
|
* The admin run list. Newest occurrence first, across every definition.
|
|
*
|
|
* `limit` is interpolated after an integer coercion rather than bound, because
|
|
* MariaDB will not take a placeholder in LIMIT on a prepared statement. It never
|
|
* reaches SQL as anything but a number.
|
|
*/
|
|
const list = async ({ definitionId = null, status = null, limit = 100 } = {}) => {
|
|
const where = []
|
|
const args = []
|
|
if (definitionId) {
|
|
where.push('r.definition_id = ?')
|
|
args.push(definitionId)
|
|
}
|
|
if (status) {
|
|
where.push('r.status = ?')
|
|
args.push(status)
|
|
}
|
|
const clause = where.length ? `WHERE ${where.join(' AND ')}` : ''
|
|
const n = Math.min(Math.max(Number(limit) || 100, 1), 500)
|
|
const rows = await query(
|
|
`${SELECT_LIST} ${clause} ORDER BY r.scheduled_for DESC, r.id DESC LIMIT ${n}`,
|
|
args,
|
|
)
|
|
return rows.map(hydrate)
|
|
}
|
|
|
|
const getById = async (id) => {
|
|
const [row] = await query(`${SELECT_LIST} WHERE r.id = ?`, [id])
|
|
return hydrate(row)
|
|
}
|
|
|
|
/**
|
|
* Materialise one occurrence. Answers the row id, or `null` when one already
|
|
* existed — which is not an error and is the ordinary answer under a tick that
|
|
* overran into the next one.
|
|
*/
|
|
const materialise = async (run) => {
|
|
const result = await query(
|
|
`INSERT IGNORE INTO event_runs
|
|
(definition_id, version_id, scope, scheduled_for, timezone, concurrency_key,
|
|
params, rehearsal, started_by)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
[
|
|
run.definition_id,
|
|
run.version_id,
|
|
run.scope || '',
|
|
run.scheduled_for,
|
|
run.timezone || 'UTC',
|
|
run.concurrency_key,
|
|
run.params === null || run.params === undefined ? null : JSON.stringify(run.params),
|
|
run.rehearsal ? 1 : 0,
|
|
run.started_by,
|
|
],
|
|
)
|
|
return Number(result?.affectedRows || 0) === 1 ? result.insertId : null
|
|
}
|
|
|
|
/** The occurrence the unique key names, whether or not this call created it. */
|
|
const findOccurrence = async (definitionId, scope, scheduledFor) => {
|
|
const [row] = await query(
|
|
`${SELECT_LIST} WHERE r.definition_id = ? AND r.scope = ? AND r.scheduled_for = ?`,
|
|
[definitionId, scope || '', scheduledFor],
|
|
)
|
|
return hydrate(row)
|
|
}
|
|
|
|
/** Is anything of this definition not yet terminal? The archive pre-check. */
|
|
const countActiveForDefinition = async (definitionId) => {
|
|
const [row] = await query(
|
|
`SELECT COUNT(*) AS n FROM event_runs
|
|
WHERE definition_id = ?
|
|
AND status IN ('scheduled','starting','running','paused','ending')`,
|
|
[definitionId],
|
|
)
|
|
return Number(row?.n || 0)
|
|
}
|
|
|
|
// ── Phase 2: the claim, the transitions and the reclaim ────────────────────
|
|
//
|
|
// Everything below is the runner's, and none of it existed in Phase 1 for a
|
|
// stated reason: a half-written claim is worse than no claim, because it reads
|
|
// as protection. It is written here now, in full.
|
|
//
|
|
// **The division of labour with the unique index has not changed.** The index one
|
|
// section up is what makes "one run per occurrence per scope" TRUE; the CAS below
|
|
// decides only WHO advances an occurrence that already exists. Neither substitutes
|
|
// for the other, and this deployment being single-instance (§N4) changes the test
|
|
// rather than the design — the same two protections are what keep a tick that
|
|
// overran into the next one from advancing a run twice.
|
|
|
|
/**
|
|
* Runs the runner should look at this tick: due, and not yet terminal.
|
|
*
|
|
* It selects rather than claims — `claimStart` and `claimTick` below are one row
|
|
* at a time — so two sweepers see the same candidates and then disagree,
|
|
* harmlessly, about which of them owns each. `idx_evrun_due (status,
|
|
* scheduled_for)` is this query.
|
|
*
|
|
* `paused` is absent from the status list on purpose. A paused run is waiting on
|
|
* a human and must not be advanced by a tick; the only thing that moves it is
|
|
* Phase 3's resume control.
|
|
*/
|
|
const findDue = async (now, limit = 50) => {
|
|
const n = Math.min(Math.max(Number(limit) || 50, 1), 500)
|
|
return (
|
|
await query(
|
|
`SELECT * FROM event_runs
|
|
WHERE status IN ('scheduled','starting','running','ending')
|
|
AND scheduled_for <= ?
|
|
ORDER BY scheduled_for, id
|
|
LIMIT ${n}`,
|
|
[now],
|
|
)
|
|
).map(hydrate)
|
|
}
|
|
|
|
/**
|
|
* Take ownership of a run that has not started: the CAS `scheduled -> starting`.
|
|
*
|
|
* Verbatim the outbox claim the org lead settled over `SELECT ... FOR UPDATE
|
|
* SKIP LOCKED` — the instance the server reports `affectedRows = 1` to owns the
|
|
* row, every other sweeper gets 0 and moves on. No transaction to hold open and
|
|
* no MariaDB version floor.
|
|
*/
|
|
async function claimStart(id, owner, leaseUntil) {
|
|
const result = await query(
|
|
`UPDATE event_runs
|
|
SET status = 'starting', claimed_by = ?, claim_expires_at = ?,
|
|
started_at = COALESCE(started_at, NOW())
|
|
WHERE id = ? AND status = 'scheduled'`,
|
|
[owner, leaseUntil, id],
|
|
)
|
|
return Number(result?.affectedRows || 0) === 1
|
|
}
|
|
|
|
/**
|
|
* Take a lease on a run already in flight, so one tick works on it at a time.
|
|
*
|
|
* Unlike `claimStart` this does not change `status` — the run is already
|
|
* `starting`, `running` or `ending`, and what is being claimed is the right to
|
|
* advance it.
|
|
*
|
|
* **A live lease is not re-enterable, not even by the process that took it**, and
|
|
* that is the whole point rather than an oversight. `setInterval` fires the next
|
|
* tick whether or not the last one has returned, so an owner-matches escape
|
|
* clause here would let one process advance one run twice at once — which is
|
|
* precisely the overrun the plan says the CAS is meant to protect against. A run
|
|
* this process still holds is a run this process is still working on; the tick
|
|
* skips it, and `releaseClaim` below is what ends that in the ordinary case.
|
|
*/
|
|
async function claimTick(id, owner, leaseUntil, now) {
|
|
const result = await query(
|
|
`UPDATE event_runs
|
|
SET claimed_by = ?, claim_expires_at = ?
|
|
WHERE id = ?
|
|
AND status IN ('starting','running','ending')
|
|
AND (claim_expires_at IS NULL OR claim_expires_at < ?)`,
|
|
[owner, leaseUntil, id, now],
|
|
)
|
|
return Number(result?.affectedRows || 0) === 1
|
|
}
|
|
|
|
/**
|
|
* Give a still-in-flight run back, so the next tick can pick it up at once.
|
|
*
|
|
* A run left parked on a GM cue, or waiting out a `core.wait`, is not finished
|
|
* and must not carry a lease: without this the run would be unadvanceable until
|
|
* the lease expired, which would turn every wait into `max(wait, leaseMs)`.
|
|
* Scoped to `claimed_by = ?` so a process can only release its own claim.
|
|
*/
|
|
async function releaseClaim(id, owner) {
|
|
const result = await query(
|
|
'UPDATE event_runs SET claimed_by = NULL, claim_expires_at = NULL WHERE id = ? AND claimed_by = ?',
|
|
[id, owner],
|
|
)
|
|
return Number(result?.affectedRows || 0) === 1
|
|
}
|
|
|
|
/**
|
|
* A guarded status transition: `from -> to`, and only from `from`.
|
|
*
|
|
* Every move the runner makes goes through here rather than through a bare
|
|
* UPDATE, so "did this transition actually happen" is answerable at each call
|
|
* site. A `false` is not an error — it is another worker, or this run having been
|
|
* cancelled from the admin surface between the read and the write, which is a
|
|
* race Phase 3's live controls make ordinary.
|
|
*/
|
|
async function transition(id, from, to, { phase, error, clearClaim = false } = {}) {
|
|
const sets = ['status = ?']
|
|
const args = [to]
|
|
if (phase !== undefined) {
|
|
sets.push('current_phase = ?')
|
|
args.push(phase)
|
|
}
|
|
if (error !== undefined) {
|
|
sets.push('last_error = ?')
|
|
args.push(error === null ? null : String(error).slice(0, 500))
|
|
}
|
|
if (TERMINAL.includes(to)) sets.push('ended_at = COALESCE(ended_at, NOW())')
|
|
if (clearClaim || TERMINAL.includes(to)) sets.push('claimed_by = NULL', 'claim_expires_at = NULL')
|
|
|
|
const froms = Array.isArray(from) ? from : [from]
|
|
const result = await query(
|
|
`UPDATE event_runs SET ${sets.join(', ')}
|
|
WHERE id = ? AND status IN (${froms.map(() => '?').join(',')})`,
|
|
[...args, id, ...froms],
|
|
)
|
|
return Number(result?.affectedRows || 0) === 1
|
|
}
|
|
|
|
/**
|
|
* Just this run's status, for a caller that must not act on a stale read.
|
|
*
|
|
* The runner drains a bounded batch of steps from one run inside a single tick,
|
|
* and Phase 3 put a pause and a cancel button in a human's hand — so between two
|
|
* steps of that batch the run may have stopped. A loop that only re-checked at
|
|
* the top of the tick would answer a pause by dispatching another two dozen
|
|
* steps, which is not a pause. One column, by primary key.
|
|
*/
|
|
const statusOf = async (id) => {
|
|
const [row] = await query('SELECT status FROM event_runs WHERE id = ?', [id])
|
|
return row?.status || null
|
|
}
|
|
|
|
/**
|
|
* Set health without touching status (§E).
|
|
*
|
|
* The two columns are separate because a run can be genuinely running and
|
|
* degraded at once — announcements landing, world writes parked — and one column
|
|
* cannot say both. Guarded on the current value so a tick that re-observes the
|
|
* same degradation does not restamp `updated_at`.
|
|
*/
|
|
async function setHealth(id, health) {
|
|
const result = await query('UPDATE event_runs SET health = ? WHERE id = ? AND health <> ?', [
|
|
health,
|
|
id,
|
|
health,
|
|
])
|
|
return Number(result?.affectedRows || 0) === 1
|
|
}
|
|
|
|
/**
|
|
* Runs whose start instant passed more than their own grace window ago (§E, §L).
|
|
*
|
|
* The window is per definition, so the comparison is against `grace_seconds` on
|
|
* the joined row rather than against a constant here: an event whose announcement
|
|
* gives a fifteen-minute window and one that must start on the second are the
|
|
* same query with different data.
|
|
*
|
|
* Only `scheduled` runs qualify. A run that reached `starting` has begun, and
|
|
* "began and then stalled" is a different fact from "never began" — conflating
|
|
* them would let `missed` describe a run that had already announced itself.
|
|
*/
|
|
const findMissed = async (now, limit = 100) => {
|
|
const n = Math.min(Math.max(Number(limit) || 100, 1), 500)
|
|
return (
|
|
await query(
|
|
`SELECT r.* FROM event_runs r
|
|
JOIN event_definitions d ON d.id = r.definition_id
|
|
WHERE r.status = 'scheduled'
|
|
AND r.scheduled_for + INTERVAL d.grace_seconds SECOND < ?
|
|
ORDER BY r.scheduled_for
|
|
LIMIT ${n}`,
|
|
[now],
|
|
)
|
|
).map(hydrate)
|
|
}
|
|
|
|
/**
|
|
* Is another run holding this concurrency key?
|
|
*
|
|
* The org lead's answer for a held key (2026-09-02) is to leave the run
|
|
* `scheduled` and let the grace window decide, so this is a READ rather than a
|
|
* claim: the caller holds off, logs which run holds the key, and tries again next
|
|
* tick. `idx_evrun_concurrency (concurrency_key, status)` is this query, and a
|
|
* NULL key is skipped by that index — which is right, because a definition with
|
|
* no key never contends.
|
|
*/
|
|
const concurrencyHolder = async (key, exceptRunId) => {
|
|
if (!key) return null
|
|
const [row] = await query(
|
|
`SELECT id, status, definition_id FROM event_runs
|
|
WHERE concurrency_key = ?
|
|
AND id <> ?
|
|
AND status IN ('starting','running','paused','ending')
|
|
ORDER BY id LIMIT 1`,
|
|
[key, exceptRunId],
|
|
)
|
|
return row || null
|
|
}
|
|
|
|
/**
|
|
* Recover runs whose claim outlived the process that took it.
|
|
*
|
|
* **It does not change status and it touches no counter.** All it releases is the
|
|
* lease; the run stays exactly where it was and the next tick picks it up through
|
|
* `findDue`. This is Engagement Phase 14's lesson applied one table over: a sweep
|
|
* that returned a stale row to its start state made the attempt ceiling
|
|
* unreachable, so the row cycled forever, never terminal, and therefore never
|
|
* eligible for any retention sweep.
|
|
*/
|
|
const reclaimStale = async (now) => {
|
|
const result = await query(
|
|
`UPDATE event_runs SET claimed_by = NULL, claim_expires_at = NULL
|
|
WHERE status IN ('starting','running','ending')
|
|
AND claim_expires_at IS NOT NULL
|
|
AND claim_expires_at < ?`,
|
|
[now],
|
|
)
|
|
return Number(result?.affectedRows || 0)
|
|
}
|
|
|
|
/** Terminal runs that ended before `before` — what the log retention sweep walks. */
|
|
const terminalBefore = async (before, limit = 500) => {
|
|
const n = Math.min(Math.max(Number(limit) || 500, 1), 5000)
|
|
return (
|
|
await query(
|
|
`SELECT id FROM event_runs
|
|
WHERE status IN (${TERMINAL.map(() => '?').join(',')})
|
|
AND COALESCE(ended_at, updated_at) < ?
|
|
ORDER BY id LIMIT ${n}`,
|
|
[...TERMINAL, before],
|
|
)
|
|
).map((r) => Number(r.id))
|
|
}
|
|
|
|
module.exports = {
|
|
list,
|
|
getById,
|
|
materialise,
|
|
findOccurrence,
|
|
countActiveForDefinition,
|
|
findDue,
|
|
findMissed,
|
|
claimStart,
|
|
claimTick,
|
|
releaseClaim,
|
|
statusOf,
|
|
transition,
|
|
setHealth,
|
|
concurrencyHolder,
|
|
reclaimStale,
|
|
terminalBefore,
|
|
TERMINAL,
|
|
}
|