wtclaude
7b570c8ea1
Three screens, an Events nav group and the six live run controls Phase 1 left
absent on purpose because nothing was in flight. An admin can now author,
publish, start and watch an event that announces things and cues a human; a
moderator can stop one that is going wrong.
Six controls, not eight. `advance` is absent because a phase today advances when
its steps go terminal — the per-step skip already does that — and Phase 5 is what
gives a phase an advance condition. Cancel takes `{ reason }`, not `{ cleanup }`,
until Phase 8's ledger exists. Every control is a compare-and-set on the status it
may act from, so a console rendered thirty seconds ago cannot act on a run that
has moved.
Fixes a defect in the Phase 2 runner: `advanceRun` drained up to
EVENT_STEPS_PER_TICK steps while only checking the run's status at the top of the
tick, so a pause pressed mid-batch did nothing for up to 24 more steps.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T6t8mrAWhZU5vnyYgZTMtL
662 lines
29 KiB
JavaScript
662 lines
29 KiB
JavaScript
// ── The runner's raw SQL, against a real MariaDB ───────────────────────────
|
||
//
|
||
// EVENTS_PLAN.md Phase 2. `eventRunner.test.js` stubs the three tables and
|
||
// exercises everything the runner DECIDES. It cannot prove the statements whose
|
||
// whole correctness is a server contract, and on this codebase that gap has
|
||
// already cost something once: engagement's cooldown claim was green against its
|
||
// stub and always allowed the send against a real server, because the connector
|
||
// defaults `foundRows: true` and a no-op UPDATE reports 1 rather than 0.
|
||
//
|
||
// So the five statements that decide who owns what run here, for real:
|
||
//
|
||
// • **`claimStart`** — the CAS `scheduled -> starting`. "Exactly one winner" is
|
||
// `affectedRows = 1` for one caller and 0 for every other, and that is a
|
||
// property of the SERVER's answer, not of the SQL's shape.
|
||
// • **`claimTick`** — the same, for a run already in flight, and with no
|
||
// owner-matches escape clause. A live lease must refuse its own holder, or
|
||
// one `setInterval` that overran advances one run twice.
|
||
// • **`transition`** — a guarded status move. The guard is the whole thing: a
|
||
// run cancelled between the read and the write must not be transitioned.
|
||
// • **`reclaimStale`** on steps — two statements in a fixed ORDER, give-up
|
||
// before hand-back. Reversing them makes `MAX_ATTEMPTS` unreachable and the
|
||
// row cycles forever (Engagement Phase 14's defect), and **neither statement
|
||
// may touch `attempts`**.
|
||
// • **`holdNext`** — `UPDATE ... ORDER BY seq LIMIT 1` with a guard, which is
|
||
// both a MariaDB-specific syntax and a correctness claim: it must move the
|
||
// next PENDING step and only ever push a due date later.
|
||
//
|
||
// **Phase 3 added four more**, and each of them is a control a staff member
|
||
// presses against a live game world:
|
||
//
|
||
// * **`confirmParked` / `skipByHuman`** - both keyed on `status = 'running'
|
||
// AND claim_expires_at IS NULL`. That pair, and only that pair, means "a cue
|
||
// waiting on a human". If the clause let a LEASED step through, confirm would
|
||
// race the process mid-dispatch on that row.
|
||
// * **`cancelOpen`** - pending steps and parked cues, never a leased one.
|
||
// * **`lastStartedSeq`** - a `MAX(seq) ... WHERE status <> 'pending'`, which is
|
||
// what decides whether retry is offered. The first draft asked for the LOWEST
|
||
// unsettled seq instead, which is a different step whenever a phase carried
|
||
// on past an `on_failure: skip` failure.
|
||
//
|
||
// Plus the two unique indexes that are load-bearing rather than tidy:
|
||
// `uq_evrun_occurrence` (which, not the claim, is what stops two runs of one
|
||
// occurrence existing) and `uq_evstep_slot` (which is what makes re-materialising
|
||
// a phase a no-op).
|
||
//
|
||
// **It SKIPS when there is no database**, deliberately: CI runs the suite with
|
||
// the pool pointed at a dead port, and a file that failed there would make every
|
||
// PR red for a reason unrelated to itself. Run it against this machine's
|
||
// container with:
|
||
//
|
||
// DB_HOST=127.0.0.1 DB_PORT=3306 DB_USER=... DB_PASSWORD=... \
|
||
// node --test test/eventRunnerSql.test.js
|
||
//
|
||
// It creates its tables in a throwaway database named after the process and
|
||
// drops it again, so it can never touch a real schema.
|
||
|
||
const { test, before, after, beforeEach } = require('node:test')
|
||
const assert = require('node:assert/strict')
|
||
const mariadb = require('mariadb')
|
||
|
||
// Trimmed to the columns these statements read or write. The ENUMs are verbatim,
|
||
// because "is `missed` a legal value" is one of the things being proved.
|
||
const SCHEMA = `
|
||
CREATE TABLE event_definitions (
|
||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||
grace_seconds INT NOT NULL DEFAULT 900
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||
CREATE TABLE event_runs (
|
||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||
definition_id INT NOT NULL,
|
||
version_id INT NOT NULL,
|
||
scope VARCHAR(190) NOT NULL DEFAULT '',
|
||
status ENUM('scheduled','starting','running','paused','ending',
|
||
'completed','cancelled','failed','missed')
|
||
NOT NULL DEFAULT 'scheduled',
|
||
health ENUM('ok','degraded','stalled') NOT NULL DEFAULT 'ok',
|
||
current_phase VARCHAR(64) NULL,
|
||
scheduled_for DATETIME NOT NULL,
|
||
concurrency_key VARCHAR(190) NULL,
|
||
started_at DATETIME NULL,
|
||
ended_at DATETIME NULL,
|
||
claimed_by VARCHAR(64) NULL,
|
||
claim_expires_at DATETIME NULL,
|
||
last_error VARCHAR(500) NULL,
|
||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||
UNIQUE KEY uq_evrun_occurrence (definition_id, scope, scheduled_for),
|
||
INDEX idx_evrun_due (status, scheduled_for),
|
||
INDEX idx_evrun_concurrency (concurrency_key, status)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||
CREATE TABLE event_run_steps (
|
||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||
run_id BIGINT NOT NULL,
|
||
phase VARCHAR(64) NOT NULL,
|
||
seq INT NOT NULL,
|
||
action_id VARCHAR(96) NOT NULL,
|
||
params JSON NULL,
|
||
action_version INT NOT NULL DEFAULT 1,
|
||
status ENUM('pending','running','done','failed','skipped','refused','cancelled')
|
||
NOT NULL DEFAULT 'pending',
|
||
due_at DATETIME NULL,
|
||
attempts INT NOT NULL DEFAULT 0,
|
||
on_failure VARCHAR(32) NOT NULL DEFAULT 'skip',
|
||
idempotency_key CHAR(40) NOT NULL,
|
||
claimed_by VARCHAR(64) NULL,
|
||
claim_expires_at DATETIME NULL,
|
||
last_error VARCHAR(500) NULL,
|
||
started_at DATETIME NULL,
|
||
finished_at DATETIME NULL,
|
||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||
UNIQUE KEY uq_evstep_slot (run_id, phase, seq),
|
||
INDEX idx_evstep_due (status, due_at)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||
`
|
||
|
||
// The statements under test, verbatim from `eventRuns.db.js` and
|
||
// `eventRunSteps.db.js`. Duplicated rather than required, because requiring the
|
||
// modules would drag in `utils/db`'s pool, which the harness has already pointed
|
||
// at a dead port. The pool below leaves `foundRows` at the connector's default,
|
||
// exactly as `utils/db.js` does — pinning it here would make this file agree with
|
||
// the code by construction and prove nothing about the pool the server runs.
|
||
const CLAIM_START = `
|
||
UPDATE event_runs
|
||
SET status = 'starting', claimed_by = ?, claim_expires_at = ?,
|
||
started_at = COALESCE(started_at, NOW())
|
||
WHERE id = ? AND status = 'scheduled'`
|
||
|
||
const CLAIM_TICK = `
|
||
UPDATE event_runs
|
||
SET claimed_by = ?, claim_expires_at = ?
|
||
WHERE id = ?
|
||
AND status IN ('starting','running','ending')
|
||
AND (claim_expires_at IS NULL OR claim_expires_at < ?)`
|
||
|
||
const TRANSITION = `
|
||
UPDATE event_runs SET status = ?, current_phase = ?
|
||
WHERE id = ? AND status IN (?)`
|
||
|
||
const CLAIM_STEP = `
|
||
UPDATE event_run_steps
|
||
SET status = 'running', attempts = attempts + 1, claimed_by = ?, claim_expires_at = ?,
|
||
started_at = COALESCE(started_at, NOW())
|
||
WHERE id = ? AND status = 'pending' AND (due_at IS NULL OR due_at <= ?)`
|
||
|
||
const STEP_GIVE_UP = `
|
||
UPDATE event_run_steps
|
||
SET status = 'failed', last_error = 'gave up after repeated interruptions',
|
||
finished_at = NOW(), claimed_by = NULL, claim_expires_at = NULL
|
||
WHERE status = 'running'
|
||
AND claim_expires_at IS NOT NULL AND claim_expires_at < ?
|
||
AND attempts >= ?`
|
||
|
||
const STEP_RECLAIM = `
|
||
UPDATE event_run_steps
|
||
SET status = 'pending', claimed_by = NULL, claim_expires_at = NULL
|
||
WHERE status = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at < ?`
|
||
|
||
const HOLD_NEXT = `
|
||
UPDATE event_run_steps
|
||
SET due_at = ?
|
||
WHERE run_id = ? AND phase = ? AND seq > ? AND status = 'pending'
|
||
AND (due_at IS NULL OR due_at < ?)
|
||
ORDER BY seq LIMIT 1`
|
||
|
||
// Phase 3's four, verbatim from `eventRunSteps.db.js`.
|
||
const CONFIRM_PARKED = `
|
||
UPDATE event_run_steps
|
||
SET status = 'done', finished_at = NOW(), claimed_by = NULL,
|
||
last_error = ?
|
||
WHERE id = ? AND status = 'running' AND claim_expires_at IS NULL`
|
||
|
||
const SKIP_BY_HUMAN = `
|
||
UPDATE event_run_steps
|
||
SET status = 'skipped', finished_at = NOW(), claimed_by = NULL,
|
||
last_error = ?
|
||
WHERE id = ?
|
||
AND (status = 'pending' OR (status = 'running' AND claim_expires_at IS NULL))`
|
||
|
||
const REQUEUE = `
|
||
UPDATE event_run_steps
|
||
SET status = 'pending', attempts = 0, due_at = NULL, last_error = NULL,
|
||
claimed_by = NULL, claim_expires_at = NULL, finished_at = NULL
|
||
WHERE id = ? AND status = 'failed'`
|
||
|
||
const CANCEL_OPEN = `
|
||
UPDATE event_run_steps
|
||
SET status = 'cancelled', finished_at = NOW(), claimed_by = NULL
|
||
WHERE run_id = ?
|
||
AND (status = 'pending' OR (status = 'running' AND claim_expires_at IS NULL))`
|
||
|
||
const LAST_STARTED_SEQ = `
|
||
SELECT MAX(seq) AS seq FROM event_run_steps
|
||
WHERE run_id = ? AND phase = ? AND status <> 'pending'`
|
||
|
||
const MATERIALISE_RUN = `
|
||
INSERT IGNORE INTO event_runs (definition_id, version_id, scope, scheduled_for, concurrency_key)
|
||
VALUES (?, ?, ?, ?, ?)`
|
||
|
||
const MATERIALISE_STEP = `
|
||
INSERT IGNORE INTO event_run_steps (run_id, phase, seq, action_id, idempotency_key)
|
||
VALUES (?, ?, ?, ?, ?)`
|
||
|
||
const FIND_MISSED = `
|
||
SELECT r.id FROM event_runs r
|
||
JOIN event_definitions d ON d.id = r.definition_id
|
||
WHERE r.status = 'scheduled'
|
||
AND r.scheduled_for + INTERVAL d.grace_seconds SECOND < ?`
|
||
|
||
const DB = `rg_events_test_${process.pid}`
|
||
let pool = null
|
||
let available = false
|
||
|
||
const poolOpts = () => ({
|
||
host: process.env.DB_HOST || '127.0.0.1',
|
||
port: Number(process.env.DB_PORT) || 3306,
|
||
user: process.env.DB_USER || 'root',
|
||
password: process.env.DB_PASSWORD || '',
|
||
})
|
||
|
||
before(async () => {
|
||
const admin = mariadb.createPool({
|
||
...poolOpts(),
|
||
connectionLimit: 1,
|
||
connectTimeout: 2000,
|
||
initializationTimeout: 2000,
|
||
multipleStatements: true,
|
||
})
|
||
try {
|
||
await admin.query(`CREATE DATABASE ${DB}`)
|
||
available = true
|
||
} catch {
|
||
available = false
|
||
} finally {
|
||
await admin.end().catch(() => {})
|
||
}
|
||
if (!available) return
|
||
|
||
pool = mariadb.createPool({
|
||
...poolOpts(),
|
||
database: DB,
|
||
connectionLimit: 3,
|
||
multipleStatements: true,
|
||
bigIntAsNumber: true,
|
||
insertIdAsNumber: true,
|
||
})
|
||
await pool.query(SCHEMA)
|
||
})
|
||
|
||
after(async () => {
|
||
if (pool) {
|
||
await pool.query(`DROP DATABASE IF EXISTS ${DB}`).catch(() => {})
|
||
await pool.end().catch(() => {})
|
||
}
|
||
})
|
||
|
||
// Checked INSIDE each test, never as a `{ skip }` option: the option is evaluated
|
||
// when the file is read, which is before `before()` has had a chance to find out
|
||
// whether there is a database. Every test skipped unconditionally is what that
|
||
// mistake looks like, and it looks exactly like a passing suite.
|
||
const SKIP = 'no database reachable - set DB_HOST/DB_PORT/DB_USER/DB_PASSWORD to run'
|
||
const needDb = (t) => {
|
||
if (available) return false
|
||
t.skip(SKIP)
|
||
return true
|
||
}
|
||
|
||
const T0 = new Date('2026-09-02T12:00:00Z')
|
||
const later = (ms) => new Date(T0.getTime() + ms)
|
||
const rows = (r) => Number(r.affectedRows)
|
||
|
||
beforeEach(async () => {
|
||
if (!available) return
|
||
await pool.query('DELETE FROM event_run_steps')
|
||
await pool.query('DELETE FROM event_runs')
|
||
await pool.query('DELETE FROM event_definitions')
|
||
})
|
||
|
||
async function seedRun(over = {}) {
|
||
const def = await pool.query('INSERT INTO event_definitions (grace_seconds) VALUES (?)', [
|
||
over.graceSeconds ?? 900,
|
||
])
|
||
const r = await pool.query(
|
||
`INSERT INTO event_runs (definition_id, version_id, scope, status, scheduled_for, concurrency_key,
|
||
claimed_by, claim_expires_at)
|
||
VALUES (?, 1, ?, ?, ?, ?, ?, ?)`,
|
||
[
|
||
def.insertId,
|
||
over.scope ?? '',
|
||
over.status ?? 'scheduled',
|
||
over.scheduledFor ?? T0,
|
||
over.concurrencyKey ?? null,
|
||
over.claimedBy ?? null,
|
||
over.claimExpiresAt ?? null,
|
||
],
|
||
)
|
||
return { runId: r.insertId, definitionId: def.insertId }
|
||
}
|
||
|
||
const seedStep = async (runId, over = {}) =>
|
||
(
|
||
await pool.query(
|
||
`INSERT INTO event_run_steps (run_id, phase, seq, action_id, status, due_at, attempts,
|
||
claimed_by, claim_expires_at, idempotency_key)
|
||
VALUES (?, ?, ?, 'test.noop', ?, ?, ?, ?, ?, ?)`,
|
||
[
|
||
runId,
|
||
over.phase ?? 'main',
|
||
over.seq ?? 0,
|
||
over.status ?? 'pending',
|
||
over.dueAt ?? null,
|
||
over.attempts ?? 0,
|
||
over.claimedBy ?? null,
|
||
over.claimExpiresAt ?? null,
|
||
over.key ?? 'k'.repeat(40),
|
||
],
|
||
)
|
||
).insertId
|
||
|
||
const stepById = async (id) => (await pool.query('SELECT * FROM event_run_steps WHERE id = ?', [id]))[0]
|
||
const runById = async (id) => (await pool.query('SELECT * FROM event_runs WHERE id = ?', [id]))[0]
|
||
|
||
// ── claimStart: exactly one winner ─────────────────────────────────────────
|
||
|
||
test('claimStart: the first caller wins and every other gets zero', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun()
|
||
|
||
const first = await pool.query(CLAIM_START, ['host:1', later(60_000), runId])
|
||
const second = await pool.query(CLAIM_START, ['host:2', later(60_000), runId])
|
||
|
||
assert.equal(rows(first), 1, 'the winner is told 1')
|
||
assert.equal(rows(second), 0, 'the loser is told 0, not 1 with foundRows')
|
||
assert.equal((await runById(runId)).claimed_by, 'host:1')
|
||
assert.equal((await runById(runId)).status, 'starting')
|
||
})
|
||
|
||
test('claimStart: started_at is stamped once and never moved', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun()
|
||
await pool.query(CLAIM_START, ['host:1', later(60_000), runId])
|
||
const first = (await runById(runId)).started_at
|
||
|
||
await pool.query("UPDATE event_runs SET status = 'scheduled' WHERE id = ?", [runId])
|
||
await pool.query(CLAIM_START, ['host:2', later(60_000), runId])
|
||
|
||
assert.deepEqual((await runById(runId)).started_at, first, 'COALESCE keeps the original instant')
|
||
})
|
||
|
||
// ── claimTick: a live lease refuses even its own holder ────────────────────
|
||
|
||
test('claimTick: a live lease is not re-enterable by the process that took it', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running', claimedBy: 'host:1', claimExpiresAt: later(60_000) })
|
||
|
||
const again = await pool.query(CLAIM_TICK, ['host:1', later(120_000), runId, T0])
|
||
assert.equal(rows(again), 0, 'a tick that overran must not advance its own run twice')
|
||
})
|
||
|
||
test('claimTick: an expired lease is takeable, by anyone', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running', claimedBy: 'host:1', claimExpiresAt: later(-60_000) })
|
||
|
||
const taken = await pool.query(CLAIM_TICK, ['host:2', later(60_000), runId, T0])
|
||
assert.equal(rows(taken), 1)
|
||
assert.equal((await runById(runId)).claimed_by, 'host:2')
|
||
})
|
||
|
||
test('claimTick: a paused run is never claimable', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'paused' })
|
||
assert.equal(rows(await pool.query(CLAIM_TICK, ['host:1', later(60_000), runId, T0])), 0)
|
||
})
|
||
|
||
// ── transition: the guard is the whole point ───────────────────────────────
|
||
|
||
test('transition: a run cancelled underneath the tick is not transitioned', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
await pool.query("UPDATE event_runs SET status = 'cancelled' WHERE id = ?", [runId])
|
||
|
||
const moved = await pool.query(TRANSITION, ['completed', 'main', runId, 'running'])
|
||
assert.equal(rows(moved), 0)
|
||
assert.equal((await runById(runId)).status, 'cancelled')
|
||
})
|
||
|
||
test('transition: running -> running is a guarded write, not a no-op', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
|
||
// This is how the runner advances `current_phase`, and `foundRows` is exactly
|
||
// what makes it report 1 despite `status` not changing — which is the answer
|
||
// the caller needs, because what it is checking is that the run is STILL
|
||
// running, not that the status moved.
|
||
const moved = await pool.query(TRANSITION, ['running', 'closing', runId, 'running'])
|
||
assert.equal(rows(moved), 1)
|
||
assert.equal((await runById(runId)).current_phase, 'closing')
|
||
})
|
||
|
||
// ── The step claim ─────────────────────────────────────────────────────────
|
||
|
||
test('the step claim: one winner, and attempts is incremented by the claim alone', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const stepId = await seedStep(runId)
|
||
|
||
assert.equal(rows(await pool.query(CLAIM_STEP, ['host:1', later(60_000), stepId, T0])), 1)
|
||
assert.equal(rows(await pool.query(CLAIM_STEP, ['host:2', later(60_000), stepId, T0])), 0)
|
||
assert.equal((await stepById(stepId)).attempts, 1, 'one claim, one attempt')
|
||
})
|
||
|
||
test('the step claim: a step held behind a core.wait is not due', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const stepId = await seedStep(runId, { dueAt: later(300_000) })
|
||
|
||
assert.equal(rows(await pool.query(CLAIM_STEP, ['host:1', later(60_000), stepId, T0])), 0)
|
||
assert.equal(rows(await pool.query(CLAIM_STEP, ['host:1', later(360_000), stepId, later(301_000)])), 1)
|
||
})
|
||
|
||
// ── The reclaim: order, and what it must not touch ─────────────────────────
|
||
|
||
test('the reclaim hands a stale step back WITHOUT resetting attempts', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const stepId = await seedStep(runId, { status: 'running', attempts: 2, claimExpiresAt: later(-1000) })
|
||
|
||
await pool.query(STEP_GIVE_UP, [T0, 3])
|
||
await pool.query(STEP_RECLAIM, [T0])
|
||
|
||
const step = await stepById(stepId)
|
||
assert.equal(step.status, 'pending')
|
||
assert.equal(step.attempts, 2, 'Engagement Phase 14: a reclaim that reset this made MAX_ATTEMPTS unreachable')
|
||
})
|
||
|
||
test('the reclaim gives up FIRST, so a spent step leaves running as failed', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const stepId = await seedStep(runId, { status: 'running', attempts: 3, claimExpiresAt: later(-1000) })
|
||
|
||
const gaveUp = await pool.query(STEP_GIVE_UP, [T0, 3])
|
||
const reclaimed = await pool.query(STEP_RECLAIM, [T0])
|
||
|
||
assert.equal(rows(gaveUp), 1)
|
||
assert.equal(rows(reclaimed), 0, 'reversing these two makes the row retry forever')
|
||
assert.equal((await stepById(stepId)).status, 'failed')
|
||
assert.equal((await stepById(stepId)).attempts, 3)
|
||
})
|
||
|
||
test('the reclaim leaves a PARKED step alone, however long it waits', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const stepId = await seedStep(runId, { status: 'running', attempts: 1, claimExpiresAt: null })
|
||
|
||
await pool.query(STEP_GIVE_UP, [later(365 * 24 * 3600 * 1000), 3])
|
||
await pool.query(STEP_RECLAIM, [later(365 * 24 * 3600 * 1000)])
|
||
|
||
const step = await stepById(stepId)
|
||
assert.equal(step.status, 'running', 'a NULL lease is a parked cue, not staleness')
|
||
assert.equal(step.attempts, 1)
|
||
})
|
||
|
||
// ── holdNext ───────────────────────────────────────────────────────────────
|
||
|
||
test('holdNext moves the next PENDING step of the phase, and only one', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
await seedStep(runId, { seq: 0, status: 'done' })
|
||
const second = await seedStep(runId, { seq: 1 })
|
||
const third = await seedStep(runId, { seq: 2 })
|
||
|
||
const moved = await pool.query(HOLD_NEXT, [later(300_000), runId, 'main', 0, later(300_000)])
|
||
assert.equal(rows(moved), 1)
|
||
assert.deepEqual((await stepById(second)).due_at, later(300_000))
|
||
assert.equal((await stepById(third)).due_at, null, 'a wait holds the next step, not the rest of the phase')
|
||
})
|
||
|
||
test('holdNext never pulls a due date earlier, so a re-dispatch cannot double the wait', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
await seedStep(runId, { seq: 0, status: 'done' })
|
||
const second = await seedStep(runId, { seq: 1, dueAt: later(600_000) })
|
||
|
||
const moved = await pool.query(HOLD_NEXT, [later(300_000), runId, 'main', 0, later(300_000)])
|
||
assert.equal(rows(moved), 0)
|
||
assert.deepEqual((await stepById(second)).due_at, later(600_000))
|
||
})
|
||
|
||
test('holdNext skips a step that is already running', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
await seedStep(runId, { seq: 0, status: 'done' })
|
||
await seedStep(runId, { seq: 1, status: 'running' })
|
||
const third = await seedStep(runId, { seq: 2 })
|
||
|
||
await pool.query(HOLD_NEXT, [later(300_000), runId, 'main', 0, later(300_000)])
|
||
assert.deepEqual((await stepById(third)).due_at, later(300_000))
|
||
})
|
||
|
||
// ── The two unique indexes that carry the weight ───────────────────────────
|
||
|
||
test('uq_evrun_occurrence, not the claim, is what stops two runs of one occurrence', async (t) => {
|
||
if (needDb(t)) return
|
||
const def = await pool.query('INSERT INTO event_definitions (grace_seconds) VALUES (900)')
|
||
|
||
const a = await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null])
|
||
const b = await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null])
|
||
|
||
assert.equal(rows(a), 1)
|
||
assert.equal(rows(b), 0, 'INSERT IGNORE answers honestly rather than raising a 1062')
|
||
const all = await pool.query('SELECT COUNT(*) AS n FROM event_runs')
|
||
assert.equal(Number(all[0].n), 1)
|
||
})
|
||
|
||
test("scope '' rather than NULL is what makes that index work at all", async (t) => {
|
||
if (needDb(t)) return
|
||
const def = await pool.query('INSERT INTO event_definitions (grace_seconds) VALUES (900)')
|
||
|
||
// The empty-string case collides, as it must. A NULL scope would NOT: multiple
|
||
// NULLs do not collide in MariaDB, which would silently permit two runs of one
|
||
// occurrence — the reason the column is NOT NULL DEFAULT ''.
|
||
await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null])
|
||
assert.equal(rows(await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null])), 0)
|
||
|
||
// Two different scopes are two different occurrences, which is what lets a
|
||
// worldwide event fan out across servers without colliding with itself.
|
||
assert.equal(rows(await pool.query(MATERIALISE_RUN, [def.insertId, 1, 'atlantic', T0, null])), 1)
|
||
})
|
||
|
||
test('uq_evstep_slot makes re-materialising a phase a no-op', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
|
||
assert.equal(rows(await pool.query(MATERIALISE_STEP, [runId, 'main', 0, 'test.noop', 'a'.repeat(40)])), 1)
|
||
assert.equal(rows(await pool.query(MATERIALISE_STEP, [runId, 'main', 0, 'test.noop', 'b'.repeat(40)])), 0)
|
||
|
||
const [step] = await pool.query('SELECT idempotency_key FROM event_run_steps WHERE run_id = ?', [runId])
|
||
assert.equal(step.idempotency_key, 'a'.repeat(40), 'a re-materialise cannot overwrite a key a dispatch already sent')
|
||
})
|
||
|
||
// ── The grace window is per definition ─────────────────────────────────────
|
||
|
||
test('findMissed compares against each definition’s own grace window', async (t) => {
|
||
if (needDb(t)) return
|
||
const tight = await seedRun({ graceSeconds: 600 })
|
||
const generous = await seedRun({ graceSeconds: 3600, scope: 'b' })
|
||
|
||
const missed = (await pool.query(FIND_MISSED, [later(30 * 60 * 1000)])).map((r) => Number(r.id))
|
||
assert.deepEqual(missed, [tight.runId])
|
||
assert.ok(!missed.includes(generous.runId), 'inside its own window a run starts late rather than being missed')
|
||
})
|
||
|
||
// -- Phase 3: the controls a human presses ----------------------------------
|
||
|
||
test('confirm resolves a parked cue and cannot touch a step being dispatched', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const parked = await seedStep(runId, { seq: 0, status: 'running', claimedBy: 'host:1', claimExpiresAt: null })
|
||
const busy = await seedStep(runId, { seq: 1, status: 'running', claimedBy: 'host:1', claimExpiresAt: later(60_000), key: 'x'.repeat(40) })
|
||
|
||
assert.equal(rows(await pool.query(CONFIRM_PARKED, ['gate opened', parked])), 1)
|
||
assert.equal(rows(await pool.query(CONFIRM_PARKED, ['nope', busy])), 0, 'a live lease is a step somebody owns')
|
||
|
||
assert.equal((await stepById(parked)).status, 'done')
|
||
assert.equal((await stepById(parked)).last_error, 'gate opened')
|
||
assert.equal((await stepById(busy)).status, 'running')
|
||
})
|
||
|
||
test('a confirm of an already-confirmed cue reports 0, not 1', async (t) => {
|
||
if (needDb(t)) return
|
||
// The engagement Phase 4a shape: a connector that defaults `foundRows: true`
|
||
// reports 1 for an UPDATE that matched and changed nothing, and a control that
|
||
// read that as success would tell a second staff member their press worked.
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const parked = await seedStep(runId, { status: 'running', claimExpiresAt: null })
|
||
|
||
assert.equal(rows(await pool.query(CONFIRM_PARKED, [null, parked])), 1)
|
||
assert.equal(rows(await pool.query(CONFIRM_PARKED, [null, parked])), 0)
|
||
})
|
||
|
||
test('skip takes a pending step and a parked cue, and refuses a leased one', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const pending = await seedStep(runId, { seq: 0, status: 'pending' })
|
||
const parked = await seedStep(runId, { seq: 1, status: 'running', claimExpiresAt: null, key: 'y'.repeat(40) })
|
||
const busy = await seedStep(runId, { seq: 2, status: 'running', claimExpiresAt: later(60_000), key: 'z'.repeat(40) })
|
||
const failed = await seedStep(runId, { seq: 3, status: 'failed', key: 'w'.repeat(40) })
|
||
|
||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, pending])), 1)
|
||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, parked])), 1)
|
||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, busy])), 0)
|
||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, failed])), 0, 'a failed step is terminal; resume carries the phase past it')
|
||
})
|
||
|
||
test('cancelOpen closes pending steps and parked cues, and leaves a leased one alone', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
const done = await seedStep(runId, { seq: 0, status: 'done' })
|
||
const busy = await seedStep(runId, { seq: 1, status: 'running', claimExpiresAt: later(60_000), key: 'p'.repeat(40) })
|
||
const parked = await seedStep(runId, { seq: 2, status: 'running', claimExpiresAt: null, key: 'q'.repeat(40) })
|
||
const pending = await seedStep(runId, { seq: 3, status: 'pending', key: 'r'.repeat(40) })
|
||
|
||
assert.equal(rows(await pool.query(CANCEL_OPEN, [runId])), 2)
|
||
|
||
assert.equal((await stepById(done)).status, 'done')
|
||
assert.equal((await stepById(busy)).status, 'running', 'nothing can recall a command already sent')
|
||
assert.equal((await stepById(parked)).status, 'cancelled', 'a cancelled run must stop claiming to wait on somebody')
|
||
assert.equal((await stepById(pending)).status, 'cancelled')
|
||
})
|
||
|
||
test('requeue only takes a failed step, and puts attempts back to zero', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'paused' })
|
||
const failed = await seedStep(runId, { seq: 0, status: 'failed', attempts: 3 })
|
||
const pending = await seedStep(runId, { seq: 1, status: 'pending', key: 's'.repeat(40) })
|
||
|
||
assert.equal(rows(await pool.query(REQUEUE, [failed])), 1)
|
||
assert.equal(rows(await pool.query(REQUEUE, [pending])), 0)
|
||
|
||
const row = await stepById(failed)
|
||
assert.equal(row.status, 'pending')
|
||
assert.equal(Number(row.attempts), 0)
|
||
assert.equal(row.due_at, null, 'a re-queued step is due now, not at the retry backoff it was left on')
|
||
})
|
||
|
||
test('lastStartedSeq names the furthest step of the phase, not the earliest unsettled one', async (t) => {
|
||
if (needDb(t)) return
|
||
// The defect this replaced: a phase that carried on past a failed step (an
|
||
// `on_failure` of `skip`) and then paused at a later one. "The lowest seq that
|
||
// is not settled" answers with the FIRST failure - a step the runner has long
|
||
// since stepped over - and retry would re-queue a row behind its own cursor.
|
||
const { runId } = await seedRun({ status: 'paused' })
|
||
await seedStep(runId, { seq: 0, status: 'failed', key: 'a'.repeat(40) })
|
||
await seedStep(runId, { seq: 1, status: 'done', key: 'b'.repeat(40) })
|
||
await seedStep(runId, { seq: 2, status: 'failed', key: 'c'.repeat(40) })
|
||
await seedStep(runId, { seq: 3, status: 'pending', key: 'd'.repeat(40) })
|
||
|
||
const [row] = await pool.query(LAST_STARTED_SEQ, [runId, 'main'])
|
||
assert.equal(Number(row.seq), 2)
|
||
})
|
||
|
||
test('lastStartedSeq is NULL for a phase nothing has touched', async (t) => {
|
||
if (needDb(t)) return
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
await seedStep(runId, { seq: 0, status: 'pending' })
|
||
|
||
const [row] = await pool.query(LAST_STARTED_SEQ, [runId, 'main'])
|
||
assert.equal(row.seq, null, 'a null must read as "nothing to retry", not as seq 0')
|
||
})
|
||
|
||
test('a guarded transition refuses a run that was cancelled underneath it', async (t) => {
|
||
if (needDb(t)) return
|
||
// What the admin cancel looks like from the runner's side, mid-tick: the
|
||
// guarded write returns 0 and the tick treats the run as taken rather than
|
||
// advancing a run somebody has just stopped.
|
||
const { runId } = await seedRun({ status: 'running' })
|
||
await pool.query("UPDATE event_runs SET status = 'cancelled' WHERE id = ?", [runId])
|
||
|
||
assert.equal(rows(await pool.query(TRANSITION, ['running', 'two', runId, 'running'])), 0)
|
||
assert.equal((await runById(runId)).status, 'cancelled')
|
||
})
|