// ── The runner's raw SQL, against a real MariaDB ─────────────────────────── // // EVENTS_PLAN.md Phase 2. `eventRunner.test.js` stubs the three tables and // exercises everything the runner DECIDES. It cannot prove the statements whose // whole correctness is a server contract, and on this codebase that gap has // already cost something once: engagement's cooldown claim was green against its // stub and always allowed the send against a real server, because the connector // defaults `foundRows: true` and a no-op UPDATE reports 1 rather than 0. // // So the five statements that decide who owns what run here, for real: // // • **`claimStart`** — the CAS `scheduled -> starting`. "Exactly one winner" is // `affectedRows = 1` for one caller and 0 for every other, and that is a // property of the SERVER's answer, not of the SQL's shape. // • **`claimTick`** — the same, for a run already in flight, and with no // owner-matches escape clause. A live lease must refuse its own holder, or // one `setInterval` that overran advances one run twice. // • **`transition`** — a guarded status move. The guard is the whole thing: a // run cancelled between the read and the write must not be transitioned. // • **`reclaimStale`** on steps — two statements in a fixed ORDER, give-up // before hand-back. Reversing them makes `MAX_ATTEMPTS` unreachable and the // row cycles forever (Engagement Phase 14's defect), and **neither statement // may touch `attempts`**. // • **`holdNext`** — `UPDATE ... ORDER BY seq LIMIT 1` with a guard, which is // both a MariaDB-specific syntax and a correctness claim: it must move the // next PENDING step and only ever push a due date later. // // Plus the two unique indexes that are load-bearing rather than tidy: // `uq_evrun_occurrence` (which, not the claim, is what stops two runs of one // occurrence existing) and `uq_evstep_slot` (which is what makes re-materialising // a phase a no-op). // // **It SKIPS when there is no database**, deliberately: CI runs the suite with // the pool pointed at a dead port, and a file that failed there would make every // PR red for a reason unrelated to itself. Run it against this machine's // container with: // // DB_HOST=127.0.0.1 DB_PORT=3306 DB_USER=... DB_PASSWORD=... \ // node --test test/eventRunnerSql.test.js // // It creates its tables in a throwaway database named after the process and // drops it again, so it can never touch a real schema. const { test, before, after, beforeEach } = require('node:test') const assert = require('node:assert/strict') const mariadb = require('mariadb') // Trimmed to the columns these statements read or write. The ENUMs are verbatim, // because "is `missed` a legal value" is one of the things being proved. const SCHEMA = ` CREATE TABLE event_definitions ( id INT AUTO_INCREMENT PRIMARY KEY, grace_seconds INT NOT NULL DEFAULT 900 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE event_runs ( id BIGINT AUTO_INCREMENT PRIMARY KEY, definition_id INT NOT NULL, version_id INT NOT NULL, scope VARCHAR(190) NOT NULL DEFAULT '', status ENUM('scheduled','starting','running','paused','ending', 'completed','cancelled','failed','missed') NOT NULL DEFAULT 'scheduled', health ENUM('ok','degraded','stalled') NOT NULL DEFAULT 'ok', current_phase VARCHAR(64) NULL, scheduled_for DATETIME NOT NULL, concurrency_key VARCHAR(190) NULL, started_at DATETIME NULL, ended_at DATETIME NULL, claimed_by VARCHAR(64) NULL, claim_expires_at DATETIME NULL, last_error VARCHAR(500) NULL, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, UNIQUE KEY uq_evrun_occurrence (definition_id, scope, scheduled_for), INDEX idx_evrun_due (status, scheduled_for), INDEX idx_evrun_concurrency (concurrency_key, status) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE event_run_steps ( id BIGINT AUTO_INCREMENT PRIMARY KEY, run_id BIGINT NOT NULL, phase VARCHAR(64) NOT NULL, seq INT NOT NULL, action_id VARCHAR(96) NOT NULL, params JSON NULL, action_version INT NOT NULL DEFAULT 1, status ENUM('pending','running','done','failed','skipped','refused','cancelled') NOT NULL DEFAULT 'pending', due_at DATETIME NULL, attempts INT NOT NULL DEFAULT 0, on_failure VARCHAR(32) NOT NULL DEFAULT 'skip', idempotency_key CHAR(40) NOT NULL, claimed_by VARCHAR(64) NULL, claim_expires_at DATETIME NULL, last_error VARCHAR(500) NULL, started_at DATETIME NULL, finished_at DATETIME NULL, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, UNIQUE KEY uq_evstep_slot (run_id, phase, seq), INDEX idx_evstep_due (status, due_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; ` // The statements under test, verbatim from `eventRuns.db.js` and // `eventRunSteps.db.js`. Duplicated rather than required, because requiring the // modules would drag in `utils/db`'s pool, which the harness has already pointed // at a dead port. The pool below leaves `foundRows` at the connector's default, // exactly as `utils/db.js` does — pinning it here would make this file agree with // the code by construction and prove nothing about the pool the server runs. const CLAIM_START = ` UPDATE event_runs SET status = 'starting', claimed_by = ?, claim_expires_at = ?, started_at = COALESCE(started_at, NOW()) WHERE id = ? AND status = 'scheduled'` const CLAIM_TICK = ` UPDATE event_runs SET claimed_by = ?, claim_expires_at = ? WHERE id = ? AND status IN ('starting','running','ending') AND (claim_expires_at IS NULL OR claim_expires_at < ?)` const TRANSITION = ` UPDATE event_runs SET status = ?, current_phase = ? WHERE id = ? AND status IN (?)` const CLAIM_STEP = ` UPDATE event_run_steps SET status = 'running', attempts = attempts + 1, claimed_by = ?, claim_expires_at = ?, started_at = COALESCE(started_at, NOW()) WHERE id = ? AND status = 'pending' AND (due_at IS NULL OR due_at <= ?)` const STEP_GIVE_UP = ` UPDATE event_run_steps SET status = 'failed', last_error = 'gave up after repeated interruptions', finished_at = NOW(), claimed_by = NULL, claim_expires_at = NULL WHERE status = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at < ? AND attempts >= ?` const STEP_RECLAIM = ` UPDATE event_run_steps SET status = 'pending', claimed_by = NULL, claim_expires_at = NULL WHERE status = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at < ?` const HOLD_NEXT = ` UPDATE event_run_steps SET due_at = ? WHERE run_id = ? AND phase = ? AND seq > ? AND status = 'pending' AND (due_at IS NULL OR due_at < ?) ORDER BY seq LIMIT 1` const MATERIALISE_RUN = ` INSERT IGNORE INTO event_runs (definition_id, version_id, scope, scheduled_for, concurrency_key) VALUES (?, ?, ?, ?, ?)` const MATERIALISE_STEP = ` INSERT IGNORE INTO event_run_steps (run_id, phase, seq, action_id, idempotency_key) VALUES (?, ?, ?, ?, ?)` const FIND_MISSED = ` SELECT r.id FROM event_runs r JOIN event_definitions d ON d.id = r.definition_id WHERE r.status = 'scheduled' AND r.scheduled_for + INTERVAL d.grace_seconds SECOND < ?` const DB = `rg_events_test_${process.pid}` let pool = null let available = false const poolOpts = () => ({ host: process.env.DB_HOST || '127.0.0.1', port: Number(process.env.DB_PORT) || 3306, user: process.env.DB_USER || 'root', password: process.env.DB_PASSWORD || '', }) before(async () => { const admin = mariadb.createPool({ ...poolOpts(), connectionLimit: 1, connectTimeout: 2000, initializationTimeout: 2000, multipleStatements: true, }) try { await admin.query(`CREATE DATABASE ${DB}`) available = true } catch { available = false } finally { await admin.end().catch(() => {}) } if (!available) return pool = mariadb.createPool({ ...poolOpts(), database: DB, connectionLimit: 3, multipleStatements: true, bigIntAsNumber: true, insertIdAsNumber: true, }) await pool.query(SCHEMA) }) after(async () => { if (pool) { await pool.query(`DROP DATABASE IF EXISTS ${DB}`).catch(() => {}) await pool.end().catch(() => {}) } }) // Checked INSIDE each test, never as a `{ skip }` option: the option is evaluated // when the file is read, which is before `before()` has had a chance to find out // whether there is a database. Every test skipped unconditionally is what that // mistake looks like, and it looks exactly like a passing suite. const SKIP = 'no database reachable - set DB_HOST/DB_PORT/DB_USER/DB_PASSWORD to run' const needDb = (t) => { if (available) return false t.skip(SKIP) return true } const T0 = new Date('2026-09-02T12:00:00Z') const later = (ms) => new Date(T0.getTime() + ms) const rows = (r) => Number(r.affectedRows) beforeEach(async () => { if (!available) return await pool.query('DELETE FROM event_run_steps') await pool.query('DELETE FROM event_runs') await pool.query('DELETE FROM event_definitions') }) async function seedRun(over = {}) { const def = await pool.query('INSERT INTO event_definitions (grace_seconds) VALUES (?)', [ over.graceSeconds ?? 900, ]) const r = await pool.query( `INSERT INTO event_runs (definition_id, version_id, scope, status, scheduled_for, concurrency_key, claimed_by, claim_expires_at) VALUES (?, 1, ?, ?, ?, ?, ?, ?)`, [ def.insertId, over.scope ?? '', over.status ?? 'scheduled', over.scheduledFor ?? T0, over.concurrencyKey ?? null, over.claimedBy ?? null, over.claimExpiresAt ?? null, ], ) return { runId: r.insertId, definitionId: def.insertId } } const seedStep = async (runId, over = {}) => ( await pool.query( `INSERT INTO event_run_steps (run_id, phase, seq, action_id, status, due_at, attempts, claimed_by, claim_expires_at, idempotency_key) VALUES (?, ?, ?, 'test.noop', ?, ?, ?, ?, ?, ?)`, [ runId, over.phase ?? 'main', over.seq ?? 0, over.status ?? 'pending', over.dueAt ?? null, over.attempts ?? 0, over.claimedBy ?? null, over.claimExpiresAt ?? null, over.key ?? 'k'.repeat(40), ], ) ).insertId const stepById = async (id) => (await pool.query('SELECT * FROM event_run_steps WHERE id = ?', [id]))[0] const runById = async (id) => (await pool.query('SELECT * FROM event_runs WHERE id = ?', [id]))[0] // ── claimStart: exactly one winner ───────────────────────────────────────── test('claimStart: the first caller wins and every other gets zero', async (t) => { if (needDb(t)) return const { runId } = await seedRun() const first = await pool.query(CLAIM_START, ['host:1', later(60_000), runId]) const second = await pool.query(CLAIM_START, ['host:2', later(60_000), runId]) assert.equal(rows(first), 1, 'the winner is told 1') assert.equal(rows(second), 0, 'the loser is told 0, not 1 with foundRows') assert.equal((await runById(runId)).claimed_by, 'host:1') assert.equal((await runById(runId)).status, 'starting') }) test('claimStart: started_at is stamped once and never moved', async (t) => { if (needDb(t)) return const { runId } = await seedRun() await pool.query(CLAIM_START, ['host:1', later(60_000), runId]) const first = (await runById(runId)).started_at await pool.query("UPDATE event_runs SET status = 'scheduled' WHERE id = ?", [runId]) await pool.query(CLAIM_START, ['host:2', later(60_000), runId]) assert.deepEqual((await runById(runId)).started_at, first, 'COALESCE keeps the original instant') }) // ── claimTick: a live lease refuses even its own holder ──────────────────── test('claimTick: a live lease is not re-enterable by the process that took it', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running', claimedBy: 'host:1', claimExpiresAt: later(60_000) }) const again = await pool.query(CLAIM_TICK, ['host:1', later(120_000), runId, T0]) assert.equal(rows(again), 0, 'a tick that overran must not advance its own run twice') }) test('claimTick: an expired lease is takeable, by anyone', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running', claimedBy: 'host:1', claimExpiresAt: later(-60_000) }) const taken = await pool.query(CLAIM_TICK, ['host:2', later(60_000), runId, T0]) assert.equal(rows(taken), 1) assert.equal((await runById(runId)).claimed_by, 'host:2') }) test('claimTick: a paused run is never claimable', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'paused' }) assert.equal(rows(await pool.query(CLAIM_TICK, ['host:1', later(60_000), runId, T0])), 0) }) // ── transition: the guard is the whole point ─────────────────────────────── test('transition: a run cancelled underneath the tick is not transitioned', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) await pool.query("UPDATE event_runs SET status = 'cancelled' WHERE id = ?", [runId]) const moved = await pool.query(TRANSITION, ['completed', 'main', runId, 'running']) assert.equal(rows(moved), 0) assert.equal((await runById(runId)).status, 'cancelled') }) test('transition: running -> running is a guarded write, not a no-op', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) // This is how the runner advances `current_phase`, and `foundRows` is exactly // what makes it report 1 despite `status` not changing — which is the answer // the caller needs, because what it is checking is that the run is STILL // running, not that the status moved. const moved = await pool.query(TRANSITION, ['running', 'closing', runId, 'running']) assert.equal(rows(moved), 1) assert.equal((await runById(runId)).current_phase, 'closing') }) // ── The step claim ───────────────────────────────────────────────────────── test('the step claim: one winner, and attempts is incremented by the claim alone', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) const stepId = await seedStep(runId) assert.equal(rows(await pool.query(CLAIM_STEP, ['host:1', later(60_000), stepId, T0])), 1) assert.equal(rows(await pool.query(CLAIM_STEP, ['host:2', later(60_000), stepId, T0])), 0) assert.equal((await stepById(stepId)).attempts, 1, 'one claim, one attempt') }) test('the step claim: a step held behind a core.wait is not due', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) const stepId = await seedStep(runId, { dueAt: later(300_000) }) assert.equal(rows(await pool.query(CLAIM_STEP, ['host:1', later(60_000), stepId, T0])), 0) assert.equal(rows(await pool.query(CLAIM_STEP, ['host:1', later(360_000), stepId, later(301_000)])), 1) }) // ── The reclaim: order, and what it must not touch ───────────────────────── test('the reclaim hands a stale step back WITHOUT resetting attempts', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) const stepId = await seedStep(runId, { status: 'running', attempts: 2, claimExpiresAt: later(-1000) }) await pool.query(STEP_GIVE_UP, [T0, 3]) await pool.query(STEP_RECLAIM, [T0]) const step = await stepById(stepId) assert.equal(step.status, 'pending') assert.equal(step.attempts, 2, 'Engagement Phase 14: a reclaim that reset this made MAX_ATTEMPTS unreachable') }) test('the reclaim gives up FIRST, so a spent step leaves running as failed', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) const stepId = await seedStep(runId, { status: 'running', attempts: 3, claimExpiresAt: later(-1000) }) const gaveUp = await pool.query(STEP_GIVE_UP, [T0, 3]) const reclaimed = await pool.query(STEP_RECLAIM, [T0]) assert.equal(rows(gaveUp), 1) assert.equal(rows(reclaimed), 0, 'reversing these two makes the row retry forever') assert.equal((await stepById(stepId)).status, 'failed') assert.equal((await stepById(stepId)).attempts, 3) }) test('the reclaim leaves a PARKED step alone, however long it waits', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) const stepId = await seedStep(runId, { status: 'running', attempts: 1, claimExpiresAt: null }) await pool.query(STEP_GIVE_UP, [later(365 * 24 * 3600 * 1000), 3]) await pool.query(STEP_RECLAIM, [later(365 * 24 * 3600 * 1000)]) const step = await stepById(stepId) assert.equal(step.status, 'running', 'a NULL lease is a parked cue, not staleness') assert.equal(step.attempts, 1) }) // ── holdNext ─────────────────────────────────────────────────────────────── test('holdNext moves the next PENDING step of the phase, and only one', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) await seedStep(runId, { seq: 0, status: 'done' }) const second = await seedStep(runId, { seq: 1 }) const third = await seedStep(runId, { seq: 2 }) const moved = await pool.query(HOLD_NEXT, [later(300_000), runId, 'main', 0, later(300_000)]) assert.equal(rows(moved), 1) assert.deepEqual((await stepById(second)).due_at, later(300_000)) assert.equal((await stepById(third)).due_at, null, 'a wait holds the next step, not the rest of the phase') }) test('holdNext never pulls a due date earlier, so a re-dispatch cannot double the wait', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) await seedStep(runId, { seq: 0, status: 'done' }) const second = await seedStep(runId, { seq: 1, dueAt: later(600_000) }) const moved = await pool.query(HOLD_NEXT, [later(300_000), runId, 'main', 0, later(300_000)]) assert.equal(rows(moved), 0) assert.deepEqual((await stepById(second)).due_at, later(600_000)) }) test('holdNext skips a step that is already running', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) await seedStep(runId, { seq: 0, status: 'done' }) await seedStep(runId, { seq: 1, status: 'running' }) const third = await seedStep(runId, { seq: 2 }) await pool.query(HOLD_NEXT, [later(300_000), runId, 'main', 0, later(300_000)]) assert.deepEqual((await stepById(third)).due_at, later(300_000)) }) // ── The two unique indexes that carry the weight ─────────────────────────── test('uq_evrun_occurrence, not the claim, is what stops two runs of one occurrence', async (t) => { if (needDb(t)) return const def = await pool.query('INSERT INTO event_definitions (grace_seconds) VALUES (900)') const a = await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null]) const b = await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null]) assert.equal(rows(a), 1) assert.equal(rows(b), 0, 'INSERT IGNORE answers honestly rather than raising a 1062') const all = await pool.query('SELECT COUNT(*) AS n FROM event_runs') assert.equal(Number(all[0].n), 1) }) test("scope '' rather than NULL is what makes that index work at all", async (t) => { if (needDb(t)) return const def = await pool.query('INSERT INTO event_definitions (grace_seconds) VALUES (900)') // The empty-string case collides, as it must. A NULL scope would NOT: multiple // NULLs do not collide in MariaDB, which would silently permit two runs of one // occurrence — the reason the column is NOT NULL DEFAULT ''. await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null]) assert.equal(rows(await pool.query(MATERIALISE_RUN, [def.insertId, 1, '', T0, null])), 0) // Two different scopes are two different occurrences, which is what lets a // worldwide event fan out across servers without colliding with itself. assert.equal(rows(await pool.query(MATERIALISE_RUN, [def.insertId, 1, 'atlantic', T0, null])), 1) }) test('uq_evstep_slot makes re-materialising a phase a no-op', async (t) => { if (needDb(t)) return const { runId } = await seedRun({ status: 'running' }) assert.equal(rows(await pool.query(MATERIALISE_STEP, [runId, 'main', 0, 'test.noop', 'a'.repeat(40)])), 1) assert.equal(rows(await pool.query(MATERIALISE_STEP, [runId, 'main', 0, 'test.noop', 'b'.repeat(40)])), 0) const [step] = await pool.query('SELECT idempotency_key FROM event_run_steps WHERE run_id = ?', [runId]) assert.equal(step.idempotency_key, 'a'.repeat(40), 'a re-materialise cannot overwrite a key a dispatch already sent') }) // ── The grace window is per definition ───────────────────────────────────── test('findMissed compares against each definition’s own grace window', async (t) => { if (needDb(t)) return const tight = await seedRun({ graceSeconds: 600 }) const generous = await seedRun({ graceSeconds: 3600, scope: 'b' }) const missed = (await pool.query(FIND_MISSED, [later(30 * 60 * 1000)])).map((r) => Number(r.id)) assert.deepEqual(missed, [tight.runId]) assert.ok(!missed.includes(generous.runId), 'inside its own window a run starts late rather than being missed') })