// ── The live run controls (EVENTS_PLAN.md Phase 3) ───────────────────────── // // Six controls, and what is tested is almost entirely the REFUSALS. A control // that works is easy; a control that works from a status it should not have // worked from is a staff member changing a live game world by pressing a button // a stale screen offered them. So each of the six is exercised from every status // it must decline, and the four that a run console could plausibly offer wrongly // get a test of their own: // // • retry on a step the run has already moved past (the `skip` disposition) — // the test that found the first draft's guard was reading the wrong end of // the phase // • confirm on a step a process is mid-dispatch on, not a parked cue // • skip on a step with a live lease // • cancel closing out a parked cue, so a cancelled run stops "waiting" // // The three tables are stubbed at the `.db` layer and the model's own logic runs // for real against them — the shape `eventRunner.test.js` uses. What a stub // cannot prove is that the five statements mean this against a real server; the // guards that are pure SQL (`status = 'running' AND claim_expires_at IS NULL` // and `lastStartedSeq`'s MAX) are proved in `eventRunnerSql.test.js`. // // Point the DB at a closed port before requiring anything. process.env.DB_HOST = '127.0.0.1' process.env.DB_PORT = '59999' const { test, beforeEach, afterEach, after } = require('node:test') const assert = require('node:assert/strict') const controls = require('../src/model/events/eventRunControls.model') const runsDb = require('../src/model/events/eventRuns.db') const stepsDb = require('../src/model/events/eventRunSteps.db') const logDb = require('../src/model/events/eventRunLog.db') const db = require('../src/utils/db') after(() => db.close()) const TERMINAL = ['completed', 'cancelled', 'failed', 'missed'] const ACTOR = 7 let store const originals = [ ['runs', runsDb, { ...runsDb }], ['steps', stepsDb, { ...stepsDb }], ['log', logDb, { ...logDb }], ] function installStubs() { store = { runs: new Map(), steps: new Map(), log: [], nextStepId: 1 } const snap = (o) => ({ ...o }) runsDb.getById = async (id) => { const r = store.runs.get(Number(id)) return r ? snap(r) : null } runsDb.transition = async (id, from, to, opts = {}) => { const r = store.runs.get(Number(id)) const froms = Array.isArray(from) ? from : [from] if (!r || !froms.includes(r.status)) return false r.status = to if (opts.phase !== undefined) r.current_phase = opts.phase if (opts.error !== undefined) r.last_error = opts.error if (TERMINAL.includes(to) || opts.clearClaim) { r.claimed_by = null r.claim_expires_at = null } return true } stepsDb.getById = async (id) => { const s = store.steps.get(Number(id)) return s ? snap(s) : null } // Each of the four mirrors its statement's WHERE clause exactly. A stub can // only ever agree with whoever wrote it, so what these buy is the model's // logic around them; the clauses themselves are checked against a real server. stepsDb.confirmParked = async (id, note) => { const s = store.steps.get(Number(id)) if (!s || s.status !== 'running' || s.claim_expires_at) return false Object.assign(s, { status: 'done', last_error: note, claimed_by: null }) return true } stepsDb.skipByHuman = async (id, reason) => { const s = store.steps.get(Number(id)) if (!s) return false const ok = s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at) if (!ok) return false Object.assign(s, { status: 'skipped', last_error: reason, claimed_by: null }) return true } stepsDb.requeue = async (id) => { const s = store.steps.get(Number(id)) if (!s || s.status !== 'failed') return false Object.assign(s, { status: 'pending', attempts: 0, due_at: null, last_error: null, claimed_by: null, claim_expires_at: null }) return true } stepsDb.cancelOpen = async (runId) => { let n = 0 for (const s of store.steps.values()) { if (s.run_id !== Number(runId)) continue if (s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at)) { s.status = 'cancelled' n += 1 } } return n } stepsDb.lastStartedSeq = async (runId, phase) => { const started = [...store.steps.values()] .filter((s) => s.run_id === Number(runId) && s.phase === phase && s.status !== 'pending') .map((s) => s.seq) return started.length ? Math.max(...started) : null } logDb.write = async (line) => { store.log.push(line) return true } } beforeEach(installStubs) afterEach(() => { for (const [, mod, fns] of originals) Object.assign(mod, fns) }) let nextRunId = 1 function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) { const id = nextRunId++ store.runs.set(id, { id, definition_id: id, version_id: id, status, health: 'ok', current_phase: phase, claimed_by: null, claim_expires_at: null, last_error: null, }) steps.forEach((s, i) => { const stepId = store.nextStepId++ store.steps.set(stepId, { id: stepId, run_id: id, phase: s.phase || phase, seq: s.seq ?? i, action_id: s.actionId || 'test.action', status: s.status || 'pending', attempts: s.attempts ?? 0, due_at: null, claimed_by: s.leased ? 'someone' : null, claim_expires_at: s.leased ? new Date(Date.now() + 60_000) : null, last_error: null, params: {}, }) }) return id } const runRow = (id) => store.runs.get(id) const stepsOf = (id) => [...store.steps.values()].filter((s) => s.run_id === id).sort((a, b) => a.seq - b.seq) const lastLog = () => store.log[store.log.length - 1] // ── pause / resume ───────────────────────────────────────────────────────── test('pause takes a run in flight and records who did it', async () => { const id = seedRun({ status: 'running' }) const result = await controls.pause(id, { reason: 'the shard is lagging' }, ACTOR) assert.equal(result.ok, true) assert.equal(runRow(id).status, 'paused') assert.deepEqual(lastLog().detail, { from: 'running', to: 'paused', control: 'pause', by: ACTOR, reason: 'the shard is lagging', }) }) test('pause drops the claim, so the next tick is not locked out of a resumed run', async () => { const id = seedRun({ status: 'running' }) Object.assign(runRow(id), { claimed_by: 'host:1', claim_expires_at: new Date(Date.now() + 900_000) }) await controls.pause(id, {}, ACTOR) assert.equal(runRow(id).claimed_by, null) assert.equal(runRow(id).claim_expires_at, null) }) test('a scheduled run cannot be paused — it is cancelled instead', async () => { // Pausing one would leave a run that is neither going to start nor visibly // abandoned, and resuming it after its grace window had passed would produce a // `missed` from a button labelled resume. const id = seedRun({ status: 'scheduled' }) const result = await controls.pause(id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.match(result.errors[0], /scheduled/) assert.equal(runRow(id).status, 'scheduled') }) test('a completed run cannot be paused', async () => { const id = seedRun({ status: 'completed' }) assert.equal((await controls.pause(id, {}, ACTOR)).ok, false) }) test('resume returns a run to running, or to starting when it never entered a phase', async () => { const withPhase = seedRun({ status: 'paused', phase: 'main' }) assert.equal((await controls.resume(withPhase, {}, ACTOR)).ok, true) assert.equal(runRow(withPhase).status, 'running') const beforePhase = seedRun({ status: 'paused', phase: null }) assert.equal((await controls.resume(beforePhase, {}, ACTOR)).ok, true) assert.equal(runRow(beforePhase).status, 'starting', 'both are in findDue; neither is a fourth column') }) test('resume clears the error it was paused over and leaves health alone', async () => { const id = seedRun({ status: 'paused' }) Object.assign(runRow(id), { last_error: 'core.spawn failed', health: 'degraded' }) await controls.resume(id, {}, ACTOR) assert.equal(runRow(id).last_error, null, 'a resolved failure must not accuse a healthy run for ever') assert.equal(runRow(id).health, 'degraded', 'that this run has already had trouble stays true') }) test('resume refuses a run that is not paused', async () => { const id = seedRun({ status: 'running' }) const result = await controls.resume(id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) }) // ── cancel ───────────────────────────────────────────────────────────────── test('cancel closes out the pending steps and the parked cue, and leaves a leased step alone', async () => { const id = seedRun({ status: 'running', steps: [ { status: 'done' }, { status: 'running', leased: true }, // mid-dispatch: nothing can recall a sent command { status: 'running' }, // parked on a human: nothing is holding it { status: 'pending' }, ], }) const result = await controls.cancel(id, { reason: 'called off' }, ACTOR) assert.equal(result.ok, true) assert.equal(runRow(id).status, 'cancelled') assert.equal(result.cancelledSteps, 2) const [done, leased, parked, pending] = stepsOf(id) assert.equal(done.status, 'done') assert.equal(leased.status, 'running', 'a step being dispatched is not touched') assert.equal(parked.status, 'cancelled', 'a cancelled run must stop claiming to wait on somebody') assert.equal(pending.status, 'cancelled') }) test('cancel is legal before a run has started', async () => { const id = seedRun({ status: 'scheduled', steps: [{ status: 'pending' }] }) assert.equal((await controls.cancel(id, {}, ACTOR)).ok, true) assert.equal(runRow(id).status, 'cancelled') }) test('cancel refuses a run that is already terminal', async () => { for (const status of TERMINAL) { const id = seedRun({ status }) const result = await controls.cancel(id, {}, ACTOR) assert.equal(result.ok, false, `${status} should not be cancellable`) assert.match(result.errors[0], new RegExp(status)) } }) // ── confirm ──────────────────────────────────────────────────────────────── test('confirm resolves a parked cue as done, keeping what the person says they did', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'running', actionId: 'core.cue' }] }) const [cue] = stepsOf(id) const result = await controls.confirmStep(id, cue.id, { note: 'gate opened, herald read' }, ACTOR) assert.equal(result.ok, true) assert.equal(stepsOf(id)[0].status, 'done', 'a person saying they did it is the step having succeeded') assert.equal(stepsOf(id)[0].last_error, 'gate opened, herald read') assert.equal(lastLog().detail.control, 'confirm') assert.equal(lastLog().detail.by, ACTOR) }) test('confirm cannot resolve a step a process is dispatching', async () => { // The whole vocabulary here is "running with a NULL lease". A live lease means // something is mid-dispatch, and confirming it would race the process that // owns the row. const id = seedRun({ status: 'running', steps: [{ status: 'running', leased: true }] }) const [busy] = stepsOf(id) const result = await controls.confirmStep(id, busy.id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.equal(stepsOf(id)[0].status, 'running') }) test('a step id from another run is a 404, not an action', async () => { const mine = seedRun({ status: 'running', steps: [{ status: 'pending' }] }) const theirs = seedRun({ status: 'running', steps: [{ status: 'running' }] }) const [theirStep] = stepsOf(theirs) const result = await controls.confirmStep(mine, theirStep.id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 404) assert.equal(stepsOf(theirs)[0].status, 'running') }) // ── skip ─────────────────────────────────────────────────────────────────── test('skip takes a pending step and a parked cue, and nothing else', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'pending' }, { status: 'running' }, { status: 'running', leased: true }, { status: 'failed' }], }) const [pending, parked, leased, failed] = stepsOf(id) assert.equal((await controls.skipStep(id, pending.id, {}, ACTOR)).ok, true) assert.equal((await controls.skipStep(id, parked.id, {}, ACTOR)).ok, true) assert.equal((await controls.skipStep(id, leased.id, {}, ACTOR)).ok, false) // A failed step does not need skipping: `nextOpenStep` already passes over it, // so resuming the run carries the phase past it. assert.equal((await controls.skipStep(id, failed.id, {}, ACTOR)).ok, false) const after = stepsOf(id) assert.equal(after[0].status, 'skipped') assert.equal(after[1].status, 'skipped') assert.equal(after[2].status, 'running') assert.equal(after[3].status, 'failed') }) test('skip refuses once the run is over', async () => { const id = seedRun({ status: 'completed', steps: [{ status: 'pending' }] }) const [step] = stepsOf(id) assert.equal((await controls.skipStep(id, step.id, {}, ACTOR)).ok, false) }) // ── retry ────────────────────────────────────────────────────────────────── test('retry re-queues the step a paused run is stopped at, and resumes in the same action', async () => { const id = seedRun({ status: 'paused', steps: [{ status: 'done' }, { status: 'failed', attempts: 3 }, { status: 'pending' }], }) const failed = stepsOf(id)[1] const result = await controls.retryStep(id, failed.id, {}, ACTOR) assert.equal(result.ok, true) assert.equal(result.resumed, true) assert.equal(stepsOf(id)[1].status, 'pending') assert.equal(stepsOf(id)[1].attempts, 0, 'the ceiling bounds the runner, not a person deciding once') assert.equal(runRow(id).status, 'running', 'there is no state in which you would want half of this') }) test('retry refuses a step the run has already moved past', async () => { // The case the guard exists for: a failed step under an `on_failure` of `skip` // is one the phase carried on from. Re-queueing it would put a pending row // behind the runner's cursor, where it would sit for ever. const id = seedRun({ status: 'paused', steps: [{ status: 'failed', attempts: 3 }, { status: 'done' }, { status: 'failed', attempts: 3 }], }) const [movedPast] = stepsOf(id) const result = await controls.retryStep(id, movedPast.id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.match(result.errors[0], /stopped at this step/) assert.equal(stepsOf(id)[0].status, 'failed') assert.equal(runRow(id).status, 'paused', 'a refused retry does not resume the run either') }) test('retry refuses a step in a phase the run has left', async () => { const id = seedRun({ status: 'paused', phase: 'two', steps: [{ phase: 'one', seq: 0, status: 'failed' }, { phase: 'two', seq: 0, status: 'pending' }], }) const [old] = stepsOf(id) const result = await controls.retryStep(id, old.id, {}, ACTOR) assert.equal(result.ok, false) assert.match(result.errors[0], /already left/) }) test('retry refuses while the run is still running', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'failed' }] }) const [failed] = stepsOf(id) const result = await controls.retryStep(id, failed.id, {}, ACTOR) assert.equal(result.ok, false) assert.match(result.errors[0], /paused/) }) test('retry refuses a step that is not failed', async () => { const id = seedRun({ status: 'paused', steps: [{ status: 'pending' }] }) const [pending] = stepsOf(id) assert.equal((await controls.retryStep(id, pending.id, {}, ACTOR)).ok, false) }) // ── the record ───────────────────────────────────────────────────────────── test('every control writes one log line carrying the actor and the control name', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'running' }, { status: 'pending' }] }) const [parked, pending] = stepsOf(id) await controls.confirmStep(id, parked.id, { note: 'done' }, ACTOR) await controls.skipStep(id, pending.id, { reason: 'not needed' }, ACTOR) await controls.pause(id, {}, ACTOR) await controls.resume(id, {}, ACTOR) await controls.cancel(id, { reason: 'over' }, ACTOR) const human = store.log.filter((l) => l.detail?.control) assert.deepEqual(human.map((l) => l.detail.control), ['confirm', 'skip', 'pause', 'resume', 'cancel']) assert.ok(human.every((l) => l.detail.by === ACTOR)) // The kinds are the ones a reader already scans for. A human transition is // still a transition; `detail.control` is what separates it from the runner's. assert.deepEqual([...new Set(human.map((l) => l.kind))].sort(), ['run.status', 'step.status']) }) test('an empty reason is stored as NULL rather than as an empty string', async () => { const id = seedRun({ status: 'running' }) await controls.pause(id, { reason: ' ' }, ACTOR) assert.equal(lastLog().detail.reason, null) })