// ── The live run controls (EVENTS_PLAN.md Phase 3) ───────────────────────── // // Seven controls as of Phase 5, and what is tested is almost entirely the // REFUSALS. A control // that works is easy; a control that works from a status it should not have // worked from is a staff member changing a live game world by pressing a button // a stale screen offered them. So each of the six is exercised from every status // it must decline, and the four that a run console could plausibly offer wrongly // get a test of their own: // // • retry on a step the run has already moved past (the `skip` disposition) — // the test that found the first draft's guard was reading the wrong end of // the phase // • confirm on a step a process is mid-dispatch on, not a parked cue // • skip on a step with a live lease // • cancel closing out a parked cue, so a cancelled run stops "waiting" // • advance on a phase that is NOT waiting on its gate — the refusal that // makes force-advance an override rather than a way to skip a phase's steps // // The three tables are stubbed at the `.db` layer and the model's own logic runs // for real against them — the shape `eventRunner.test.js` uses. What a stub // cannot prove is that the five statements mean this against a real server; the // guards that are pure SQL (`status = 'running' AND claim_expires_at IS NULL` // and `lastStartedSeq`'s MAX) are proved in `eventRunnerSql.test.js`. // // Point the DB at a closed port before requiring anything. process.env.DB_HOST = '127.0.0.1' process.env.DB_PORT = '59999' const { test, beforeEach, afterEach, after } = require('node:test') const assert = require('node:assert/strict') const controls = require('../src/model/events/eventRunControls.model') const runsDb = require('../src/model/events/eventRuns.db') const stepsDb = require('../src/model/events/eventRunSteps.db') const logDb = require('../src/model/events/eventRunLog.db') const gatesDb = require('../src/model/events/eventPhaseGates.db') const db = require('../src/utils/db') after(() => db.close()) const TERMINAL = ['completed', 'cancelled', 'failed', 'missed'] const ACTOR = 7 let store const originals = [ ['runs', runsDb, { ...runsDb }], ['steps', stepsDb, { ...stepsDb }], ['log', logDb, { ...logDb }], ['gates', gatesDb, { ...gatesDb }], ] function installStubs() { store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), nextStepId: 1, nextGateId: 1 } const snap = (o) => ({ ...o }) runsDb.getById = async (id) => { const r = store.runs.get(Number(id)) return r ? snap(r) : null } runsDb.transition = async (id, from, to, opts = {}) => { const r = store.runs.get(Number(id)) const froms = Array.isArray(from) ? from : [from] if (!r || !froms.includes(r.status)) return false r.status = to if (opts.phase !== undefined) r.current_phase = opts.phase if (opts.error !== undefined) r.last_error = opts.error if (TERMINAL.includes(to) || opts.clearClaim) { r.claimed_by = null r.claim_expires_at = null } return true } stepsDb.getById = async (id) => { const s = store.steps.get(Number(id)) return s ? snap(s) : null } // Each of the four mirrors its statement's WHERE clause exactly. A stub can // only ever agree with whoever wrote it, so what these buy is the model's // logic around them; the clauses themselves are checked against a real server. stepsDb.confirmParked = async (id, note) => { const s = store.steps.get(Number(id)) if (!s || s.status !== 'running' || s.claim_expires_at) return false Object.assign(s, { status: 'done', last_error: note, claimed_by: null }) return true } stepsDb.skipByHuman = async (id, reason) => { const s = store.steps.get(Number(id)) if (!s) return false const ok = s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at) if (!ok) return false Object.assign(s, { status: 'skipped', last_error: reason, claimed_by: null }) return true } stepsDb.requeue = async (id) => { const s = store.steps.get(Number(id)) if (!s || s.status !== 'failed') return false Object.assign(s, { status: 'pending', attempts: 0, due_at: null, last_error: null, claimed_by: null, claim_expires_at: null }) return true } stepsDb.cancelOpen = async (runId) => { let n = 0 for (const s of store.steps.values()) { if (s.run_id !== Number(runId)) continue if (s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at)) { s.status = 'cancelled' n += 1 } } return n } // Phase 5's advance guard reads this. Unstubbed it is a real query, and this // file points the pool at a closed port — three tests would hang for ten // seconds each and then fail with ECONNREFUSED rather than saying anything // about the control. Same shape as the statement: pending and running only, // lowest seq first. stepsDb.nextOpenStep = async (runId, phase) => { const s = [...store.steps.values()] .filter((x) => x.run_id === Number(runId) && x.phase === phase && ['pending', 'running'].includes(x.status)) .sort((a, b) => a.seq - b.seq || a.id - b.id)[0] return s ? { ...s } : null } stepsDb.lastStartedSeq = async (runId, phase) => { const started = [...store.steps.values()] .filter((s) => s.run_id === Number(runId) && s.phase === phase && s.status !== 'pending') .map((s) => s.seq) return started.length ? Math.max(...started) : null } logDb.write = async (line) => { store.log.push(line) return true } // Phase 5. `satisfy` carries `WHERE satisfied_at IS NULL`, and the stub keeps // it: a gate that could be forced twice would log two advances of one phase. gatesDb.forPhase = async (runId, phase) => { const g = store.gates.get(`${Number(runId)}|${phase}`) return g ? { ...g } : null } gatesDb.satisfy = async (id, by, { userId = null, now = new Date() } = {}) => { const g = [...store.gates.values()].find((x) => x.id === id) if (!g || g.satisfied_at) return false Object.assign(g, { satisfied_at: now, satisfied_by: by, forced_by: userId }) return true } } beforeEach(installStubs) afterEach(() => { for (const [, mod, fns] of originals) Object.assign(mod, fns) }) let nextRunId = 1 function seedGate(runId, phase, { kind = 'on', trigger = 'test.trigger', needed = 1, tally = 0, minutesAgo = 5 } = {}) { const g = { id: store.nextGateId++, run_id: runId, phase, kind, after_seconds: kind === 'after' ? 1800 : null, trigger_id: kind === 'on' ? trigger : null, conditions: null, needed, tally, entered_at: new Date(Date.now() - minutesAgo * 60_000), due_at: null, last_event: null, last_event_at: null, satisfied_at: null, satisfied_by: null, forced_by: null, } store.gates.set(`${runId}|${phase}`, g) return g } function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) { const id = nextRunId++ store.runs.set(id, { id, definition_id: id, version_id: id, status, health: 'ok', current_phase: phase, claimed_by: null, claim_expires_at: null, last_error: null, }) steps.forEach((s, i) => { const stepId = store.nextStepId++ store.steps.set(stepId, { id: stepId, run_id: id, phase: s.phase || phase, seq: s.seq ?? i, action_id: s.actionId || 'test.action', status: s.status || 'pending', attempts: s.attempts ?? 0, due_at: null, claimed_by: s.leased ? 'someone' : null, claim_expires_at: s.leased ? new Date(Date.now() + 60_000) : null, last_error: null, params: {}, }) }) return id } const runRow = (id) => store.runs.get(id) const stepsOf = (id) => [...store.steps.values()].filter((s) => s.run_id === id).sort((a, b) => a.seq - b.seq) const lastLog = () => store.log[store.log.length - 1] // ── advance (Phase 5) ────────────────────────────────────────────────────── test('advance releases a phase that is genuinely waiting on its gate', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'done' }] }) const gate = seedGate(id, 'main', { needed: 3, tally: 1, minutesAgo: 68 }) const result = await controls.advancePhase(id, { reason: 'the boss never spawned' }, ACTOR) assert.equal(result.ok, true) assert.equal(result.phase, 'main') assert.equal(gate.satisfied_by, 'forced') assert.equal(gate.forced_by, ACTOR) // The run is NOT transitioned here: the next tick does the phase boundary, // exactly as it does after resume, so there is one implementation of what a // phase boundary is rather than two. assert.equal(runRow(id).current_phase, 'main') const line = lastLog() assert.equal(line.kind, 'phase.advanced') assert.equal(line.detail.because, 'forced') assert.equal(line.detail.by, ACTOR) assert.equal(line.detail.reason, 'the boss never spawned') assert.equal(line.detail.seen, 1) assert.equal(line.detail.needed, 3, 'and the log says what it was still waiting for') assert.ok(line.detail.waitedSeconds > 4000) }) test('advance is refused on a phase that is waiting on a STEP, not on its gate', async () => { // The refusal that makes this an override rather than a way to skip work: a // phase with an open step is held by the step, and skip is its control. const id = seedRun({ status: 'running', steps: [{ status: 'done' }, { status: 'pending', actionId: 'test.slow' }] }) seedGate(id, 'main') const result = await controls.advancePhase(id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.match(result.errors[0], /waiting on step 1 \(test\.slow\)/) assert.equal(store.log.length, 0, 'and nothing is logged for a refusal') }) test('advance is refused when the phase has no advance condition at all', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'done' }] }) const result = await controls.advancePhase(id, {}, ACTOR) assert.equal(result.ok, false) assert.match(result.errors[0], /has no advance condition; skip its steps instead/) }) test('advance is refused from every status that is not `running`', async () => { for (const status of ['scheduled', 'starting', 'paused', 'ending', ...TERMINAL]) { const id = seedRun({ status, steps: [{ status: 'done' }] }) seedGate(id, 'main') const result = await controls.advancePhase(id, {}, ACTOR) assert.equal(result.ok, false, `a ${status} run should not be advanceable`) assert.match(result.errors[0], new RegExp(`a ${status} run has no phase to advance`)) } }) test('advance twice is refused the second time', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'done' }] }) seedGate(id, 'main') assert.equal((await controls.advancePhase(id, {}, ACTOR)).ok, true) const second = await controls.advancePhase(id, {}, ACTOR) assert.equal(second.ok, false) assert.match(second.errors[0], /already past its advance condition/) }) // ── pause / resume ───────────────────────────────────────────────────────── test('pause takes a run in flight and records who did it', async () => { const id = seedRun({ status: 'running' }) const result = await controls.pause(id, { reason: 'the shard is lagging' }, ACTOR) assert.equal(result.ok, true) assert.equal(runRow(id).status, 'paused') assert.deepEqual(lastLog().detail, { from: 'running', to: 'paused', control: 'pause', by: ACTOR, reason: 'the shard is lagging', }) }) test('pause drops the claim, so the next tick is not locked out of a resumed run', async () => { const id = seedRun({ status: 'running' }) Object.assign(runRow(id), { claimed_by: 'host:1', claim_expires_at: new Date(Date.now() + 900_000) }) await controls.pause(id, {}, ACTOR) assert.equal(runRow(id).claimed_by, null) assert.equal(runRow(id).claim_expires_at, null) }) test('a scheduled run cannot be paused — it is cancelled instead', async () => { // Pausing one would leave a run that is neither going to start nor visibly // abandoned, and resuming it after its grace window had passed would produce a // `missed` from a button labelled resume. const id = seedRun({ status: 'scheduled' }) const result = await controls.pause(id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.match(result.errors[0], /scheduled/) assert.equal(runRow(id).status, 'scheduled') }) test('a completed run cannot be paused', async () => { const id = seedRun({ status: 'completed' }) assert.equal((await controls.pause(id, {}, ACTOR)).ok, false) }) test('resume returns a run to running, or to starting when it never entered a phase', async () => { const withPhase = seedRun({ status: 'paused', phase: 'main' }) assert.equal((await controls.resume(withPhase, {}, ACTOR)).ok, true) assert.equal(runRow(withPhase).status, 'running') const beforePhase = seedRun({ status: 'paused', phase: null }) assert.equal((await controls.resume(beforePhase, {}, ACTOR)).ok, true) assert.equal(runRow(beforePhase).status, 'starting', 'both are in findDue; neither is a fourth column') }) test('resume clears the error it was paused over and leaves health alone', async () => { const id = seedRun({ status: 'paused' }) Object.assign(runRow(id), { last_error: 'core.spawn failed', health: 'degraded' }) await controls.resume(id, {}, ACTOR) assert.equal(runRow(id).last_error, null, 'a resolved failure must not accuse a healthy run for ever') assert.equal(runRow(id).health, 'degraded', 'that this run has already had trouble stays true') }) test('resume refuses a run that is not paused', async () => { const id = seedRun({ status: 'running' }) const result = await controls.resume(id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) }) // ── cancel ───────────────────────────────────────────────────────────────── test('cancel closes out the pending steps and the parked cue, and leaves a leased step alone', async () => { const id = seedRun({ status: 'running', steps: [ { status: 'done' }, { status: 'running', leased: true }, // mid-dispatch: nothing can recall a sent command { status: 'running' }, // parked on a human: nothing is holding it { status: 'pending' }, ], }) const result = await controls.cancel(id, { reason: 'called off' }, ACTOR) assert.equal(result.ok, true) assert.equal(runRow(id).status, 'cancelled') assert.equal(result.cancelledSteps, 2) const [done, leased, parked, pending] = stepsOf(id) assert.equal(done.status, 'done') assert.equal(leased.status, 'running', 'a step being dispatched is not touched') assert.equal(parked.status, 'cancelled', 'a cancelled run must stop claiming to wait on somebody') assert.equal(pending.status, 'cancelled') }) test('cancel is legal before a run has started', async () => { const id = seedRun({ status: 'scheduled', steps: [{ status: 'pending' }] }) assert.equal((await controls.cancel(id, {}, ACTOR)).ok, true) assert.equal(runRow(id).status, 'cancelled') }) test('cancel refuses a run that is already terminal', async () => { for (const status of TERMINAL) { const id = seedRun({ status }) const result = await controls.cancel(id, {}, ACTOR) assert.equal(result.ok, false, `${status} should not be cancellable`) assert.match(result.errors[0], new RegExp(status)) } }) // ── confirm ──────────────────────────────────────────────────────────────── test('confirm resolves a parked cue as done, keeping what the person says they did', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'running', actionId: 'core.cue' }] }) const [cue] = stepsOf(id) const result = await controls.confirmStep(id, cue.id, { note: 'gate opened, herald read' }, ACTOR) assert.equal(result.ok, true) assert.equal(stepsOf(id)[0].status, 'done', 'a person saying they did it is the step having succeeded') assert.equal(stepsOf(id)[0].last_error, 'gate opened, herald read') assert.equal(lastLog().detail.control, 'confirm') assert.equal(lastLog().detail.by, ACTOR) }) test('confirm cannot resolve a step a process is dispatching', async () => { // The whole vocabulary here is "running with a NULL lease". A live lease means // something is mid-dispatch, and confirming it would race the process that // owns the row. const id = seedRun({ status: 'running', steps: [{ status: 'running', leased: true }] }) const [busy] = stepsOf(id) const result = await controls.confirmStep(id, busy.id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.equal(stepsOf(id)[0].status, 'running') }) test('a step id from another run is a 404, not an action', async () => { const mine = seedRun({ status: 'running', steps: [{ status: 'pending' }] }) const theirs = seedRun({ status: 'running', steps: [{ status: 'running' }] }) const [theirStep] = stepsOf(theirs) const result = await controls.confirmStep(mine, theirStep.id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 404) assert.equal(stepsOf(theirs)[0].status, 'running') }) // ── skip ─────────────────────────────────────────────────────────────────── test('skip takes a pending step and a parked cue, and nothing else', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'pending' }, { status: 'running' }, { status: 'running', leased: true }, { status: 'failed' }], }) const [pending, parked, leased, failed] = stepsOf(id) assert.equal((await controls.skipStep(id, pending.id, {}, ACTOR)).ok, true) assert.equal((await controls.skipStep(id, parked.id, {}, ACTOR)).ok, true) assert.equal((await controls.skipStep(id, leased.id, {}, ACTOR)).ok, false) // A failed step does not need skipping: `nextOpenStep` already passes over it, // so resuming the run carries the phase past it. assert.equal((await controls.skipStep(id, failed.id, {}, ACTOR)).ok, false) const after = stepsOf(id) assert.equal(after[0].status, 'skipped') assert.equal(after[1].status, 'skipped') assert.equal(after[2].status, 'running') assert.equal(after[3].status, 'failed') }) test('skip refuses once the run is over', async () => { const id = seedRun({ status: 'completed', steps: [{ status: 'pending' }] }) const [step] = stepsOf(id) assert.equal((await controls.skipStep(id, step.id, {}, ACTOR)).ok, false) }) // ── retry ────────────────────────────────────────────────────────────────── test('retry re-queues the step a paused run is stopped at, and resumes in the same action', async () => { const id = seedRun({ status: 'paused', steps: [{ status: 'done' }, { status: 'failed', attempts: 3 }, { status: 'pending' }], }) const failed = stepsOf(id)[1] const result = await controls.retryStep(id, failed.id, {}, ACTOR) assert.equal(result.ok, true) assert.equal(result.resumed, true) assert.equal(stepsOf(id)[1].status, 'pending') assert.equal(stepsOf(id)[1].attempts, 0, 'the ceiling bounds the runner, not a person deciding once') assert.equal(runRow(id).status, 'running', 'there is no state in which you would want half of this') }) test('retry refuses a step the run has already moved past', async () => { // The case the guard exists for: a failed step under an `on_failure` of `skip` // is one the phase carried on from. Re-queueing it would put a pending row // behind the runner's cursor, where it would sit for ever. const id = seedRun({ status: 'paused', steps: [{ status: 'failed', attempts: 3 }, { status: 'done' }, { status: 'failed', attempts: 3 }], }) const [movedPast] = stepsOf(id) const result = await controls.retryStep(id, movedPast.id, {}, ACTOR) assert.equal(result.ok, false) assert.equal(result.status, 409) assert.match(result.errors[0], /stopped at this step/) assert.equal(stepsOf(id)[0].status, 'failed') assert.equal(runRow(id).status, 'paused', 'a refused retry does not resume the run either') }) test('retry refuses a step in a phase the run has left', async () => { const id = seedRun({ status: 'paused', phase: 'two', steps: [{ phase: 'one', seq: 0, status: 'failed' }, { phase: 'two', seq: 0, status: 'pending' }], }) const [old] = stepsOf(id) const result = await controls.retryStep(id, old.id, {}, ACTOR) assert.equal(result.ok, false) assert.match(result.errors[0], /already left/) }) test('retry refuses while the run is still running', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'failed' }] }) const [failed] = stepsOf(id) const result = await controls.retryStep(id, failed.id, {}, ACTOR) assert.equal(result.ok, false) assert.match(result.errors[0], /paused/) }) test('retry refuses a step that is not failed', async () => { const id = seedRun({ status: 'paused', steps: [{ status: 'pending' }] }) const [pending] = stepsOf(id) assert.equal((await controls.retryStep(id, pending.id, {}, ACTOR)).ok, false) }) // ── the record ───────────────────────────────────────────────────────────── test('every control writes one log line carrying the actor and the control name', async () => { const id = seedRun({ status: 'running', steps: [{ status: 'running' }, { status: 'pending' }] }) const [parked, pending] = stepsOf(id) await controls.confirmStep(id, parked.id, { note: 'done' }, ACTOR) await controls.skipStep(id, pending.id, { reason: 'not needed' }, ACTOR) await controls.pause(id, {}, ACTOR) await controls.resume(id, {}, ACTOR) await controls.cancel(id, { reason: 'over' }, ACTOR) const human = store.log.filter((l) => l.detail?.control) assert.deepEqual(human.map((l) => l.detail.control), ['confirm', 'skip', 'pause', 'resume', 'cancel']) assert.ok(human.every((l) => l.detail.by === ACTOR)) // The kinds are the ones a reader already scans for. A human transition is // still a transition; `detail.control` is what separates it from the runner's. assert.deepEqual([...new Set(human.map((l) => l.kind))].sort(), ['run.status', 'step.status']) }) test('an empty reason is stored as NULL rather than as an empty string', async () => { const id = seedRun({ status: 'running' }) await controls.pause(id, { reason: ' ' }, ACTOR) assert.equal(lastLog().detail.reason, null) })