feat(events): the minimal admin surface (Phase 3)
Three screens, an Events nav group and the six live run controls Phase 1 left
absent on purpose because nothing was in flight. An admin can now author,
publish, start and watch an event that announces things and cues a human; a
moderator can stop one that is going wrong.
Six controls, not eight. `advance` is absent because a phase today advances when
its steps go terminal — the per-step skip already does that — and Phase 5 is what
gives a phase an advance condition. Cancel takes `{ reason }`, not `{ cleanup }`,
until Phase 8's ledger exists. Every control is a compare-and-set on the status it
may act from, so a console rendered thirty seconds ago cannot act on a run that
has moved.
Fixes a defect in the Phase 2 runner: `advanceRun` drained up to
EVENT_STEPS_PER_TICK steps while only checking the run's status at the top of the
tick, so a pause pressed mid-batch did nothing for up to 24 more steps.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T6t8mrAWhZU5vnyYgZTMtL
This commit is contained in:
442
server/test/eventRunControls.test.js
Normal file
442
server/test/eventRunControls.test.js
Normal file
@@ -0,0 +1,442 @@
|
||||
// ── The live run controls (EVENTS_PLAN.md Phase 3) ─────────────────────────
|
||||
//
|
||||
// Six controls, and what is tested is almost entirely the REFUSALS. A control
|
||||
// that works is easy; a control that works from a status it should not have
|
||||
// worked from is a staff member changing a live game world by pressing a button
|
||||
// a stale screen offered them. So each of the six is exercised from every status
|
||||
// it must decline, and the four that a run console could plausibly offer wrongly
|
||||
// get a test of their own:
|
||||
//
|
||||
// • retry on a step the run has already moved past (the `skip` disposition) —
|
||||
// the test that found the first draft's guard was reading the wrong end of
|
||||
// the phase
|
||||
// • confirm on a step a process is mid-dispatch on, not a parked cue
|
||||
// • skip on a step with a live lease
|
||||
// • cancel closing out a parked cue, so a cancelled run stops "waiting"
|
||||
//
|
||||
// The three tables are stubbed at the `.db` layer and the model's own logic runs
|
||||
// for real against them — the shape `eventRunner.test.js` uses. What a stub
|
||||
// cannot prove is that the five statements mean this against a real server; the
|
||||
// guards that are pure SQL (`status = 'running' AND claim_expires_at IS NULL`
|
||||
// and `lastStartedSeq`'s MAX) are proved in `eventRunnerSql.test.js`.
|
||||
//
|
||||
// Point the DB at a closed port before requiring anything.
|
||||
process.env.DB_HOST = '127.0.0.1'
|
||||
process.env.DB_PORT = '59999'
|
||||
|
||||
const { test, beforeEach, afterEach, after } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
|
||||
const controls = require('../src/model/events/eventRunControls.model')
|
||||
const runsDb = require('../src/model/events/eventRuns.db')
|
||||
const stepsDb = require('../src/model/events/eventRunSteps.db')
|
||||
const logDb = require('../src/model/events/eventRunLog.db')
|
||||
const db = require('../src/utils/db')
|
||||
|
||||
after(() => db.close())
|
||||
|
||||
const TERMINAL = ['completed', 'cancelled', 'failed', 'missed']
|
||||
const ACTOR = 7
|
||||
|
||||
let store
|
||||
const originals = [
|
||||
['runs', runsDb, { ...runsDb }],
|
||||
['steps', stepsDb, { ...stepsDb }],
|
||||
['log', logDb, { ...logDb }],
|
||||
]
|
||||
|
||||
function installStubs() {
|
||||
store = { runs: new Map(), steps: new Map(), log: [], nextStepId: 1 }
|
||||
const snap = (o) => ({ ...o })
|
||||
|
||||
runsDb.getById = async (id) => {
|
||||
const r = store.runs.get(Number(id))
|
||||
return r ? snap(r) : null
|
||||
}
|
||||
|
||||
runsDb.transition = async (id, from, to, opts = {}) => {
|
||||
const r = store.runs.get(Number(id))
|
||||
const froms = Array.isArray(from) ? from : [from]
|
||||
if (!r || !froms.includes(r.status)) return false
|
||||
r.status = to
|
||||
if (opts.phase !== undefined) r.current_phase = opts.phase
|
||||
if (opts.error !== undefined) r.last_error = opts.error
|
||||
if (TERMINAL.includes(to) || opts.clearClaim) {
|
||||
r.claimed_by = null
|
||||
r.claim_expires_at = null
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
stepsDb.getById = async (id) => {
|
||||
const s = store.steps.get(Number(id))
|
||||
return s ? snap(s) : null
|
||||
}
|
||||
|
||||
// Each of the four mirrors its statement's WHERE clause exactly. A stub can
|
||||
// only ever agree with whoever wrote it, so what these buy is the model's
|
||||
// logic around them; the clauses themselves are checked against a real server.
|
||||
stepsDb.confirmParked = async (id, note) => {
|
||||
const s = store.steps.get(Number(id))
|
||||
if (!s || s.status !== 'running' || s.claim_expires_at) return false
|
||||
Object.assign(s, { status: 'done', last_error: note, claimed_by: null })
|
||||
return true
|
||||
}
|
||||
|
||||
stepsDb.skipByHuman = async (id, reason) => {
|
||||
const s = store.steps.get(Number(id))
|
||||
if (!s) return false
|
||||
const ok = s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at)
|
||||
if (!ok) return false
|
||||
Object.assign(s, { status: 'skipped', last_error: reason, claimed_by: null })
|
||||
return true
|
||||
}
|
||||
|
||||
stepsDb.requeue = async (id) => {
|
||||
const s = store.steps.get(Number(id))
|
||||
if (!s || s.status !== 'failed') return false
|
||||
Object.assign(s, { status: 'pending', attempts: 0, due_at: null, last_error: null, claimed_by: null, claim_expires_at: null })
|
||||
return true
|
||||
}
|
||||
|
||||
stepsDb.cancelOpen = async (runId) => {
|
||||
let n = 0
|
||||
for (const s of store.steps.values()) {
|
||||
if (s.run_id !== Number(runId)) continue
|
||||
if (s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at)) {
|
||||
s.status = 'cancelled'
|
||||
n += 1
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
stepsDb.lastStartedSeq = async (runId, phase) => {
|
||||
const started = [...store.steps.values()]
|
||||
.filter((s) => s.run_id === Number(runId) && s.phase === phase && s.status !== 'pending')
|
||||
.map((s) => s.seq)
|
||||
return started.length ? Math.max(...started) : null
|
||||
}
|
||||
|
||||
logDb.write = async (line) => {
|
||||
store.log.push(line)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(installStubs)
|
||||
afterEach(() => {
|
||||
for (const [, mod, fns] of originals) Object.assign(mod, fns)
|
||||
})
|
||||
|
||||
let nextRunId = 1
|
||||
|
||||
function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) {
|
||||
const id = nextRunId++
|
||||
store.runs.set(id, {
|
||||
id,
|
||||
definition_id: id,
|
||||
version_id: id,
|
||||
status,
|
||||
health: 'ok',
|
||||
current_phase: phase,
|
||||
claimed_by: null,
|
||||
claim_expires_at: null,
|
||||
last_error: null,
|
||||
})
|
||||
steps.forEach((s, i) => {
|
||||
const stepId = store.nextStepId++
|
||||
store.steps.set(stepId, {
|
||||
id: stepId,
|
||||
run_id: id,
|
||||
phase: s.phase || phase,
|
||||
seq: s.seq ?? i,
|
||||
action_id: s.actionId || 'test.action',
|
||||
status: s.status || 'pending',
|
||||
attempts: s.attempts ?? 0,
|
||||
due_at: null,
|
||||
claimed_by: s.leased ? 'someone' : null,
|
||||
claim_expires_at: s.leased ? new Date(Date.now() + 60_000) : null,
|
||||
last_error: null,
|
||||
params: {},
|
||||
})
|
||||
})
|
||||
return id
|
||||
}
|
||||
|
||||
const runRow = (id) => store.runs.get(id)
|
||||
const stepsOf = (id) => [...store.steps.values()].filter((s) => s.run_id === id).sort((a, b) => a.seq - b.seq)
|
||||
const lastLog = () => store.log[store.log.length - 1]
|
||||
|
||||
// ── pause / resume ─────────────────────────────────────────────────────────
|
||||
|
||||
test('pause takes a run in flight and records who did it', async () => {
|
||||
const id = seedRun({ status: 'running' })
|
||||
const result = await controls.pause(id, { reason: 'the shard is lagging' }, ACTOR)
|
||||
|
||||
assert.equal(result.ok, true)
|
||||
assert.equal(runRow(id).status, 'paused')
|
||||
assert.deepEqual(lastLog().detail, {
|
||||
from: 'running',
|
||||
to: 'paused',
|
||||
control: 'pause',
|
||||
by: ACTOR,
|
||||
reason: 'the shard is lagging',
|
||||
})
|
||||
})
|
||||
|
||||
test('pause drops the claim, so the next tick is not locked out of a resumed run', async () => {
|
||||
const id = seedRun({ status: 'running' })
|
||||
Object.assign(runRow(id), { claimed_by: 'host:1', claim_expires_at: new Date(Date.now() + 900_000) })
|
||||
|
||||
await controls.pause(id, {}, ACTOR)
|
||||
|
||||
assert.equal(runRow(id).claimed_by, null)
|
||||
assert.equal(runRow(id).claim_expires_at, null)
|
||||
})
|
||||
|
||||
test('a scheduled run cannot be paused — it is cancelled instead', async () => {
|
||||
// Pausing one would leave a run that is neither going to start nor visibly
|
||||
// abandoned, and resuming it after its grace window had passed would produce a
|
||||
// `missed` from a button labelled resume.
|
||||
const id = seedRun({ status: 'scheduled' })
|
||||
const result = await controls.pause(id, {}, ACTOR)
|
||||
|
||||
assert.equal(result.ok, false)
|
||||
assert.equal(result.status, 409)
|
||||
assert.match(result.errors[0], /scheduled/)
|
||||
assert.equal(runRow(id).status, 'scheduled')
|
||||
})
|
||||
|
||||
test('a completed run cannot be paused', async () => {
|
||||
const id = seedRun({ status: 'completed' })
|
||||
assert.equal((await controls.pause(id, {}, ACTOR)).ok, false)
|
||||
})
|
||||
|
||||
test('resume returns a run to running, or to starting when it never entered a phase', async () => {
|
||||
const withPhase = seedRun({ status: 'paused', phase: 'main' })
|
||||
assert.equal((await controls.resume(withPhase, {}, ACTOR)).ok, true)
|
||||
assert.equal(runRow(withPhase).status, 'running')
|
||||
|
||||
const beforePhase = seedRun({ status: 'paused', phase: null })
|
||||
assert.equal((await controls.resume(beforePhase, {}, ACTOR)).ok, true)
|
||||
assert.equal(runRow(beforePhase).status, 'starting', 'both are in findDue; neither is a fourth column')
|
||||
})
|
||||
|
||||
test('resume clears the error it was paused over and leaves health alone', async () => {
|
||||
const id = seedRun({ status: 'paused' })
|
||||
Object.assign(runRow(id), { last_error: 'core.spawn failed', health: 'degraded' })
|
||||
|
||||
await controls.resume(id, {}, ACTOR)
|
||||
|
||||
assert.equal(runRow(id).last_error, null, 'a resolved failure must not accuse a healthy run for ever')
|
||||
assert.equal(runRow(id).health, 'degraded', 'that this run has already had trouble stays true')
|
||||
})
|
||||
|
||||
test('resume refuses a run that is not paused', async () => {
|
||||
const id = seedRun({ status: 'running' })
|
||||
const result = await controls.resume(id, {}, ACTOR)
|
||||
assert.equal(result.ok, false)
|
||||
assert.equal(result.status, 409)
|
||||
})
|
||||
|
||||
// ── cancel ─────────────────────────────────────────────────────────────────
|
||||
|
||||
test('cancel closes out the pending steps and the parked cue, and leaves a leased step alone', async () => {
|
||||
const id = seedRun({
|
||||
status: 'running',
|
||||
steps: [
|
||||
{ status: 'done' },
|
||||
{ status: 'running', leased: true }, // mid-dispatch: nothing can recall a sent command
|
||||
{ status: 'running' }, // parked on a human: nothing is holding it
|
||||
{ status: 'pending' },
|
||||
],
|
||||
})
|
||||
|
||||
const result = await controls.cancel(id, { reason: 'called off' }, ACTOR)
|
||||
|
||||
assert.equal(result.ok, true)
|
||||
assert.equal(runRow(id).status, 'cancelled')
|
||||
assert.equal(result.cancelledSteps, 2)
|
||||
const [done, leased, parked, pending] = stepsOf(id)
|
||||
assert.equal(done.status, 'done')
|
||||
assert.equal(leased.status, 'running', 'a step being dispatched is not touched')
|
||||
assert.equal(parked.status, 'cancelled', 'a cancelled run must stop claiming to wait on somebody')
|
||||
assert.equal(pending.status, 'cancelled')
|
||||
})
|
||||
|
||||
test('cancel is legal before a run has started', async () => {
|
||||
const id = seedRun({ status: 'scheduled', steps: [{ status: 'pending' }] })
|
||||
assert.equal((await controls.cancel(id, {}, ACTOR)).ok, true)
|
||||
assert.equal(runRow(id).status, 'cancelled')
|
||||
})
|
||||
|
||||
test('cancel refuses a run that is already terminal', async () => {
|
||||
for (const status of TERMINAL) {
|
||||
const id = seedRun({ status })
|
||||
const result = await controls.cancel(id, {}, ACTOR)
|
||||
assert.equal(result.ok, false, `${status} should not be cancellable`)
|
||||
assert.match(result.errors[0], new RegExp(status))
|
||||
}
|
||||
})
|
||||
|
||||
// ── confirm ────────────────────────────────────────────────────────────────
|
||||
|
||||
test('confirm resolves a parked cue as done, keeping what the person says they did', async () => {
|
||||
const id = seedRun({ status: 'running', steps: [{ status: 'running', actionId: 'core.cue' }] })
|
||||
const [cue] = stepsOf(id)
|
||||
|
||||
const result = await controls.confirmStep(id, cue.id, { note: 'gate opened, herald read' }, ACTOR)
|
||||
|
||||
assert.equal(result.ok, true)
|
||||
assert.equal(stepsOf(id)[0].status, 'done', 'a person saying they did it is the step having succeeded')
|
||||
assert.equal(stepsOf(id)[0].last_error, 'gate opened, herald read')
|
||||
assert.equal(lastLog().detail.control, 'confirm')
|
||||
assert.equal(lastLog().detail.by, ACTOR)
|
||||
})
|
||||
|
||||
test('confirm cannot resolve a step a process is dispatching', async () => {
|
||||
// The whole vocabulary here is "running with a NULL lease". A live lease means
|
||||
// something is mid-dispatch, and confirming it would race the process that
|
||||
// owns the row.
|
||||
const id = seedRun({ status: 'running', steps: [{ status: 'running', leased: true }] })
|
||||
const [busy] = stepsOf(id)
|
||||
|
||||
const result = await controls.confirmStep(id, busy.id, {}, ACTOR)
|
||||
|
||||
assert.equal(result.ok, false)
|
||||
assert.equal(result.status, 409)
|
||||
assert.equal(stepsOf(id)[0].status, 'running')
|
||||
})
|
||||
|
||||
test('a step id from another run is a 404, not an action', async () => {
|
||||
const mine = seedRun({ status: 'running', steps: [{ status: 'pending' }] })
|
||||
const theirs = seedRun({ status: 'running', steps: [{ status: 'running' }] })
|
||||
const [theirStep] = stepsOf(theirs)
|
||||
|
||||
const result = await controls.confirmStep(mine, theirStep.id, {}, ACTOR)
|
||||
|
||||
assert.equal(result.ok, false)
|
||||
assert.equal(result.status, 404)
|
||||
assert.equal(stepsOf(theirs)[0].status, 'running')
|
||||
})
|
||||
|
||||
// ── skip ───────────────────────────────────────────────────────────────────
|
||||
|
||||
test('skip takes a pending step and a parked cue, and nothing else', async () => {
|
||||
const id = seedRun({
|
||||
status: 'running',
|
||||
steps: [{ status: 'pending' }, { status: 'running' }, { status: 'running', leased: true }, { status: 'failed' }],
|
||||
})
|
||||
const [pending, parked, leased, failed] = stepsOf(id)
|
||||
|
||||
assert.equal((await controls.skipStep(id, pending.id, {}, ACTOR)).ok, true)
|
||||
assert.equal((await controls.skipStep(id, parked.id, {}, ACTOR)).ok, true)
|
||||
assert.equal((await controls.skipStep(id, leased.id, {}, ACTOR)).ok, false)
|
||||
// A failed step does not need skipping: `nextOpenStep` already passes over it,
|
||||
// so resuming the run carries the phase past it.
|
||||
assert.equal((await controls.skipStep(id, failed.id, {}, ACTOR)).ok, false)
|
||||
|
||||
const after = stepsOf(id)
|
||||
assert.equal(after[0].status, 'skipped')
|
||||
assert.equal(after[1].status, 'skipped')
|
||||
assert.equal(after[2].status, 'running')
|
||||
assert.equal(after[3].status, 'failed')
|
||||
})
|
||||
|
||||
test('skip refuses once the run is over', async () => {
|
||||
const id = seedRun({ status: 'completed', steps: [{ status: 'pending' }] })
|
||||
const [step] = stepsOf(id)
|
||||
assert.equal((await controls.skipStep(id, step.id, {}, ACTOR)).ok, false)
|
||||
})
|
||||
|
||||
// ── retry ──────────────────────────────────────────────────────────────────
|
||||
|
||||
test('retry re-queues the step a paused run is stopped at, and resumes in the same action', async () => {
|
||||
const id = seedRun({
|
||||
status: 'paused',
|
||||
steps: [{ status: 'done' }, { status: 'failed', attempts: 3 }, { status: 'pending' }],
|
||||
})
|
||||
const failed = stepsOf(id)[1]
|
||||
|
||||
const result = await controls.retryStep(id, failed.id, {}, ACTOR)
|
||||
|
||||
assert.equal(result.ok, true)
|
||||
assert.equal(result.resumed, true)
|
||||
assert.equal(stepsOf(id)[1].status, 'pending')
|
||||
assert.equal(stepsOf(id)[1].attempts, 0, 'the ceiling bounds the runner, not a person deciding once')
|
||||
assert.equal(runRow(id).status, 'running', 'there is no state in which you would want half of this')
|
||||
})
|
||||
|
||||
test('retry refuses a step the run has already moved past', async () => {
|
||||
// The case the guard exists for: a failed step under an `on_failure` of `skip`
|
||||
// is one the phase carried on from. Re-queueing it would put a pending row
|
||||
// behind the runner's cursor, where it would sit for ever.
|
||||
const id = seedRun({
|
||||
status: 'paused',
|
||||
steps: [{ status: 'failed', attempts: 3 }, { status: 'done' }, { status: 'failed', attempts: 3 }],
|
||||
})
|
||||
const [movedPast] = stepsOf(id)
|
||||
|
||||
const result = await controls.retryStep(id, movedPast.id, {}, ACTOR)
|
||||
|
||||
assert.equal(result.ok, false)
|
||||
assert.equal(result.status, 409)
|
||||
assert.match(result.errors[0], /stopped at this step/)
|
||||
assert.equal(stepsOf(id)[0].status, 'failed')
|
||||
assert.equal(runRow(id).status, 'paused', 'a refused retry does not resume the run either')
|
||||
})
|
||||
|
||||
test('retry refuses a step in a phase the run has left', async () => {
|
||||
const id = seedRun({
|
||||
status: 'paused',
|
||||
phase: 'two',
|
||||
steps: [{ phase: 'one', seq: 0, status: 'failed' }, { phase: 'two', seq: 0, status: 'pending' }],
|
||||
})
|
||||
const [old] = stepsOf(id)
|
||||
|
||||
const result = await controls.retryStep(id, old.id, {}, ACTOR)
|
||||
assert.equal(result.ok, false)
|
||||
assert.match(result.errors[0], /already left/)
|
||||
})
|
||||
|
||||
test('retry refuses while the run is still running', async () => {
|
||||
const id = seedRun({ status: 'running', steps: [{ status: 'failed' }] })
|
||||
const [failed] = stepsOf(id)
|
||||
|
||||
const result = await controls.retryStep(id, failed.id, {}, ACTOR)
|
||||
assert.equal(result.ok, false)
|
||||
assert.match(result.errors[0], /paused/)
|
||||
})
|
||||
|
||||
test('retry refuses a step that is not failed', async () => {
|
||||
const id = seedRun({ status: 'paused', steps: [{ status: 'pending' }] })
|
||||
const [pending] = stepsOf(id)
|
||||
assert.equal((await controls.retryStep(id, pending.id, {}, ACTOR)).ok, false)
|
||||
})
|
||||
|
||||
// ── the record ─────────────────────────────────────────────────────────────
|
||||
|
||||
test('every control writes one log line carrying the actor and the control name', async () => {
|
||||
const id = seedRun({ status: 'running', steps: [{ status: 'running' }, { status: 'pending' }] })
|
||||
const [parked, pending] = stepsOf(id)
|
||||
|
||||
await controls.confirmStep(id, parked.id, { note: 'done' }, ACTOR)
|
||||
await controls.skipStep(id, pending.id, { reason: 'not needed' }, ACTOR)
|
||||
await controls.pause(id, {}, ACTOR)
|
||||
await controls.resume(id, {}, ACTOR)
|
||||
await controls.cancel(id, { reason: 'over' }, ACTOR)
|
||||
|
||||
const human = store.log.filter((l) => l.detail?.control)
|
||||
assert.deepEqual(human.map((l) => l.detail.control), ['confirm', 'skip', 'pause', 'resume', 'cancel'])
|
||||
assert.ok(human.every((l) => l.detail.by === ACTOR))
|
||||
// The kinds are the ones a reader already scans for. A human transition is
|
||||
// still a transition; `detail.control` is what separates it from the runner's.
|
||||
assert.deepEqual([...new Set(human.map((l) => l.kind))].sort(), ['run.status', 'step.status'])
|
||||
})
|
||||
|
||||
test('an empty reason is stored as NULL rather than as an empty string', async () => {
|
||||
const id = seedRun({ status: 'running' })
|
||||
await controls.pause(id, { reason: ' ' }, ACTOR)
|
||||
assert.equal(lastLog().detail.reason, null)
|
||||
})
|
||||
@@ -131,6 +131,8 @@ function installStubs() {
|
||||
return true
|
||||
}
|
||||
|
||||
runsDb.statusOf = async (id) => store.runs.get(id)?.status || null
|
||||
|
||||
runsDb.setHealth = async (id, health) => {
|
||||
const r = store.runs.get(id)
|
||||
if (!r || r.health === health) return false
|
||||
@@ -687,3 +689,51 @@ test('the dispatch envelope carries what §F says it carries', async () => {
|
||||
assert.equal(envelope.idempotencyKey.length, 40)
|
||||
assert.deepEqual(Object.keys(envelope).sort(), ['actor', 'idempotencyKey', 'params', 'runId', 'scope', 'stepId', 'verify'])
|
||||
})
|
||||
|
||||
test('a run paused mid-batch stops there rather than draining the rest of the phase', async () => {
|
||||
// The whole value of a pause is that it takes effect NOW. `advanceRun` drains
|
||||
// up to STEPS_PER_TICK steps from one run inside a single tick, so a status
|
||||
// re-read only at the top of the tick would answer a pause by dispatching
|
||||
// another two dozen steps. Written by pausing from inside an action's own
|
||||
// `perform`, which is the only moment that race is reproducible.
|
||||
register([
|
||||
scriptedAction('test.pauser', {
|
||||
perform: async ({ runId }) => {
|
||||
await runsDb.transition(runId, ['starting', 'running'], 'paused', { clearClaim: true })
|
||||
return { ok: true }
|
||||
},
|
||||
}),
|
||||
scriptedAction('test.after'),
|
||||
])
|
||||
|
||||
const id = seedRun([
|
||||
{
|
||||
key: 'main',
|
||||
label: 'Main',
|
||||
steps: [step('test.pauser'), step('test.after'), step('test.after')],
|
||||
},
|
||||
])
|
||||
|
||||
await runner.tick(T0)
|
||||
|
||||
assert.equal(run(id).status, 'paused')
|
||||
const [first, second, third] = stepsOf(id)
|
||||
assert.equal(first.status, 'done', 'the step that was already dispatched finishes')
|
||||
assert.equal(second.status, 'pending', 'nothing after it ran')
|
||||
assert.equal(third.status, 'pending')
|
||||
assert.equal(scripted['test.after'], undefined, 'the later action was never called')
|
||||
})
|
||||
|
||||
test('a resumed run picks up from the step it stopped at', async () => {
|
||||
register([scriptedAction('test.a'), scriptedAction('test.b')])
|
||||
|
||||
const id = seedRun([{ key: 'main', label: 'Main', steps: [step('test.a'), step('test.b')] }])
|
||||
await runner.tick(T0)
|
||||
assert.equal(run(id).status, 'completed')
|
||||
|
||||
// And the mirror of it: a run parked at `paused` is not picked up at all, which
|
||||
// is what `findDue`'s omission of the status buys.
|
||||
const other = seedRun([{ key: 'main', label: 'Main', steps: [step('test.a')] }], { status: 'paused' })
|
||||
await runner.tick(T0)
|
||||
assert.equal(stepsOf(other)[0].status, 'pending', 'a paused run is not swept')
|
||||
})
|
||||
|
||||
@@ -25,6 +25,19 @@
|
||||
// both a MariaDB-specific syntax and a correctness claim: it must move the
|
||||
// next PENDING step and only ever push a due date later.
|
||||
//
|
||||
// **Phase 3 added four more**, and each of them is a control a staff member
|
||||
// presses against a live game world:
|
||||
//
|
||||
// * **`confirmParked` / `skipByHuman`** - both keyed on `status = 'running'
|
||||
// AND claim_expires_at IS NULL`. That pair, and only that pair, means "a cue
|
||||
// waiting on a human". If the clause let a LEASED step through, confirm would
|
||||
// race the process mid-dispatch on that row.
|
||||
// * **`cancelOpen`** - pending steps and parked cues, never a leased one.
|
||||
// * **`lastStartedSeq`** - a `MAX(seq) ... WHERE status <> 'pending'`, which is
|
||||
// what decides whether retry is offered. The first draft asked for the LOWEST
|
||||
// unsettled seq instead, which is a different step whenever a phase carried
|
||||
// on past an `on_failure: skip` failure.
|
||||
//
|
||||
// Plus the two unique indexes that are load-bearing rather than tidy:
|
||||
// `uq_evrun_occurrence` (which, not the claim, is what stops two runs of one
|
||||
// occurrence existing) and `uq_evstep_slot` (which is what makes re-materialising
|
||||
@@ -150,6 +163,36 @@ UPDATE event_run_steps
|
||||
AND (due_at IS NULL OR due_at < ?)
|
||||
ORDER BY seq LIMIT 1`
|
||||
|
||||
// Phase 3's four, verbatim from `eventRunSteps.db.js`.
|
||||
const CONFIRM_PARKED = `
|
||||
UPDATE event_run_steps
|
||||
SET status = 'done', finished_at = NOW(), claimed_by = NULL,
|
||||
last_error = ?
|
||||
WHERE id = ? AND status = 'running' AND claim_expires_at IS NULL`
|
||||
|
||||
const SKIP_BY_HUMAN = `
|
||||
UPDATE event_run_steps
|
||||
SET status = 'skipped', finished_at = NOW(), claimed_by = NULL,
|
||||
last_error = ?
|
||||
WHERE id = ?
|
||||
AND (status = 'pending' OR (status = 'running' AND claim_expires_at IS NULL))`
|
||||
|
||||
const REQUEUE = `
|
||||
UPDATE event_run_steps
|
||||
SET status = 'pending', attempts = 0, due_at = NULL, last_error = NULL,
|
||||
claimed_by = NULL, claim_expires_at = NULL, finished_at = NULL
|
||||
WHERE id = ? AND status = 'failed'`
|
||||
|
||||
const CANCEL_OPEN = `
|
||||
UPDATE event_run_steps
|
||||
SET status = 'cancelled', finished_at = NOW(), claimed_by = NULL
|
||||
WHERE run_id = ?
|
||||
AND (status = 'pending' OR (status = 'running' AND claim_expires_at IS NULL))`
|
||||
|
||||
const LAST_STARTED_SEQ = `
|
||||
SELECT MAX(seq) AS seq FROM event_run_steps
|
||||
WHERE run_id = ? AND phase = ? AND status <> 'pending'`
|
||||
|
||||
const MATERIALISE_RUN = `
|
||||
INSERT IGNORE INTO event_runs (definition_id, version_id, scope, scheduled_for, concurrency_key)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
@@ -506,3 +549,113 @@ test('findMissed compares against each definition’s own grace window', async (
|
||||
assert.deepEqual(missed, [tight.runId])
|
||||
assert.ok(!missed.includes(generous.runId), 'inside its own window a run starts late rather than being missed')
|
||||
})
|
||||
|
||||
// -- Phase 3: the controls a human presses ----------------------------------
|
||||
|
||||
test('confirm resolves a parked cue and cannot touch a step being dispatched', async (t) => {
|
||||
if (needDb(t)) return
|
||||
const { runId } = await seedRun({ status: 'running' })
|
||||
const parked = await seedStep(runId, { seq: 0, status: 'running', claimedBy: 'host:1', claimExpiresAt: null })
|
||||
const busy = await seedStep(runId, { seq: 1, status: 'running', claimedBy: 'host:1', claimExpiresAt: later(60_000), key: 'x'.repeat(40) })
|
||||
|
||||
assert.equal(rows(await pool.query(CONFIRM_PARKED, ['gate opened', parked])), 1)
|
||||
assert.equal(rows(await pool.query(CONFIRM_PARKED, ['nope', busy])), 0, 'a live lease is a step somebody owns')
|
||||
|
||||
assert.equal((await stepById(parked)).status, 'done')
|
||||
assert.equal((await stepById(parked)).last_error, 'gate opened')
|
||||
assert.equal((await stepById(busy)).status, 'running')
|
||||
})
|
||||
|
||||
test('a confirm of an already-confirmed cue reports 0, not 1', async (t) => {
|
||||
if (needDb(t)) return
|
||||
// The engagement Phase 4a shape: a connector that defaults `foundRows: true`
|
||||
// reports 1 for an UPDATE that matched and changed nothing, and a control that
|
||||
// read that as success would tell a second staff member their press worked.
|
||||
const { runId } = await seedRun({ status: 'running' })
|
||||
const parked = await seedStep(runId, { status: 'running', claimExpiresAt: null })
|
||||
|
||||
assert.equal(rows(await pool.query(CONFIRM_PARKED, [null, parked])), 1)
|
||||
assert.equal(rows(await pool.query(CONFIRM_PARKED, [null, parked])), 0)
|
||||
})
|
||||
|
||||
test('skip takes a pending step and a parked cue, and refuses a leased one', async (t) => {
|
||||
if (needDb(t)) return
|
||||
const { runId } = await seedRun({ status: 'running' })
|
||||
const pending = await seedStep(runId, { seq: 0, status: 'pending' })
|
||||
const parked = await seedStep(runId, { seq: 1, status: 'running', claimExpiresAt: null, key: 'y'.repeat(40) })
|
||||
const busy = await seedStep(runId, { seq: 2, status: 'running', claimExpiresAt: later(60_000), key: 'z'.repeat(40) })
|
||||
const failed = await seedStep(runId, { seq: 3, status: 'failed', key: 'w'.repeat(40) })
|
||||
|
||||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, pending])), 1)
|
||||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, parked])), 1)
|
||||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, busy])), 0)
|
||||
assert.equal(rows(await pool.query(SKIP_BY_HUMAN, [null, failed])), 0, 'a failed step is terminal; resume carries the phase past it')
|
||||
})
|
||||
|
||||
test('cancelOpen closes pending steps and parked cues, and leaves a leased one alone', async (t) => {
|
||||
if (needDb(t)) return
|
||||
const { runId } = await seedRun({ status: 'running' })
|
||||
const done = await seedStep(runId, { seq: 0, status: 'done' })
|
||||
const busy = await seedStep(runId, { seq: 1, status: 'running', claimExpiresAt: later(60_000), key: 'p'.repeat(40) })
|
||||
const parked = await seedStep(runId, { seq: 2, status: 'running', claimExpiresAt: null, key: 'q'.repeat(40) })
|
||||
const pending = await seedStep(runId, { seq: 3, status: 'pending', key: 'r'.repeat(40) })
|
||||
|
||||
assert.equal(rows(await pool.query(CANCEL_OPEN, [runId])), 2)
|
||||
|
||||
assert.equal((await stepById(done)).status, 'done')
|
||||
assert.equal((await stepById(busy)).status, 'running', 'nothing can recall a command already sent')
|
||||
assert.equal((await stepById(parked)).status, 'cancelled', 'a cancelled run must stop claiming to wait on somebody')
|
||||
assert.equal((await stepById(pending)).status, 'cancelled')
|
||||
})
|
||||
|
||||
test('requeue only takes a failed step, and puts attempts back to zero', async (t) => {
|
||||
if (needDb(t)) return
|
||||
const { runId } = await seedRun({ status: 'paused' })
|
||||
const failed = await seedStep(runId, { seq: 0, status: 'failed', attempts: 3 })
|
||||
const pending = await seedStep(runId, { seq: 1, status: 'pending', key: 's'.repeat(40) })
|
||||
|
||||
assert.equal(rows(await pool.query(REQUEUE, [failed])), 1)
|
||||
assert.equal(rows(await pool.query(REQUEUE, [pending])), 0)
|
||||
|
||||
const row = await stepById(failed)
|
||||
assert.equal(row.status, 'pending')
|
||||
assert.equal(Number(row.attempts), 0)
|
||||
assert.equal(row.due_at, null, 'a re-queued step is due now, not at the retry backoff it was left on')
|
||||
})
|
||||
|
||||
test('lastStartedSeq names the furthest step of the phase, not the earliest unsettled one', async (t) => {
|
||||
if (needDb(t)) return
|
||||
// The defect this replaced: a phase that carried on past a failed step (an
|
||||
// `on_failure` of `skip`) and then paused at a later one. "The lowest seq that
|
||||
// is not settled" answers with the FIRST failure - a step the runner has long
|
||||
// since stepped over - and retry would re-queue a row behind its own cursor.
|
||||
const { runId } = await seedRun({ status: 'paused' })
|
||||
await seedStep(runId, { seq: 0, status: 'failed', key: 'a'.repeat(40) })
|
||||
await seedStep(runId, { seq: 1, status: 'done', key: 'b'.repeat(40) })
|
||||
await seedStep(runId, { seq: 2, status: 'failed', key: 'c'.repeat(40) })
|
||||
await seedStep(runId, { seq: 3, status: 'pending', key: 'd'.repeat(40) })
|
||||
|
||||
const [row] = await pool.query(LAST_STARTED_SEQ, [runId, 'main'])
|
||||
assert.equal(Number(row.seq), 2)
|
||||
})
|
||||
|
||||
test('lastStartedSeq is NULL for a phase nothing has touched', async (t) => {
|
||||
if (needDb(t)) return
|
||||
const { runId } = await seedRun({ status: 'running' })
|
||||
await seedStep(runId, { seq: 0, status: 'pending' })
|
||||
|
||||
const [row] = await pool.query(LAST_STARTED_SEQ, [runId, 'main'])
|
||||
assert.equal(row.seq, null, 'a null must read as "nothing to retry", not as seq 0')
|
||||
})
|
||||
|
||||
test('a guarded transition refuses a run that was cancelled underneath it', async (t) => {
|
||||
if (needDb(t)) return
|
||||
// What the admin cancel looks like from the runner's side, mid-tick: the
|
||||
// guarded write returns 0 and the tick treats the run as taken rather than
|
||||
// advancing a run somebody has just stopped.
|
||||
const { runId } = await seedRun({ status: 'running' })
|
||||
await pool.query("UPDATE event_runs SET status = 'cancelled' WHERE id = ?", [runId])
|
||||
|
||||
assert.equal(rows(await pool.query(TRANSITION, ['running', 'two', runId, 'running'])), 0)
|
||||
assert.equal((await runById(runId)).status, 'cancelled')
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user