website/server/test/eventRunControls.test.js

// ── The live run controls (EVENTS_PLAN.md Phase 3) ─────────────────────────
//
// Seven controls as of Phase 5, and what is tested is almost entirely the
// REFUSALS. A control
// that works is easy; a control that works from a status it should not have
// worked from is a staff member changing a live game world by pressing a button
// a stale screen offered them. So each of the six is exercised from every status
// it must decline, and the four that a run console could plausibly offer wrongly
// get a test of their own:
//
//   • retry on a step the run has already moved past (the `skip` disposition) —
//     the test that found the first draft's guard was reading the wrong end of
//     the phase
//   • confirm on a step a process is mid-dispatch on, not a parked cue
//   • skip on a step with a live lease
//   • cancel closing out a parked cue, so a cancelled run stops "waiting"
//   • advance on a phase that is NOT waiting on its gate — the refusal that
//     makes force-advance an override rather than a way to skip a phase's steps
//
// The three tables are stubbed at the `.db` layer and the model's own logic runs
// for real against them — the shape `eventRunner.test.js` uses. What a stub
// cannot prove is that the five statements mean this against a real server; the
// guards that are pure SQL (`status = 'running' AND claim_expires_at IS NULL`
// and `lastStartedSeq`'s MAX) are proved in `eventRunnerSql.test.js`.
//
// Point the DB at a closed port before requiring anything.
process.env.DB_HOST = '127.0.0.1'
process.env.DB_PORT = '59999'

const { test, beforeEach, afterEach, after } = require('node:test')
const assert = require('node:assert/strict')

const controls = require('../src/model/events/eventRunControls.model')
const runsDb = require('../src/model/events/eventRuns.db')
const stepsDb = require('../src/model/events/eventRunSteps.db')
const logDb = require('../src/model/events/eventRunLog.db')
const gatesDb = require('../src/model/events/eventPhaseGates.db')
const db = require('../src/utils/db')

after(() => db.close())

const TERMINAL = ['completed', 'cancelled', 'failed', 'missed']
const ACTOR = 7

let store
const originals = [
  ['runs', runsDb, { ...runsDb }],
  ['steps', stepsDb, { ...stepsDb }],
  ['log', logDb, { ...logDb }],
  ['gates', gatesDb, { ...gatesDb }],
]

function installStubs() {
  store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), nextStepId: 1, nextGateId: 1 }
  const snap = (o) => ({ ...o })

  runsDb.getById = async (id) => {
    const r = store.runs.get(Number(id))
    return r ? snap(r) : null
  }

  runsDb.transition = async (id, from, to, opts = {}) => {
    const r = store.runs.get(Number(id))
    const froms = Array.isArray(from) ? from : [from]
    if (!r || !froms.includes(r.status)) return false
    r.status = to
    if (opts.phase !== undefined) r.current_phase = opts.phase
    if (opts.error !== undefined) r.last_error = opts.error
    if (TERMINAL.includes(to) || opts.clearClaim) {
      r.claimed_by = null
      r.claim_expires_at = null
    }
    return true
  }

  stepsDb.getById = async (id) => {
    const s = store.steps.get(Number(id))
    return s ? snap(s) : null
  }

  // Each of the four mirrors its statement's WHERE clause exactly. A stub can
  // only ever agree with whoever wrote it, so what these buy is the model's
  // logic around them; the clauses themselves are checked against a real server.
  stepsDb.confirmParked = async (id, note) => {
    const s = store.steps.get(Number(id))
    if (!s || s.status !== 'running' || s.claim_expires_at) return false
    Object.assign(s, { status: 'done', last_error: note, claimed_by: null })
    return true
  }

  stepsDb.skipByHuman = async (id, reason) => {
    const s = store.steps.get(Number(id))
    if (!s) return false
    const ok = s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at)
    if (!ok) return false
    Object.assign(s, { status: 'skipped', last_error: reason, claimed_by: null })
    return true
  }

  stepsDb.requeue = async (id) => {
    const s = store.steps.get(Number(id))
    if (!s || s.status !== 'failed') return false
    Object.assign(s, { status: 'pending', attempts: 0, due_at: null, last_error: null, claimed_by: null, claim_expires_at: null })
    return true
  }

  stepsDb.cancelOpen = async (runId) => {
    let n = 0
    for (const s of store.steps.values()) {
      if (s.run_id !== Number(runId)) continue
      if (s.status === 'pending' || (s.status === 'running' && !s.claim_expires_at)) {
        s.status = 'cancelled'
        n += 1
      }
    }
    return n
  }

  // Phase 5's advance guard reads this. Unstubbed it is a real query, and this
  // file points the pool at a closed port — three tests would hang for ten
  // seconds each and then fail with ECONNREFUSED rather than saying anything
  // about the control. Same shape as the statement: pending and running only,
  // lowest seq first.
  stepsDb.nextOpenStep = async (runId, phase) => {
    const s = [...store.steps.values()]
      .filter((x) => x.run_id === Number(runId) && x.phase === phase && ['pending', 'running'].includes(x.status))
      .sort((a, b) => a.seq - b.seq || a.id - b.id)[0]
    return s ? { ...s } : null
  }

  stepsDb.lastStartedSeq = async (runId, phase) => {
    const started = [...store.steps.values()]
      .filter((s) => s.run_id === Number(runId) && s.phase === phase && s.status !== 'pending')
      .map((s) => s.seq)
    return started.length ? Math.max(...started) : null
  }

  logDb.write = async (line) => {
    store.log.push(line)
    return true
  }

  // Phase 5. `satisfy` carries `WHERE satisfied_at IS NULL`, and the stub keeps
  // it: a gate that could be forced twice would log two advances of one phase.
  gatesDb.forPhase = async (runId, phase) => {
    const g = store.gates.get(`${Number(runId)}|${phase}`)
    return g ? { ...g } : null
  }
  gatesDb.satisfy = async (id, by, { userId = null, now = new Date() } = {}) => {
    const g = [...store.gates.values()].find((x) => x.id === id)
    if (!g || g.satisfied_at) return false
    Object.assign(g, { satisfied_at: now, satisfied_by: by, forced_by: userId })
    return true
  }
}

beforeEach(installStubs)
afterEach(() => {
  for (const [, mod, fns] of originals) Object.assign(mod, fns)
})

let nextRunId = 1

function seedGate(runId, phase, { kind = 'on', trigger = 'test.trigger', needed = 1, tally = 0, minutesAgo = 5 } = {}) {
  const g = {
    id: store.nextGateId++,
    run_id: runId,
    phase,
    kind,
    after_seconds: kind === 'after' ? 1800 : null,
    trigger_id: kind === 'on' ? trigger : null,
    conditions: null,
    needed,
    tally,
    entered_at: new Date(Date.now() - minutesAgo * 60_000),
    due_at: null,
    last_event: null,
    last_event_at: null,
    satisfied_at: null,
    satisfied_by: null,
    forced_by: null,
  }
  store.gates.set(`${runId}|${phase}`, g)
  return g
}

function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) {
  const id = nextRunId++
  store.runs.set(id, {
    id,
    definition_id: id,
    version_id: id,
    status,
    health: 'ok',
    current_phase: phase,
    claimed_by: null,
    claim_expires_at: null,
    last_error: null,
  })
  steps.forEach((s, i) => {
    const stepId = store.nextStepId++
    store.steps.set(stepId, {
      id: stepId,
      run_id: id,
      phase: s.phase || phase,
      seq: s.seq ?? i,
      action_id: s.actionId || 'test.action',
      status: s.status || 'pending',
      attempts: s.attempts ?? 0,
      due_at: null,
      claimed_by: s.leased ? 'someone' : null,
      claim_expires_at: s.leased ? new Date(Date.now() + 60_000) : null,
      last_error: null,
      params: {},
    })
  })
  return id
}

const runRow = (id) => store.runs.get(id)
const stepsOf = (id) => [...store.steps.values()].filter((s) => s.run_id === id).sort((a, b) => a.seq - b.seq)
const lastLog = () => store.log[store.log.length - 1]

// ── advance (Phase 5) ──────────────────────────────────────────────────────

test('advance releases a phase that is genuinely waiting on its gate', async () => {
  const id = seedRun({ status: 'running', steps: [{ status: 'done' }] })
  const gate = seedGate(id, 'main', { needed: 3, tally: 1, minutesAgo: 68 })

  const result = await controls.advancePhase(id, { reason: 'the boss never spawned' }, ACTOR)
  assert.equal(result.ok, true)
  assert.equal(result.phase, 'main')
  assert.equal(gate.satisfied_by, 'forced')
  assert.equal(gate.forced_by, ACTOR)

  // The run is NOT transitioned here: the next tick does the phase boundary,
  // exactly as it does after resume, so there is one implementation of what a
  // phase boundary is rather than two.
  assert.equal(runRow(id).current_phase, 'main')

  const line = lastLog()
  assert.equal(line.kind, 'phase.advanced')
  assert.equal(line.detail.because, 'forced')
  assert.equal(line.detail.by, ACTOR)
  assert.equal(line.detail.reason, 'the boss never spawned')
  assert.equal(line.detail.seen, 1)
  assert.equal(line.detail.needed, 3, 'and the log says what it was still waiting for')
  assert.ok(line.detail.waitedSeconds > 4000)
})

test('advance is refused on a phase that is waiting on a STEP, not on its gate', async () => {
  // The refusal that makes this an override rather than a way to skip work: a
  // phase with an open step is held by the step, and skip is its control.
  const id = seedRun({ status: 'running', steps: [{ status: 'done' }, { status: 'pending', actionId: 'test.slow' }] })
  seedGate(id, 'main')

  const result = await controls.advancePhase(id, {}, ACTOR)
  assert.equal(result.ok, false)
  assert.equal(result.status, 409)
  assert.match(result.errors[0], /waiting on step 1 \(test\.slow\)/)
  assert.equal(store.log.length, 0, 'and nothing is logged for a refusal')
})

test('advance is refused when the phase has no advance condition at all', async () => {
  const id = seedRun({ status: 'running', steps: [{ status: 'done' }] })
  const result = await controls.advancePhase(id, {}, ACTOR)
  assert.equal(result.ok, false)
  assert.match(result.errors[0], /has no advance condition; skip its steps instead/)
})

test('advance is refused from every status that is not `running`', async () => {
  for (const status of ['scheduled', 'starting', 'paused', 'ending', ...TERMINAL]) {
    const id = seedRun({ status, steps: [{ status: 'done' }] })
    seedGate(id, 'main')
    const result = await controls.advancePhase(id, {}, ACTOR)
    assert.equal(result.ok, false, `a ${status} run should not be advanceable`)
    assert.match(result.errors[0], new RegExp(`a ${status} run has no phase to advance`))
  }
})

test('advance twice is refused the second time', async () => {
  const id = seedRun({ status: 'running', steps: [{ status: 'done' }] })
  seedGate(id, 'main')
  assert.equal((await controls.advancePhase(id, {}, ACTOR)).ok, true)
  const second = await controls.advancePhase(id, {}, ACTOR)
  assert.equal(second.ok, false)
  assert.match(second.errors[0], /already past its advance condition/)
})

// ── pause / resume ─────────────────────────────────────────────────────────

test('pause takes a run in flight and records who did it', async () => {
  const id = seedRun({ status: 'running' })
  const result = await controls.pause(id, { reason: 'the shard is lagging' }, ACTOR)

  assert.equal(result.ok, true)
  assert.equal(runRow(id).status, 'paused')
  assert.deepEqual(lastLog().detail, {
    from: 'running',
    to: 'paused',
    control: 'pause',
    by: ACTOR,
    reason: 'the shard is lagging',
  })
})

test('pause drops the claim, so the next tick is not locked out of a resumed run', async () => {
  const id = seedRun({ status: 'running' })
  Object.assign(runRow(id), { claimed_by: 'host:1', claim_expires_at: new Date(Date.now() + 900_000) })

  await controls.pause(id, {}, ACTOR)

  assert.equal(runRow(id).claimed_by, null)
  assert.equal(runRow(id).claim_expires_at, null)
})

test('a scheduled run cannot be paused — it is cancelled instead', async () => {
  // Pausing one would leave a run that is neither going to start nor visibly
  // abandoned, and resuming it after its grace window had passed would produce a
  // `missed` from a button labelled resume.
  const id = seedRun({ status: 'scheduled' })
  const result = await controls.pause(id, {}, ACTOR)

  assert.equal(result.ok, false)
  assert.equal(result.status, 409)
  assert.match(result.errors[0], /scheduled/)
  assert.equal(runRow(id).status, 'scheduled')
})

test('a completed run cannot be paused', async () => {
  const id = seedRun({ status: 'completed' })
  assert.equal((await controls.pause(id, {}, ACTOR)).ok, false)
})

test('resume returns a run to running, or to starting when it never entered a phase', async () => {
  const withPhase = seedRun({ status: 'paused', phase: 'main' })
  assert.equal((await controls.resume(withPhase, {}, ACTOR)).ok, true)
  assert.equal(runRow(withPhase).status, 'running')

  const beforePhase = seedRun({ status: 'paused', phase: null })
  assert.equal((await controls.resume(beforePhase, {}, ACTOR)).ok, true)
  assert.equal(runRow(beforePhase).status, 'starting', 'both are in findDue; neither is a fourth column')
})

test('resume clears the error it was paused over and leaves health alone', async () => {
  const id = seedRun({ status: 'paused' })
  Object.assign(runRow(id), { last_error: 'core.spawn failed', health: 'degraded' })

  await controls.resume(id, {}, ACTOR)

  assert.equal(runRow(id).last_error, null, 'a resolved failure must not accuse a healthy run for ever')
  assert.equal(runRow(id).health, 'degraded', 'that this run has already had trouble stays true')
})

test('resume refuses a run that is not paused', async () => {
  const id = seedRun({ status: 'running' })
  const result = await controls.resume(id, {}, ACTOR)
  assert.equal(result.ok, false)
  assert.equal(result.status, 409)
})

// ── cancel ─────────────────────────────────────────────────────────────────

test('cancel closes out the pending steps and the parked cue, and leaves a leased step alone', async () => {
  const id = seedRun({
    status: 'running',
    steps: [
      { status: 'done' },
      { status: 'running', leased: true }, // mid-dispatch: nothing can recall a sent command
      { status: 'running' }, // parked on a human: nothing is holding it
      { status: 'pending' },
    ],
  })

  const result = await controls.cancel(id, { reason: 'called off' }, ACTOR)

  assert.equal(result.ok, true)
  assert.equal(runRow(id).status, 'cancelled')
  assert.equal(result.cancelledSteps, 2)
  const [done, leased, parked, pending] = stepsOf(id)
  assert.equal(done.status, 'done')
  assert.equal(leased.status, 'running', 'a step being dispatched is not touched')
  assert.equal(parked.status, 'cancelled', 'a cancelled run must stop claiming to wait on somebody')
  assert.equal(pending.status, 'cancelled')
})

test('cancel is legal before a run has started', async () => {
  const id = seedRun({ status: 'scheduled', steps: [{ status: 'pending' }] })
  assert.equal((await controls.cancel(id, {}, ACTOR)).ok, true)
  assert.equal(runRow(id).status, 'cancelled')
})

test('cancel refuses a run that is already terminal', async () => {
  for (const status of TERMINAL) {
    const id = seedRun({ status })
    const result = await controls.cancel(id, {}, ACTOR)
    assert.equal(result.ok, false, `${status} should not be cancellable`)
    assert.match(result.errors[0], new RegExp(status))
  }
})

// ── confirm ────────────────────────────────────────────────────────────────

test('confirm resolves a parked cue as done, keeping what the person says they did', async () => {
  const id = seedRun({ status: 'running', steps: [{ status: 'running', actionId: 'core.cue' }] })
  const [cue] = stepsOf(id)

  const result = await controls.confirmStep(id, cue.id, { note: 'gate opened, herald read' }, ACTOR)

  assert.equal(result.ok, true)
  assert.equal(stepsOf(id)[0].status, 'done', 'a person saying they did it is the step having succeeded')
  assert.equal(stepsOf(id)[0].last_error, 'gate opened, herald read')
  assert.equal(lastLog().detail.control, 'confirm')
  assert.equal(lastLog().detail.by, ACTOR)
})

test('confirm cannot resolve a step a process is dispatching', async () => {
  // The whole vocabulary here is "running with a NULL lease". A live lease means
  // something is mid-dispatch, and confirming it would race the process that
  // owns the row.
  const id = seedRun({ status: 'running', steps: [{ status: 'running', leased: true }] })
  const [busy] = stepsOf(id)

  const result = await controls.confirmStep(id, busy.id, {}, ACTOR)

  assert.equal(result.ok, false)
  assert.equal(result.status, 409)
  assert.equal(stepsOf(id)[0].status, 'running')
})

test('a step id from another run is a 404, not an action', async () => {
  const mine = seedRun({ status: 'running', steps: [{ status: 'pending' }] })
  const theirs = seedRun({ status: 'running', steps: [{ status: 'running' }] })
  const [theirStep] = stepsOf(theirs)

  const result = await controls.confirmStep(mine, theirStep.id, {}, ACTOR)

  assert.equal(result.ok, false)
  assert.equal(result.status, 404)
  assert.equal(stepsOf(theirs)[0].status, 'running')
})

// ── skip ───────────────────────────────────────────────────────────────────

test('skip takes a pending step and a parked cue, and nothing else', async () => {
  const id = seedRun({
    status: 'running',
    steps: [{ status: 'pending' }, { status: 'running' }, { status: 'running', leased: true }, { status: 'failed' }],
  })
  const [pending, parked, leased, failed] = stepsOf(id)

  assert.equal((await controls.skipStep(id, pending.id, {}, ACTOR)).ok, true)
  assert.equal((await controls.skipStep(id, parked.id, {}, ACTOR)).ok, true)
  assert.equal((await controls.skipStep(id, leased.id, {}, ACTOR)).ok, false)
  // A failed step does not need skipping: `nextOpenStep` already passes over it,
  // so resuming the run carries the phase past it.
  assert.equal((await controls.skipStep(id, failed.id, {}, ACTOR)).ok, false)

  const after = stepsOf(id)
  assert.equal(after[0].status, 'skipped')
  assert.equal(after[1].status, 'skipped')
  assert.equal(after[2].status, 'running')
  assert.equal(after[3].status, 'failed')
})

test('skip refuses once the run is over', async () => {
  const id = seedRun({ status: 'completed', steps: [{ status: 'pending' }] })
  const [step] = stepsOf(id)
  assert.equal((await controls.skipStep(id, step.id, {}, ACTOR)).ok, false)
})

// ── retry ──────────────────────────────────────────────────────────────────

test('retry re-queues the step a paused run is stopped at, and resumes in the same action', async () => {
  const id = seedRun({
    status: 'paused',
    steps: [{ status: 'done' }, { status: 'failed', attempts: 3 }, { status: 'pending' }],
  })
  const failed = stepsOf(id)[1]

  const result = await controls.retryStep(id, failed.id, {}, ACTOR)

  assert.equal(result.ok, true)
  assert.equal(result.resumed, true)
  assert.equal(stepsOf(id)[1].status, 'pending')
  assert.equal(stepsOf(id)[1].attempts, 0, 'the ceiling bounds the runner, not a person deciding once')
  assert.equal(runRow(id).status, 'running', 'there is no state in which you would want half of this')
})

test('retry refuses a step the run has already moved past', async () => {
  // The case the guard exists for: a failed step under an `on_failure` of `skip`
  // is one the phase carried on from. Re-queueing it would put a pending row
  // behind the runner's cursor, where it would sit for ever.
  const id = seedRun({
    status: 'paused',
    steps: [{ status: 'failed', attempts: 3 }, { status: 'done' }, { status: 'failed', attempts: 3 }],
  })
  const [movedPast] = stepsOf(id)

  const result = await controls.retryStep(id, movedPast.id, {}, ACTOR)

  assert.equal(result.ok, false)
  assert.equal(result.status, 409)
  assert.match(result.errors[0], /stopped at this step/)
  assert.equal(stepsOf(id)[0].status, 'failed')
  assert.equal(runRow(id).status, 'paused', 'a refused retry does not resume the run either')
})

test('retry refuses a step in a phase the run has left', async () => {
  const id = seedRun({
    status: 'paused',
    phase: 'two',
    steps: [{ phase: 'one', seq: 0, status: 'failed' }, { phase: 'two', seq: 0, status: 'pending' }],
  })
  const [old] = stepsOf(id)

  const result = await controls.retryStep(id, old.id, {}, ACTOR)
  assert.equal(result.ok, false)
  assert.match(result.errors[0], /already left/)
})

test('retry refuses while the run is still running', async () => {
  const id = seedRun({ status: 'running', steps: [{ status: 'failed' }] })
  const [failed] = stepsOf(id)

  const result = await controls.retryStep(id, failed.id, {}, ACTOR)
  assert.equal(result.ok, false)
  assert.match(result.errors[0], /paused/)
})

test('retry refuses a step that is not failed', async () => {
  const id = seedRun({ status: 'paused', steps: [{ status: 'pending' }] })
  const [pending] = stepsOf(id)
  assert.equal((await controls.retryStep(id, pending.id, {}, ACTOR)).ok, false)
})

// ── the record ─────────────────────────────────────────────────────────────

test('every control writes one log line carrying the actor and the control name', async () => {
  const id = seedRun({ status: 'running', steps: [{ status: 'running' }, { status: 'pending' }] })
  const [parked, pending] = stepsOf(id)

  await controls.confirmStep(id, parked.id, { note: 'done' }, ACTOR)
  await controls.skipStep(id, pending.id, { reason: 'not needed' }, ACTOR)
  await controls.pause(id, {}, ACTOR)
  await controls.resume(id, {}, ACTOR)
  await controls.cancel(id, { reason: 'over' }, ACTOR)

  const human = store.log.filter((l) => l.detail?.control)
  assert.deepEqual(human.map((l) => l.detail.control), ['confirm', 'skip', 'pause', 'resume', 'cancel'])
  assert.ok(human.every((l) => l.detail.by === ACTOR))
  // The kinds are the ones a reader already scans for. A human transition is
  // still a transition; `detail.control` is what separates it from the runner's.
  assert.deepEqual([...new Set(human.map((l) => l.kind))].sort(), ['run.status', 'step.status'])
})

test('an empty reason is stored as NULL rather than as an empty string', async () => {
  const id = seedRun({ status: 'running' })
  await controls.pause(id, { reason: '   ' }, ACTOR)
  assert.equal(lastLog().detail.reason, null)
})