website/server/src/model/events/eventRunControls.model.js

// ── The live run controls ──────────────────────────────────────────────────
//
// EVENTS.md §I ("live controls that are honest"), §K and §L. Six of them: pause,
// resume and cancel act on a run; confirm, skip and retry act on one step. They
// arrive in Phase 3 because Phase 2 is what gave them something to act on — a
// run that announces, waits and completes on its own is exactly the run that
// needs no control, and a run that paused on a failed world write is the one
// that does.
//
// **`advance` is the seventh, and it arrived in Phase 5 rather than Phase 3
// because that is when it started meaning something.** A phase used to advance
// when its steps went terminal and on nothing else, so "force it anyway" named
// no state an operator could be in; a phase with a gate can wait for a boss that
// will never spawn, and then it names exactly one. It is the other half of the
// diagnosis panel: a screen that explains why a phase has not started, beside a
// control that does something about it.
//
// **`cleanup` is the eighth, and Phase 8 is what gave it a ledger to work over.**
// It re-runs the teardown across every resource a run has not given back, and it
// is `admin` where the other seven are `admin` + `moderator`: it is not incident
// response, it is asking core to write to the world again. Its partner is
// cancel's new `cleanup: false`, which is §L's "cancelling WITHOUT cleanup is a
// separate, logged, admin-only action" — deliberately the flag that has to be
// asked for, because the safe default is to give back what the run took.
//
// **Every control is guarded on the status it may act from, and the guard is a
// WHERE clause rather than a read-then-write.** A run console rendered thirty
// seconds ago describes a run that has since moved — the runner ticks every
// fifteen — so a control that checked in JavaScript and then wrote would race
// the tick it exists to interrupt. `transition()` and the four step statements
// are all compare-and-set, and a `false` from one of them is reported as a 409
// naming the status the run is actually in.
//
// **Who may press them is `admin` + `moderator` (§K, §N2), and it is the widest
// gate in this feature on purpose.** Starting a run commits the deployment to
// everything the definition contains, unattended — that wants the narrowest gate
// there is. Stopping one is incident response at 2am, and it wants the widest.

const runsDb = require('./eventRuns.db')
const stepsDb = require('./eventRunSteps.db')
const logDb = require('./eventRunLog.db')
const announce = require('../../events/announce')
const gatesDb = require('./eventPhaseGates.db')
const resourcesDb = require('./eventRunResources.db')
const gates = require('../../events/gates')

const MAX_REASON = 500

const clean = (raw) => {
  const text = typeof raw === 'string' ? raw.trim() : ''
  return text ? text.slice(0, MAX_REASON) : null
}

const conflict = (message) => ({ ok: false, status: 409, errors: [message] })

/** The run, or a 404 shaped the way every other model here shapes one. */
async function loadRun(runId) {
  const run = await runsDb.getById(runId)
  return run || null
}

/**
 * A step of THIS run, or null.
 *
 * Scoped to the run rather than fetched by id alone: the step id arrives from a
 * URL under a run id, and a control that acted on a step belonging to a
 * different run would be a real one — the console's step ids are not secret and
 * the two paths would otherwise never be compared.
 */
async function loadStep(runId, stepId) {
  const step = await stepsDb.getById(stepId)
  if (!step || Number(step.run_id) !== Number(runId)) return null
  return step
}

// ── Run-level ─────────────────────────────────────────────────────────────

/**
 * Pause a run in flight.
 *
 * `starting` and `running` only — §K's "live control of a run **in flight**". A
 * `scheduled` run has not begun, and the thing to do with an occurrence that
 * should not happen is cancel it: pausing one would leave a run that is neither
 * going to start nor visibly abandoned, and resuming it after its grace window
 * had passed would produce a `missed` from a button labelled resume.
 *
 * The claim is cleared with the transition. A tick may be working the run at
 * this exact moment; it will find its guarded writes returning zero rows and
 * hand back a lease it no longer holds, both of which are no-ops. What it will
 * NOT do is dispatch the rest of its batch — `advanceRun` re-reads the status
 * between steps precisely so this control means what it says.
 */
async function pause(runId, { reason } = {}, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (run.status === 'paused') return conflict('this run is already paused')

  const note = clean(reason)
  if (!(await runsDb.transition(run.id, ['starting', 'running'], 'paused', { clearClaim: true }))) {
    return conflict(`a ${run.status} run cannot be paused`)
  }

  await logDb.write({
    runId: run.id,
    kind: 'run.status',
    phase: run.current_phase,
    detail: { from: run.status, to: 'paused', control: 'pause', by: userId, reason: note },
  })
  return { ok: true, run: await runsDb.getById(run.id) }
}

/**
 * Resume a paused run.
 *
 * Where it goes back to is derived rather than remembered: `current_phase` is
 * set by the transition into `running` and by nothing else, so a paused run that
 * has one was running and a paused run that has none never got past `starting`.
 * Both statuses are in `findDue`, so the next tick picks the run up either way,
 * and there is no fourth column recording what a run was paused *from* — a
 * column that could disagree with the run's own history.
 *
 * **`last_error` is cleared and `health` is not.** The error is what the pause
 * was about and an operator has just dealt with it; leaving it on the banner
 * would have a healthy run permanently accused of a failure that is in the log
 * where it belongs. Health is a different claim — that this run has already had
 * trouble — and it stays true no matter who pressed resume.
 */
async function resume(runId, options = {}, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (run.status !== 'paused') return conflict(`a ${run.status} run is not paused`)

  const to = run.current_phase ? 'running' : 'starting'
  if (!(await runsDb.transition(run.id, 'paused', to, { error: null }))) {
    return conflict('this run stopped being paused')
  }

  await logDb.write({
    runId: run.id,
    kind: 'run.status',
    phase: run.current_phase,
    detail: { from: 'paused', to, control: 'resume', by: userId },
  })
  return { ok: true, run: await runsDb.getById(run.id) }
}

/**
 * Cancel a run.
 *
 * Legal from every non-terminal status including `scheduled`, because "this
 * event is not happening" is a decision an operator makes before it starts as
 * often as during it.
 *
 * `cancelOpen` then closes out the steps that will never run — the pending ones
 * and any parked cue. A step with a LIVE lease is left exactly where it is:
 * something is dispatching it, nothing can recall a command already sent (§L),
 * and a second writer on that row would race the process that owns it. It
 * finishes into a cancelled run, which is honest.
 */
async function cancel(runId, { reason, cleanup = true } = {}, userId = null, { isAdmin = true } = {}) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is already ${run.status}`)

  // §L: cancelling WITHOUT cleanup is a separate, logged, ADMIN-only action. The
  // route itself is `admin` + `moderator`, so the narrower gate cannot live in
  // middleware — which of the two you have to be depends on what is in the body,
  // exactly as the authoring role floor does (§K).
  if (cleanup === false && !isAdmin) {
    return {
      ok: false,
      status: 403,
      errors: ['leaving a run\'s world changes in place is an administrator\'s decision'],
    }
  }

  const note = clean(reason)
  const from = ['scheduled', 'starting', 'running', 'paused', 'ending']
  if (!(await runsDb.transition(run.id, from, 'cancelled', { error: note || 'cancelled by staff' }))) {
    return conflict('this run is no longer cancellable')
  }

  const closed = await stepsDb.cancelOpen(run.id)
  await logDb.write({
    runId: run.id,
    kind: 'run.status',
    phase: run.current_phase,
    detail: {
      from: run.status,
      to: 'cancelled',
      control: 'cancel',
      by: userId,
      reason: note,
      cancelledSteps: closed,
      cleanup: cleanup !== false,
    },
  })

  // **After the guarded transition, so exactly one caller announces** (Phase
  // 10). Two moderators pressing cancel in the same second both reach the log
  // write; only one of them wins `transition`, and the loser has already
  // returned a 409 above.
  //
  // The operator's `reason`, not the run's `last_error` — `cancel` takes a
  // sentence a human typed for other humans, and the diagnostic string that
  // ends up in `last_error` would read as gibberish in a mail.
  await announce.runCancelled(run, note)

  // **The teardown is not done here, and the request does not wait for it.**
  // Cleanup is one leg of the runner's tick over terminal runs (§L), which is
  // what makes it survive a process that dies halfway through it — and a cancel
  // pressed at two in the morning must answer at once rather than after a dozen
  // round trips to a shard that may be the reason it is being cancelled. The run
  // is terminal the moment this returns, so the very next tick picks its ledger
  // up.
  //
  // `cleanup: false` is the operator saying leave it. The resources stay
  // unresolved and the run carries `incomplete`, which is the truthful value: the
  // world changes are still up, they are listed on the console, and the log line
  // above records who decided that.
  let cleanupStatus = run.cleanup_status
  if (cleanup === false && (await resourcesDb.unresolvedCount(run.id)) > 0) {
    // `incomplete` is what takes the run out of the cleanup leg's scan, and it is
    // the truthful value: the world changes are still up, they are listed on the
    // console, and the log line above records who decided that.
    //
    // **The first draft spent every row's `revert_attempts` instead**, to stop the
    // sweep by the same mechanism a failed retry does. It worked and it made the
    // console lie: the run page rendered "3 attempts" beside resources nothing had
    // ever tried, which reads as "core tried three times and could not". Found by
    // opening the page. A counter that means two things is a counter a screen
    // cannot render.
    await runsDb.setCleanupStatus(run.id, 'incomplete')
    cleanupStatus = 'incomplete'
  }

  return {
    ok: true,
    run: await runsDb.getById(run.id),
    cancelledSteps: closed,
    cleanup: cleanup !== false,
    cleanupStatus,
  }
}

/**
 * Re-run cleanup over everything a run has not given back.
 *
 * The manual retry §L promises, and the only thing that clears
 * `revert_attempts`. That licence is the same one a human's step retry has, and
 * it is deliberately not extended to the automatic sweep: Engagement Phase 14's
 * defect was exactly a sweep that reset every stale row, which made the attempt
 * ceiling unreachable and left the row cycling for ever.
 *
 * Legal on a TERMINAL run only. A run still in flight has a ledger that is still
 * growing, and reverting a resource the next step is about to use would be core
 * undoing an event while it is happening.
 */
async function cleanupRun(runId, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (!runsDb.TERMINAL.includes(run.status)) {
    return conflict(`this run is still ${run.status}; cancel it before cleaning up after it`)
  }
  if (run.cleanup_status === 'not_required') {
    return conflict('this run recorded no resources, so there is nothing to give back')
  }

  // eslint-disable-next-line global-require
  const summary = await require('../../events/cleanup').cleanupRun(run, {
    resetAttempts: true,
    actor: userId,
  })
  return { ok: true, run: await runsDb.getById(run.id), summary }
}

/**
 * Force a phase forward: open its gate without the condition that would have.
 *
 * **It is legal only when the phase is actually waiting on a gate**, and the
 * three refusals are the whole design. A run that is not `running` is not
 * waiting on anything (409 naming what it is). A phase with no gate advances on
 * its steps and always has, so forcing it would be a control that duplicated the
 * runner rather than overriding it. And a phase whose steps have not all gone
 * terminal is not being held by its gate — it is being held by a step, and the
 * step-level skip is the honest control for that, one step at a time. A force
 * that swept past pending steps would be a cancel of half a phase under a button
 * labelled advance.
 *
 * **It satisfies the gate and stops.** The next tick advances the run, exactly
 * as it does after `resume` — the phase transition, the next phase's
 * materialisation, its own gate and the log lines are one sequence in
 * `advanceRun`, and a second copy of it here would be a second opinion about
 * what a phase boundary is. The response says which phase was released, so the
 * console can say so before the tick lands.
 */
async function advancePhase(runId, { reason } = {}, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (run.status !== 'running') return conflict(`a ${run.status} run has no phase to advance`)
  if (!run.current_phase) return conflict('this run has not entered a phase yet')

  const gate = await gatesDb.forPhase(run.id, run.current_phase)
  if (!gate) return conflict(`phase "${run.current_phase}" has no advance condition; skip its steps instead`)
  if (gate.satisfied_at) return conflict(`phase "${run.current_phase}" is already past its advance condition`)

  const openStep = await stepsDb.nextOpenStep(run.id, run.current_phase)
  if (openStep) {
    return conflict(
      `phase "${run.current_phase}" is waiting on step ${openStep.seq} (${openStep.action_id}), not on its advance condition`,
    )
  }

  const note = clean(reason)
  if (!(await gatesDb.satisfy(gate.id, 'forced', { userId }))) {
    return conflict('this phase stopped waiting on its advance condition')
  }

  const described = gates.describe(gate)
  await logDb.write({
    runId: run.id,
    kind: 'phase.advanced',
    phase: run.current_phase,
    detail: {
      because: 'forced',
      control: 'advance',
      by: userId,
      reason: note,
      waitedSeconds: described.elapsedSeconds,
      ...(gate.kind === 'on'
        ? { trigger: gate.trigger_id, seen: gate.tally, needed: gate.needed }
        : { after: gate.after_seconds }),
    },
  })
  return { ok: true, run: await runsDb.getById(run.id), phase: run.current_phase }
}

// ── Step-level ────────────────────────────────────────────────────────────

/**
 * Confirm a parked step — the GM cue's other half.
 *
 * `core.cue` posts an instruction and parks: the step stays `running` with a
 * NULL lease, genuinely in flight with nothing holding it, so no sweep takes it
 * back and a cue posted on Friday is still waiting on Monday. This is what ends
 * it, and it is the control that makes the whole system useful before any module
 * automates anything — a GM does the target-driven part in-client and says so
 * here.
 *
 * The outcome is `done`, not `skipped`: a person saying they did the thing is
 * the step having succeeded. The note is what they did, and it is kept.
 */
async function confirmStep(runId, stepId, { note } = {}, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  const step = await loadStep(runId, stepId)
  if (!step) return { ok: false, status: 404, errors: ['no such step on this run'] }

  const text = clean(note)
  if (!(await stepsDb.confirmParked(step.id, text))) {
    return conflict(`this step is ${step.status} and is not waiting on anyone`)
  }

  await logDb.write({
    runId: run.id,
    stepId: step.id,
    kind: 'step.status',
    phase: step.phase,
    detail: { to: 'done', action: step.action_id, control: 'confirm', by: userId, note: text },
  })
  return { ok: true, step: await stepsDb.getById(step.id) }
}

/**
 * Skip a step: one that has not started, or a parked cue nobody is going to do.
 *
 * This is what the `skipped` status was reserved for (§L) — which is also why
 * the three `on_failure` dispositions all write `failed` instead. A status
 * meaning both "a human decided against this" and "this was attempted three
 * times and never worked" would make the console's summary line unreadable.
 *
 * A `failed` step is not skippable and does not need to be: `nextOpenStep`
 * already passes over one, so resuming a run carries the phase past it.
 */
async function skipStep(runId, stepId, { reason } = {}, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is ${run.status}`)
  const step = await loadStep(runId, stepId)
  if (!step) return { ok: false, status: 404, errors: ['no such step on this run'] }

  const note = clean(reason)
  if (!(await stepsDb.skipByHuman(step.id, note))) {
    return conflict(`a ${step.status} step cannot be skipped`)
  }

  await logDb.write({
    runId: run.id,
    stepId: step.id,
    kind: 'step.status',
    phase: step.phase,
    detail: { to: 'skipped', action: step.action_id, control: 'skip', by: userId, reason: note },
  })
  return { ok: true, step: await stepsDb.getById(step.id) }
}

/**
 * Re-queue the failed step a run is stopped at, and resume the run — one action.
 *
 * **The two halves are one control because there is no state in which you would
 * want half of it.** Retry is legal only from `paused`, and a paused run is
 * paused *at* this step; re-queueing without resuming would leave the run in
 * precisely the state it was already in, with a button the operator now has to
 * find. Splitting them would read as honesty and behave as a trap.
 *
 * Two guards, and the second is the one worth explaining. The step must be the
 * furthest one its phase has reached — `lastStartedSeq` — because a `failed`
 * step under an `on_failure` of `skip` is one the run has already moved PAST.
 * `nextOpenStep` selects `pending` and `running` only, so the runner steps over
 * a failed row; re-queueing an earlier one puts a `pending` step behind the
 * cursor, where it sits for ever.
 */
async function retryStep(runId, stepId, options = {}, userId = null) {
  const run = await loadRun(runId)
  if (!run) return { ok: false, status: 404, errors: ['no such run'] }
  if (run.status !== 'paused') {
    return conflict(`a step can only be retried while its run is paused; this run is ${run.status}`)
  }
  const step = await loadStep(runId, stepId)
  if (!step) return { ok: false, status: 404, errors: ['no such step on this run'] }
  if (step.status !== 'failed') return conflict(`a ${step.status} step cannot be retried`)
  if (step.phase !== run.current_phase) {
    return conflict('this step belongs to a phase the run has already left')
  }

  const furthest = await stepsDb.lastStartedSeq(run.id, step.phase)
  if (furthest === null || Number(furthest) !== Number(step.seq)) {
    return conflict('the run is not stopped at this step; only the step a phase is stopped at can be retried')
  }

  if (!(await stepsDb.requeue(step.id))) return conflict('this step is no longer failed')

  await logDb.write({
    runId: run.id,
    stepId: step.id,
    kind: 'step.status',
    phase: step.phase,
    detail: { to: 'pending', action: step.action_id, control: 'retry', by: userId, attemptsReset: step.attempts },
  })

  const resumed = await resume(runId, {}, userId)
  return {
    ok: true,
    step: await stepsDb.getById(step.id),
    // A resume that did not take is reported rather than swallowed: the step IS
    // re-queued either way, and an operator told "retried" about a run that is
    // still paused would be told something false.
    resumed: Boolean(resumed.ok),
    run: resumed.run || (await runsDb.getById(run.id)),
  }
}

module.exports = { pause, resume, cancel, cleanupRun, advancePhase, confirmStep, skipStep, retryStep }