// ── The live run controls ────────────────────────────────────────────────── // // EVENTS.md §I ("live controls that are honest"), §K and §L. Six of them: pause, // resume and cancel act on a run; confirm, skip and retry act on one step. They // arrive in Phase 3 because Phase 2 is what gave them something to act on — a // run that announces, waits and completes on its own is exactly the run that // needs no control, and a run that paused on a failed world write is the one // that does. // // **`advance` is the seventh, and it arrived in Phase 5 rather than Phase 3 // because that is when it started meaning something.** A phase used to advance // when its steps went terminal and on nothing else, so "force it anyway" named // no state an operator could be in; a phase with a gate can wait for a boss that // will never spawn, and then it names exactly one. It is the other half of the // diagnosis panel: a screen that explains why a phase has not started, beside a // control that does something about it. // // **`cleanup` is the eighth, and Phase 8 is what gave it a ledger to work over.** // It re-runs the teardown across every resource a run has not given back, and it // is `admin` where the other seven are `admin` + `moderator`: it is not incident // response, it is asking core to write to the world again. Its partner is // cancel's new `cleanup: false`, which is §L's "cancelling WITHOUT cleanup is a // separate, logged, admin-only action" — deliberately the flag that has to be // asked for, because the safe default is to give back what the run took. // // **Every control is guarded on the status it may act from, and the guard is a // WHERE clause rather than a read-then-write.** A run console rendered thirty // seconds ago describes a run that has since moved — the runner ticks every // fifteen — so a control that checked in JavaScript and then wrote would race // the tick it exists to interrupt. `transition()` and the four step statements // are all compare-and-set, and a `false` from one of them is reported as a 409 // naming the status the run is actually in. // // **Who may press them is `admin` + `moderator` (§K, §N2), and it is the widest // gate in this feature on purpose.** Starting a run commits the deployment to // everything the definition contains, unattended — that wants the narrowest gate // there is. Stopping one is incident response at 2am, and it wants the widest. const runsDb = require('./eventRuns.db') const stepsDb = require('./eventRunSteps.db') const logDb = require('./eventRunLog.db') const announce = require('../../events/announce') const gatesDb = require('./eventPhaseGates.db') const resourcesDb = require('./eventRunResources.db') const gates = require('../../events/gates') const MAX_REASON = 500 const clean = (raw) => { const text = typeof raw === 'string' ? raw.trim() : '' return text ? text.slice(0, MAX_REASON) : null } const conflict = (message) => ({ ok: false, status: 409, errors: [message] }) /** The run, or a 404 shaped the way every other model here shapes one. */ async function loadRun(runId) { const run = await runsDb.getById(runId) return run || null } /** * A step of THIS run, or null. * * Scoped to the run rather than fetched by id alone: the step id arrives from a * URL under a run id, and a control that acted on a step belonging to a * different run would be a real one — the console's step ids are not secret and * the two paths would otherwise never be compared. */ async function loadStep(runId, stepId) { const step = await stepsDb.getById(stepId) if (!step || Number(step.run_id) !== Number(runId)) return null return step } // ── Run-level ───────────────────────────────────────────────────────────── /** * Pause a run in flight. * * `starting` and `running` only — §K's "live control of a run **in flight**". A * `scheduled` run has not begun, and the thing to do with an occurrence that * should not happen is cancel it: pausing one would leave a run that is neither * going to start nor visibly abandoned, and resuming it after its grace window * had passed would produce a `missed` from a button labelled resume. * * The claim is cleared with the transition. A tick may be working the run at * this exact moment; it will find its guarded writes returning zero rows and * hand back a lease it no longer holds, both of which are no-ops. What it will * NOT do is dispatch the rest of its batch — `advanceRun` re-reads the status * between steps precisely so this control means what it says. */ async function pause(runId, { reason } = {}, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (run.status === 'paused') return conflict('this run is already paused') const note = clean(reason) if (!(await runsDb.transition(run.id, ['starting', 'running'], 'paused', { clearClaim: true }))) { return conflict(`a ${run.status} run cannot be paused`) } await logDb.write({ runId: run.id, kind: 'run.status', phase: run.current_phase, detail: { from: run.status, to: 'paused', control: 'pause', by: userId, reason: note }, }) return { ok: true, run: await runsDb.getById(run.id) } } /** * Resume a paused run. * * Where it goes back to is derived rather than remembered: `current_phase` is * set by the transition into `running` and by nothing else, so a paused run that * has one was running and a paused run that has none never got past `starting`. * Both statuses are in `findDue`, so the next tick picks the run up either way, * and there is no fourth column recording what a run was paused *from* — a * column that could disagree with the run's own history. * * **`last_error` is cleared and `health` is not.** The error is what the pause * was about and an operator has just dealt with it; leaving it on the banner * would have a healthy run permanently accused of a failure that is in the log * where it belongs. Health is a different claim — that this run has already had * trouble — and it stays true no matter who pressed resume. */ async function resume(runId, options = {}, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (run.status !== 'paused') return conflict(`a ${run.status} run is not paused`) const to = run.current_phase ? 'running' : 'starting' if (!(await runsDb.transition(run.id, 'paused', to, { error: null }))) { return conflict('this run stopped being paused') } await logDb.write({ runId: run.id, kind: 'run.status', phase: run.current_phase, detail: { from: 'paused', to, control: 'resume', by: userId }, }) return { ok: true, run: await runsDb.getById(run.id) } } /** * Cancel a run. * * Legal from every non-terminal status including `scheduled`, because "this * event is not happening" is a decision an operator makes before it starts as * often as during it. * * `cancelOpen` then closes out the steps that will never run — the pending ones * and any parked cue. A step with a LIVE lease is left exactly where it is: * something is dispatching it, nothing can recall a command already sent (§L), * and a second writer on that row would race the process that owns it. It * finishes into a cancelled run, which is honest. */ async function cancel(runId, { reason, cleanup = true } = {}, userId = null, { isAdmin = true } = {}) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is already ${run.status}`) // §L: cancelling WITHOUT cleanup is a separate, logged, ADMIN-only action. The // route itself is `admin` + `moderator`, so the narrower gate cannot live in // middleware — which of the two you have to be depends on what is in the body, // exactly as the authoring role floor does (§K). if (cleanup === false && !isAdmin) { return { ok: false, status: 403, errors: ['leaving a run\'s world changes in place is an administrator\'s decision'], } } const note = clean(reason) const from = ['scheduled', 'starting', 'running', 'paused', 'ending'] if (!(await runsDb.transition(run.id, from, 'cancelled', { error: note || 'cancelled by staff' }))) { return conflict('this run is no longer cancellable') } const closed = await stepsDb.cancelOpen(run.id) await logDb.write({ runId: run.id, kind: 'run.status', phase: run.current_phase, detail: { from: run.status, to: 'cancelled', control: 'cancel', by: userId, reason: note, cancelledSteps: closed, cleanup: cleanup !== false, }, }) // **After the guarded transition, so exactly one caller announces** (Phase // 10). Two moderators pressing cancel in the same second both reach the log // write; only one of them wins `transition`, and the loser has already // returned a 409 above. // // The operator's `reason`, not the run's `last_error` — `cancel` takes a // sentence a human typed for other humans, and the diagnostic string that // ends up in `last_error` would read as gibberish in a mail. await announce.runCancelled(run, note) // **The teardown is not done here, and the request does not wait for it.** // Cleanup is one leg of the runner's tick over terminal runs (§L), which is // what makes it survive a process that dies halfway through it — and a cancel // pressed at two in the morning must answer at once rather than after a dozen // round trips to a shard that may be the reason it is being cancelled. The run // is terminal the moment this returns, so the very next tick picks its ledger // up. // // `cleanup: false` is the operator saying leave it. The resources stay // unresolved and the run carries `incomplete`, which is the truthful value: the // world changes are still up, they are listed on the console, and the log line // above records who decided that. let cleanupStatus = run.cleanup_status if (cleanup === false && (await resourcesDb.unresolvedCount(run.id)) > 0) { // `incomplete` is what takes the run out of the cleanup leg's scan, and it is // the truthful value: the world changes are still up, they are listed on the // console, and the log line above records who decided that. // // **The first draft spent every row's `revert_attempts` instead**, to stop the // sweep by the same mechanism a failed retry does. It worked and it made the // console lie: the run page rendered "3 attempts" beside resources nothing had // ever tried, which reads as "core tried three times and could not". Found by // opening the page. A counter that means two things is a counter a screen // cannot render. await runsDb.setCleanupStatus(run.id, 'incomplete') cleanupStatus = 'incomplete' } return { ok: true, run: await runsDb.getById(run.id), cancelledSteps: closed, cleanup: cleanup !== false, cleanupStatus, } } /** * Re-run cleanup over everything a run has not given back. * * The manual retry §L promises, and the only thing that clears * `revert_attempts`. That licence is the same one a human's step retry has, and * it is deliberately not extended to the automatic sweep: Engagement Phase 14's * defect was exactly a sweep that reset every stale row, which made the attempt * ceiling unreachable and left the row cycling for ever. * * Legal on a TERMINAL run only. A run still in flight has a ledger that is still * growing, and reverting a resource the next step is about to use would be core * undoing an event while it is happening. */ async function cleanupRun(runId, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (!runsDb.TERMINAL.includes(run.status)) { return conflict(`this run is still ${run.status}; cancel it before cleaning up after it`) } if (run.cleanup_status === 'not_required') { return conflict('this run recorded no resources, so there is nothing to give back') } // eslint-disable-next-line global-require const summary = await require('../../events/cleanup').cleanupRun(run, { resetAttempts: true, actor: userId, }) return { ok: true, run: await runsDb.getById(run.id), summary } } /** * Force a phase forward: open its gate without the condition that would have. * * **It is legal only when the phase is actually waiting on a gate**, and the * three refusals are the whole design. A run that is not `running` is not * waiting on anything (409 naming what it is). A phase with no gate advances on * its steps and always has, so forcing it would be a control that duplicated the * runner rather than overriding it. And a phase whose steps have not all gone * terminal is not being held by its gate — it is being held by a step, and the * step-level skip is the honest control for that, one step at a time. A force * that swept past pending steps would be a cancel of half a phase under a button * labelled advance. * * **It satisfies the gate and stops.** The next tick advances the run, exactly * as it does after `resume` — the phase transition, the next phase's * materialisation, its own gate and the log lines are one sequence in * `advanceRun`, and a second copy of it here would be a second opinion about * what a phase boundary is. The response says which phase was released, so the * console can say so before the tick lands. */ async function advancePhase(runId, { reason } = {}, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (run.status !== 'running') return conflict(`a ${run.status} run has no phase to advance`) if (!run.current_phase) return conflict('this run has not entered a phase yet') const gate = await gatesDb.forPhase(run.id, run.current_phase) if (!gate) return conflict(`phase "${run.current_phase}" has no advance condition; skip its steps instead`) if (gate.satisfied_at) return conflict(`phase "${run.current_phase}" is already past its advance condition`) const openStep = await stepsDb.nextOpenStep(run.id, run.current_phase) if (openStep) { return conflict( `phase "${run.current_phase}" is waiting on step ${openStep.seq} (${openStep.action_id}), not on its advance condition`, ) } const note = clean(reason) if (!(await gatesDb.satisfy(gate.id, 'forced', { userId }))) { return conflict('this phase stopped waiting on its advance condition') } const described = gates.describe(gate) await logDb.write({ runId: run.id, kind: 'phase.advanced', phase: run.current_phase, detail: { because: 'forced', control: 'advance', by: userId, reason: note, waitedSeconds: described.elapsedSeconds, ...(gate.kind === 'on' ? { trigger: gate.trigger_id, seen: gate.tally, needed: gate.needed } : { after: gate.after_seconds }), }, }) return { ok: true, run: await runsDb.getById(run.id), phase: run.current_phase } } // ── Step-level ──────────────────────────────────────────────────────────── /** * Confirm a parked step — the GM cue's other half. * * `core.cue` posts an instruction and parks: the step stays `running` with a * NULL lease, genuinely in flight with nothing holding it, so no sweep takes it * back and a cue posted on Friday is still waiting on Monday. This is what ends * it, and it is the control that makes the whole system useful before any module * automates anything — a GM does the target-driven part in-client and says so * here. * * The outcome is `done`, not `skipped`: a person saying they did the thing is * the step having succeeded. The note is what they did, and it is kept. */ async function confirmStep(runId, stepId, { note } = {}, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } const step = await loadStep(runId, stepId) if (!step) return { ok: false, status: 404, errors: ['no such step on this run'] } const text = clean(note) if (!(await stepsDb.confirmParked(step.id, text))) { return conflict(`this step is ${step.status} and is not waiting on anyone`) } await logDb.write({ runId: run.id, stepId: step.id, kind: 'step.status', phase: step.phase, detail: { to: 'done', action: step.action_id, control: 'confirm', by: userId, note: text }, }) return { ok: true, step: await stepsDb.getById(step.id) } } /** * Skip a step: one that has not started, or a parked cue nobody is going to do. * * This is what the `skipped` status was reserved for (§L) — which is also why * the three `on_failure` dispositions all write `failed` instead. A status * meaning both "a human decided against this" and "this was attempted three * times and never worked" would make the console's summary line unreadable. * * A `failed` step is not skippable and does not need to be: `nextOpenStep` * already passes over one, so resuming a run carries the phase past it. */ async function skipStep(runId, stepId, { reason } = {}, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is ${run.status}`) const step = await loadStep(runId, stepId) if (!step) return { ok: false, status: 404, errors: ['no such step on this run'] } const note = clean(reason) if (!(await stepsDb.skipByHuman(step.id, note))) { return conflict(`a ${step.status} step cannot be skipped`) } await logDb.write({ runId: run.id, stepId: step.id, kind: 'step.status', phase: step.phase, detail: { to: 'skipped', action: step.action_id, control: 'skip', by: userId, reason: note }, }) return { ok: true, step: await stepsDb.getById(step.id) } } /** * Re-queue the failed step a run is stopped at, and resume the run — one action. * * **The two halves are one control because there is no state in which you would * want half of it.** Retry is legal only from `paused`, and a paused run is * paused *at* this step; re-queueing without resuming would leave the run in * precisely the state it was already in, with a button the operator now has to * find. Splitting them would read as honesty and behave as a trap. * * Two guards, and the second is the one worth explaining. The step must be the * furthest one its phase has reached — `lastStartedSeq` — because a `failed` * step under an `on_failure` of `skip` is one the run has already moved PAST. * `nextOpenStep` selects `pending` and `running` only, so the runner steps over * a failed row; re-queueing an earlier one puts a `pending` step behind the * cursor, where it sits for ever. */ async function retryStep(runId, stepId, options = {}, userId = null) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (run.status !== 'paused') { return conflict(`a step can only be retried while its run is paused; this run is ${run.status}`) } const step = await loadStep(runId, stepId) if (!step) return { ok: false, status: 404, errors: ['no such step on this run'] } if (step.status !== 'failed') return conflict(`a ${step.status} step cannot be retried`) if (step.phase !== run.current_phase) { return conflict('this step belongs to a phase the run has already left') } const furthest = await stepsDb.lastStartedSeq(run.id, step.phase) if (furthest === null || Number(furthest) !== Number(step.seq)) { return conflict('the run is not stopped at this step; only the step a phase is stopped at can be retried') } if (!(await stepsDb.requeue(step.id))) return conflict('this step is no longer failed') await logDb.write({ runId: run.id, stepId: step.id, kind: 'step.status', phase: step.phase, detail: { to: 'pending', action: step.action_id, control: 'retry', by: userId, attemptsReset: step.attempts }, }) const resumed = await resume(runId, {}, userId) return { ok: true, step: await stepsDb.getById(step.id), // A resume that did not take is reported rather than swallowed: the step IS // re-queued either way, and an operator told "retried" about a run that is // still paused would be told something false. resumed: Boolean(resumed.ok), run: resumed.run || (await runsDb.getById(run.id)), } } module.exports = { pause, resume, cancel, cleanupRun, advancePhase, confirmStep, skipStep, retryStep }