RunicGateway/website
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Activity

feat(events): the resource ledger, leases and cleanup (Phase 8) #190

Merged
whitlocktech merged 1 commits from feature/events-phase-8 into edge 2026-09-04 00:12:21 -05:00
Conversation 1 Commits 1 Files Changed 29 +3928 -72
29 changed files with 3928 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
client/src/api/client.js
View File

@@ -538,8 +538,12 @@ export const api = {
pauseEventRun: (runId, reason) =>
req(`/admin/events/runs/${runId}/pause`, { method: 'POST', body: { reason } }),
resumeEventRun: (runId) => req(`/admin/events/runs/${runId}/resume`, { method: 'POST' }),
cancelEventRun: (runId, reason) =>
req(`/admin/events/runs/${runId}/cancel`, { method: 'POST', body: { reason } }),
// `cleanup` defaults to true server-side and has to be asked out of: EVENTS.md
// §L makes cancelling WITHOUT cleanup the separate, admin-only, logged action,
// so an absent flag means "give back what this run took".
cancelEventRun: (runId, reason, cleanup = true) =>
req(`/admin/events/runs/${runId}/cancel`, { method: 'POST', body: { reason, cleanup } }),
cleanupEventRun: (runId) => req(`/admin/events/runs/${runId}/cleanup`, { method: 'POST' }),
advanceEventRun: (runId, reason) =>
req(`/admin/events/runs/${runId}/advance`, { method: 'POST', body: { reason } }),
confirmEventStep: (runId, stepId, note) =>

122
client/src/routes/admin/views/EventRun.jsx
View File

@@ -38,6 +38,15 @@ import {
// SERVER'S. `gates[].where` arrives already rendered, because those labels are
// defined in the condition grammar and a second renderer in the browser would
// be a second opinion about what `gte` reads as.
//
// **Phase 8 gave it a third, and it is the one that outlives the event.** The
// resource ledger is what this run changed in the world and what became of it,
// and its unresolved rows are the reason a `completed` run can still need a
// person — EVENTS.md §L: a run reaches `completed` with `cleanup_status =
// 'incomplete'` rather than being held open, because a tidy `completed` row over
// a shard full of orphaned monsters is the failure that would end this feature's
// credibility on its first bad night. The panel is shown on finished runs for
// exactly that reason, and it is the only panel here whose empty state matters.
const POLL_MS = 5000
@@ -50,6 +59,29 @@ const STATUS_COLOR = {
completed: '#8fc79a',
}
// The six ledger statuses, in the two groups that matter to a reader: green is
// resolved, amber wants a person. `orphaned` and `drifted` are amber rather than
// red because neither is a fault — one thing vanished, the other was taken by
// somebody with every right to take it — and red is reserved for "this did not
// come back and core kept asking".
const RESOURCE_COLOR = {
reverted: '#8fc79a',
confirmed: '#d9c184',
pending: '#d9c184',
reverting: '#d9c184',
drifted: '#d9c184',
orphaned: '#d9c184',
}
const RESOURCE_WORD = {
pending: 'recorded, unconfirmed',
confirmed: 'still out there',
reverting: 'being given back',
reverted: 'given back',
orphaned: 'gone',
drifted: 'someone else moved it',
}
const STEP_COLOR = {
done: '#8fc79a',
failed: '#d98b84',
@@ -148,6 +180,9 @@ export default function EventRun() {
// into the run when it was created, so this is what THIS run is allowed rather
// than what the switchboard says today.
const [budget, setBudget] = useState([])
// What this run created or borrowed, and what became of each (Phase 8).
const [resources, setResources] = useState([])
const [unresolved, setUnresolved] = useState(0)
const [lines, setLines] = useState([])
const [loading, setLoading] = useState(true)
const [error, setError] = useState(null)
@@ -168,6 +203,8 @@ export default function EventRun() {
setCounts(detail.counts || {})
setGates(detail.gates || [])
setBudget(detail.budget || [])
setResources(detail.resources || [])
setUnresolved(detail.unresolvedResources || 0)
setLines(log.log || [])
}, [runId])
@@ -308,6 +345,18 @@ export default function EventRun() {
onClick={() => act(() => api.admin.cancelEventRun(run.id, reason))}>
Cancel run
</button>
{/* The separate, admin-only decision (§L). It is a second button rather
than a checkbox on the first because the two are not variants of one
action: one gives the world back, the other deliberately leaves it
changed. A checkbox next to Cancel is a thing an operator unticks by
accident at two in the morning. The server refuses this to a
moderator, and the refusal arrives as a sentence in `problem`. */}
{controls.cancel && (
<button type="button" className="pill" style={{ fontSize: '0.74rem' }} disabled={busy}
onClick={() => act(() => api.admin.cancelEventRun(run.id, reason, false))}>
Cancel, leave changes up
</button>
)}
</div>
{isTerminalRun(run.status) && (
@@ -400,6 +449,79 @@ export default function EventRun() {
</div>
)}
{/* ── What this run changed in the world (Phase 8) ──
The WHOLE ledger, reverted rows included: "how much did last night's
invasion actually spawn, and did all of it come back" is one question
with two halves, and a list of only the failures answers neither.
Shown on finished runs for the same reason the caps meter is. */}
{(resources.length > 0 || run.cleanupStatus === 'incomplete') && (
<div
className="panel-flat"
style={{
padding: 14,
marginBottom: 14,
borderLeft: `3px solid ${unresolved > 0 ? '#d9c184' : 'var(--rule)'}`,
}}
>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'baseline', gap: 12, flexWrap: 'wrap' }}>
<h3 className="sans" style={{ margin: '0 0 4px', fontSize: '0.92rem' }}>
What this run changed
</h3>
{/* The manual retry. Offered only on a terminal run, because a run
still in flight has a ledger that is still growing and reverting a
resource the next step is about to use would be undoing an event
while it is happening. */}
{isTerminalRun(run.status) && unresolved > 0 && (
<button type="button" className="pill" style={{ fontSize: '0.74rem' }} disabled={busy}
onClick={() => act(() => api.admin.cleanupEventRun(run.id))}>
Try cleanup again
</button>
)}
</div>
<p className="sans dim" style={{ margin: '0 0 10px', fontSize: '0.8rem' }}>
{unresolved > 0 ? (
<>
{unresolved} of these {unresolved === 1 ? 'is' : 'are'} still unresolved. The runner
gives them back on its own and stops asking after a few tries;{' '}
<em>Try cleanup again</em> clears that count and asks once more.
</>
) : (
'Everything this run created or borrowed has been given back.'
)}
</p>
{resources.length === 0 ? (
<p className="sans dim" style={{ margin: 0, fontSize: '0.8rem' }}>
Nothing named — a step changed the world and its answer never arrived, so core kept the
record it wrote beforehand and will ask the module to undo it by key.
</p>
) : (
<table className="sans" style={{ fontSize: '0.82rem', borderCollapse: 'collapse', width: '100%' }}>
<tbody>
{resources.map((r) => (
<tr key={r.id}>
<td style={{ padding: '3px 12px 3px 0', whiteSpace: 'nowrap' }}>
<code style={{ fontSize: '0.78rem' }}>{r.kind}</code>{' '}
<code className="dim" style={{ fontSize: '0.78rem' }}>{r.ref}</code>
</td>
<td style={{ padding: '3px 12px 3px 0', whiteSpace: 'nowrap', color: RESOURCE_COLOR[r.status] }}>
{RESOURCE_WORD[r.status] || r.status}
</td>
<td className="dim" style={{ padding: '3px 12px 3px 0', whiteSpace: 'nowrap', fontSize: '0.78rem' }}>
{r.module}
{r.leaseUntil ? ` · until ${clock(r.leaseUntil)}` : ''}
{r.revertAttempts > 0 ? ` · ${r.revertAttempts} attempt${r.revertAttempts === 1 ? '' : 's'}` : ''}
</td>
<td className="dim" style={{ width: '100%', padding: '3px 0', fontSize: '0.78rem' }}>
{r.lastError || ''}
</td>
</tr>
))}
</tbody>
</table>
)}
</div>
)}
{/* ── Waiting on a person ── */}
{parked.length > 0 && (
<div className="panel-flat" style={{ padding: 14, marginBottom: 14, borderLeft: '3px solid #d9c184' }}>

97
server/db/schema.sql
View File

@@ -2090,7 +2090,7 @@ CREATE TABLE IF NOT EXISTS engagement_suppressions (
-- module contract. The rest arrive with the phases that give them a writer
-- rather than as empty tables nothing reads -- `event_run_phase_gates` in P5,
-- `event_action_settings` and `event_run_budget` in P6, `event_run_resources` in
-- P8 and `event_run_participants` in P10.
-- P8 (below) and `event_run_participants` in P10.
--
-- Core tables, so no module prefix, and no game vocabulary anywhere below: an
-- action id, a scope, a resource kind and a budget dimension are all opaque
@@ -2492,6 +2492,101 @@ CREATE TABLE IF NOT EXISTS event_run_budget (
UNIQUE KEY uq_evbud_dim (run_id, dimension)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- The cleanup ledger: everything one run created or leased, and what became of
-- it (EVENTS.md §D, §L "The ledger's two rules"; Phase 8).
--
-- **It holds both kinds of thing an event owns.** An OBJECT it created is
-- `kind: 'creature'` with `ref` a serial, reverted by its own action's
-- `revert()`. A VALUE it leased is `kind: 'override'` with `ref` the lease id and
-- `payload` carrying the baseline and what was applied, restored by the lease's
-- own `restore()`. One table, because cleanup asks both the same question --
-- what is still out there, and did putting it back work.
--
-- **Rule 1: a resource is recorded BEFORE it is confirmed.** A spawn's serial
-- does not exist until the module answers, so what is written before the dispatch
-- is a PLACEHOLDER keyed by the step's idempotency key (`kind` = the reserved
-- '@step', `ref` = that key). On the answer the reported resources are inserted
-- `confirmed` and the placeholder is resolved. If the acknowledgement is lost the
-- placeholder survives, and cleanup calls `revert()` with the idempotency key and
-- no resources -- which is why §F's `revert({ runId, resources, idempotencyKey })`
-- takes the key at all. Recording afterwards instead would make every object
-- whose ack was lost invisible to cleanup for ever.
--
-- **Rule 2: revert is idempotent, and its failure is loud and sticky.** A row
-- that never reverts stays visible -- the run reaches `completed` with
-- `cleanup_status = 'incomplete'` rather than being held `running`, because a
-- tidy `completed` over a shard full of orphaned monsters is the failure that
-- would end this feature's credibility on its first bad night.
--
-- **The unique key is what stops two events leasing one target**, and it must
-- hold among LIVE rows only: last week's finished event must not keep this
-- week's from leasing the same rate. MariaDB has no partial index, so the
-- encoding is a STORED generated column that is NULL once the row is no longer
-- ours -- and multiple NULLs do not collide in a unique index. It is derived from
-- `status` ALONE and the opaque columns stay in the KEY, which is the shape
-- TEAMS.md §2.5 had to be corrected into: MariaDB refuses ON DELETE SET NULL on a
-- foreign key whose column is a base column of a stored generated column
-- (error 1901), so `step_id` must not appear in the expression.
--
-- **The key is held by the three statuses that mean "core still believes this is
-- ours"** -- `pending`, `confirmed`, `reverting` -- and released by the three that
-- mean it is not. §D says "among non-reverted rows", which was written before the
-- six statuses had their meanings; taken literally it makes `drifted` and
-- `orphaned` hold a target for ever, so one bad night would disable a lease
-- permanently with no control able to clear it. `drifted` means somebody else has
-- hold of the value and this run has deliberately let go of it; `orphaned` means
-- it vanished. Neither is a claim on the target, and both stay LOUD by another
-- mechanism -- `cleanup_status = 'incomplete'` and a row on the run console --
-- which is what §L's rule 2 actually asks for. Amended 2026-09-03.
CREATE TABLE IF NOT EXISTS event_run_resources (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
run_id BIGINT NOT NULL,
-- Which step made it. It is how cleanup finds the ACTION to call `revert()` on:
-- the row records the module and the opaque names, and the step records the
-- verb. SET NULL rather than CASCADE, for `engagement_sends`' reason -- a record
-- of what was changed in the world must outlive the row that scheduled it.
step_id BIGINT NULL,
-- The registering module, copied at record time rather than derived from the
-- action id, so an uninstalled module still names itself on the console.
owner_module VARCHAR(64) NOT NULL,
-- Both module-opaque, stored verbatim, never interpreted -- `ctx.teams.activity.push`'s
-- treatment. '@step' is the one reserved `kind` and core owns it.
kind VARCHAR(64) NOT NULL,
ref VARCHAR(190) NOT NULL,
payload JSON NULL,
-- A lease's deadline, and NULL for an owned object. It goes DOWN THE WIRE as
-- well: the game side restores baseline when it passes, without being asked
-- again, which is the fail-safe that makes an unattended world change
-- defensible. This column is core's copy of that promise, for the console and
-- for the boot-time check.
lease_until DATETIME NULL,
status ENUM('pending','confirmed','reverting','reverted','orphaned','drifted')
NOT NULL DEFAULT 'pending',
-- Bounded like a step's `attempts`, and for the same reason: a revert that can
-- never succeed must become visible rather than cycling for ever. Engagement
-- Phase 14's rule -- only a terminal row is ever retention-eligible -- is what
-- makes an unbounded counter a row nothing can ever sweep.
revert_attempts INT NOT NULL DEFAULT 0,
last_error VARCHAR(500) NULL,
-- Optional, and module-opaque like the rest: who received it, for a granted
-- reward that results should be able to name. `event_run_participants` joins on
-- the same key in Phase 10.
member_key VARCHAR(190) NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- 1 while core still believes this resource is this run's, NULL once it is not.
-- See the unique key below; derived from `status` alone, deliberately.
live_marker TINYINT AS (IF(status IN ('pending','confirmed','reverting'), 1, NULL)) STORED,
CONSTRAINT fk_evres_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE,
CONSTRAINT fk_evres_step FOREIGN KEY (step_id) REFERENCES event_run_steps(id) ON DELETE SET NULL,
-- "Two events cannot hold a lease on one target", among non-reverted rows.
UNIQUE KEY uq_evres_target (owner_module, kind, ref, live_marker),
-- The run console, and the cleanup sweep's read: one run's ledger in order.
INDEX idx_evres_run (run_id, status),
-- The cleanup leg's scan across runs, and the boot-time lease self-check.
INDEX idx_evres_live (status, lease_until)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- §K's last bound: "a scheduled definition that has never been verified is the
-- case worth refusing to start". A version is immutable, so a dry run that passed
-- against it stays true — which is what makes the pass a property of the VERSION

9
server/routes.guards.json
View File

@@ -581,6 +581,15 @@
"requireAuth"
]
},
{
"method": "POST",
"path": "/api/v1/admin/events/runs/:runId/cleanup",
"handlers": 2,
"gates": [
"noindex",
"requireAuth"
]
},
{
"method": "GET",
"path": "/api/v1/admin/events/runs/:runId/log",

4
server/routes.manifest.json
View File

@@ -257,6 +257,10 @@
"method": "POST",
"path": "/api/v1/admin/events/runs/:runId/cancel"
},
{
"method": "POST",
"path": "/api/v1/admin/events/runs/:runId/cleanup"
},
{
"method": "GET",
"path": "/api/v1/admin/events/runs/:runId/log"

211
server/src/config/coreEventActions.js
View File

@@ -13,6 +13,15 @@
// a human to go and do something. A deployment with no game module installed has
// a working event system made of exactly these.
//
// **Phase 8 added a fourth, and it is the odd one out on purpose.** `core.lease`
// names no game noun either — it borrows a value some module declared — but
// unlike the other three it genuinely changes the world, so it is `risk: 'change'`
// and therefore default-off, admin-only and cap-checked like any module verb.
// It is CORE's rather than each module's because §F puts the duration bound and
// the two-events-one-target conflict check on core's side of the seam: a lease
// verb per module would be that bound re-implemented once per module, advisory
// everywhere, and wrong in the first one that forgot it.
//
// **Phase 2 gave all three real bodies**, and between them they exercise every
// shape §F's envelope can take: `core.announce` does work and finishes,
// `core.wait` finishes while deferring what follows it, and `core.cue` succeeds
@@ -24,10 +33,42 @@
// which runs under `routeManifest.js` and `swagger.js` against a dead pool
// (MODULE_API.md §2.2). Nothing below runs at require time; the announce leg is
// looked up inside `perform()`, per call, which is also what makes a leg
// registered by a module that booted later reachable at all.
// registered by a module that booted later reachable at all. `core.lease` is the
// one action here that reaches a table, and it requires the model INSIDE
// `perform()` for the same reason — a top-level require would make this file
// build a pool during route-manifest generation.
const registries = require('../modules/registries')
/**
* Turn the `value` param's text into whatever the named lease says it holds.
*
* The range check is here too, and it is REQUIRED on the numeric types for the
* reason §F gives: unlike a cap, a bad lease value is in force the moment it is
* applied, so "0.5 to 5" is not advice.
*/
function coerceLeaseValue(lease, raw) {
const text = String(raw === undefined || raw === null ? '' : raw).trim()
if (lease.type === 'string') return { ok: true, value: text }
if (lease.type === 'bool') {
if (['true', '1', 'yes', 'on'].includes(text.toLowerCase())) return { ok: true, value: true }
if (['false', '0', 'no', 'off'].includes(text.toLowerCase())) return { ok: true, value: false }
return { ok: false, error: `"${raw}" is not a yes or no value for ${lease.label}` }
}
const n = Number(text)
if (text === '' || !Number.isFinite(n)) {
return { ok: false, error: `"${raw}" is not a number, and ${lease.label} holds one` }
}
if (lease.type === 'int' && !Number.isInteger(n)) {
return { ok: false, error: `${lease.label} holds a whole number, and "${raw}" is not one` }
}
if (n < lease.min || n > lease.max) {
return { ok: false, error: `${lease.label} accepts ${lease.min} to ${lease.max}, and "${raw}" is outside that` }
}
return { ok: true, value: n }
}
const ACTIONS = [
{
id: 'core.announce',
@@ -214,6 +255,164 @@ const ACTIONS = [
return { ok: true, await: 'human' }
},
},
{
id: 'core.lease',
label: 'Borrow a value',
description:
'Hold a module-declared value at a new setting for a bounded time, and put the old one back at teardown.',
// The world changes and it changes back, so `change` rather than
// `irreversible` — and `change`'s default `on_failure` is `pause`, which is
// the right stop for a run that failed halfway through altering the world.
risk: 'change',
// The one action core ships in this class. `override` is what tells the
// cleanup sweep to restore through the LEASE registry rather than through an
// action's `revert()`, which is why this action needs no `revert()` of its own
// and why the registry refuses one on it.
reversible: 'override',
version: 1,
params: [
{
name: 'lease',
type: 'string',
required: true,
example: 'uo.rate.skillgain',
source: 'core.options.leases',
description: 'Which declared value to borrow.',
},
{
// **A string, and the coercion is here rather than in the type system.**
// A param declares ONE type; a lease declares its own, and they are four
// different ones. Typing this `float` would make a boolean lease
// unauthorable and a string lease nonsense, so the field takes text and
// this action turns it into whatever the named lease said it holds — the
// one place that knows both halves.
name: 'value',
type: 'string',
required: true,
example: '3.0',
description: 'What to hold it at, in whatever type the lease declares.',
},
{
name: 'minutes',
type: 'int',
required: true,
example: 120,
description: 'How long to hold it. Core refuses more than the lease allows.',
},
],
// What a lease costs is the LEASE's business to bound, not a budget's:
// `maxDurationMs` and the numeric range are declared beside the callables and
// enforced below. A cap dimension here would be core inventing an accounting
// unit for something a module already bounds — and `registerEventBudgets`
// refuses a dimension nobody declared, which is exactly the rule that would
// then bite core's own action.
/**
* Read the baseline, reserve the target, apply the value.
*
* **This is rule 1 in its strongest form.** Unlike a spawn, a lease's target
* is knowable before the dispatch — it is the lease id the step names — so
* the ledger row is written with its real `kind` and `ref` BEFORE anything
* touches the world, and the two-events-one-target refusal comes from the
* unique index at that moment rather than from a check that read and then
* wrote. A second run asking for a lease another run holds comes back
* `refused`, in the same words a cap breach uses and for the same reason:
* nothing is broken, the deployment already has that value spoken for.
*
* The order is read then reserve then apply, and a failure at each stage
* undoes the one before it: a reservation whose `apply` refuses is released
* here rather than left for the sweep, because there is nothing out there to
* give back and a shard that is merely down must not lock a lease out for the
* length of a retry cycle.
*/
async perform({ runId, stepId, params, verify }) {
// eslint-disable-next-line global-require
const resourcesDb = require('../model/events/eventRunResources.db')
const lease = registries.eventLease(params.lease)
if (!lease) {
return { ok: false, retry: false, error: `no module registers the lease "${params.lease}"` }
}
const coerced = coerceLeaseValue(lease, params.value)
if (!coerced.ok) return { ok: false, retry: false, error: coerced.error }
const minutes = Number(params.minutes)
if (!Number.isFinite(minutes) || minutes <= 0) {
return { ok: false, retry: false, error: `"${params.minutes}" is not a number of minutes` }
}
const ms = Math.round(minutes * 60_000)
if (ms > lease.maxDurationMs) {
return {
ok: false,
retry: false,
error: `${lease.label} may be held for at most ${Math.floor(lease.maxDurationMs / 60_000)} minutes, not ${minutes}`,
}
}
// **The dry run stops here, and it has still checked everything worth
// checking**: the lease exists, the value is in range and the duration is
// allowed. What it deliberately does not do is reserve the target — a
// verify that took a lease would be a dry run that changed something, and
// it would then refuse the real run that followed it.
if (verify) return { ok: true }
const baseline = await lease.read()
if (!baseline || baseline.ok !== true) {
return { ok: false, error: `could not read the current value of ${lease.label}` }
}
const until = new Date(Date.now() + ms)
const reserved = await resourcesDb.reserve({
runId,
stepId,
owner: lease.owner || 'core',
kind: 'override',
ref: lease.id,
payload: { target: lease.id, baseline: baseline.value, applied: coerced.value, until: until.toISOString() },
leaseUntil: until,
})
if (!reserved.ok) {
const heldBy = reserved.holder ? ` (run ${reserved.holder.run_id})` : ''
return {
ok: false,
retry: false,
error: `${lease.label} is already leased by another run${heldBy}`,
}
}
// **`until` goes down the wire** (§F). The module passes it to its sidecar
// and the game side restores baseline when it passes, without being asked
// again — the fail-safe that makes an unattended, scheduled world change
// defensible, because the worst case is a world back at baseline early
// rather than one stuck changed indefinitely.
let applied
try {
applied = await lease.apply(coerced.value, until)
} catch (err) {
applied = { ok: false, error: err.message }
}
if (!applied || applied.ok !== true) {
await resourcesDb.markReverted(reserved.id)
return { ok: false, error: applied && applied.error ? String(applied.error) : `${lease.label} refused the new value` }
}
await resourcesDb.confirm(reserved.id)
// **The run now owes the world something, and something has to say so.**
// The generic path marks a run dirty when it records a module's reported
// resources; this action reserves its own row and never goes through it, so
// a run whose only resource was a lease would have kept `cleanup_status =
// 'not_required'` and never been swept. Found by the live walk, and the
// cleanup leg's own scan was widened to make the class impossible rather
// than only this instance.
// eslint-disable-next-line global-require
await require('../events/ledger').markRunDirty(runId)
return { ok: true }
},
},
]
// ── Core's own param option sources (§F, Phase 7) ──────────────────
@@ -237,6 +436,16 @@ const OPTION_SOURCES = [
return registries.announceLegs().map((l) => ({ value: l.leg, label: l.label || l.leg }))
},
},
{
id: 'core.options.leases',
label: 'Borrowable values',
description: 'Every value a module has declared this deployment may lease.',
async resolve() {
return registries
.allEventLeases()
.map((l) => ({ value: l.id, label: l.label, group: l.id.split('.')[0] }))
},
},
]
module.exports = { ACTIONS, OPTION_SOURCES }

427
server/src/events/cleanup.js Normal file
View File

@@ -0,0 +1,427 @@
// ── Giving back what a run took ────────────────────────────────────────────
//
// EVENTS.md §C ("Cleanup is generated, never authored"), §L and its two ledger
// rules, and Phase 8 of EVENTS_PLAN.md. `events/ledger.js` is the write half;
// this is the undo half, plus the reconcile that answers "is any of it still
// there?" after something outside core restarted.
//
// **Cleanup is derived from the ledger, never authored.** An operator cannot be
// relied on to write the undo, and an aborted run never reaches the phase they
// wrote it in — so there is no cleanup phase in a spec and no `on_teardown` on an
// action. There is one function, it reads rows, and it runs on EVERY terminal
// path: completion, cancellation and abort alike.
//
// **It is not built out of `event_run_steps` rows** (org lead, 2026-09-03). The
// plan's phrase is "cleanup steps are generated from the ledger", and the
// tempting reading is a synthetic phase of real step rows so the console's
// per-step retry comes free. It is the wrong shape here for one concrete reason:
// `event_run_resources` already carries `revert_attempts` and `last_error`, so
// synthetic steps would put a second retry counter beside the first and the two
// would disagree the first time a step reverted three of its four resources.
// The manual retry the API surface promises is a route over the ledger —
// `POST /admin/events/runs/:runId/cleanup` — rather than a step control.
//
// **Where it runs from.** One place: the runner's cleanup leg, which finds
// terminal runs that still owe the world something and works their rows. Hooking
// each terminal path instead would be four call sites, three of which are inside
// a request, and none of which would survive the process dying mid-cleanup. The
// leg is ordered AFTER advance in the tick, so a run that completes in one tick
// is cleaned in the same one.
//
// **The scan's WHERE clause cost two live-walk findings, in opposite
// directions.** A run whose only resource was a LEASE never went through
// `ledger.markRunDirty` — `core.lease` reserves its own row — so its
// `cleanup_status` stayed `not_required` and the lease was never given back at
// all. And a run whose first sweep failed was moved to `incomplete` by that very
// sweep, so it was never picked up again: `MAX_REVERT_ATTEMPTS` meant ONE attempt
// rather than three. The first is why `not_required` is in the scan; the second
// is why `incomplete` is written HERE only once nothing retryable is left.
//
// **Rule 2 is what the bounds are for.** A revert that never succeeds must stay
// visible rather than cycle: `MAX_REVERT_ATTEMPTS` stops the automatic retry, the
// run reaches `completed` with `cleanup_status = 'incomplete'`, and the rows stay
// on the console with their last error. Only a human's cleanup clears the
// counter — Engagement Phase 14's rule, whose defect was a sweep that reset every
// stale row and made the ceiling unreachable for ever.
const resourcesDb = require('../model/events/eventRunResources.db')
const runsDb = require('../model/events/eventRuns.db')
const logDb = require('../model/events/eventRunLog.db')
const stepsDb = require('../model/events/eventRunSteps.db')
const registries = require('../modules/registries')
const { withDeadline } = require('./dispatch')
const log = require('../utils/logger')('events')
// How many times the automatic sweep will ask before leaving a resource for a
// human. Three, like a step's, and for the same reason: a fourth attempt against
// a shard that has answered the same way three times is not new information.
const MAX_REVERT_ATTEMPTS = Number(process.env.EVENT_REVERT_MAX_ATTEMPTS) || 3
// The bound on one revert call, when the action that made the resource is gone
// and there is no `budgetMs` to read. A restore is a round trip like any other.
const DEFAULT_REVERT_BUDGET_MS = 10_000
// How many runs one cleanup leg looks at, and how many resource groups it works
// per run. Bounds rather than targets, exactly like `RUN_BATCH`: the tick runs
// again, and an unbounded teardown is how one run's bad night stalls every other.
const CLEANUP_RUN_BATCH = Number(process.env.EVENT_CLEANUP_RUN_BATCH) || 10
const CLEANUP_GROUPS_PER_RUN = Number(process.env.EVENT_CLEANUP_GROUPS_PER_RUN) || 25
/**
* Classify one revert answer, with `dispatch.classify`'s posture: no shape a
* failure can take may read as success.
*
* The extra value here is `drifted`. It is NOT an error — the module did exactly
* what it was asked and found somebody else's value in place — so it is a third
* outcome rather than a failure with a flag, and the row it produces is the one
* §L wants surfaced beside the unreverted ones.
*/
function classifyRevert(raw, what) {
if (raw && raw.__timedOut) return { outcome: 'retry', error: raw.error }
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) {
return { outcome: 'retry', error: `${what} answered with no envelope` }
}
if (raw.ok === true) {
// §L, and the Rust wipe: "gone, and that is fine" is a successful revert. A
// module never has to distinguish "I deleted it" from "it was not there".
return { outcome: 'done', failed: Array.isArray(raw.failed) ? raw.failed.map(String) : [] }
}
if (raw.drifted === true) {
return {
outcome: 'drifted',
error: `the value is now ${JSON.stringify(raw.current)} rather than what this run applied, so it was left alone`,
}
}
return {
outcome: raw.retry === false ? 'terminal' : 'retry',
error: raw.error ? String(raw.error) : `${what} refused`,
}
}
/** Call a lease's `restore`, under a deadline, never throwing. */
async function restoreLease(row) {
const lease = registries.eventLease(row.ref)
if (!lease) {
// The module that owned it is uninstalled or failed to boot. Not a failure to
// retry away — nothing will change until an operator reinstalls it — and not
// an orphan either, because core has no idea whether the value is still
// applied. It stays unresolved with the reason on it, which is exactly what
// `cleanup_status = 'incomplete'` is for.
return { outcome: 'terminal', error: `no module registers the lease "${row.ref}"` }
}
const payload = row.payload || {}
let raw
try {
raw = await withDeadline(
() => lease.restore(payload.baseline, { expected: payload.applied, runId: row.run_id }),
DEFAULT_REVERT_BUDGET_MS,
row.ref,
)
} catch (err) {
return { outcome: 'retry', error: err.message }
}
return classifyRevert(raw, row.ref)
}
/** Call an action's `revert` over a group of its rows, under a deadline, never throwing. */
async function revertGroup(actionId, rows, idempotencyKey) {
const action = registries.eventAction(actionId)
if (!action || typeof action.revert !== 'function') {
return {
outcome: 'terminal',
error: action
? `${actionId} declares no revert()`
: `no module registers "${actionId}", so its resources cannot be given back`,
}
}
const payload = rows
.filter((r) => r.kind !== resourcesDb.STEP_KIND)
.map((r) => ({ kind: r.kind, ref: r.ref, payload: r.payload || null, memberKey: r.member_key || null }))
let raw
try {
raw = await withDeadline(
() => action.revert({ runId: rows[0].run_id, resources: payload, idempotencyKey }),
action.budgetMs || DEFAULT_REVERT_BUDGET_MS,
actionId,
)
} catch (err) {
// A module should not throw from `revert` any more than from `perform`, and
// one that does has produced a transient failure rather than a crashed sweep.
log.warn('event revert threw', { action: actionId, message: err.message })
return { outcome: 'retry', error: err.message }
}
return classifyRevert(raw, actionId)
}
/**
* Work one run's ledger once.
*
* Answers what it found and what it managed, and sets `cleanup_status` from the
* rows that are left rather than from what it did — the two differ whenever
* another writer touched the run, and the rows are the truth.
*
* `resetAttempts` is the human's flag. It is never set by the automatic leg.
*/
async function cleanupRun(run, { resetAttempts = false, actor = null } = {}) {
const summary = { attempted: 0, reverted: 0, drifted: 0, failed: 0, remaining: 0 }
if (resetAttempts) {
const cleared = await resourcesDb.resetAttempts(run.id)
await runsDb.setCleanupStatus(run.id, 'pending')
if (cleared) {
await logDb.write({
runId: run.id,
kind: 'cleanup.retry',
detail: { resources: cleared, by: actor },
})
}
}
const rows = await resourcesDb.unresolvedForRun(run.id, {
maxAttempts: resetAttempts ? null : MAX_REVERT_ATTEMPTS,
})
// **Grouped by the step that made them**, because that is what names the verb:
// the resource row records the module and the opaque names, the step records
// the action, and `revert()` takes a LIST so one round trip can give back
// twelve creatures. A lease is its own group of one — core restores it through
// the lease registry rather than through any action, which is the split §F
// draws and the reason `core.lease` needs no `revert()` of its own.
const leases = rows.filter((r) => r.kind === 'override')
const byStep = new Map()
for (const row of rows) {
if (row.kind === 'override') continue
const key = row.step_id === null ? `orphan:${row.id}` : `step:${row.step_id}`
if (!byStep.has(key)) byStep.set(key, [])
byStep.get(key).push(row)
}
const groups = [...leases.map((r) => ({ lease: r })), ...[...byStep.values()].map((rs) => ({ rows: rs }))]
for (const group of groups.slice(0, CLEANUP_GROUPS_PER_RUN)) {
if (group.lease) {
const row = group.lease
if (!(await resourcesDb.claimRevert(row.id))) continue
summary.attempted += 1
const verdict = await restoreLease(row)
await applyVerdict(run, [row], verdict, summary, row.ref)
continue
}
const rs = group.rows
// The step is what names the action and carries the idempotency key. A row
// whose step was deleted keeps the action id in its own payload, which is why
// the placeholder writes one.
const step = rs[0].step_id === null ? null : await stepsDb.getById(rs[0].step_id)
const actionId = step?.action_id || rs[0].payload?.action || null
if (!actionId) {
await noteUnrevertable(run, rs, 'nothing records which action created this', summary)
continue
}
const claimed = []
for (const row of rs) if (await resourcesDb.claimRevert(row.id)) claimed.push(row)
if (!claimed.length) continue
summary.attempted += claimed.length
const verdict = await revertGroup(actionId, claimed, step?.idempotency_key || rs[0].ref)
await applyVerdict(run, claimed, verdict, summary, actionId)
}
summary.remaining = await resourcesDb.unresolvedCount(run.id)
// **`incomplete` means "finished with, and not finished"**, so it is written
// only once there is nothing left this sweep will try. Writing it after the
// FIRST failure — which is what the first draft did — took the run straight out
// of the leg's own scan, and `MAX_REVERT_ATTEMPTS` quietly meant one attempt
// rather than three. Found by the live walk, watching `revert_attempts` sit at
// 1 through half a minute of ticks.
const retryable = await resourcesDb.unresolvedForRun(run.id, { maxAttempts: MAX_REVERT_ATTEMPTS })
const status = summary.remaining === 0 ? 'complete' : retryable.length ? 'pending' : 'incomplete'
await runsDb.setCleanupStatus(run.id, status)
if (summary.attempted > 0) {
await logDb.write({
runId: run.id,
kind: 'cleanup.swept',
detail: { ...summary, by: actor },
})
}
return summary
}
/** Write one verdict across the rows it covers, and count it. */
async function applyVerdict(run, rows, verdict, summary, what) {
for (const row of rows) {
if (verdict.outcome === 'done' && !(verdict.failed || []).includes(row.ref)) {
await resourcesDb.markReverted(row.id)
summary.reverted += 1
continue
}
if (verdict.outcome === 'drifted') {
await resourcesDb.failRevert(row.id, verdict.error, 'drifted')
summary.drifted += 1
continue
}
const error =
verdict.outcome === 'done'
? `${what} could not give "${row.ref}" back`
: verdict.error
await resourcesDb.failRevert(row.id, error, 'confirmed')
summary.failed += 1
}
await logDb.write({
runId: run.id,
kind: verdict.outcome === 'done' ? 'cleanup.reverted' : 'cleanup.failed',
detail: {
what,
outcome: verdict.outcome,
resources: rows.map((r) => `${r.kind}:${r.ref}`),
...(verdict.error ? { error: verdict.error } : {}),
},
})
}
/** A group core cannot even name a verb for. Counted as failed, and said once. */
async function noteUnrevertable(run, rows, reason, summary) {
for (const row of rows) {
if (!(await resourcesDb.claimRevert(row.id))) continue
await resourcesDb.failRevert(row.id, reason, 'confirmed')
summary.attempted += 1
summary.failed += 1
}
await logDb.write({
runId: run.id,
kind: 'cleanup.failed',
detail: { what: null, outcome: 'terminal', resources: rows.map((r) => `${r.kind}:${r.ref}`), error: reason },
})
}
/**
* The cleanup leg of the tick: every TERMINAL run with something left to give
* back.
*
* Terminal only. A run still in flight has a ledger that is still growing, and
* reverting a resource the next step is about to use would be core undoing an
* event while it is happening.
*/
async function sweep() {
// The ceiling goes INTO the query, so a run whose rows are all spent is not
// selected, worked over and found to have nothing to do on every tick for the
// rest of its life. It is also what excludes a run an admin cancelled without
// cleanup, whose counters were spent deliberately.
const candidates = await resourcesDb.runsNeedingCleanup(CLEANUP_RUN_BATCH, MAX_REVERT_ATTEMPTS)
let swept = 0
for (const candidate of candidates) {
if (!runsDb.TERMINAL.includes(candidate.status)) continue
try {
await cleanupRun(candidate)
swept += 1
} catch (err) {
log.error('event cleanup failed', { run: candidate.id, message: err.message })
}
}
return swept
}
/**
* Ask one module which of its ledgered resources the game still has (§L, and
* §N7's "the shard stays stateless about events").
*
* **Core cannot know when to ask**, and that is not an omission: §F says core has
* no concept of the game being up, because a module with six sidecars cannot
* answer that question in the singular. So the module triggers this, through
* `ctx.events.reconcile()`, when it sees its own reconnect — module-uo already
* watches `bootId` for exactly that. Core also asks once at boot, for its own
* restart.
*
* **A resource the module no longer has becomes `orphaned`, never `reverted`.**
* Reverting it would be core recording that it put something back when what
* actually happened is that the thing vanished while nobody was looking, and the
* two are different sentences to the operator reading the console afterwards.
*
* A module with no `reconcile()` on the action is not broken: core keeps
* believing its own ledger, which is precisely the behaviour before this phase.
*/
async function reconcileModule(owner) {
const rows = await resourcesDb.liveForModule(owner)
const summary = { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 }
if (!rows.length) return summary
const byStep = new Map()
for (const row of rows) {
if (row.kind === resourcesDb.STEP_KIND) continue // nothing to ask about yet
const key = row.step_id === null ? `orphan:${row.id}` : `step:${row.step_id}`
if (!byStep.has(key)) byStep.set(key, [])
byStep.get(key).push(row)
}
for (const group of byStep.values()) {
const step = group[0].step_id === null ? null : await stepsDb.getById(group[0].step_id)
const actionId = step?.action_id || group[0].payload?.action || null
const action = actionId ? registries.eventAction(actionId) : null
if (!action || typeof action.reconcile !== 'function') {
summary.unanswered += group.length
continue
}
summary.asked += group.length
let raw
try {
raw = await withDeadline(
() =>
action.reconcile({
runId: group[0].run_id,
resources: group.map((r) => ({ kind: r.kind, ref: r.ref, payload: r.payload || null })),
}),
action.budgetMs || DEFAULT_REVERT_BUDGET_MS,
actionId,
)
} catch (err) {
log.warn('event reconcile threw', { action: actionId, message: err.message })
raw = null
}
// Same posture as everywhere else: nothing that is not an explicit answer
// counts as one. A module that could not answer leaves the ledger alone,
// because "I do not know" must never be read as "it is gone".
if (!raw || raw.__timedOut || raw.ok !== true || !Array.isArray(raw.inForce)) {
summary.unanswered += group.length
continue
}
const held = new Set(raw.inForce.map(String))
for (const row of group) {
if (held.has(row.ref)) {
summary.inForce += 1
continue
}
await resourcesDb.markOrphaned(row.id, 'the module reports this is no longer in force')
summary.orphaned += 1
await logDb.write({
runId: row.run_id,
kind: 'resource.orphaned',
detail: { module: owner, resource: `${row.kind}:${row.ref}`, action: actionId },
})
}
}
return summary
}
/** Ask every module that owns a live row. Core's own boot-time sweep. */
async function reconcileAll() {
const owners = await resourcesDb.modulesWithLiveRows()
const out = {}
for (const owner of owners) {
try {
out[owner] = await reconcileModule(owner)
} catch (err) {
log.error('event reconcile failed', { module: owner, message: err.message })
}
}
return out
}
module.exports = {
MAX_REVERT_ATTEMPTS,
classifyRevert,
cleanupRun,
sweep,
reconcileModule,
reconcileAll,
}

221
server/src/events/ledger.js Normal file
View File

@@ -0,0 +1,221 @@
// ── Recording what a run changed in the world ──────────────────────────────
//
// EVENTS.md §D and §L, and Phase 8 of EVENTS_PLAN.md. The write half of the
// resource ledger; `events/cleanup.js` is the read-and-undo half.
//
// **Rule 1 is the whole reason this file is not two lines inside `drainStep`.**
// A resource is recorded BEFORE it is confirmed. The obstacle is that a spawn's
// serial does not exist until the module answers, so there is nothing to write a
// row about yet — which is why what goes in before the dispatch is a PLACEHOLDER
// keyed by the step's idempotency key rather than by the object:
//
// pre-dispatch INSERT pending { kind: '@step', ref: <idempotency key> }
// answer INSERT confirmed { kind: 'creature', ref: '0x40001234' } × n
// resolve the placeholder
// ack lost the placeholder is still `pending`
// cleanup revert({ idempotencyKey, resources: [] })
//
// That last line is why §F's `revert({ runId, resources, idempotencyKey })` takes
// the key at all. A module that half-ran and never answered is reachable by its
// key and by nothing else, and Phase 11's plugin-side key ledger is what makes
// answering it exact. Until then the contract is still honest, because §L
// requires reverting something that does not exist to be a SUCCESS.
//
// **Recording is idempotent, and the database is what makes it so.** A retry
// re-dispatches the same idempotency key, and a module that answers with the same
// resources twice must not produce two rows. `uq_evres_target` refuses the second
// insert, and this file reads that refusal as "already recorded" rather than as an
// error — the same posture `materialisePhase`'s INSERT IGNORE takes.
//
// **A lease does not use the placeholder.** Its target is knowable before the
// dispatch — it is the lease id the step names — so `core.lease` reserves the
// real row first, which is both a stronger form of rule 1 and the only place the
// two-events-one-target refusal can happen before the world has been written to.
const resourcesDb = require('../model/events/eventRunResources.db')
const runsDb = require('../model/events/eventRuns.db')
const registries = require('../modules/registries')
const log = require('../utils/logger')('events')
// Which reversible classes get a pre-dispatch placeholder. `none` is gone once
// done and `self` undoes itself, so neither has anything core could come back
// for; `override` reserves its own target instead (see the header). That leaves
// `ledger` — the class that declares `revert()`, which is exactly the class whose
// refs core cannot know until the module speaks.
const PLACEHOLDER_CLASSES = ['ledger']
// A resource `kind` may be anything a module likes except core's own reserved
// one. Bounded to the column, and refused rather than truncated: a truncated ref
// is a cleanup call naming the wrong object.
const MAX_KIND = 64
const MAX_REF = 190
const MAX_MEMBER_KEY = 190
/** Does this action produce anything core will have to come back for? */
const ledgers = (action) => action && (action.reversible === 'ledger' || action.reversible === 'override')
/**
* Turn one entry of a module's `resources` array into a row, or say why not.
*
* Every failure here is the module's mistake rather than the world's, so none of
* them is a retry: a badly shaped resource will be just as badly shaped on the
* second attempt. They are logged and dropped, and the step still counts as done
* — because it IS done; something happened in the world, and refusing to record
* it would be the one outcome worse than recording it imperfectly.
*/
function normalise(entry, actionId) {
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
return { ok: false, reason: `${actionId} reported a resource that is not an object` }
}
const kind = String(entry.kind || '')
const ref = String(entry.ref === undefined || entry.ref === null ? '' : entry.ref)
if (!kind || kind.length > MAX_KIND) {
return { ok: false, reason: `${actionId} reported a resource with a bad kind "${entry.kind}"` }
}
if (kind === resourcesDb.STEP_KIND) {
// Core's own. A module that could write one would be a module that could make
// its own step's placeholder look resolved.
return { ok: false, reason: `${actionId} reported a resource of the reserved kind "${kind}"` }
}
if (!ref || ref.length > MAX_REF) {
return { ok: false, reason: `${actionId} reported a resource with a bad ref "${entry.ref}"` }
}
const memberKey = entry.memberKey === undefined || entry.memberKey === null ? null : String(entry.memberKey)
if (memberKey !== null && memberKey.length > MAX_MEMBER_KEY) {
return { ok: false, reason: `${actionId} reported a resource with an over-long memberKey` }
}
let leaseUntil = null
if (entry.until !== undefined && entry.until !== null) {
const at = new Date(entry.until)
if (Number.isNaN(at.getTime())) {
return { ok: false, reason: `${actionId} reported a resource with a bad until "${entry.until}"` }
}
leaseUntil = at
}
// A borrowed value must name a lease core knows how to give back. Core restores
// an `override` through the lease registry — that is the split §F draws — so a
// ref naming nothing registered is a resource core would be recording with no
// way to undo it, which is the promise rule 2 exists to stop core making.
if (kind === 'override' && !registries.eventLease(ref)) {
return { ok: false, reason: `${actionId} reported a lease "${ref}" no module registers` }
}
return {
ok: true,
row: {
kind,
ref,
payload: entry.payload === undefined ? null : entry.payload,
leaseUntil,
memberKey,
},
}
}
/**
* Write the pre-dispatch placeholder for a step that is about to change the
* world. Answers the row id, or null when this action ledgers nothing.
*
* **A duplicate is not a failure.** A retry re-uses its step's idempotency key, so
* the second attempt's placeholder collides with the first's — and finding it
* already there is the correct answer, not an error. The existing row is reused.
*/
async function reserveStep(run, step, action) {
if (!ledgers(action) || !PLACEHOLDER_CLASSES.includes(action.reversible)) return null
const owner = action.owner || 'core'
const reserved = await resourcesDb.reserve({
runId: run.id,
stepId: step.id,
owner,
kind: resourcesDb.STEP_KIND,
ref: step.idempotency_key,
payload: { action: action.id, phase: step.phase, seq: step.seq },
})
if (reserved.ok) {
await markRunDirty(run.id)
return reserved.id
}
// The only way a '@step' row collides is with this step's own earlier attempt,
// because an idempotency key is minted once per step and never varies by
// attempt (§E). Reuse it.
const existing = await resourcesDb.findByTarget(owner, resourcesDb.STEP_KIND, step.idempotency_key)
return existing ? existing.id : null
}
/**
* Record what a module said it made, and close out the placeholder.
*
* Answers `{ recorded, rejected }` — how many rows went in, and the reasons any
* entry was dropped. Never throws: a step that changed the world has changed it,
* and a ledger that threw would turn a bookkeeping problem into a failed step and
* then into a retry of a world write that already happened.
*/
async function recordAnswer({ run, step, action, placeholderId, resources }) {
const out = { recorded: 0, rejected: [] }
if (!ledgers(action)) return out
const owner = action.owner || 'core'
const list = Array.isArray(resources) ? resources : []
for (const entry of list) {
const parsed = normalise(entry, action.id)
if (!parsed.ok) {
out.rejected.push(parsed.reason)
log.warn('event resource rejected', { run: run.id, step: step.id, reason: parsed.reason })
continue
}
try {
const reserved = await resourcesDb.reserve({
runId: run.id,
stepId: step.id,
owner,
kind: parsed.row.kind,
ref: parsed.row.ref,
payload: parsed.row.payload,
leaseUntil: parsed.row.leaseUntil,
memberKey: parsed.row.memberKey,
})
if (!reserved.ok) {
// Already ledgered — by this step's own earlier attempt, or (a module bug
// rather than a race) by another run that still holds the same target.
// Either way there is a live row for it and a second would be the double
// cleanup the unique key exists to prevent.
if (reserved.holder && reserved.holder.run_id !== run.id) {
out.rejected.push(`${parsed.row.kind} "${parsed.row.ref}" is already held by run ${reserved.holder.run_id}`)
}
continue
}
await resourcesDb.confirm(reserved.id)
out.recorded += 1
} catch (err) {
// Bookkeeping must not become the step's control flow.
out.rejected.push(err.message)
log.error('event resource insert failed', { run: run.id, step: step.id, message: err.message })
}
}
if (out.recorded > 0) await markRunDirty(run.id)
// The placeholder's job is over the moment the real rows exist. It is resolved
// even when the module reported nothing at all — an action that ledgers and
// then answers `ok` with an empty list is saying "I made nothing", and holding
// its placeholder open would make cleanup call `revert()` for a step that has
// nothing to give back on every terminal path for ever.
if (placeholderId) await resourcesDb.resolvePlaceholder(placeholderId)
return out
}
/**
* There is now something to clean up. Idempotent and guarded, so it can never
* walk a run back from `complete` or `incomplete` to `pending` — only a human's
* cleanup does that, and it does it deliberately.
*/
async function markRunDirty(runId) {
await runsDb.setCleanupStatus(runId, 'pending', ['not_required'])
}
module.exports = { ledgers, normalise, reserveStep, recordAnswer, markRunDirty, PLACEHOLDER_CLASSES }

106
server/src/model/events/eventRunControls.model.js
View File

@@ -15,10 +15,13 @@
// diagnosis panel: a screen that explains why a phase has not started, beside a
// control that does something about it.
//
// **One of §I's controls is still not here.** `cleanup` needs Phase 8's resource
// ledger; there is nothing to revert, so cancel takes `{ reason }` and gains
// `cleanup` when there is something for it to do. Absent rather than inert,
// which is the posture Phase 1 set and every phase since has kept.
// **`cleanup` is the eighth, and Phase 8 is what gave it a ledger to work over.**
// It re-runs the teardown across every resource a run has not given back, and it
// is `admin` where the other seven are `admin` + `moderator`: it is not incident
// response, it is asking core to write to the world again. Its partner is
// cancel's new `cleanup: false`, which is §L's "cancelling WITHOUT cleanup is a
// separate, logged, admin-only action" — deliberately the flag that has to be
// asked for, because the safe default is to give back what the run took.
//
// **Every control is guarded on the status it may act from, and the guard is a
// WHERE clause rather than a read-then-write.** A run console rendered thirty
@@ -37,6 +40,7 @@ const runsDb = require('./eventRuns.db')
const stepsDb = require('./eventRunSteps.db')
const logDb = require('./eventRunLog.db')
const gatesDb = require('./eventPhaseGates.db')
const resourcesDb = require('./eventRunResources.db')
const gates = require('../../events/gates')
const MAX_REASON = 500
@@ -152,11 +156,23 @@ async function resume(runId, options = {}, userId = null) {
* and a second writer on that row would race the process that owns it. It
* finishes into a cancelled run, which is honest.
*/
async function cancel(runId, { reason } = {}, userId = null) {
async function cancel(runId, { reason, cleanup = true } = {}, userId = null, { isAdmin = true } = {}) {
const run = await loadRun(runId)
if (!run) return { ok: false, status: 404, errors: ['no such run'] }
if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is already ${run.status}`)
// §L: cancelling WITHOUT cleanup is a separate, logged, ADMIN-only action. The
// route itself is `admin` + `moderator`, so the narrower gate cannot live in
// middleware — which of the two you have to be depends on what is in the body,
// exactly as the authoring role floor does (§K).
if (cleanup === false && !isAdmin) {
return {
ok: false,
status: 403,
errors: ['leaving a run\'s world changes in place is an administrator\'s decision'],
}
}
const note = clean(reason)
const from = ['scheduled', 'starting', 'running', 'paused', 'ending']
if (!(await runsDb.transition(run.id, from, 'cancelled', { error: note || 'cancelled by staff' }))) {
@@ -168,9 +184,83 @@ async function cancel(runId, { reason } = {}, userId = null) {
runId: run.id,
kind: 'run.status',
phase: run.current_phase,
detail: { from: run.status, to: 'cancelled', control: 'cancel', by: userId, reason: note, cancelledSteps: closed },
detail: {
from: run.status,
to: 'cancelled',
control: 'cancel',
by: userId,
reason: note,
cancelledSteps: closed,
cleanup: cleanup !== false,
},
})
return { ok: true, run: await runsDb.getById(run.id), cancelledSteps: closed }
// **The teardown is not done here, and the request does not wait for it.**
// Cleanup is one leg of the runner's tick over terminal runs (§L), which is
// what makes it survive a process that dies halfway through it — and a cancel
// pressed at two in the morning must answer at once rather than after a dozen
// round trips to a shard that may be the reason it is being cancelled. The run
// is terminal the moment this returns, so the very next tick picks its ledger
// up.
//
// `cleanup: false` is the operator saying leave it. The resources stay
// unresolved and the run carries `incomplete`, which is the truthful value: the
// world changes are still up, they are listed on the console, and the log line
// above records who decided that.
let cleanupStatus = run.cleanup_status
if (cleanup === false && (await resourcesDb.unresolvedCount(run.id)) > 0) {
// `incomplete` is what takes the run out of the cleanup leg's scan, and it is
// the truthful value: the world changes are still up, they are listed on the
// console, and the log line above records who decided that.
//
// **The first draft spent every row's `revert_attempts` instead**, to stop the
// sweep by the same mechanism a failed retry does. It worked and it made the
// console lie: the run page rendered "3 attempts" beside resources nothing had
// ever tried, which reads as "core tried three times and could not". Found by
// opening the page. A counter that means two things is a counter a screen
// cannot render.
await runsDb.setCleanupStatus(run.id, 'incomplete')
cleanupStatus = 'incomplete'
}
return {
ok: true,
run: await runsDb.getById(run.id),
cancelledSteps: closed,
cleanup: cleanup !== false,
cleanupStatus,
}
}
/**
* Re-run cleanup over everything a run has not given back.
*
* The manual retry §L promises, and the only thing that clears
* `revert_attempts`. That licence is the same one a human's step retry has, and
* it is deliberately not extended to the automatic sweep: Engagement Phase 14's
* defect was exactly a sweep that reset every stale row, which made the attempt
* ceiling unreachable and left the row cycling for ever.
*
* Legal on a TERMINAL run only. A run still in flight has a ledger that is still
* growing, and reverting a resource the next step is about to use would be core
* undoing an event while it is happening.
*/
async function cleanupRun(runId, userId = null) {
const run = await loadRun(runId)
if (!run) return { ok: false, status: 404, errors: ['no such run'] }
if (!runsDb.TERMINAL.includes(run.status)) {
return conflict(`this run is still ${run.status}; cancel it before cleaning up after it`)
}
if (run.cleanup_status === 'not_required') {
return conflict('this run recorded no resources, so there is nothing to give back')
}
// eslint-disable-next-line global-require
const summary = await require('../../events/cleanup').cleanupRun(run, {
resetAttempts: true,
actor: userId,
})
return { ok: true, run: await runsDb.getById(run.id), summary }
}
/**
@@ -359,4 +449,4 @@ async function retryStep(runId, stepId, options = {}, userId = null) {
}
}
module.exports = { pause, resume, cancel, advancePhase, confirmStep, skipStep, retryStep }
module.exports = { pause, resume, cancel, cleanupRun, advancePhase, confirmStep, skipStep, retryStep }

10
server/src/model/events/eventRunLog.db.js
View File

@@ -47,6 +47,16 @@ const KINDS = [
'run.budget', // the caps this run was seeded with, and which switch set each
'step.refused', // a step was not permitted: disabled, or over a cap
'version.verified', // a dry run passed against a version, unlocking scheduled starts
// Phase 8's six, and every one of them is an answer to "what did this event
// leave behind". `resource.recorded` is written at the ANSWER rather than at
// the placeholder, because a placeholder is a promise and the operator's
// question is about the world.
'resource.recorded', // a step reported what it created or borrowed, and it is ledgered
'resource.orphaned', // a module reports a ledgered resource is no longer in force
'cleanup.reverted', // a group of resources came back
'cleanup.failed', // a group did not, with the reason and how it was left
'cleanup.swept', // one pass over a run's ledger, and what it found
'cleanup.retry', // a human cleared the attempt counter and asked again
]
const hydrate = (row) => row && { ...row, detail: parseJson(row.detail, null) }

353
server/src/model/events/eventRunResources.db.js Normal file
View File

@@ -0,0 +1,353 @@
// ── event_run_resources — SQL only ─────────────────────────────────────────
//
// EVENTS.md §D and §L ("The ledger's two rules"), and Phase 8 of EVENTS_PLAN.md.
// Everything one run created or leased, and what became of it.
//
// **Rule 1 lives in `reserve()`.** A resource is recorded BEFORE it is
// confirmed, so the placeholder this writes is the row that exists while the
// dispatch is in flight — and the row that SURVIVES when the acknowledgement is
// lost. Recording on the answer instead would make every object whose ack went
// missing invisible to cleanup for ever.
//
// **Rule 2 lives in the status column and in `failRevert()`.** A revert that
// never succeeds leaves its row unreverted, with the error on it, and the run
// completes with `cleanup_status = 'incomplete'` rather than being held open.
// Loud and sticky.
//
// **The unique key is enforced by the database, not by a read.** `reserve()`
// answers `{ ok: false, code: 'held' }` on a duplicate key rather than checking
// first and then inserting — two runs entering the same tick would both pass the
// check. It is the argument `event_run_budget.spend()` makes about the cap and
// `runsDb.transition` makes about a status, in the third place it applies.
const { query } = require('../../utils/db')
const { parseJson } = require('./eventJson')
// The one `kind` core owns. A module's kinds are opaque and stored verbatim; this
// one is core's own, and `registries` refuses a module resource that claims it.
const STEP_KIND = '@step'
// The statuses that mean "core still believes this resource is this run's". They
// are exactly the ones the `live_marker` generated column keeps non-NULL, so the
// unique target key holds while a row is in one of them and releases when it
// leaves. Duplicated here as a JavaScript list because the sweeps read by it too,
// and a second copy that can drift is better than a query that cannot express it.
const HELD = ['pending', 'confirmed', 'reverting']
// Every status that still wants a human or a retry: `HELD` plus the two that mean
// "we let go, and not cleanly". This is what "unreverted" means everywhere in
// this feature — the console's list, `cleanup_status`, and the manual retry.
const UNRESOLVED = [...HELD, 'orphaned', 'drifted']
const COLUMNS = `id, run_id, step_id, owner_module, kind, ref, payload, lease_until,
status, revert_attempts, last_error, member_key, created_at, updated_at`
// `payload` is opaque to core and stored verbatim, but it comes back as a string
// from the driver and every caller wants the object — the cleanup sweep reads a
// lease's baseline out of it, and the console renders it. Hydrated here for the
// same reason a step's params are: one place rather than at each read.
const hydrate = (row) => row && { ...row, payload: parseJson(row.payload, null) }
/**
* Record a resource that does not exist yet.
*
* Answers `{ ok: true, id }`, or `{ ok: false, code: 'held', holder }` when the
* target is already someone's — which is the lease conflict, surfaced as a
* refusal rather than a failure because nothing is wrong with the system: another
* run has the thing.
*
* **`ER_DUP_ENTRY` is the check.** The holder is looked up only to name it in the
* refusal, and only after the insert has already lost the race.
*/
async function reserve({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) {
try {
const result = await query(
`INSERT INTO event_run_resources
(run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')`,
[runId, stepId, owner, kind, ref, payload === null ? null : JSON.stringify(payload), leaseUntil, memberKey],
)
return { ok: true, id: Number(result.insertId) }
} catch (err) {
if (err && (err.code === 'ER_DUP_ENTRY' || err.errno === 1062)) {
const [holder] = await query(
`SELECT run_id, status FROM event_run_resources
WHERE owner_module = ? AND kind = ? AND ref = ? AND status IN (?, ?, ?)
LIMIT 1`,
[owner, kind, ref, ...HELD],
)
return { ok: false, code: 'held', holder: holder || null }
}
throw err
}
}
/**
* Promote a reserved row to `confirmed`, optionally attaching what the module
* finally said about it.
*
* Guarded on `pending` so a late answer cannot un-revert a row cleanup has
* already dealt with — the same reason every other write in this feature is a
* compare-and-set rather than a read-then-write.
*/
async function confirm(id, { payload, leaseUntil, memberKey } = {}) {
const sets = ["status = 'confirmed'"]
const params = []
if (payload !== undefined) {
sets.push('payload = ?')
params.push(payload === null ? null : JSON.stringify(payload))
}
if (leaseUntil !== undefined) {
sets.push('lease_until = ?')
params.push(leaseUntil)
}
if (memberKey !== undefined) {
sets.push('member_key = ?')
params.push(memberKey)
}
const result = await query(
`UPDATE event_run_resources SET ${sets.join(', ')} WHERE id = ? AND status = 'pending'`,
[...params, id],
)
return (result.affectedRows || 0) > 0
}
/**
* Resolve a step placeholder once the module has named what it actually made.
*
* The placeholder's whole job is over at this point: the real rows exist, so the
* `@step` row must stop being one of the things cleanup will try to revert.
* `reverted` is the honest terminal state for it — there is nothing left to undo
* that the rows it stood in for do not now cover — and it releases the
* idempotency key for a later run, which matters because keys are per step and a
* re-materialised step reuses its own.
*/
async function resolvePlaceholder(id) {
const result = await query(
`UPDATE event_run_resources
SET status = 'reverted', last_error = NULL
WHERE id = ? AND kind = ? AND status IN ('pending', 'confirmed')`,
[id, STEP_KIND],
)
return (result.affectedRows || 0) > 0
}
/**
* One row by its target, live or not — how a caller that lost the insert race
* finds the row it meant to write. Newest first, so a target that has been held
* and released several times answers with the current holder.
*/
async function findByTarget(owner, kind, ref) {
const [row] = await query(
`SELECT ${COLUMNS} FROM event_run_resources
WHERE owner_module = ? AND kind = ? AND ref = ?
ORDER BY id DESC LIMIT 1`,
[owner, kind, ref],
)
return hydrate(row) || null
}
/** One run's whole ledger, oldest first — the console's read. */
async function forRun(runId) {
const rows = await query(
`SELECT ${COLUMNS} FROM event_run_resources WHERE run_id = ? ORDER BY id`,
[runId],
)
return rows.map(hydrate)
}
/** The rows of one run that still want something: the cleanup sweep's input. */
async function unresolvedForRun(runId, { maxAttempts = null } = {}) {
const params = [runId, ...UNRESOLVED]
const attemptClause = maxAttempts === null ? '' : ' AND revert_attempts < ?'
if (maxAttempts !== null) params.push(maxAttempts)
const rows = await query(
`SELECT ${COLUMNS} FROM event_run_resources
WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)${attemptClause}
ORDER BY id`,
params,
)
return rows.map(hydrate)
}
/** How many of one run's rows are still unresolved — what `cleanup_status` is derived from. */
async function unresolvedCount(runId) {
const [row] = await query(
`SELECT COUNT(*) AS n FROM event_run_resources
WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`,
[runId, ...UNRESOLVED],
)
return Number(row?.n || 0)
}
/** Unresolved counts for several runs at once, keyed by run id — the run LIST's read. */
async function unresolvedCounts(runIds) {
const ids = [...new Set(runIds || [])].filter(Boolean)
if (!ids.length) return new Map()
const rows = await query(
`SELECT run_id, COUNT(*) AS n FROM event_run_resources
WHERE run_id IN (${ids.map(() => '?').join(',')}) AND status IN (?, ?, ?, ?, ?)
GROUP BY run_id`,
[...ids, ...UNRESOLVED],
)
return new Map(rows.map((r) => [r.run_id, Number(r.n)]))
}
/**
* Claim one row for a revert: `pending | confirmed | orphaned | drifted → reverting`.
*
* The compare-and-set that keeps the cleanup leg and the manual cleanup route off
* each other's rows. `reverting` is deliberately not claimable — a row another
* pass is mid-revert on is left alone, exactly as a step with a live claim is.
*/
async function claimRevert(id) {
const result = await query(
`UPDATE event_run_resources
SET status = 'reverting'
WHERE id = ? AND status IN ('pending', 'confirmed', 'orphaned', 'drifted')`,
[id],
)
return (result.affectedRows || 0) > 0
}
/** The revert worked. `reverted` is terminal and releases the target. */
async function markReverted(id) {
await query(
`UPDATE event_run_resources SET status = 'reverted', last_error = NULL WHERE id = ?`,
[id],
)
}
/**
* The revert did not work, and the row goes back to being unresolved.
*
* `revert_attempts` is incremented here and NOWHERE else, and it is never reset by
* a sweep — Engagement Phase 14's rule, whose defect was a reclaim that returned
* every stale row to its start state and made the attempt ceiling unreachable, so
* the row cycled for ever and was never eligible for any retention sweep. The one
* thing that may reset it is a human pressing cleanup, which is the same licence
* a human's step retry has.
*
* `restoreTo` is where the row lands: `drifted` when the module says somebody else
* moved the value, `orphaned` when it says the thing is gone, and `confirmed`
* otherwise — still ours, still out there, try again.
*/
async function failRevert(id, error, restoreTo = 'confirmed') {
await query(
`UPDATE event_run_resources
SET status = ?, revert_attempts = revert_attempts + 1, last_error = ?
WHERE id = ?`,
[restoreTo, String(error || 'the revert did not answer').slice(0, 500), id],
)
}
/**
* A human is trying again: clear the attempt counter on one run's unresolved rows.
*
* Only ever called from the cleanup route with an actor behind it. The automatic
* leg must never do this (see `failRevert`).
*/
async function resetAttempts(runId) {
const result = await query(
`UPDATE event_run_resources
SET revert_attempts = 0
WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`,
[runId, ...UNRESOLVED],
)
return result.affectedRows || 0
}
/** Every live row one module owns, for the reconcile sweep. */
async function liveForModule(owner, { limit = 500 } = {}) {
const rows = await query(
`SELECT ${COLUMNS} FROM event_run_resources
WHERE owner_module = ? AND status IN ('pending', 'confirmed')
ORDER BY id LIMIT ?`,
[owner, Number(limit)],
)
return rows.map(hydrate)
}
/** Every module that currently owns a live row — who the reconcile sweep asks. */
async function modulesWithLiveRows() {
const rows = await query(
`SELECT DISTINCT owner_module FROM event_run_resources
WHERE status IN ('pending', 'confirmed')`,
)
return rows.map((r) => r.owner_module)
}
/**
* The game no longer has it. Never reached by a revert — a revert that finds
* nothing there is a SUCCESS (§L, and it is what a Rust wipe needs) — only by
* reconcile, which is a different question: nobody asked for this to go.
*/
async function markOrphaned(id, detail = null) {
await query(
`UPDATE event_run_resources
SET status = 'orphaned', last_error = ?
WHERE id = ? AND status IN ('pending', 'confirmed', 'reverting')`,
[detail === null ? null : String(detail).slice(0, 500), id],
)
}
/**
* Terminal runs that still owe the world something — the cleanup leg's scan.
*
* **Both halves of the WHERE were live-walk findings, and they are opposite
* mistakes.**
*
* `cleanup_status = 'pending'` alone missed a run whose only resource was a
* LEASE: `core.lease` reserves its own row and never goes through the ledger's
* `markRunDirty`, so the flag stayed `not_required` and the lease was never given
* back at all. Hence `not_required` is in the list — a terminal run with an
* unresolved row has something to do whatever any summary column says, and
* treating that combination as work is the fail-safe direction.
*
* And the run status filter alone made `MAX_REVERT_ATTEMPTS` mean ONE attempt,
* because the first failing sweep set `incomplete` and nothing looked at the run
* again. That is fixed in `cleanupRun`, which now only writes `incomplete` once
* there is nothing left it will try — so `incomplete` genuinely means "finished
* with, and not finished", which is exactly what excludes both a run whose
* retries are spent and a run an admin cancelled without cleanup.
*
* The attempt bound is in the join for a different reason: without it a run whose
* rows are all spent would be selected, worked over and found to have nothing to
* do on every tick for the rest of its life.
*/
async function runsNeedingCleanup(limit = 25, maxAttempts = 3) {
return query(
`SELECT DISTINCT r.id, r.status, r.cleanup_status, r.version_id, r.definition_id, r.scope
FROM event_runs r
JOIN event_run_resources res ON res.run_id = r.id
WHERE r.status IN ('completed', 'cancelled', 'failed', 'missed')
AND r.cleanup_status IN ('pending', 'not_required')
AND res.status IN (?, ?, ?, ?, ?)
AND res.revert_attempts < ?
ORDER BY r.id
LIMIT ?`,
[...UNRESOLVED, Number(maxAttempts), Number(limit)],
)
}
module.exports = {
STEP_KIND,
HELD,
UNRESOLVED,
reserve,
confirm,
resolvePlaceholder,
findByTarget,
forRun,
unresolvedForRun,
unresolvedCount,
unresolvedCounts,
claimRevert,
markReverted,
failRevert,
resetAttempts,
liveForModule,
modulesWithLiveRows,
markOrphaned,
runsNeedingCleanup,
}

27
server/src/model/events/eventRuns.db.js
View File

@@ -386,6 +386,32 @@ async function setHealth(id, health) {
return Number(result?.affectedRows || 0) === 1
}
/**
* Set `cleanup_status`, optionally guarded on where it is now (Phase 8).
*
* Four values and three writers, which is why the guard is a parameter rather
* than baked in. The ledger stamps `pending` the first time a run records
* anything, and it must do so only over `not_required` — a run already marked
* `complete` must not be walked back to `pending` by a late resource, and a
* `incomplete` one must not be silently tidied. The cleanup sweep sets `complete`
* or `incomplete` from what it found, unguarded, because the sweep IS the
* authority on that. A human's cleanup route re-opens `pending` deliberately, and
* says so in the log with the actor.
*
* **`pending` on a run that is still running is not a bug and reads correctly**:
* there is something to clean up and it has not happened yet. The alternative -
* a fifth value meaning "there will be something later" - is a state nothing
* would ever branch on.
*/
async function setCleanupStatus(id, to, from = null) {
const guard = from === null ? '' : ` AND cleanup_status IN (${from.map(() => '?').join(',')})`
const result = await query(
`UPDATE event_runs SET cleanup_status = ? WHERE id = ?${guard}`,
from === null ? [to, id] : [to, id, ...from],
)
return Number(result?.affectedRows || 0) === 1
}
/**
* Runs whose start instant passed more than their own grace window ago (§E, §L).
*
@@ -488,6 +514,7 @@ module.exports = {
statusOf,
transition,
setHealth,
setCleanupStatus,
concurrencyHolder,
reclaimStale,
terminalBefore,

35
server/src/model/events/eventRuns.model.js
View File

@@ -27,6 +27,7 @@ const definitionsDb = require('./eventDefinitions.db')
const versionsDb = require('./eventVersions.db')
const settingsDb = require('./eventActionSettings.db')
const budgetDb = require('./eventRunBudget.db')
const resourcesDb = require('./eventRunResources.db')
const authorize = require('../../events/authorize')
const MAX_SCOPE = 190
@@ -204,11 +205,12 @@ async function create(
async function detail(runId) {
const run = await db.getById(runId)
if (!run) return null
const [steps, counts, gateRows, budget] = await Promise.all([
const [steps, counts, gateRows, budget, resources] = await Promise.all([
stepsDb.listForRun(runId),
stepsDb.statusCounts(runId),
gatesDb.listForRun(runId),
budgetDb.forRun(runId),
resourcesDb.forRun(runId),
])
const now = new Date()
return {
@@ -226,6 +228,37 @@ async function detail(runId) {
cap: b.cap,
from: b.effective_from,
})),
// What this run changed in the world, and what became of it (Phase 8). The
// WHOLE ledger, reverted rows included, because "what did last night's
// invasion actually spawn, and did all of it come back" is the question this
// panel exists for and a list of only the failures cannot answer the second
// half of it.
//
// **The `@step` placeholders are filtered out.** They are core's own
// bookkeeping — a row that says "a dispatch is in flight and may have made
// something" — and the console's list is of things in the world. One left in
// would read as a resource nobody can name, which is exactly the confusion it
// exists to prevent internally.
resources: resources
.filter((r) => r.kind !== resourcesDb.STEP_KIND)
.map((r) => ({
id: r.id,
stepId: r.step_id,
module: r.owner_module,
kind: r.kind,
ref: r.ref,
payload: r.payload,
leaseUntil: r.lease_until,
status: r.status,
revertAttempts: r.revert_attempts,
lastError: r.last_error,
memberKey: r.member_key,
createdAt: r.created_at,
})),
// How many rows are still unresolved, counted over the WHOLE ledger rather
// than over the list above — a placeholder left standing by a lost
// acknowledgement is exactly the case `cleanup_status` must not call clean.
unresolvedResources: resources.filter((r) => resourcesDb.UNRESOLVED.includes(r.status)).length,
}
}

33
server/src/modules/loader.js
View File

@@ -228,6 +228,39 @@ function buildCtx(id, moduleRoot) {
emit: (triggerId, envelope) => {
engagementEmit.emit(id, triggerId, envelope)
},
// EVENTS.md §L, and the resource ledger (Phase 8). "On reconnect the runner
// asks each ledgered resource's module to reconcile" — and this is how the
// runner learns there has BEEN a reconnect.
//
// **Core cannot decide when to call this, and that is the contract rather
// than a gap.** §F: core has no concept of the game being up, because a
// module with six sidecars cannot answer that question in the singular. So
// the module says so, when it sees its own — module-uo already watches
// `bootId` to tell a shard restart from a sidecar reconnect, which is
// exactly the moment a ledger of live spawns has become a claim about a
// world that no longer exists.
//
// `id` is bound here and never taken from the arguments, like `emit` and
// `teams.activity.push` before it: a module reconciles its OWN ledger, and
// without the binding this would be a way to have core mark another
// module's resources orphaned.
//
// Fire-and-forget and returns undefined, for the third time and the same
// reason: this is called from inside a connection handler, and there is
// nothing a module could correctly do with a failure of core's bookkeeping.
reconcile: () => {
// eslint-disable-next-line global-require
require('../events/cleanup')
.reconcileModule(id)
.then(
(summary) => {
if (summary && summary.orphaned) {
log.warn('event resources orphaned on reconcile', { module: id, ...summary })
}
},
(err) => { log.error('ctx.events.reconcile failed', { module: id, message: err.message }) },
)
},
},
// The in-app sink (§5.1) — a module writing the inbox directly, without a
// rule. Live from Phase 7; it threw until the `user_notifications` table

46
server/src/modules/registries.js
View File

@@ -157,11 +157,11 @@ const eventBudgets = new Map()
// lease id → { owner, id, label, type, min, max, maxDurationMs, description,
// read, apply, restore } (§F "Leases: one more declaration", Phase 7).
//
// **Phase 7 registers a lease and nothing acquires one.** Core owns the duration
// and the conflict check, the module owns reading the current value and writing a
// new one — and both halves of that live in the resource ledger, which is Phase
// 8's. What is here is the declaration, its validation and its catalog entry, so
// that the module contract is one version rather than two.
// **Core owns the duration and the conflict check; the module owns reading the
// current value and writing a new one.** Phase 7 registered a lease and nothing
// acquired one; Phase 8 gave it a verb — `core.lease`, a CORE action, so the
// bound and the two-events-one-target refusal are enforced in one place rather
// than re-implemented by every module that ships a lease.
const eventLeases = new Map()
// source id → { owner, id, label, description, resolve } (§F "Param option
@@ -435,14 +435,14 @@ async function resolveAudience(id, params = {}) {
/**
* Every declaration WITHOUT its callables — what the admin catalog serves.
*
* `perform`, `revert` and `cost` are stripped for the same reason `resolve` is
* `perform`, `revert`, `reconcile` and `cost` are stripped for the same reason `resolve` is
* stripped from an audience and `handler` from a slash command: this is the
* object that leaves the process, and the browser's whole relationship with an
* action is naming one by id. §F's "a module registers actions server-side and
* adds no routes for them" is only true if the functions never ride out.
*/
const allEventActions = () =>
[...eventActions.values()].map(({ perform, revert, cost, ...rest }) => rest)
[...eventActions.values()].map(({ perform, revert, reconcile, cost, ...rest }) => rest)
/** One declaration, callables included. The runner's lookup (Phase 2). */
const eventAction = (id) => eventActions.get(id) || null
@@ -485,7 +485,7 @@ const isEventBudget = (id) => eventBudgets.has(id)
const allEventLeases = () =>
[...eventLeases.values()].map(({ read, apply: applyValue, restore, ...rest }) => rest)
/** One lease, callables included. Phase 8's lookup; nothing calls it yet. */
/** One lease, callables included. `core.lease` and the cleanup sweep read it. */
const eventLease = (id) => eventLeases.get(id) || null
/** Every option source WITHOUT its resolver — the authoring form's list. */
@@ -985,7 +985,7 @@ function checkActionParam(actionId, entry, seen) {
}
/**
* `registerEventActions([{ id, label, risk, reversible, version, budgetMs, cost, params, perform, revert }])`.
* `registerEventActions([{ id, label, risk, reversible, version, budgetMs, cost, params, perform, revert, reconcile }])`.
*
* A typed verb core may ask a registrant to carry out. Everything decidable from
* the argument alone is decided here, at the call; the collision — is this id
@@ -1042,6 +1042,23 @@ function checkEventActionShape(entry) {
`registerEventActions: ${a.id} declares revert() but is reversible: '${a.reversible}'`,
)
}
// §L's reconnect row, and it is OPTIONAL where `revert` is required (Phase 8).
// `revert` is how a run gives a resource back; `reconcile` is how a module says
// which of them the game still has after something outside core restarted. A
// module that cannot answer that question is not broken -- core simply keeps
// believing its own ledger, which is the pre-Phase-8 behaviour -- whereas a
// module that created something and cannot undo it has made a promise core has
// no way to keep. Only meaningful for an action that ledgers anything.
if (a.reconcile !== undefined) {
if (typeof a.reconcile !== 'function') {
throw new Error(`registerEventActions: ${a.id} reconcile must be a function`)
}
if (a.reversible === 'none' || a.reversible === 'self') {
throw new Error(
`registerEventActions: ${a.id} declares reconcile() but is reversible: '${a.reversible}' and ledgers nothing`,
)
}
}
if (a.cost !== undefined && typeof a.cost !== 'function') {
throw new Error(`registerEventActions: ${a.id} cost must be a function of its params`)
}
@@ -1076,6 +1093,7 @@ function checkEventActionShape(entry) {
cost: a.cost || null,
perform: a.perform,
revert: a.revert || null,
reconcile: a.reconcile || null,
}
}
@@ -1130,10 +1148,12 @@ function checkEventBudgetShape(entry) {
* thing a module must not be allowed to skip. A lease whose restore writes blindly
* is a lease that silently reverts an operator's manual fix.
*
* **Nothing acquires a lease in Phase 7.** This registers, validates and serves
* one; the ledger that holds it, the deadline that goes down the wire and the
* drift answer are Phase 8's. Declaring it now is what keeps the module contract
* one version rather than two.
* **A lease is acquired by `core.lease` and by nothing else** (Phase 8). The step
* names a lease id, a value and a duration; core reads the baseline, reserves the
* target in `event_run_resources` — which is where the two-events-one-target
* refusal comes from — applies the value with the deadline, and restores it at
* teardown through the same `restore()` the drift check lives in. A module ships
* the three callables and never has to own any of that.
*/
function checkEventLeaseShape(entry) {
const l = entry || {}

59
server/src/router/v1/admin/events.controller.js
View File

@@ -9,13 +9,12 @@
// this screen does.
//
// **Phase 3 added the live run controls** at the bottom of this file: pause,
// resume, cancel, and a step's confirm, skip and retry. What is still absent is
// `advance`, `cleanup` and the action switchboard — `advance` has no honest
// meaning until Phase 5 gives a phase an advance condition, `cleanup` has no
// ledger to work over until Phase 8, and the switchboard is Phase 6's. Each of
// them is absent rather than stubbed, for the reason the whole set was in Phase
// 1: a control that returns 200 and does nothing is worse than one that is not
// there.
// resume, cancel, and a step's confirm, skip and retry. `advance` joined them in
// Phase 5, the action switchboard in Phase 6, and **`cleanup` in Phase 8** —
// each when the phase that gave it something to act on landed, and each absent
// rather than stubbed until then, for the reason the whole set was in Phase 1: a
// control that returns 200 and does nothing is worse than one that is not there.
// Nothing in the § API surface table is absent any more.
const registries = require('../../../modules/registries')
const spec = require('../../../events/spec')
@@ -315,6 +314,12 @@ exports.getRun = async (req, res) => {
// rather than "what is allowed now" — which is the question that survives
// an admin moving a switch tomorrow.
budget: found.budget,
// The resource ledger (Phase 8): everything this run created or borrowed, and
// what became of each. The WHOLE ledger, reverted rows included — "how much
// did last night's invasion spawn, and did all of it come back" is one
// question with two halves, and a list of only the failures answers neither.
resources: found.resources,
unresolvedResources: found.unresolvedResources,
})
}
@@ -678,14 +683,48 @@ exports.advanceRunPhase = async (req, res) => {
exports.cancelRun = async (req, res) => {
const runId = asId(req.params.runId)
if (!runId) return res.status(400).json({ error: 'bad run id' })
const result = await controls.cancel(runId, { reason: req.body?.reason }, req.user.id)
// **`cleanup` defaults to true and has to be asked out of.** §L makes cancelling
// WITHOUT cleanup the separate, admin-only, logged action, so an absent flag
// must mean "give back what this run took" — the safe direction, and the one a
// moderator's cancel at two in the morning takes without having to know the
// flag exists.
const withCleanup = req.body?.cleanup !== false
const result = await controls.cancel(
runId,
{ reason: req.body?.reason, cleanup: withCleanup },
req.user.id,
{ isAdmin: req.user.role === 'admin' },
)
if (!result.ok) return res.status(result.status || 409).json({ errors: result.errors })
await activity.log({
req,
action: 'event.run.cancelled',
detail: { runId, reason: req.body?.reason || null, cancelledSteps: result.cancelledSteps },
detail: {
runId,
reason: req.body?.reason || null,
cancelledSteps: result.cancelledSteps,
cleanup: result.cleanup,
},
})
return res.json({ run: shapeRun(result.run), cancelledSteps: result.cancelledSteps })
return res.json({
run: shapeRun(result.run),
cancelledSteps: result.cancelledSteps,
cleanup: result.cleanup,
})
}
/** POST /api/v1/admin/events/runs/:runId/cleanup */
exports.cleanupRun = async (req, res) => {
const runId = asId(req.params.runId)
if (!runId) return res.status(400).json({ error: 'bad run id' })
const result = await controls.cleanupRun(runId, req.user.id)
if (!result.ok) return res.status(result.status || 409).json({ errors: result.errors })
await activity.log({ req, action: 'event.run.cleaned', detail: { runId, ...result.summary } })
// **A 200 whatever the sweep found.** The request succeeded; some resources may
// still be out there, and answering 4xx would make "the shard refused to delete
// three of these" indistinguishable from "you sent a bad run id" — the same
// argument the dry run's findings make.
return res.json({ run: shapeRun(result.run), summary: result.summary })
}
/** POST /api/v1/admin/events/runs/:runId/steps/:stepId/confirm */

31
server/src/router/v1/admin/events.router.js
View File

@@ -17,8 +17,10 @@
// 5; **`verify` and the action switchboard arrived in Phase 6** — `verify` at
// `admin, editor` because a dry run dispatches nothing, and both halves of
// `/actions` at `admin`, because §K puts the switchboard in the same row as the
// world-changing actions it governs. `cleanup` is still absent rather than
// stubbed: there is no resource ledger until Phase 8.
// world-changing actions it governs. **`cleanup` completed the set in Phase 8**,
// and it is `admin` rather than admin+moderator for the same §K reason: it asks
// core to write to the world again, which is not incident response. There is no
// route in the § API surface table left absent.
//
// **Literal paths are declared before `/:id`**, so `/catalog`, `/series`,
// `/calendar` and `/runs` are never read as an event id.
@@ -207,9 +209,9 @@ eventsRouter.get(
'/runs/:runId',
// #swagger.tags = ['Admin · Events']
// #swagger.summary = 'One run: its status, health, cleanup state and every step with its params and idempotency key'
// #swagger.description = 'The run console. `counts` summarises the step list by status. Steps carry the idempotency key core minted at materialisation — stable across every attempt, which is what lets the game side recognise a repeat. `gates` is the diagnosis panel (Phase 5): one entry per phase that authored an advance condition, already rendered in the condition builder's own words — `gte` as "is at least", `present` as "is present" — with the tally, how long it has waited, and the last related firing whether or not it matched. A phase is waiting on its gate only once every one of its steps is terminal; `stalled` means an `on` gate has waited past EVENT_PHASE_STALL_MS, which is visibility and never a timeout — nothing advances a phase but its condition or a human.'
// #swagger.description = 'The run console. `counts` summarises the step list by status. Steps carry the idempotency key core minted at materialisation — stable across every attempt, which is what lets the game side recognise a repeat. `gates` is the diagnosis panel (Phase 5): one entry per phase that authored an advance condition, already rendered in the condition builder's own words — `gte` as "is at least", `present` as "is present" — with the tally, how long it has waited, and the last related firing whether or not it matched. A phase is waiting on its gate only once every one of its steps is terminal; `stalled` means an `on` gate has waited past EVENT_PHASE_STALL_MS, which is visibility and never a timeout — nothing advances a phase but its condition or a human. `budget` is the cap meter (Phase 6), and `resources` is the cleanup ledger (Phase 8): every object this run created and every value it borrowed, with what became of each — `confirmed` is still out there, `reverted` came back, `drifted` means somebody moved it and core left it alone, and `orphaned` means the module reports it is gone. `unresolvedResources` counts the ones still wanting something, including a placeholder left standing by a lost acknowledgement, which is why it can exceed the length of the list.'
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
/* #swagger.responses[200] = { description: 'The run, its steps, the status counts and the phase gates', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, steps: { type: "array", items: { type: "object", additionalProperties: true } }, counts: { type: "object", additionalProperties: true }, gates: { type: "array", items: { type: "object", additionalProperties: true } } } } } } } */
/* #swagger.responses[200] = { description: 'The run, its steps, the status counts, the phase gates, the cap meter and the resource ledger', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, steps: { type: "array", items: { type: "object", additionalProperties: true } }, counts: { type: "object", additionalProperties: true }, gates: { type: "array", items: { type: "object", additionalProperties: true } }, budget: { type: "array", items: { type: "object", additionalProperties: true } }, resources: { type: "array", items: { type: "object", additionalProperties: true } }, unresolvedResources: { type: "integer" } } } } } } */
/* #swagger.responses[404] = { description: 'No such run', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
controller.getRun,
)
@@ -273,16 +275,29 @@ eventsRouter.post(
'/runs/:runId/cancel',
// #swagger.tags = ['Admin · Events']
// #swagger.summary = 'Cancel a run'
// #swagger.description = 'Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` is not a parameter yet — the resource ledger it would work over arrives in Phase 8, and a flag that changes nothing is worse than one that is not there.'
// #swagger.description = 'Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` arrived in Phase 8 and DEFAULTS TO TRUE: what the run created or borrowed is given back by the runner cleanup leg on its next tick, which is why this answers at once rather than after a round trip per resource. Sending `cleanup: false` deliberately leaves the world changes in place — that is admin-only even though the route is admin+moderator, because which of the two you have to be depends on what is in the body — and the run then carries `cleanup_status: incomplete` with every unreverted row listed on its console.'
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
/* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { reason: { type: "string", description: "Why. Recorded on the run and in its log, with the actor." } } } } } } */
/* #swagger.responses[200] = { description: 'The cancelled run and how many steps were closed out with it', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, cancelledSteps: { type: "integer" } } } } } } */
/* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { reason: { type: "string", description: "Why. Recorded on the run and in its log, with the actor." }, cleanup: { type: "boolean", description: "Default true. False leaves the world changes from this run in place, and is admin-only." } } } } } } */
/* #swagger.responses[200] = { description: 'The cancelled run, how many steps were closed out with it, and whether cleanup was asked for', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, cancelledSteps: { type: "integer" }, cleanup: { type: "boolean" } } } } } } */
/* #swagger.responses[409] = { description: 'The run has already reached a terminal status', content: { "application/json": { schema: { type: "object", properties: { errors: { type: "array", items: { type: "string" } } } } } } } */
/* #swagger.responses[403] = { description: 'Not an admin or moderator', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
/* #swagger.responses[403] = { description: 'Not an admin or moderator, or a moderator asking to skip cleanup', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
liveControl,
controller.cancelRun,
)
eventsRouter.post(
'/runs/:runId/cleanup',
// #swagger.tags = ['Admin · Events']
// #swagger.summary = 'Re-run cleanup over everything this run has not given back'
// #swagger.description = 'The manual retry EVENTS.md §L promises, and the only thing that clears a resource attempt counter — the automatic sweep never does, because a sweep that reset every stale row is what made an attempt ceiling unreachable in the engagement workstream. Legal on a TERMINAL run only: a run still in flight has a ledger that is still growing, and reverting a resource the next step is about to use would be core undoing an event while it is happening. `admin` rather than admin+moderator, unlike the seven live controls beside it, because this is not incident response — it asks core to write to the world again, which §K puts in the same row as the world-changing actions themselves. Answers 200 whatever it found: some resources may still be out there, and a 4xx would make that indistinguishable from a bad run id.'
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
/* #swagger.responses[200] = { description: 'The run and what the sweep managed', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, summary: { type: "object", properties: { attempted: { type: "integer" }, reverted: { type: "integer" }, drifted: { type: "integer" }, failed: { type: "integer" }, remaining: { type: "integer" } } } } } } } } */
/* #swagger.responses[409] = { description: 'The run is still in flight, or recorded no resources at all', content: { "application/json": { schema: { type: "object", properties: { errors: { type: "array", items: { type: "string" } } } } } } } */
/* #swagger.responses[403] = { description: 'Not an admin', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
adminOnly,
controller.cleanupRun,
)
eventsRouter.post(
'/runs/:runId/advance',
// #swagger.tags = ['Admin · Events']

21
server/src/server.js
View File

@@ -16,6 +16,7 @@ const teamForumUploadSweep = require('./utils/teamForumUploadSweep')
const teamDigestWorker = require('./utils/teamDigestWorker')
const engagementWorker = require('./utils/engagementWorker')
const eventRunner = require('./utils/eventRunner')
const eventCleanup = require('./events/cleanup')
const { ensureSchema, close } = require('./utils/db')
const { seedDefaults, createInitialAdminFromEnv } = require('../db/seed')
const settings = require('./model/settings/settings.model')
@@ -170,11 +171,25 @@ async function start() {
// enables a rule: core seeds none and `enabled` defaults to 0.
engagementWorker.start()
// Advance scheduled events (EVENTS.md §E). Materialise, advance, drain, and the
// run-log retention sweep. No-op until an admin publishes a definition and
// starts a run: core ships no event definitions.
// Advance scheduled events (EVENTS.md §E). Materialise, advance, drain, clean
// up and the run-log retention sweep. No-op until an admin publishes a
// definition and starts a run: core ships no event definitions.
eventRunner.start()
// **Core's own restart is the one reconnect core can see** (EVENTS.md §L,
// Phase 8). Every other one belongs to a module, which reports it through
// `ctx.events.reconcile()`; this is the case where the thing that restarted was
// this process, and the ledger it wakes up holding may describe a world that
// moved on while it was down. Awaited by nobody and never fatal: a module that
// cannot answer leaves its rows alone, which is the pre-Phase-8 behaviour.
eventCleanup
.reconcileAll()
.then((summaries) => {
const orphaned = Object.values(summaries).reduce((n, x) => n + (x.orphaned || 0), 0)
if (orphaned) log.warn('event resources orphaned at boot', { orphaned, summaries })
})
.catch((err) => log.error('boot reconcile failed', { message: err.message }))
setupShutdown(server, internalServer)
}

88
server/src/utils/eventRunner.js
View File

@@ -10,7 +10,16 @@
// 1. **reclaim** — release leases whose holder died, never touching `attempts`
// 2. **materialise** — sweep occurrences past their grace window into `missed`
// 3. **advance** — claim each due run and move it through its phases
// 4. **prune** — the `event_run_log` retention sweep, on its own long clock
// 4. **cleanup** — give back what a TERMINAL run still holds (Phase 8)
// 5. **prune** — the `event_run_log` retention sweep, on its own long clock
//
// **Cleanup is a leg rather than a limb of `advanceRun`**, and its position in
// that list is load-bearing: it runs after advance, so a run that completes in
// one tick is torn down in the same one, and it is one place rather than four, so
// a process that dies mid-teardown resumes on the next tick instead of leaving a
// world half-restored with nothing scheduled to finish it. §L's "cleanup steps
// are generated from the ledger and run" on cancellation and abort as well as on
// completion is one query here rather than a hook on each of the three.
//
// **What a phase advances on, as of Phase 5.** Every step terminal, and — if the
// phase authored one — its GATE open as well. The gate is an ADDITIONAL
@@ -69,6 +78,8 @@ const gates = require('../events/gates')
const spec = require('../events/spec')
const registries = require('../modules/registries')
const { dispatchStep } = require('../events/dispatch')
const ledger = require('../events/ledger')
const cleanup = require('../events/cleanup')
const authorize = require('../events/authorize')
const log = require('./logger')('event-runner')
@@ -261,6 +272,27 @@ async function drainStep(run, step, now, carry = {}) {
}
}
// ── Rule 1: record BEFORE the dispatch, not after ──
//
// §D. A step that ledgers gets a placeholder keyed by its idempotency key,
// written before anything reaches the module, so an answer that never comes
// back still leaves cleanup something to act on. Recording afterwards would
// make every object whose acknowledgement was lost invisible for ever, which is
// the one failure the whole world-write half cannot tolerate.
//
// It is deliberately AFTER the permission check: a refused step never reaches
// the module, so it has created nothing and must ledger nothing.
let placeholderId = null
try {
placeholderId = await ledger.reserveStep(run, step, action)
} catch (err) {
// The ledger is what makes a world write recoverable, so a step that cannot
// be recorded must not be dispatched. Transient — the next attempt tries the
// insert again — because the alternative is an unrecorded world change.
log.error('could not reserve the ledger row', { run: run.id, step: step.id, message: err.message })
return applyFailure(run, step, `the resource ledger could not record this step: ${err.message}`)
}
const result = await dispatchStep(step, { run })
if (result.actionVersionDrift) {
@@ -273,6 +305,36 @@ async function drainStep(run, step, now, carry = {}) {
})
}
// ── …and promote it on the answer ──
//
// On both success shapes, because `await: 'human'` is a SUCCESS: the module did
// its part and something outside the system has to happen next, and a cue's
// confirm finishes the step without a second dispatch — so this is the only
// moment its resources can be recorded. A failure records nothing and leaves the
// placeholder standing, which is the whole point of writing one.
if (result.outcome === 'done' || result.outcome === 'parked') {
const recorded = await ledger.recordAnswer({
run,
step,
action,
placeholderId,
resources: result.resources,
})
if (recorded.recorded > 0 || recorded.rejected.length > 0) {
await logDb.write({
runId: run.id,
stepId: step.id,
kind: 'resource.recorded',
phase: step.phase,
detail: {
action: step.action_id,
recorded: recorded.recorded,
...(recorded.rejected.length ? { rejected: recorded.rejected } : {}),
},
})
}
}
if (result.outcome === 'parked') {
// The GM cue. The step stays `running` with a NULL lease: genuinely in
// flight, nothing holding it, so the stale reclaim passes it by and a cue
@@ -459,9 +521,11 @@ async function advanceRun(run, now) {
}
if (run.status === 'ending') {
// A run that reached the wind-down and then lost its process. Phase 8 puts
// cleanup here; until then `ending` is a state a run passes through rather
// than one it does work in, and completing it is the whole recovery.
// A run that reached the wind-down and then lost its process. Completing it is
// still the whole recovery even with a ledger in the picture: cleanup is a leg
// of the tick over TERMINAL runs, so the row this leaves behind is exactly
// what that leg is looking for, and doing the teardown here as well would be
// the second call site the leg exists to avoid.
await runsDb.transition(run.id, 'ending', 'completed')
await logDb.write({ runId: run.id, kind: 'run.status', detail: { from: 'ending', to: 'completed' } })
return 'completed'
@@ -536,8 +600,9 @@ async function advanceRun(run, now) {
const next = phases[phaseIndex + 1]
if (!next) {
// §E's `ending` exists for the reason `sending` does in the outbox — it is
// what a claim sets — so the run passes through it even though Phase 2 has
// no cleanup to do there. Phase 8 is what gives it work.
// what a claim sets. The run still passes straight through it: the ledger's
// teardown is the tick's cleanup leg, which runs after this one in the same
// tick and takes the run as it now is.
if (!(await runsDb.transition(run.id, 'running', 'ending'))) return 'taken'
await logDb.write({ runId: run.id, kind: 'run.status', phase: phaseKey, detail: { from: 'running', to: 'ending' } })
await runsDb.transition(run.id, 'ending', 'completed')
@@ -790,6 +855,17 @@ async function tick(now = new Date()) {
}
if (due && due.length) log.info('event runs swept', { due: due.length, ...counts })
// **After advance, deliberately.** A run that reached `completed` two lines ago
// is torn down in this tick rather than the next, so §L's "a run reaches
// `completed` with `cleanup_status = 'incomplete'`" is what an operator sees
// instead of a completed run that briefly claims it has cleanup pending.
try {
const swept = await cleanup.sweep()
if (swept) log.info('event cleanup swept', { runs: swept })
} catch (err) {
log.error('cleanup sweep failed', { message: err.message })
}
try {
await prune(now)
} catch (err) {

128
server/swagger/swagger-output.json
View File

@@ -4137,7 +4137,7 @@
],
"responses": {
"200": {
"description": "The run, its steps, the status counts and the phase gates",
"description": "The run, its steps, the status counts, the phase gates, the cap meter and the resource ledger",
"content": {
"application/json": {
"schema": {
@@ -4164,6 +4164,23 @@
"type": "object",
"additionalProperties": true
}
},
"budget": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": true
}
},
"resources": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": true
}
},
"unresolvedResources": {
"type": "integer"
}
}
}
@@ -4295,7 +4312,7 @@
"Admin · Events"
],
"summary": "Cancel a run",
"description": "Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` is not a parameter yet — the resource ledger it would work over arrives in Phase 8, and a flag that changes nothing is worse than one that is not there.",
"description": "Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` arrived in Phase 8 and DEFAULTS TO TRUE: what the run created or borrowed is given back by the runner cleanup leg on its next tick, which is why this answers at once rather than after a round trip per resource. Sending `cleanup: false` deliberately leaves the world changes in place — that is admin-only even though the route is admin+moderator, because which of the two you have to be depends on what is in the body — and the run then carries `cleanup_status: incomplete` with every unreverted row listed on its console.",
"parameters": [
{
"name": "runId",
@@ -4308,7 +4325,7 @@
],
"responses": {
"200": {
"description": "The cancelled run and how many steps were closed out with it",
"description": "The cancelled run, how many steps were closed out with it, and whether cleanup was asked for",
"content": {
"application/json": {
"schema": {
@@ -4320,6 +4337,9 @@
},
"cancelledSteps": {
"type": "integer"
},
"cleanup": {
"type": "boolean"
}
}
}
@@ -4330,7 +4350,7 @@
"description": "Bad Request"
},
"403": {
"description": "Not an admin or moderator",
"description": "Not an admin or moderator, or a moderator asking to skip cleanup",
"content": {
"application/json": {
"schema": {
@@ -4376,6 +4396,10 @@
"reason": {
"type": "string",
"description": "Why. Recorded on the run and in its log, with the actor."
},
"cleanup": {
"type": "boolean",
"description": "Default true. False leaves the world changes from this run in place, and is admin-only."
}
}
}
@@ -4384,6 +4408,102 @@
}
}
},
"/api/v1/admin/events/runs/{runId}/cleanup": {
"post": {
"tags": [
"Admin · Events"
],
"summary": "Re-run cleanup over everything this run has not given back",
"description": "The manual retry EVENTS.md §L promises, and the only thing that clears a resource attempt counter — the automatic sweep never does, because a sweep that reset every stale row is what made an attempt ceiling unreachable in the engagement workstream. Legal on a TERMINAL run only: a run still in flight has a ledger that is still growing, and reverting a resource the next step is about to use would be core undoing an event while it is happening. `admin` rather than admin+moderator, unlike the seven live controls beside it, because this is not incident response — it asks core to write to the world again, which §K puts in the same row as the world-changing actions themselves. Answers 200 whatever it found: some resources may still be out there, and a 4xx would make that indistinguishable from a bad run id.",
"parameters": [
{
"name": "runId",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "The run and what the sweep managed",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"run": {
"type": "object",
"additionalProperties": true
},
"summary": {
"type": "object",
"properties": {
"attempted": {
"type": "integer"
},
"reverted": {
"type": "integer"
},
"drifted": {
"type": "integer"
},
"failed": {
"type": "integer"
},
"remaining": {
"type": "integer"
}
}
}
}
}
}
}
},
"400": {
"description": "Bad Request"
},
"403": {
"description": "Not an admin",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
},
"409": {
"description": "The run is still in flight, or recorded no resources at all",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"errors": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
}
}
},
"security": [
{
"cookieAuth": []
},
{
"bearerAuth": []
}
]
}
},
"/api/v1/admin/events/runs/{runId}/log": {
"get": {
"tags": [

85
server/test/eventActionRegistry.test.js
View File

@@ -44,10 +44,15 @@ const register = (owner, entries) => {
registries.apply(api.staged)
}
test('core registers its three actions on every boot', () => {
test('core registers its four actions on every boot', () => {
registries.registerCore()
const ids = registries.allEventActions().map((a) => a.id)
assert.deepEqual(ids, ['core.announce', 'core.wait', 'core.cue'])
// `core.lease` joined the three in Phase 8, and it is the only one of the four
// that genuinely changes the world — which is why it is core's rather than each
// module's: §F puts the duration bound and the two-events-one-target conflict
// check on core's side of the seam, and a lease verb per module would be that
// bound re-implemented once per module and advisory everywhere.
assert.deepEqual(ids, ['core.announce', 'core.wait', 'core.cue', 'core.lease'])
assert.equal(ids.length, coreEventActions.ACTIONS.length)
})
@@ -56,6 +61,7 @@ test('the catalog carries no callable', () => {
for (const action of registries.allEventActions()) {
assert.equal(action.perform, undefined, `${action.id} leaked perform`)
assert.equal(action.revert, undefined, `${action.id} leaked revert`)
assert.equal(action.reconcile, undefined, `${action.id} leaked reconcile`)
assert.equal(action.cost, undefined, `${action.id} leaked cost`)
}
// …and the runner's own lookup still has it, which is the half that makes the
@@ -242,7 +248,7 @@ test('a whole batch is refused or taken, never half', () => {
test('_reset() hands the process back', () => {
registries.registerCore()
assert.equal(registries.allEventActions().length, 3)
assert.equal(registries.allEventActions().length, 4)
registries._reset()
assert.equal(registries.allEventActions().length, 0)
assert.equal(registries.isEventAction('core.wait'), false)
@@ -366,9 +372,80 @@ test('an option source needs a resolver, and core registers one of its own', ()
// Core through the same door (Phase 7): `core.announce`'s `leg` param names a
// source, and the announce legs are already a registry with labels in them.
registries.registerCore()
assert.deepEqual(registries.allEventOptionSources().map((s) => s.id), ['core.options.legs'])
assert.deepEqual(registries.allEventOptionSources().map((s) => s.id), [
'core.options.legs',
// Phase 8's, and it is the same argument one phase on: `core.lease`'s `lease`
// param would otherwise be a free-text box whose typo is caught at dispatch,
// mid-run — and the leases are already a registry with labels in them.
'core.options.leases',
])
const leg = registries.eventAction('core.announce').params.find((p) => p.name === 'leg')
assert.equal(leg.source, 'core.options.legs')
const which = registries.eventAction('core.lease').params.find((p) => p.name === 'lease')
assert.equal(which.source, 'core.options.leases')
})
// ── `reconcile`, the one member Phase 8 added to the action shape ──────────
//
// Optional where `revert` is required, and the asymmetry is the design: a module
// that cannot say what the game still has is not broken — core keeps believing
// its own ledger, which is the behaviour before this phase — whereas a module
// that created something and cannot undo it has made a promise core has no way
// to keep.
test('reconcile is optional, must be a function, and only on an action that ledgers', () => {
const ledgering = {
id: 'demo.spawn',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
perform: async () => ({ ok: true }),
revert: async () => ({ ok: true }),
}
// Absent is legal, and it lands as an explicit null rather than as a missing
// key — the same shape `revert` and `cost` take, so the catalog's strip list
// and the sweep's `typeof` check both have something to look at.
register('demo', [ledgering])
assert.equal(registries.eventAction('demo.spawn').reconcile, null)
registries._reset()
assert.throws(
() => register('demo', [{ ...ledgering, reconcile: 'yes please' }]),
/reconcile must be a function/,
)
// The mirror check `revert` already has. An action that ledgers nothing has no
// rows for core to ask about, so a `reconcile` on one is an author who believes
// something is being tracked and a sweep that will never call it.
assert.throws(
() =>
register('demo', [
{
id: 'demo.shout',
label: 'Shout',
risk: 'notify',
reversible: 'none',
perform: async () => ({ ok: true }),
reconcile: async () => ({ ok: true, inForce: [] }),
},
]),
/declares reconcile\(\) but is reversible: 'none' and ledgers nothing/,
)
})
test('an override action may reconcile, because a lease is ledgered too', () => {
register('demo', [
{
id: 'demo.borrow',
label: 'Borrow',
risk: 'change',
reversible: 'override',
perform: async () => ({ ok: true }),
reconcile: async () => ({ ok: true, inForce: [] }),
},
])
assert.equal(typeof registries.eventAction('demo.borrow').reconcile, 'function')
})
test('_reset() hands back the three new registries too', () => {

500
server/test/eventCleanup.test.js Normal file
View File

@@ -0,0 +1,500 @@
// ── Giving back what a run took (EVENTS_PLAN.md Phase 8) ───────────────────
//
// The phase's shipped claim: **cleanup is generated from the ledger and runs on
// every terminal path.** An operator cannot be relied on to write the undo, and
// an aborted run never reaches the phase they wrote it in — so there is no
// cleanup phase in a spec, no `on_teardown` on an action, and one function that
// reads rows.
//
// The properties around it are §L's, and two of them are ones this codebase has
// already paid for once:
//
// • **a revert that never succeeds stays visible rather than cycling** — rule 2,
// and `MAX_REVERT_ATTEMPTS` is what stops the automatic retry. Only a human
// clears the counter, which is Engagement Phase 14's rule stated a third time
// • **reverting something that does not exist is a SUCCESS** — §L, and what a
// Rust wipe needs
// • **drift is not an error.** The module did exactly what it was asked and
// found somebody else's value in place. A restore that wrote anyway would
// silently revert an operator's manual fix
// • **a resource the module no longer has becomes `orphaned`, never `reverted`** —
// reverting it would be core recording that it put something back when what
// happened is that the thing vanished
// • **"I do not know" is never read as "it is gone".** Every unanswerable
// reconcile leaves the ledger alone
//
// The sweep is driven against a stubbed db layer, exactly as `eventRunner.test.js`
// drives the runner: what a stub cannot prove is the SQL, and the unique key that
// makes two events unable to lease one target runs against a real MariaDB in
// `eventRunnerSql.test.js`.
process.env.DB_HOST = '127.0.0.1'
process.env.DB_PORT = '59999'
const { test, beforeEach, afterEach, after } = require('node:test')
const assert = require('node:assert/strict')
const registries = require('../src/modules/registries')
const cleanup = require('../src/events/cleanup')
const resourcesDb = require('../src/model/events/eventRunResources.db')
const runsDb = require('../src/model/events/eventRuns.db')
const stepsDb = require('../src/model/events/eventRunSteps.db')
const logDb = require('../src/model/events/eventRunLog.db')
const db = require('../src/utils/db')
after(() => db.close())
const UNRESOLVED = ['pending', 'confirmed', 'reverting', 'orphaned', 'drifted']
let store
const originals = {
resourcesDb: { ...resourcesDb },
runsDb: { ...runsDb },
stepsDb: { ...stepsDb },
logDb: { ...logDb },
}
beforeEach(() => {
registries._reset()
store = { rows: new Map(), steps: new Map(), log: [], next: 1, cleanupStatus: 'pending' }
resourcesDb.unresolvedForRun = async (runId, { maxAttempts = null } = {}) =>
[...store.rows.values()]
.filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status))
.filter((r) => maxAttempts === null || r.revert_attempts < maxAttempts)
.map((r) => ({ ...r }))
resourcesDb.unresolvedCount = async (runId) =>
[...store.rows.values()].filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status)).length
resourcesDb.claimRevert = async (id) => {
const r = store.rows.get(id)
if (!r || !['pending', 'confirmed', 'orphaned', 'drifted'].includes(r.status)) return false
r.status = 'reverting'
return true
}
resourcesDb.markReverted = async (id) => {
const r = store.rows.get(id)
if (r) Object.assign(r, { status: 'reverted', last_error: null })
}
resourcesDb.failRevert = async (id, error, restoreTo = 'confirmed') => {
const r = store.rows.get(id)
if (r) Object.assign(r, { status: restoreTo, revert_attempts: r.revert_attempts + 1, last_error: String(error) })
}
resourcesDb.resetAttempts = async (runId) => {
let n = 0
for (const r of store.rows.values()) {
if (r.run_id === runId && UNRESOLVED.includes(r.status)) {
r.revert_attempts = 0
n += 1
}
}
return n
}
resourcesDb.markOrphaned = async (id, detail = null) => {
const r = store.rows.get(id)
if (r && ['pending', 'confirmed', 'reverting'].includes(r.status)) {
Object.assign(r, { status: 'orphaned', last_error: detail })
}
}
resourcesDb.liveForModule = async (owner) =>
[...store.rows.values()].filter((r) => r.owner_module === owner && ['pending', 'confirmed'].includes(r.status)).map((r) => ({ ...r }))
resourcesDb.modulesWithLiveRows = async () => [
...new Set([...store.rows.values()].filter((r) => ['pending', 'confirmed'].includes(r.status)).map((r) => r.owner_module)),
]
resourcesDb.runsNeedingCleanup = async () => store.candidates || []
runsDb.setCleanupStatus = async (id, to, from = null) => {
if (from && !from.includes(store.cleanupStatus)) return false
store.cleanupStatus = to
return true
}
stepsDb.getById = async (id) => store.steps.get(id) || null
logDb.write = async (entry) => {
store.log.push(entry)
}
})
afterEach(() => {
Object.assign(resourcesDb, originals.resourcesDb)
Object.assign(runsDb, originals.runsDb)
Object.assign(stepsDb, originals.stepsDb)
Object.assign(logDb, originals.logDb)
registries._reset()
})
const RUN = { id: 7, status: 'completed', cleanup_status: 'pending' }
function addStep(id, actionId, key = 'k'.repeat(40)) {
store.steps.set(id, { id, run_id: RUN.id, action_id: actionId, idempotency_key: key, phase: 'p', seq: 0 })
}
function addResource(over = {}) {
const id = store.next++
const row = {
id,
run_id: RUN.id,
step_id: 1,
owner_module: 'demo',
kind: 'creature',
ref: `0x${id}`,
payload: null,
lease_until: null,
status: 'confirmed',
revert_attempts: 0,
last_error: null,
member_key: null,
...over,
}
store.rows.set(id, row)
return row
}
function registerAction(over = {}) {
const api = registries.stage('demo')
api.registerEventActions([
{
id: 'demo.spawn',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
budgetMs: 500,
perform: async () => ({ ok: true }),
revert: async () => ({ ok: true }),
...over,
},
])
registries.apply(api.staged)
}
function registerLease(over = {}) {
const api = registries.stage('demo')
api.registerEventLeases([
{
id: 'demo.rate',
label: 'Gather rate',
type: 'float',
min: 0.5,
max: 5,
maxDurationMs: 3_600_000,
read: async () => ({ ok: true, value: 1 }),
apply: async () => ({ ok: true }),
restore: async () => ({ ok: true }),
...over,
},
])
registries.apply(api.staged)
}
const kinds = () => store.log.map((e) => e.kind)
// ── The shipped claim ──────────────────────────────────────────────────────
test('a run\'s ledger is given back in one call per step, and the run goes complete', async () => {
// `revert` takes a LIST because that is what makes twelve creatures one round
// trip rather than twelve. The grouping key is the STEP, because the resource
// row records the module and the opaque names while the step records the verb.
let seen = null
registerAction({ revert: async (arg) => { seen = arg; return { ok: true } } })
addStep(1, 'demo.spawn', 'key-one')
addResource({ ref: '0xA' })
addResource({ ref: '0xB' })
const summary = await cleanup.cleanupRun(RUN)
assert.deepEqual(summary, { attempted: 2, reverted: 2, drifted: 0, failed: 0, remaining: 0 })
assert.equal(seen.runId, RUN.id)
assert.equal(seen.idempotencyKey, 'key-one')
assert.deepEqual(seen.resources.map((r) => r.ref), ['0xA', '0xB'])
assert.equal(store.cleanupStatus, 'complete')
})
test('two steps are two calls, and one failing does not take the other down', async () => {
const calls = []
registerAction({
revert: async ({ idempotencyKey, resources }) => {
calls.push(idempotencyKey)
return idempotencyKey === 'bad' ? { ok: false, error: 'the shard did not answer' } : { ok: true, resources }
},
})
addStep(1, 'demo.spawn', 'good')
addStep(2, 'demo.spawn', 'bad')
addResource({ step_id: 1 })
addResource({ step_id: 2 })
const summary = await cleanup.cleanupRun(RUN)
assert.equal(calls.length, 2)
assert.equal(summary.reverted, 1)
assert.equal(summary.failed, 1)
assert.equal(summary.remaining, 1)
// **Still `pending`, because the failed row has retries left.** `incomplete`
// means "finished with, and not finished" — it is what takes a run out of the
// sweep's own scan, so writing it after the FIRST failure made
// `MAX_REVERT_ATTEMPTS` quietly mean one attempt. Found by watching
// `revert_attempts` sit at 1 through half a minute of live ticks.
assert.equal(store.cleanupStatus, 'pending')
})
test('reverting something that does not exist is a success', async () => {
// §L, and the Rust wipe: "gone, and that is fine". The module never has to
// distinguish "I deleted it" from "it was not there" — which is also what makes
// a placeholder for an object that may never have existed safe to write.
registerAction({ revert: async () => ({ ok: true, detail: 'resource no longer exists' }) })
addStep(1, 'demo.spawn')
addResource()
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.reverted, 1)
assert.equal(store.cleanupStatus, 'complete')
})
test('a module may name the ones that did not come back', async () => {
// Partial cleanup is the ordinary case — eleven of twelve creatures deleted —
// and it is why the ledger is a row per object rather than a row per step.
registerAction({ revert: async () => ({ ok: true, failed: ['0x2'] }) })
addStep(1, 'demo.spawn')
addResource({ ref: '0x1' })
addResource({ ref: '0x2' })
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.reverted, 1)
assert.equal(summary.failed, 1)
assert.match([...store.rows.values()].find((r) => r.ref === '0x2').last_error, /could not give "0x2" back/)
})
// ── Rule 2: loud and sticky ────────────────────────────────────────────────
test('a revert that never works stops retrying and stays visible', async () => {
let calls = 0
registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } })
addStep(1, 'demo.spawn')
addResource()
for (let i = 0; i < 6; i++) await cleanup.cleanupRun(RUN)
// Bounded at MAX_REVERT_ATTEMPTS, exactly like a step's attempts. A fourth ask
// of a shard that has answered the same way three times is not new information,
// and an unbounded counter is a row nothing can ever sweep.
assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS)
const row = [...store.rows.values()][0]
assert.equal(row.revert_attempts, cleanup.MAX_REVERT_ATTEMPTS)
assert.equal(row.status, 'confirmed')
assert.equal(row.last_error, 'nope')
// And ONLY now, with nothing left to try, does the run stop being the sweep's
// business. §L: it does not stay `running` — an event whose world changes are
// still up is a real state, and pretending the event is in progress hides it.
assert.equal(store.cleanupStatus, 'incomplete')
})
test('the run goes back to pending each time it still has an attempt left', async () => {
// The other half of the same rule, watched one pass at a time rather than at
// the end. Each of the first two sweeps leaves the run in the scan; the third
// takes it out. A test that only looked at the end state would pass against the
// defect this replaced.
registerAction({ revert: async () => ({ ok: false, error: 'nope' }) })
addStep(1, 'demo.spawn')
addResource()
const seen = []
for (let i = 0; i < 3; i++) {
await cleanup.cleanupRun(RUN)
seen.push(store.cleanupStatus)
}
assert.deepEqual(seen, ['pending', 'pending', 'incomplete'])
})
test('only a human clears the attempt counter', async () => {
// Engagement Phase 14's defect, stated a third time: a SWEEP that returned every
// stale row to its start state made the attempt ceiling unreachable, so the row
// cycled for ever and was never eligible for any retention sweep. The automatic
// leg must never do this; the cleanup route may, because a person asked.
let calls = 0
registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } })
addStep(1, 'demo.spawn')
addResource()
for (let i = 0; i < 5; i++) await cleanup.cleanupRun(RUN)
assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS)
await cleanup.cleanupRun(RUN, { resetAttempts: true, actor: 9 })
assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS + 1)
assert.ok(store.log.some((e) => e.kind === 'cleanup.retry' && e.detail.by === 9))
})
test('a module that throws from revert is a transient failure, not a crashed sweep', async () => {
registerAction({ revert: async () => { throw new Error('socket hung up') } })
addStep(1, 'demo.spawn')
addResource()
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.failed, 1)
assert.match([...store.rows.values()][0].last_error, /socket hung up/)
})
test('no shape a revert failure can take reads as success', () => {
// `dispatch.classify`'s rule, applied to the other direction of the contract.
// The expensive mistake here is the mirror of the one there: recording that a
// world change was UNDONE when it was not.
for (const raw of [null, undefined, 'ok', [], {}, { ok: 'yes' }, { ok: 1 }]) {
assert.notEqual(cleanup.classifyRevert(raw, 'demo.spawn').outcome, 'done', JSON.stringify(raw))
}
assert.equal(cleanup.classifyRevert({ __timedOut: true, error: 'slow' }, 'x').outcome, 'retry')
assert.equal(cleanup.classifyRevert({ ok: false, retry: false, error: 'never' }, 'x').outcome, 'terminal')
assert.equal(cleanup.classifyRevert({ ok: true }, 'x').outcome, 'done')
})
test('an action whose module is gone leaves its rows unresolved with the reason', async () => {
// Not a retry — nothing will change until an operator reinstalls it — and not
// an orphan either, because core has no idea whether the thing is still there.
addStep(1, 'demo.spawn')
addResource()
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.failed, 1)
assert.match([...store.rows.values()][0].last_error, /no module registers "demo.spawn"/)
// Retried like any other failure rather than given up on at once, and that is
// the right uniformity here: "nothing registers this" stops being true the
// moment an operator reinstalls the module, and three registry lookups cost
// nothing. So it is `pending` until the attempts are spent.
assert.equal(store.cleanupStatus, 'pending')
})
// ── Leases ─────────────────────────────────────────────────────────────────
test('a lease is restored through the LEASE registry, not through any action', async () => {
// The split §F draws: core owns the duration and the conflict check, the module
// owns reading and writing. It is why `core.lease` needs no `revert()` of its
// own, and why an `override` row routes here rather than to its step's action.
let seen = null
registerLease({ restore: async (baseline, opts) => { seen = { baseline, opts }; return { ok: true } } })
addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } })
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.reverted, 1)
assert.equal(seen.baseline, 1)
// The drift check's input. `restore` MUST verify current === expected before
// writing, and a lease whose restore wrote blindly would silently revert an
// operator's manual fix.
assert.equal(seen.opts.expected, 3)
})
test('drift is not an error: the world is left alone and the row says so', async () => {
registerLease({ restore: async () => ({ ok: false, drifted: true, current: 4.5 }) })
addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } })
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.drifted, 1)
assert.equal(summary.failed, 0)
const row = [...store.rows.values()][0]
assert.equal(row.status, 'drifted')
assert.match(row.last_error, /now 4\.5 rather than what this run applied/)
// Still surfaced. §L: "surfaced beside the unreverted ones" — the run does not
// get to call itself clean because somebody else took the value. It is `pending`
// rather than `incomplete` for one more reason worth keeping: drift is retried
// like any other failure, because a GM who puts the value back between two ticks
// should have the lease close cleanly.
assert.equal(store.cleanupStatus, 'pending')
})
test('a lease whose module is uninstalled is unresolved, never assumed restored', async () => {
addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } })
const summary = await cleanup.cleanupRun(RUN)
assert.equal(summary.failed, 1)
assert.match([...store.rows.values()][0].last_error, /no module registers the lease "demo.rate"/)
})
// ── The sweep ──────────────────────────────────────────────────────────────
test('the sweep only touches TERMINAL runs', async () => {
// A run still in flight has a ledger that is still growing, and reverting a
// resource the next step is about to use would be core undoing an event while
// it is happening.
registerAction()
addStep(1, 'demo.spawn')
addResource()
store.candidates = [{ id: RUN.id, status: 'running', cleanup_status: 'pending' }]
assert.equal(await cleanup.sweep(), 0)
assert.equal([...store.rows.values()][0].status, 'confirmed')
store.candidates = [{ id: RUN.id, status: 'cancelled', cleanup_status: 'pending' }]
assert.equal(await cleanup.sweep(), 1)
assert.equal([...store.rows.values()][0].status, 'reverted')
})
// ── Reconcile ──────────────────────────────────────────────────────────────
test('a resource the module no longer has becomes orphaned, never reverted', async () => {
// §L, and the distinction matters to the operator reading the console
// afterwards: `reverted` says core put something back, `orphaned` says the
// thing vanished while nobody was looking. Recording the second as the first
// would be core claiming credit for a shard restart.
registerAction({ reconcile: async () => ({ ok: true, inForce: ['0x1'] }) })
addStep(1, 'demo.spawn')
addResource({ ref: '0x1' })
addResource({ ref: '0x2' })
const summary = await cleanup.reconcileModule('demo')
assert.deepEqual(summary, { asked: 2, inForce: 1, orphaned: 1, unanswered: 0 })
assert.equal([...store.rows.values()].find((r) => r.ref === '0x2').status, 'orphaned')
assert.equal([...store.rows.values()].find((r) => r.ref === '0x1').status, 'confirmed')
assert.ok(store.log.some((e) => e.kind === 'resource.orphaned'))
})
test('"I do not know" is never read as "it is gone"', async () => {
// Every unanswerable shape leaves the ledger exactly as it was. A reconcile
// that read silence as absence would orphan a whole shard's worth of live
// spawns the first time a sidecar was slow.
for (const answer of [null, undefined, { ok: false }, { ok: true }, { ok: true, inForce: 'all' }, 'yes']) {
store.rows.clear()
store.log.length = 0
registries._reset()
registerAction({ reconcile: async () => answer })
addStep(1, 'demo.spawn')
addResource({ ref: '0x1' })
const summary = await cleanup.reconcileModule('demo')
assert.equal(summary.orphaned, 0, JSON.stringify(answer))
assert.equal(summary.unanswered, 1, JSON.stringify(answer))
assert.equal([...store.rows.values()][0].status, 'confirmed')
}
})
test('a module with no reconcile is not broken; core keeps believing its ledger', async () => {
// Optional where `revert` is required. A module that cannot answer leaves core
// exactly where it was before this phase, which is a capability its deployment
// does without rather than a boot it fails.
registerAction()
addStep(1, 'demo.spawn')
addResource()
const summary = await cleanup.reconcileModule('demo')
assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 1 })
assert.equal([...store.rows.values()][0].status, 'confirmed')
})
test('a reconcile that throws orphans nothing', async () => {
registerAction({ reconcile: async () => { throw new Error('sidecar gone') } })
addStep(1, 'demo.spawn')
addResource()
const summary = await cleanup.reconcileModule('demo')
assert.equal(summary.unanswered, 1)
assert.equal([...store.rows.values()][0].status, 'confirmed')
})
test('placeholders are not asked about, because there is nothing to ask yet', async () => {
// A `@step` row names no object — it says "a dispatch was in flight and may
// have made something". Asking a module whether it is in force is a question
// with no answer, and reading a shrug as absence would resolve the one row whose
// survival is the safety property.
registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) })
addStep(1, 'demo.spawn')
addResource({ kind: resourcesDb.STEP_KIND, ref: 'a'.repeat(40), status: 'pending' })
const summary = await cleanup.reconcileModule('demo')
assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 })
assert.equal([...store.rows.values()][0].status, 'pending')
})
test('reconcileAll asks every module that owns a live row', async () => {
registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) })
addStep(1, 'demo.spawn')
addStep(2, 'other.thing')
addResource({ owner_module: 'demo' })
addResource({ owner_module: 'other', step_id: 2 })
const out = await cleanup.reconcileAll()
assert.deepEqual(Object.keys(out).sort(), ['demo', 'other'])
assert.equal(out.demo.orphaned, 1)
// The other module registers nothing, so its row is left alone rather than
// orphaned by a module that is not there to be asked.
assert.equal(out.other.unanswered, 1)
})

337
server/test/eventLedger.test.js Normal file
View File

@@ -0,0 +1,337 @@
// ── The resource ledger's write half (EVENTS_PLAN.md Phase 8) ──────────────
//
// §D's two rules, and rule 1 is the one this file exists for: **a resource is
// recorded BEFORE it is confirmed.** The obstacle it works around is that a
// spawn's serial does not exist until the module answers, so what goes in before
// the dispatch is a placeholder keyed by the step's idempotency key — and the
// property worth a test is that the placeholder SURVIVES an answer that never
// comes, because that is the case where recording afterwards would have lost the
// object for ever.
//
// The other rules here are about what core will and will not write down on a
// module's say-so. Every one of them is fail-closed in a specific direction:
//
// • the reserved `@step` kind is core's and a module may not claim it
// • an `override` must name a lease core knows how to give back, or core would
// be recording something it has no way to restore
// • a duplicate is "already recorded", not an error — a retry re-sends the same
// idempotency key and a module may honestly report the same resources twice
// • a badly shaped resource is dropped and LOGGED, never a failed step: the
// step changed the world, and turning bookkeeping into a retry would re-run
// a world write that already happened
//
// The db layer is stubbed with a store that enforces `uq_evres_target`, because
// that refusal is behaviour the callers branch on rather than an implementation
// detail. The SQL itself is `eventRunnerSql.test.js`'s, against a real MariaDB.
process.env.DB_HOST = '127.0.0.1'
process.env.DB_PORT = '59999'
const { test, beforeEach, afterEach, after } = require('node:test')
const assert = require('node:assert/strict')
const registries = require('../src/modules/registries')
const ledger = require('../src/events/ledger')
const resourcesDb = require('../src/model/events/eventRunResources.db')
const runsDb = require('../src/model/events/eventRuns.db')
const db = require('../src/utils/db')
after(() => db.close())
const HELD = ['pending', 'confirmed', 'reverting']
let store
const originals = { resourcesDb: { ...resourcesDb }, runsDb: { ...runsDb } }
beforeEach(() => {
registries._reset()
store = { rows: new Map(), next: 1, cleanupStatus: 'not_required' }
resourcesDb.reserve = async ({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) => {
const holder = [...store.rows.values()].find(
(r) => r.owner_module === owner && r.kind === kind && r.ref === ref && HELD.includes(r.status),
)
if (holder) return { ok: false, code: 'held', holder: { run_id: holder.run_id, status: holder.status } }
const id = store.next++
store.rows.set(id, {
id,
run_id: runId,
step_id: stepId,
owner_module: owner,
kind,
ref,
payload,
lease_until: leaseUntil,
status: 'pending',
revert_attempts: 0,
last_error: null,
member_key: memberKey,
})
return { ok: true, id }
}
resourcesDb.confirm = async (id) => {
const r = store.rows.get(id)
if (!r || r.status !== 'pending') return false
r.status = 'confirmed'
return true
}
resourcesDb.resolvePlaceholder = async (id) => {
const r = store.rows.get(id)
if (!r || r.kind !== resourcesDb.STEP_KIND) return false
r.status = 'reverted'
return true
}
resourcesDb.findByTarget = async (owner, kind, ref) =>
[...store.rows.values()].reverse().find((r) => r.owner_module === owner && r.kind === kind && r.ref === ref) || null
runsDb.setCleanupStatus = async (id, to, from = null) => {
if (from && !from.includes(store.cleanupStatus)) return false
store.cleanupStatus = to
return true
}
})
afterEach(() => {
Object.assign(resourcesDb, originals.resourcesDb)
Object.assign(runsDb, originals.runsDb)
registries._reset()
})
const RUN = { id: 7 }
const STEP = { id: 42, phase: 'invasion', seq: 0, idempotency_key: 'a'.repeat(40) }
const action = (over = {}) => ({
id: 'demo.spawn',
owner: 'demo',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
budgetMs: 1000,
...over,
})
const rows = () => [...store.rows.values()]
// ── Which actions ledger at all ────────────────────────────────────────────
test('only the two reversible classes core has to come back for are ledgered', () => {
// `none` is gone once done, `self` undoes itself. Neither has anything core
// could revert, and giving one a placeholder would put a row in the ledger that
// teardown could never resolve — the exact reason `core.announce` is declared
// `none` rather than `ledger`.
assert.equal(ledger.ledgers(action({ reversible: 'ledger' })), true)
assert.equal(ledger.ledgers(action({ reversible: 'override' })), true)
assert.equal(ledger.ledgers(action({ reversible: 'none' })), false)
assert.equal(ledger.ledgers(action({ reversible: 'self' })), false)
})
test('only a ledger action gets a placeholder; an override reserves its own target', async () => {
// The asymmetry is the design. A spawn's ref is unknown until the module
// answers, so the placeholder stands in for it; a lease's target is the lease
// id the step already names, so `core.lease` writes the real row before it
// touches the world — which is rule 1 in a stronger form, and the only place
// the two-events-one-target refusal can happen before the world has changed.
assert.equal(typeof (await ledger.reserveStep(RUN, STEP, action())), 'number')
assert.equal(await ledger.reserveStep(RUN, STEP, action({ id: 'demo.b', reversible: 'override' })), null)
assert.equal(await ledger.reserveStep(RUN, STEP, action({ id: 'demo.c', reversible: 'none' })), null)
assert.equal(rows().length, 1)
assert.equal(rows()[0].kind, '@step')
assert.equal(rows()[0].ref, STEP.idempotency_key)
})
test('the first ledger row is what makes a run dirty, and only from not_required', async () => {
assert.equal(store.cleanupStatus, 'not_required')
await ledger.reserveStep(RUN, STEP, action())
assert.equal(store.cleanupStatus, 'pending')
// A run whose sweep has already finished must not be walked back to `pending`
// by a late row: only a human's cleanup re-opens it, and it does so
// deliberately and with an actor on the log line.
store.cleanupStatus = 'complete'
await ledger.recordAnswer({
run: RUN,
step: { ...STEP, id: 43, idempotency_key: 'b'.repeat(40) },
action: action(),
placeholderId: null,
resources: [{ kind: 'creature', ref: '0x1' }],
})
assert.equal(store.cleanupStatus, 'complete')
})
// ── Rule 1 ─────────────────────────────────────────────────────────────────
test('a lost acknowledgement leaves the placeholder standing, which is the whole point', async () => {
const placeholderId = await ledger.reserveStep(RUN, STEP, action())
// The dispatch timed out: no answer, so `recordAnswer` is never reached. This
// is the case rule 1 exists for — record afterwards and the object the module
// may well have created is invisible to cleanup for ever.
assert.equal(rows()[0].status, 'pending')
assert.equal(rows()[0].payload.action, 'demo.spawn')
assert.ok(placeholderId)
})
test('a retry reuses its own placeholder rather than writing a second', async () => {
// An idempotency key is minted once per step and does not vary by attempt (§E),
// so the second attempt's insert collides with the first attempt's row. Finding
// it already there is the correct answer, and a second row would be a second
// thing for cleanup to revert.
const first = await ledger.reserveStep(RUN, STEP, action())
const second = await ledger.reserveStep(RUN, STEP, action())
assert.equal(first, second)
assert.equal(rows().length, 1)
})
test('the placeholder is resolved once the real rows exist', async () => {
const placeholderId = await ledger.reserveStep(RUN, STEP, action())
const out = await ledger.recordAnswer({
run: RUN,
step: STEP,
action: action(),
placeholderId,
resources: [
{ kind: 'creature', ref: '0x40001234' },
{ kind: 'creature', ref: '0x40001235' },
],
})
assert.equal(out.recorded, 2)
assert.deepEqual(out.rejected, [])
assert.equal(store.rows.get(placeholderId).status, 'reverted')
assert.deepEqual(
rows().filter((r) => r.kind === 'creature').map((r) => [r.ref, r.status]),
[['0x40001234', 'confirmed'], ['0x40001235', 'confirmed']],
)
})
test('an action that ledgers and reports nothing still resolves its placeholder', async () => {
// "I made nothing" is a real answer. Holding the placeholder open for it would
// make cleanup call `revert()` on every terminal path, for ever, for a step that
// has nothing to give back.
const placeholderId = await ledger.reserveStep(RUN, STEP, action())
const out = await ledger.recordAnswer({ run: RUN, step: STEP, action: action(), placeholderId, resources: [] })
assert.equal(out.recorded, 0)
assert.equal(store.rows.get(placeholderId).status, 'reverted')
})
test('a module reporting the same resources twice produces one row', async () => {
// The database is what makes recording idempotent: `uq_evres_target` refuses
// the second insert and this file reads that as "already recorded". Without it
// a retry against a module that honestly re-reports its work would double every
// row cleanup then has to revert.
const args = { run: RUN, step: STEP, action: action(), placeholderId: null, resources: [{ kind: 'creature', ref: '0x1' }] }
await ledger.recordAnswer(args)
const again = await ledger.recordAnswer(args)
assert.equal(again.recorded, 0)
assert.deepEqual(again.rejected, [])
assert.equal(rows().filter((r) => r.kind === 'creature').length, 1)
})
test('a target another RUN holds is rejected by name rather than silently skipped', async () => {
await ledger.recordAnswer({
run: { id: 1 },
step: STEP,
action: action(),
placeholderId: null,
resources: [{ kind: 'creature', ref: '0x1' }],
})
const out = await ledger.recordAnswer({
run: { id: 2 },
step: { ...STEP, id: 99 },
action: action(),
placeholderId: null,
resources: [{ kind: 'creature', ref: '0x1' }],
})
assert.equal(out.recorded, 0)
assert.match(out.rejected.join('\n'), /already held by run 1/)
})
// ── What core will not write down ──────────────────────────────────────────
test('a module may not claim core\'s reserved kind', () => {
// A module that could write a `@step` row could make its own step's placeholder
// look resolved — which is the one row whose survival is the safety property.
const bad = ledger.normalise({ kind: '@step', ref: 'x' }, 'demo.spawn')
assert.equal(bad.ok, false)
assert.match(bad.reason, /reserved kind/)
})
test('an override must name a lease core knows how to give back', () => {
// Core restores an `override` through the LEASE registry — that is the split §F
// draws — so a ref naming nothing registered is a resource core would be
// recording with no way to undo it. Refusing to record it is the fail-closed
// direction: rule 2 is a promise core must not make and then break.
assert.equal(ledger.normalise({ kind: 'override', ref: 'demo.rate' }, 'demo.x').ok, false)
const api = registries.stage('demo')
api.registerEventLeases([
{
id: 'demo.rate',
label: 'Rate',
type: 'float',
min: 0.5,
max: 5,
maxDurationMs: 3_600_000,
read: async () => ({ ok: true, value: 1 }),
apply: async () => ({ ok: true }),
restore: async () => ({ ok: true }),
},
])
registries.apply(api.staged)
assert.equal(ledger.normalise({ kind: 'override', ref: 'demo.rate' }, 'demo.x').ok, true)
})
test('every bad shape is refused, and none of them is a retry', () => {
// A badly shaped resource is the module's mistake rather than the world's, and
// it will be just as badly shaped on the second attempt. They are dropped and
// reported; the STEP still counts as done, because it is — something happened
// in the world, and refusing to record it would be the one outcome worse than
// recording it imperfectly.
const bad = [
null,
'a string',
['an array'],
{ ref: 'x' }, // no kind
{ kind: 'creature' }, // no ref
{ kind: 'creature', ref: 'x'.repeat(200) },
{ kind: 'k'.repeat(80), ref: 'x' },
{ kind: 'creature', ref: 'x', memberKey: 'm'.repeat(200) },
{ kind: 'creature', ref: 'x', until: 'not a date' },
]
for (const entry of bad) {
assert.equal(ledger.normalise(entry, 'demo.spawn').ok, false, JSON.stringify(entry))
}
})
test('a bad resource never fails the step it came from', async () => {
const placeholderId = await ledger.reserveStep(RUN, STEP, action())
const out = await ledger.recordAnswer({
run: RUN,
step: STEP,
action: action(),
placeholderId,
resources: [{ kind: 'creature', ref: '0x1' }, { nonsense: true }],
})
assert.equal(out.recorded, 1)
assert.equal(out.rejected.length, 1)
// And the placeholder is still resolved: the good row exists, and leaving the
// placeholder open would ask the module to undo the step a second time.
assert.equal(store.rows.get(placeholderId).status, 'reverted')
})
test('a lease deadline and a member key ride through verbatim', async () => {
const until = new Date('2026-09-04T00:00:00Z')
await ledger.recordAnswer({
run: RUN,
step: STEP,
action: action(),
placeholderId: null,
resources: [{ kind: 'reward', ref: 'item-1', memberKey: 'Darrow', until, payload: { cliloc: 1234 } }],
})
const row = rows()[0]
assert.equal(row.member_key, 'Darrow')
assert.equal(row.lease_until.getTime(), until.getTime())
assert.deepEqual(row.payload, { cliloc: 1234 })
// Opaque: core stores what the module said and never interprets it, which is
// `ctx.teams.activity.push`'s exact treatment one registry along.
assert.equal(row.kind, 'reward')
assert.equal(row.owner_module, 'demo')
})

239
server/test/eventModuleContract.test.js
View File

@@ -513,3 +513,242 @@ test('an action whose module is gone goes dormant, and a step naming it fails te
assert.equal(result.dormant, true)
assert.match(result.error, /no module registers "demo\.summon"/)
})
// ── Phase 8: the ledger's two callables, from a module ─────────────────────
//
// `revert` was already required at registration for `reversible: 'ledger'` —
// Phase 1 put that check in. What Phase 8 added is a caller for it, and
// `reconcile` beside it. Both are proved here through the REAL loader for the
// same reason the four registrations are: a `revert` a test called directly is a
// `revert` core might still have no way to reach.
test('a module\'s revert is reached by the cleanup sweep, resources and key in hand', async () => {
const record = loadModule('demo', `
let seen = null
module.exports = (ctx, api) => {
api.registerEventActions([{
id: 'demo.spawn',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
params: [],
async perform() { return { ok: true, resources: [{ kind: 'creature', ref: '0xA' }] } },
async revert(arg) { seen = arg; return { ok: true } },
async reconcile() { return { ok: true, inForce: [] } },
}])
api.registerEventOptionSources([
{ id: 'demo.options.seen', label: 'seen', async resolve() { return [{ value: JSON.stringify(seen), label: 'seen' }] } },
])
}
`)
assertRegistered(record)
const action = registries.eventAction('demo.spawn')
// Both callables survived the registration copy — which is not a given: that
// copy is explicit rather than a spread, precisely so nothing rides along, and
// a member added to the contract without being added to it is a member that
// silently does not exist.
assert.equal(typeof action.revert, 'function')
assert.equal(typeof action.reconcile, 'function')
assert.equal(action.owner, 'demo')
const answer = await action.revert({
runId: 3,
resources: [{ kind: 'creature', ref: '0xA', payload: null, memberKey: null }],
idempotencyKey: 'k-1',
})
assert.deepEqual(answer, { ok: true })
// Read back through the module's own option source rather than out of a
// closure this file holds: the point is that what core PASSED is what the
// module SAW, across the seam.
const seen = JSON.parse((await registries.resolveOptionSource('demo.options.seen')).options[0].value)
assert.equal(seen.runId, 3)
assert.equal(seen.idempotencyKey, 'k-1')
assert.deepEqual(seen.resources, [{ kind: 'creature', ref: '0xA', payload: null, memberKey: null }])
})
test('reconcile is optional, and a module without one still registers', () => {
// The asymmetry with `revert`, from the loader's side. A module that cannot say
// what the game still has is not broken — core keeps believing its own ledger,
// which is the behaviour before this phase — whereas one that creates something
// and cannot undo it has made a promise core has no way to keep.
const withNone = loadModule('quiet', `module.exports = (ctx, api) => {
api.registerEventActions([{
id: 'quiet.spawn',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
params: [],
async perform() { return { ok: true } },
async revert() { return { ok: true } },
}])
}`)
assertRegistered(withNone)
assert.equal(registries.eventAction('quiet.spawn').reconcile, null)
const withoutRevert = loadModule('broken', `module.exports = (ctx, api) => {
api.registerEventActions([{
id: 'broken.spawn',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
params: [],
async perform() { return { ok: true } },
}])
}`)
assert.equal(withoutRevert.state, 'startup_failed')
assert.match(withoutRevert.reason, /reversible: 'ledger' but has no revert\(\)/)
})
test('a module cannot claim core\'s reserved resource kind', async () => {
// `@step` is the placeholder's kind, and the placeholder is the row whose
// survival is the safety property: a module able to write one could make its
// own step look already accounted for. Refused at recording, and the STEP still
// succeeds — because it did.
const record = loadModule('sneaky', `module.exports = (ctx, api) => {
api.registerEventActions([{
id: 'sneaky.spawn',
label: 'Spawn',
risk: 'change',
reversible: 'ledger',
params: [],
async perform() { return { ok: true, resources: [{ kind: '@step', ref: 'anything' }] } },
async revert() { return { ok: true } },
}])
}`)
assertRegistered(record)
// eslint-disable-next-line global-require
const ledger = require('../src/events/ledger')
const result = await dispatch.dispatchStep(step('sneaky.spawn'), { run: RUN })
assert.equal(result.outcome, 'done')
const parsed = ledger.normalise(result.resources[0], 'sneaky.spawn')
assert.equal(parsed.ok, false)
assert.match(parsed.reason, /reserved kind/)
})
test('core registers the lease VERB and a module registers the lease', async () => {
// The seam working the way round it is meant to (Phase 8). A module ships the
// three callables; the verb an author puts in a step is `core.lease`, so the
// duration bound and the two-events-one-target conflict check live in one place
// rather than being re-implemented once per module and advisory everywhere.
const record = loadModule('demo', `module.exports = (ctx, api) => {
api.registerEventLeases([{
id: 'demo.rate.gain',
label: 'Gain rate',
type: 'float',
min: 0.5,
max: 5,
maxDurationMs: 3600000,
async read() { return { ok: true, value: 1 } },
async apply() { return { ok: true } },
async restore() { return { ok: true } },
}])
}`)
assertRegistered(record)
registries.registerCore()
// The module registers no ACTION at all, and its lease is still reachable.
assert.equal(registries.eventAction('demo.lease'), null)
assert.equal(registries.eventAction('core.lease').reversible, 'override')
// And the dropdown behind `core.lease`'s first param is answered by what the
// module declared — resolved per request, so a module that booted later is
// still in the list.
const options = await registries.resolveOptionSource('core.options.leases')
assert.equal(options.ok, true)
assert.deepEqual(options.options, [{ value: 'demo.rate.gain', label: 'Gain rate', group: 'demo' }])
})
test('core refuses a lease held longer than the module allows', async () => {
// The bound is the MODULE's number and the enforcement is CORE's, which is the
// §F split stated as one assertion. `retry: false` because a duration that is
// too long will still be too long in sixty seconds: it is an authoring error,
// not an outage.
const record = loadModule('demo', `module.exports = (ctx, api) => {
api.registerEventLeases([{
id: 'demo.rate.gain',
label: 'Gain rate',
type: 'float',
min: 0.5,
max: 5,
maxDurationMs: 3600000,
async read() { return { ok: true, value: 1 } },
async apply() { return { ok: true } },
async restore() { return { ok: true } },
}])
}`)
assertRegistered(record)
registries.registerCore()
const tooLong = await dispatch.dispatchStep(
step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 120 }),
{ run: RUN },
)
assert.equal(tooLong.outcome, 'terminal')
assert.match(tooLong.error, /at most 60 minutes, not 120/)
// The same for a value outside the declared range. Unlike a cap, a bad lease
// value is in force the moment it is applied, which is why min/max are required
// on the numeric types rather than advisory.
const tooBig = await dispatch.dispatchStep(
step('core.lease', { lease: 'demo.rate.gain', value: '9', minutes: 10 }),
{ run: RUN },
)
assert.equal(tooBig.outcome, 'terminal')
assert.match(tooBig.error, /accepts 0\.5 to 5/)
// And a lease nobody registers, which is the dormancy rule one registry along.
const missing = await dispatch.dispatchStep(
step('core.lease', { lease: 'demo.nope', value: '3', minutes: 10 }),
{ run: RUN },
)
assert.equal(missing.outcome, 'terminal')
assert.match(missing.error, /no module registers the lease "demo\.nope"/)
})
test('a dry run of core.lease checks everything and takes nothing', async () => {
// `verify: true` must change nothing and must answer honestly (§F). A verify
// that reserved the target would be a dry run that changed something — and it
// would then refuse the real run that followed it, which is the worst of both.
let applied = 0
const record = loadModule('demo', `
let applied = 0
module.exports = (ctx, api) => {
api.registerEventLeases([{
id: 'demo.rate.gain',
label: 'Gain rate',
type: 'float',
min: 0.5,
max: 5,
maxDurationMs: 3600000,
async read() { return { ok: true, value: 1 } },
async apply() { applied += 1; return { ok: true } },
async restore() { return { ok: true } },
}])
api.registerEventOptionSources([
{ id: 'demo.options.applied', label: 'applied', async resolve() { return [{ value: String(applied), label: 'n' }] } },
])
}
`)
assertRegistered(record)
registries.registerCore()
void applied
const ok = await dispatch.dispatchStep(
step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 10 }),
{ run: RUN, verify: true },
)
assert.equal(ok.outcome, 'done')
assert.equal((await registries.resolveOptionSource('demo.options.applied')).options[0].value, '0')
// A dry run that is still a real check: the bad duration is caught with
// `verify: true` as well, which is the whole value of the switchboard's
// "find out before you schedule it".
const bad = await dispatch.dispatchStep(
step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 999 }),
{ run: RUN, verify: true },
)
assert.equal(bad.outcome, 'terminal')
})

145
server/test/eventRunControls.test.js
View File

@@ -35,6 +35,11 @@ const runsDb = require('../src/model/events/eventRuns.db')
const stepsDb = require('../src/model/events/eventRunSteps.db')
const logDb = require('../src/model/events/eventRunLog.db')
const gatesDb = require('../src/model/events/eventPhaseGates.db')
// Phase 8: cancel now decides what happens to the run's world changes, and
// `cleanupRun` is the eighth control. Same rule as every phase since the fourth —
// a new leg under a model needs a stub in every file that stubs that layer.
const resourcesDb = require('../src/model/events/eventRunResources.db')
const eventCleanup = require('../src/events/cleanup')
const db = require('../src/utils/db')
after(() => db.close())
@@ -48,12 +53,35 @@ const originals = [
['steps', stepsDb, { ...stepsDb }],
['log', logDb, { ...logDb }],
['gates', gatesDb, { ...gatesDb }],
['resources', resourcesDb, { ...resourcesDb }],
['cleanup', eventCleanup, { ...eventCleanup }],
]
function installStubs() {
store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), nextStepId: 1, nextGateId: 1 }
store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), sweeps: [], unresolved: {}, nextStepId: 1, nextGateId: 1 }
const snap = (o) => ({ ...o })
runsDb.setCleanupStatus = async (id, to, from = null) => {
const r = store.runs.get(Number(id))
if (!r) return false
if (from && !from.includes(r.cleanup_status)) return false
r.cleanup_status = to
return true
}
// The sweep itself is `eventCleanup.test.js`'s subject. What this file is
// about is which control calls it, with what, and whether it is allowed to.
eventCleanup.cleanupRun = async (run, opts = {}) => {
store.sweeps.push({ runId: run.id, ...opts })
return { attempted: 1, reverted: 1, drifted: 0, failed: 0, remaining: 0 }
}
// Cancel asks the LEDGER whether the run owes the world anything, so that
// `cleanup: false` on a run with nothing recorded does not stamp `incomplete`
// over `not_required`. Unstubbed this is the ten-second dead-port wait, for the
// fifth time in this feature.
resourcesDb.unresolvedCount = async (runId) => store.unresolved[runId] ?? 0
runsDb.getById = async (id) => {
const r = store.runs.get(Number(id))
return r ? snap(r) : null
@@ -184,7 +212,7 @@ function seedGate(runId, phase, { kind = 'on', trigger = 'test.trigger', needed
return g
}
function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) {
function seedRun({ status = 'running', phase = 'main', steps = [], cleanupStatus = 'not_required' } = {}) {
const id = nextRunId++
store.runs.set(id, {
id,
@@ -192,6 +220,7 @@ function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) {
version_id: id,
status,
health: 'ok',
cleanup_status: cleanupStatus,
current_phase: phase,
claimed_by: null,
claim_expires_at: null,
@@ -559,3 +588,115 @@ test('an empty reason is stored as NULL rather than as an empty string', async (
await controls.pause(id, { reason: ' ' }, ACTOR)
assert.equal(lastLog().detail.reason, null)
})
// ── cancel decides what happens to the world (Phase 8) ─────────────────────
test('cancel gives back what the run took, by default and without waiting for it', async () => {
// The teardown is the runner cleanup leg over TERMINAL runs, not this request.
// Two reasons, and both are why the control answers at once: a cancel pressed
// at two in the morning must not block on a dozen round trips to the shard
// that may BE the reason it is being cancelled, and a process that dies
// halfway through a teardown has to resume rather than leave a world half
// restored with nothing scheduled to finish it.
const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] })
store.unresolved[id] = 3
const result = await controls.cancel(id, { reason: 'called off' }, ACTOR)
assert.equal(result.ok, true)
assert.equal(result.cleanup, true)
assert.deepEqual(store.sweeps, [], 'the request must not do the teardown itself')
// Still `pending`, which is what the leg looks for. The run is terminal the
// moment this returns, so the very next tick picks its ledger up.
assert.equal(runRow(id).cleanup_status, 'pending')
assert.equal(store.log.at(-1).detail.cleanup, true)
})
test('cancel WITHOUT cleanup is admin-only, even though the route is wider', async () => {
// §L: "cancelling without cleanup is a separate, logged, admin-only action."
// The route is `admin` + `moderator`, so the narrower gate cannot live in
// middleware — WHICH of the two you have to be depends on what is in the body,
// exactly as the authoring role floor does (§K).
const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] })
store.unresolved[id] = 3
const refused = await controls.cancel(id, { cleanup: false }, ACTOR, { isAdmin: false })
assert.equal(refused.ok, false)
assert.equal(refused.status, 403)
assert.equal(runRow(id).status, 'running', 'and the run is not cancelled either')
// A moderator asking for the ordinary cancel is fine: the safe direction is
// the default, so the widest gate keeps the button it exists for.
const allowed = await controls.cancel(id, {}, ACTOR, { isAdmin: false })
assert.equal(allowed.ok, true)
assert.equal(allowed.cleanup, true)
})
test('cancel without cleanup leaves the world changes up, and says so on the run', async () => {
// `incomplete` is the truthful value rather than a tidy one: the changes are
// still up, they are listed on the console, and the log line records who
// decided that. A `complete` here would be the "tidy completed row over a shard
// full of orphaned monsters" §L names as the failure that ends this feature's
// credibility.
const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] })
store.unresolved[id] = 3
const result = await controls.cancel(id, { cleanup: false, reason: 'leave it up' }, ACTOR)
assert.equal(result.ok, true)
assert.equal(result.cleanup, false)
assert.equal(runRow(id).cleanup_status, 'incomplete')
assert.equal(store.log.at(-1).detail.cleanup, false)
assert.equal(store.log.at(-1).detail.by, ACTOR)
})
test('a run that recorded nothing is unaffected by either flag', async () => {
// `not_required` is not walked to `incomplete` by a cancel that skipped a
// teardown there was nothing to do — and it is the LEDGER that says so, not the
// status column, because `not_required` is also what a run holding only a lease
// wrongly carried before the live walk found it.
const id = seedRun({ status: 'running', cleanupStatus: 'not_required', steps: [{ status: 'pending' }] })
store.unresolved[id] = 0
await controls.cancel(id, { cleanup: false }, ACTOR)
assert.equal(runRow(id).cleanup_status, 'not_required')
})
// ── cleanup, the eighth control ────────────────────────────────────────────
test('cleanup re-runs the teardown and clears the attempt counter', async () => {
// The manual retry §L promises. `resetAttempts` is the licence a human has and
// the automatic sweep does not — Engagement Phase 14's rule, whose defect was
// a sweep that reset every stale row and made the attempt ceiling unreachable.
const id = seedRun({ status: 'completed', cleanupStatus: 'incomplete' })
const result = await controls.cleanupRun(id, ACTOR)
assert.equal(result.ok, true)
assert.deepEqual(store.sweeps, [{ runId: id, resetAttempts: true, actor: ACTOR }])
assert.equal(result.summary.reverted, 1)
})
test('cleanup refuses a run that is still in flight', async () => {
// A run still going has a ledger that is still growing, and reverting a
// resource the next step is about to use would be core undoing an event while
// it is happening. Cancel is the control for a run that should stop.
for (const status of ['scheduled', 'starting', 'running', 'paused', 'ending']) {
const id = seedRun({ status, cleanupStatus: 'pending' })
const result = await controls.cleanupRun(id, ACTOR)
assert.equal(result.ok, false, status)
assert.match(result.errors[0], /cancel it before cleaning up after it/)
}
assert.deepEqual(store.sweeps, [])
})
test('cleanup refuses a run that recorded no resources', async () => {
const id = seedRun({ status: 'completed', cleanupStatus: 'not_required' })
const result = await controls.cleanupRun(id, ACTOR)
assert.equal(result.ok, false)
assert.match(result.errors[0], /nothing to give back/)
})
test('cleanup on an unknown run is a 404, not a 409', async () => {
const result = await controls.cleanupRun(9999, ACTOR)
assert.equal(result.status, 404)
})

232
server/test/eventRunner.test.js
View File

@@ -45,6 +45,11 @@ const gatesDb = require('../src/model/events/eventPhaseGates.db')
// every file that stubs the layer under it needs the stub.
const settingsDb = require('../src/model/events/eventActionSettings.db')
const budgetDb = require('../src/model/events/eventRunBudget.db')
// Phase 8 put a ledger write in front of every world-changing dispatch and a
// cleanup leg at the end of the tick. **Fourth time, same rule, and this file's
// own note is what caught it**: unstubbed, one of these is not a wrong answer,
// it is a ten-second wait on the dead port.
const resourcesDb = require('../src/model/events/eventRunResources.db')
const gates = require('../src/events/gates')
const db = require('../src/utils/db')
@@ -58,7 +63,7 @@ const later = (ms) => new Date(T0.getTime() + ms)
let store
const originals = {}
for (const [name, mod] of [['runsDb', runsDb], ['stepsDb', stepsDb], ['logDb', logDb], ['versionsDb', versionsDb], ['definitionsDb', definitionsDb], ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb]]) {
for (const [name, mod] of [['runsDb', runsDb], ['stepsDb', stepsDb], ['logDb', logDb], ['versionsDb', versionsDb], ['definitionsDb', definitionsDb], ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb], ['resourcesDb', resourcesDb]]) {
originals[name] = { mod, fns: { ...mod } }
}
@@ -79,8 +84,10 @@ function installStubs() {
gates: new Map(),
settings: new Map(),
budget: new Map(),
resources: new Map(),
nextStepId: 1,
nextGateId: 1,
nextResourceId: 1,
}
// Phase 4 put a schedule-expansion leg in front of the tick. This file is
@@ -91,6 +98,69 @@ function installStubs() {
// connection timeout.
Object.assign(definitionsDb, { findSchedulable: async () => [] })
// ── The resource ledger (Phase 8) ──
//
// `reserve` enforces `uq_evres_target` in the stub, because the refusal it
// produces is BEHAVIOUR the runner branches on rather than an implementation
// detail: a placeholder that collides is this step's own earlier attempt and is
// reused, and a lease that collides is another run holding the target. A stub
// that let both inserts through would make the retry path grow a second row and
// the conflict test pass for no reason.
const HELD_STATUSES = ['pending', 'confirmed', 'reverting']
resourcesDb.reserve = async ({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) => {
const holder = [...store.resources.values()].find(
(r) => r.owner_module === owner && r.kind === kind && r.ref === ref && HELD_STATUSES.includes(r.status),
)
if (holder) return { ok: false, code: 'held', holder: { run_id: holder.run_id, status: holder.status } }
const id = store.nextResourceId++
store.resources.set(id, {
id,
run_id: runId,
step_id: stepId,
owner_module: owner,
kind,
ref,
payload,
lease_until: leaseUntil,
status: 'pending',
revert_attempts: 0,
last_error: null,
member_key: memberKey,
})
return { ok: true, id }
}
resourcesDb.confirm = async (id) => {
const r = store.resources.get(id)
if (!r || r.status !== 'pending') return false
r.status = 'confirmed'
return true
}
resourcesDb.resolvePlaceholder = async (id) => {
const r = store.resources.get(id)
if (!r || r.kind !== resourcesDb.STEP_KIND || !['pending', 'confirmed'].includes(r.status)) return false
r.status = 'reverted'
return true
}
resourcesDb.findByTarget = async (owner, kind, ref) =>
[...store.resources.values()].reverse().find((r) => r.owner_module === owner && r.kind === kind && r.ref === ref) || null
resourcesDb.forRun = async (runId) => [...store.resources.values()].filter((r) => r.run_id === runId)
resourcesDb.markReverted = async (id) => {
const r = store.resources.get(id)
if (r) r.status = 'reverted'
}
// The cleanup leg's scan. This file is about the runner's own legs, and the
// sweep has its own file — answering with nothing is what keeps every `tick()`
// here measuring the runner rather than a teardown.
resourcesDb.runsNeedingCleanup = async () => []
runsDb.setCleanupStatus = async (id, to, from = null) => {
const r = store.runs.get(id)
if (!r) return false
if (from && !from.includes(r.cleanup_status)) return false
r.cleanup_status = to
return true
}
// Snapshots, not live references. A SQL SELECT hands back a copy, and the
// runner reads `step.attempts` as the value BEFORE its own claim incremented
// it — returning references here would make the retry budget off by one in the
@@ -1319,3 +1389,163 @@ test('the runner never re-checks the role of whoever started the run', async ()
assert.equal(stepsOf(id)[0].status, 'done')
})
// ── The resource ledger, from the runner's side (Phase 8) ──────────────────
//
// `eventLedger.test.js` owns the recording RULES and `eventCleanup.test.js` owns
// the undo. What belongs here is the ORDER — that the placeholder is written
// before the module is reached and after the permission check, and that a step
// which never answers leaves it standing. Those are properties of `drainStep`,
// and nothing below the runner can observe them.
const ledgerRows = (id) => [...store.resources.values()].filter((r) => r.run_id === id)
test('a world-changing step is recorded BEFORE it is dispatched', async () => {
// §D rule 1. The assertion is made from INSIDE `perform()`, which is the only
// place that can tell "recorded first" from "recorded at all" — and the
// difference between the two is every object whose acknowledgement is lost.
let seenDuringDispatch = null
register([scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) })])
setSwitch('test.spawn', true)
const id = seedRun([{ key: 'main', steps: [step('test.spawn')] }])
scripted['test.spawn'] = {
calls: [],
answer: () => {
seenDuringDispatch = ledgerRows(id).map((r) => [r.kind, r.status])
return { ok: true, resources: [{ kind: 'creature', ref: '0x40001234' }] }
},
}
await runner.tick(T0)
assert.deepEqual(seenDuringDispatch, [['@step', 'pending']])
// And on the answer the real row exists and the placeholder is done with.
assert.deepEqual(
ledgerRows(id).map((r) => [r.kind, r.ref, r.status]),
[
['@step', stepsOf(id)[0].idempotency_key, 'reverted'],
['creature', '0x40001234', 'confirmed'],
],
)
assert.equal(run(id).cleanup_status, 'pending')
assert.ok(kinds(id).includes('resource.recorded'))
})
test('a step that never answers leaves its placeholder pending', async () => {
// The whole reason the placeholder exists. The module timed out, so core does
// not know whether anything was created — and the row that survives is what
// lets cleanup ask it by idempotency key later. Record on the answer instead
// and this run finishes looking spotless over a shard full of orphans.
register([
scriptedAction('test.spawn', {
risk: 'change',
reversible: 'ledger',
budgetMs: 5,
revert: async () => ({ ok: true }),
perform: () => new Promise(() => {}),
}),
])
setSwitch('test.spawn', true)
const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }])
await runner.tick(T0)
assert.deepEqual(ledgerRows(id).map((r) => [r.kind, r.status]), [['@step', 'pending']])
})
test('a refused step ledgers nothing, because it never reached the module', async () => {
// The placeholder is written AFTER the permission check, deliberately. A step
// refused by a cap or by a switch created nothing, and a ledger row for it
// would be core asking a module to undo something it was never asked to do.
register([scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) })])
// No `setSwitch`, so it is default-off: §K's world-changing default.
const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }])
await runner.tick(T0)
assert.equal(stepsOf(id)[0].status, 'refused')
assert.deepEqual(ledgerRows(id), [])
assert.equal(run(id).cleanup_status, 'not_required')
})
test('an announce step ledgers nothing at all', async () => {
// `reversible: 'none'`, so there is nothing core could come back for. A
// placeholder here would be a row teardown could never resolve — which is the
// reason `core.announce` is declared `none` rather than `ledger` even though a
// sent message cannot be unsent.
register([scriptedAction('test.say')])
const id = seedRun([{ key: 'main', steps: [step('test.say')] }])
await runner.tick(T0)
assert.equal(run(id).status, 'completed')
assert.deepEqual(ledgerRows(id), [])
assert.equal(run(id).cleanup_status, 'not_required')
})
test('a parked step records what it made, because its confirm never dispatches again', async () => {
// `await: 'human'` is a SUCCESS: the module did its part and something outside
// the system has to happen next. The cue's confirm finishes the step without a
// second dispatch, so this is the only moment its resources can be recorded.
register([
scriptedAction('test.stage', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }),
])
setSwitch('test.stage', true)
const id = seedRun([{ key: 'main', steps: [step('test.stage')] }])
scripted['test.stage'] = {
calls: [],
answer: { ok: true, await: 'human', resources: [{ kind: 'prop', ref: 'gate-1' }] },
}
await runner.tick(T0)
assert.equal(stepsOf(id)[0].status, 'running')
assert.deepEqual(
ledgerRows(id).map((r) => [r.kind, r.status]),
[['@step', 'reverted'], ['prop', 'confirmed']],
)
})
test('a retry re-uses its placeholder and records the resources once', async () => {
// An idempotency key does not vary by attempt (§E), so the second attempt's
// placeholder insert collides with the first attempt's row — and a module that
// honestly re-reports the same creature must not produce a second thing for
// cleanup to revert.
register([
scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }),
])
setSwitch('test.spawn', true)
const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }])
scripted['test.spawn'] = {
calls: [],
answers: [
{ ok: false, error: 'the shard did not answer' },
{ ok: true, resources: [{ kind: 'creature', ref: '0xFEED' }] },
],
}
await runner.tick(T0)
await runner.tick(later(runner.RETRY_MS + 1000))
assert.equal(stepsOf(id)[0].status, 'done')
assert.equal(ledgerRows(id).filter((r) => r.kind === '@step').length, 1)
assert.equal(ledgerRows(id).filter((r) => r.kind === 'creature').length, 1)
})
test('a ledger write that fails stops the dispatch rather than losing the record', async () => {
// The ledger is what makes a world write recoverable, so a step that cannot be
// recorded must not be sent. Transient, because the alternative is an
// unrecorded world change — the one outcome §D rule 1 exists to make impossible.
register([
scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }),
])
setSwitch('test.spawn', true)
const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }])
scripted['test.spawn'] = { calls: [], answer: { ok: true } }
resourcesDb.reserve = async () => {
throw new Error('the ledger is unreachable')
}
await runner.tick(T0)
assert.equal(scripted['test.spawn'].calls.length, 0, 'the module must not have been reached')
assert.match(stepsOf(id)[0].last_error, /the resource ledger could not record this step/)
})

371
server/test/eventRunnerSql.test.js
View File

@@ -80,6 +80,18 @@
// that died between entering a phase and opening its gate opens no second
// one on the next tick.
//
// **Phase 8 added the resource ledger**, and its unique key is the single most
// server-dependent thing in this feature. "Two events cannot hold a lease on one
// target" has to hold among LIVE rows only — last week's finished event must not
// keep this week's from leasing the same rate — and MariaDB has no partial index,
// so the encoding is a STORED generated column that goes NULL once the row is no
// longer ours. Whether multiple NULLs collide in a unique index is a property of
// the server and of nothing else, and TEAMS.md §2.5 already had to be corrected
// once on this exact shape: MariaDB refuses ON DELETE SET NULL on a foreign key
// whose column is a base column of a stored generated column (error 1901), which
// is why the expression reads `status` alone and `step_id` stays a SET NULL FK.
// Both halves of that are proved below rather than believed.
//
// Plus the two unique indexes that are load-bearing rather than tidy:
// `uq_evrun_occurrence` (which, not the claim, is what stops two runs of one
// occurrence existing) and `uq_evstep_slot` (which is what makes re-materialising
@@ -208,6 +220,29 @@ CREATE TABLE event_run_budget (
CONSTRAINT fk_evbud_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE,
UNIQUE KEY uq_evbud_dim (run_id, dimension)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE event_run_resources (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
run_id BIGINT NOT NULL,
step_id BIGINT NULL,
owner_module VARCHAR(64) NOT NULL,
kind VARCHAR(64) NOT NULL,
ref VARCHAR(190) NOT NULL,
payload JSON NULL,
lease_until DATETIME NULL,
status ENUM('pending','confirmed','reverting','reverted','orphaned','drifted')
NOT NULL DEFAULT 'pending',
revert_attempts INT NOT NULL DEFAULT 0,
last_error VARCHAR(500) NULL,
member_key VARCHAR(190) NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
live_marker TINYINT AS (IF(status IN ('pending','confirmed','reverting'), 1, NULL)) STORED,
CONSTRAINT fk_evres_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE,
CONSTRAINT fk_evres_step FOREIGN KEY (step_id) REFERENCES event_run_steps(id) ON DELETE SET NULL,
UNIQUE KEY uq_evres_target (owner_module, kind, ref, live_marker),
INDEX idx_evres_run (run_id, status),
INDEX idx_evres_live (status, lease_until)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE event_action_settings (
action_id VARCHAR(96) NOT NULL PRIMARY KEY,
enabled TINYINT(1) NOT NULL DEFAULT 0,
@@ -438,6 +473,7 @@ const rows = (r) => Number(r.affectedRows)
beforeEach(async () => {
if (!available) return
await pool.query('DELETE FROM event_run_phase_gates')
await pool.query('DELETE FROM event_run_resources')
await pool.query('DELETE FROM event_run_steps')
await pool.query('DELETE FROM event_runs')
await pool.query('DELETE FROM event_definitions')
@@ -1139,7 +1175,21 @@ test('a gate goes with its run', async (t) => {
// live server, so the pool holds open connections and the process never exits —
// 49 green tests and a file that hangs until the harness kills it. Every other
// event test file already closes it in an `after`; this one now has a reason to.
//
// **And that second pool has to be aimed at the throwaway database**, which it
// was not before Phase 8 noticed. `utils/db` builds its pool at REQUIRE time from
// `DB_NAME`, and its own `dotenv.config()` reads `server/.env` — so a model test
// run on a developer's machine was reaching that developer's real schema while
// the fixtures it was asserting against were being written next door. It passed
// only because the two tables happened to exist in both. `dotenv` does not
// overwrite a variable that already exists, so setting it here, before the
// require below, is what beats the file. This file drops its database in `after`,
// so aiming the model pool at it is also what keeps the whole run disposable.
process.env.DB_NAME = DB
const budgetDb = require('../src/model/events/eventRunBudget.db')
const resourcesDb = require('../src/model/events/eventRunResources.db')
const runsDb = require('../src/model/events/eventRuns.db')
const appDb = require('../src/utils/db')
after(() => appDb.close())
@@ -1303,3 +1353,324 @@ test('a non-positive spend never reaches the database', async (t) => {
assert.equal(await budgetDb.spend(runId, 'uo.creatures', 0), true)
assert.equal(await consumedOf(runId), 0)
})
// ── The resource ledger's unique key (Phase 8) ─────────────────────────────
//
// Every test here is about a property of the SERVER. A stub can enforce whatever
// rule its author had in mind; only MariaDB can say whether this encoding of
// "unique among live rows" actually is one.
const insertResource = async (runId, over = {}) =>
(
await pool.query(
`INSERT INTO event_run_resources (run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
runId,
over.stepId ?? null,
over.owner ?? 'demo',
over.kind ?? 'creature',
over.ref ?? '0xA',
over.payload ?? null,
over.leaseUntil ?? null,
over.memberKey ?? null,
over.status ?? 'pending',
],
)
).insertId
const dup = async (fn) => {
try {
await fn()
return null
} catch (err) {
return err.code || String(err.errno)
}
}
test('two runs cannot hold one target, and the refusal comes from the server', async (t) => {
if (needDb(t)) return
// §D: "the unique index is what stops two events leasing one target." Not a
// read-then-insert — two runs entering the same tick would both pass the check
// — so the whole conflict story is this one constraint answering.
const a = await seedRun()
const b = await seedRun()
await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' })
const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' }))
assert.equal(code, 'ER_DUP_ENTRY')
})
test('the key is released by the three statuses that mean it is no longer ours', async (t) => {
if (needDb(t)) return
// Amended 2026-09-03. §D says "among non-reverted rows", which was written
// before the six statuses had their meanings; taken literally it makes
// `drifted` and `orphaned` hold a target for ever, so one bad night would
// disable a lease permanently with no control able to clear it. `drifted` means
// somebody else has hold of the value and this run has let go; `orphaned` means
// it vanished. Neither is a claim on the target.
for (const status of ['reverted', 'drifted', 'orphaned']) {
await pool.query('DELETE FROM event_run_resources')
const a = await seedRun()
const b = await seedRun()
await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status })
const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' }))
assert.equal(code, null, `a ${status} row must not hold the target`)
}
})
test('the key is HELD by the three that mean core still believes it is ours', async (t) => {
if (needDb(t)) return
for (const status of ['pending', 'confirmed', 'reverting']) {
await pool.query('DELETE FROM event_run_resources')
const a = await seedRun()
const b = await seedRun()
await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status })
const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' }))
assert.equal(code, 'ER_DUP_ENTRY', `a ${status} row must hold the target`)
}
})
test('many released rows on one target coexist, which is the whole encoding', async (t) => {
if (needDb(t)) return
// The property the NULL depends on: multiple NULLs do not collide in a unique
// index. A weekly event that leases the same rate every Saturday accumulates one
// released row per week, and the fiftieth must not fail to insert.
const a = await seedRun()
for (let i = 0; i < 5; i++) {
await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'reverted' })
}
const held = await pool.query(
'SELECT COUNT(*) AS n FROM event_run_resources WHERE ref = ? AND live_marker IS NULL',
['demo.rate'],
)
assert.equal(Number(held[0].n), 5)
// And a live one still goes on top of them.
assert.equal(await dup(() => insertResource(a.runId, { kind: 'override', ref: 'demo.rate' })), null)
})
test('an UPDATE that releases a row frees the target at once', async (t) => {
if (needDb(t)) return
// The generated column is STORED, so this is really asking whether MariaDB
// recomputes it on UPDATE and re-indexes. Cleanup depends on it entirely: the
// moment a lease is restored, the next event may take it.
const a = await seedRun()
const b = await seedRun()
const id = await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' })
assert.equal(await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })), 'ER_DUP_ENTRY')
await pool.query("UPDATE event_run_resources SET status = 'reverted' WHERE id = ?", [id])
assert.equal(await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })), null)
})
test('the key is per owner and per kind, not per ref', async (t) => {
if (needDb(t)) return
// `kind` and `ref` are module-opaque strings core stores verbatim, so two
// modules using the same word for different things must not collide — and one
// module's `creature:0xA` and `item:0xA` are two objects.
const a = await seedRun()
await insertResource(a.runId, { owner: 'demo', kind: 'creature', ref: '0xA', status: 'confirmed' })
assert.equal(await dup(() => insertResource(a.runId, { owner: 'other', kind: 'creature', ref: '0xA' })), null)
assert.equal(await dup(() => insertResource(a.runId, { owner: 'demo', kind: 'item', ref: '0xA' })), null)
assert.equal(await dup(() => insertResource(a.runId, { owner: 'demo', kind: 'creature', ref: '0xB' })), null)
})
test('a deleted step leaves its resources behind, and a deleted run does not', async (t) => {
if (needDb(t)) return
// TEAMS.md §2.5's correction, held as a test: `step_id` is SET NULL and it only
// works because the generated column reads `status` alone. If `step_id` were in
// that expression MariaDB would refuse the constraint outright (error 1901), and
// the migration would fail on a fresh install rather than here.
//
// The two directions are different on purpose. A record of what was changed in
// the WORLD must outlive the row that scheduled it — `engagement_sends`'
// argument — while a deleted RUN takes its ledger with it, because the ledger
// exists to answer questions about a run.
const a = await seedRun()
const stepId = await seedStep(a.runId)
const id = await insertResource(a.runId, { stepId, status: 'confirmed' })
await pool.query('DELETE FROM event_run_steps WHERE id = ?', [stepId])
const orphaned = (await pool.query('SELECT step_id, status FROM event_run_resources WHERE id = ?', [id]))[0]
assert.equal(orphaned.step_id, null)
assert.equal(orphaned.status, 'confirmed')
await pool.query('DELETE FROM event_runs WHERE id = ?', [a.runId])
const gone = await pool.query('SELECT id FROM event_run_resources WHERE id = ?', [id])
assert.equal(gone.length, 0)
})
test('the ledger model reserves, confirms, reverts and refuses for real', async (t) => {
if (needDb(t)) return
// Through the shipping module rather than a copy of its statements, like the
// budget tests above: `reserve` reads ER_DUP_ENTRY as a refusal and looks the
// holder up to name it, and both halves of that are the connector's behaviour
// rather than this file's.
const a = await seedRun()
const b = await seedRun()
const first = await resourcesDb.reserve({ runId: a.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' })
assert.equal(first.ok, true)
await resourcesDb.confirm(first.id)
const second = await resourcesDb.reserve({ runId: b.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' })
assert.equal(second.ok, false)
assert.equal(second.code, 'held')
assert.equal(Number(second.holder.run_id), Number(a.runId))
assert.equal(second.holder.status, 'confirmed')
// And once it is given back the second run gets it.
await resourcesDb.markReverted(first.id)
const third = await resourcesDb.reserve({ runId: b.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' })
assert.equal(third.ok, true)
})
test('failRevert increments and NEVER resets, and only a human clears it', async (t) => {
if (needDb(t)) return
// Engagement Phase 14's rule at the statement level: `revert_attempts =
// revert_attempts + 1` is written in one place, and the reset is a separate
// statement with an actor behind it. A `SET revert_attempts = ?` anywhere in
// the sweep would make the ceiling unreachable and the row cycle for ever.
const a = await seedRun()
const id = await resourcesDb
.reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' })
.then((r) => r.id)
await resourcesDb.confirm(id)
await resourcesDb.failRevert(id, 'the shard did not answer')
await resourcesDb.failRevert(id, 'still nothing')
let row = (await pool.query('SELECT * FROM event_run_resources WHERE id = ?', [id]))[0]
assert.equal(Number(row.revert_attempts), 2)
assert.equal(row.status, 'confirmed')
assert.equal(row.last_error, 'still nothing')
assert.equal(await resourcesDb.resetAttempts(a.runId), 1)
row = (await pool.query('SELECT * FROM event_run_resources WHERE id = ?', [id]))[0]
assert.equal(Number(row.revert_attempts), 0)
})
test('claimRevert is a compare-and-set, and reverting is not re-claimable', async (t) => {
if (needDb(t)) return
// The cleanup leg and the manual cleanup route can both be working one run at
// once. `reverting` is deliberately not claimable — a row another pass is
// mid-revert on is left alone, exactly as a step with a live claim is.
const a = await seedRun()
const id = await resourcesDb
.reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' })
.then((r) => r.id)
await resourcesDb.confirm(id)
assert.equal(await resourcesDb.claimRevert(id), true)
assert.equal(await resourcesDb.claimRevert(id), false, 'a second pass must not take a row mid-revert')
await resourcesDb.markReverted(id)
assert.equal(await resourcesDb.claimRevert(id), false, 'and a reverted row is finished')
})
test('the placeholder is resolvable exactly once, and only if it is a placeholder', async (t) => {
if (needDb(t)) return
// The `kind = '@step'` guard in the statement, not in JavaScript. It is what
// stops a bug elsewhere resolving a real resource — which would be core marking
// a live creature as given back without asking anyone.
const a = await seedRun()
const placeholder = await resourcesDb
.reserve({ runId: a.runId, owner: 'demo', kind: resourcesDb.STEP_KIND, ref: 'k'.repeat(40) })
.then((r) => r.id)
const real = await resourcesDb
.reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' })
.then((r) => r.id)
assert.equal(await resourcesDb.resolvePlaceholder(placeholder), true)
assert.equal(await resourcesDb.resolvePlaceholder(placeholder), false)
assert.equal(await resourcesDb.resolvePlaceholder(real), false)
const stillThere = (await pool.query('SELECT status FROM event_run_resources WHERE id = ?', [real]))[0]
assert.equal(stillThere.status, 'pending')
})
test('the unresolved reads see the five statuses that still want something', async (t) => {
if (needDb(t)) return
// `cleanup_status` is derived from this count, so what it includes IS the
// definition of "clean". A `drifted` row left out of it would let a run whose
// lease somebody else took call itself complete.
const a = await seedRun()
for (const status of ['pending', 'confirmed', 'reverting', 'reverted', 'orphaned', 'drifted']) {
await insertResource(a.runId, { ref: `ref-${status}`, status })
}
assert.equal(await resourcesDb.unresolvedCount(a.runId), 5)
const rows = await resourcesDb.unresolvedForRun(a.runId)
assert.deepEqual(
rows.map((r) => r.status).sort(),
['confirmed', 'drifted', 'orphaned', 'pending', 'reverting'],
)
const counts = await resourcesDb.unresolvedCounts([a.runId])
assert.equal(counts.get(a.runId) ?? counts.get(String(a.runId)), 5)
})
test('payload comes back as an object rather than as a string', async (t) => {
if (needDb(t)) return
// A lease's baseline lives in here and the cleanup sweep reads it out to pass
// to `restore`. The connector hands JSON back as text, so a missing hydration
// is a `restore(undefined)` — a lease put back to nothing, silently.
const a = await seedRun()
const id = await resourcesDb
.reserve({
runId: a.runId,
owner: 'demo',
kind: 'override',
ref: 'demo.rate',
payload: { baseline: 1, applied: 3 },
})
.then((r) => r.id)
void id
const [row] = await resourcesDb.forRun(a.runId)
assert.deepEqual(row.payload, { baseline: 1, applied: 3 })
})
test('the cleanup scan finds a run that owes something, whatever its status column says', async (t) => {
if (needDb(t)) return
await pool.query('ALTER TABLE event_runs ADD COLUMN IF NOT EXISTS cleanup_status ' +
"ENUM('not_required','pending','complete','incomplete') NOT NULL DEFAULT 'not_required'")
// **`not_required` is in the scan because of a live-walk defect**, not for
// tidiness. A run whose only resource was a LEASE never went through the
// ledger's `markRunDirty` — `core.lease` reserves its own row — so its column
// stayed `not_required` and the lease was never given back at all. A terminal
// run with an unresolved row has work to do whatever any summary column says.
const lease = await seedRun({ status: 'completed' })
await insertResource(lease.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' })
const marked = await seedRun({ status: 'cancelled' })
await insertResource(marked.runId, { ref: '0xB', status: 'pending' })
await pool.query("UPDATE event_runs SET cleanup_status = 'pending' WHERE id = ?", [marked.runId])
// Three that must NOT be selected, one per reason.
const running = await seedRun({ status: 'running' })
await insertResource(running.runId, { ref: '0xC', status: 'confirmed' })
const done = await seedRun({ status: 'completed' })
await insertResource(done.runId, { ref: '0xD', status: 'reverted' })
const givenUp = await seedRun({ status: 'completed' })
await insertResource(givenUp.runId, { ref: '0xE', status: 'confirmed' })
await pool.query("UPDATE event_runs SET cleanup_status = 'incomplete' WHERE id = ?", [givenUp.runId])
const found = (await resourcesDb.runsNeedingCleanup(10, 3)).map((r) => Number(r.id))
assert.deepEqual(found.sort(), [Number(lease.runId), Number(marked.runId)].sort())
})
test('the scan stops selecting a run whose attempts are spent', async (t) => {
if (needDb(t)) return
// Without the bound in the join, a run whose rows are all spent would be
// selected, worked over and found to have nothing to do on every tick for the
// rest of its life.
const a = await seedRun({ status: 'completed' })
const id = await insertResource(a.runId, { status: 'confirmed' })
await pool.query("UPDATE event_runs SET cleanup_status = 'pending' WHERE id = ?", [a.runId])
assert.equal((await resourcesDb.runsNeedingCleanup(10, 3)).length, 1)
await pool.query('UPDATE event_run_resources SET revert_attempts = 3 WHERE id = ?', [id])
assert.equal((await resourcesDb.runsNeedingCleanup(10, 3)).length, 0)
})
test('setCleanupStatus is guarded, which is what stops a late row re-opening a swept run', async (t) => {
if (needDb(t)) return
const a = await seedRun({ status: 'completed' })
assert.equal(await runsDb.setCleanupStatus(a.runId, 'complete'), true)
assert.equal(await runsDb.setCleanupStatus(a.runId, 'pending', ['not_required']), false)
assert.equal(await runsDb.setCleanupStatus(a.runId, 'pending'), true)
})

37
server/test/eventsAdmin.test.js
View File

@@ -42,6 +42,13 @@ const logDb = require('../src/model/events/eventRunLog.db')
// model needs a stub in every file that stubs that layer.
const settingsDb = require('../src/model/events/eventActionSettings.db')
const budgetDb = require('../src/model/events/eventRunBudget.db')
// Phase 8: the run console reads the resource ledger. The SAME rule, for the
// fourth time in this feature -- Phase 4's expansion leg, Phase 5's gate read,
// Phase 6's settings read and now this one. **Unstubbed it is not a wrong
// answer, it is a ten-second ECONNREFUSED against the dead port**, which is why
// one missing stub here cost the run detail test ten seconds and said nothing
// about the route it was testing.
const resourcesDb = require('../src/model/events/eventRunResources.db')
const seriesDb = require('../src/model/events/eventSeries.db')
const gatesDb = require('../src/model/events/eventPhaseGates.db')
const activity = require('../src/model/activity/activity.model')
@@ -63,6 +70,7 @@ for (const [name, mod] of [
['gatesDb', gatesDb],
['settingsDb', settingsDb],
['budgetDb', budgetDb],
['resourcesDb', resourcesDb],
['activity', activity],
]) {
originals[name] = { mod, fns: { ...mod } }
@@ -88,6 +96,7 @@ function installStubs() {
gates: [],
settings: new Map(),
budget: new Map(),
resources: [],
occurrences: new Set(),
nextDefinition: 1,
nextVersion: 1,
@@ -341,6 +350,9 @@ function installStubs() {
[...store.budget.values()]
.filter((b) => Number(b.run_id) === Number(runId))
.sort((a, b) => a.dimension.localeCompare(b.dimension))
resourcesDb.forRun = async (runId) =>
store.resources.filter((r) => Number(r.run_id) === Number(runId))
}
// ── Fixtures ───────────────────────────────────────────────────────────────
@@ -415,23 +427,26 @@ test('the catalog serves the registry, callables stripped, with its vocabularies
assert.equal(res.statusCode, 200)
assert.deepEqual(
res.body.actions.map((a) => a.id),
['core.announce', 'core.wait', 'core.cue'],
['core.announce', 'core.wait', 'core.cue', 'core.lease'],
)
for (const action of res.body.actions) assert.equal(action.perform, undefined)
assert.deepEqual(res.body.risks, ['notify', 'inspect', 'change', 'irreversible'])
assert.deepEqual(res.body.onFailure, ['skip', 'pause', 'abort_run'])
// The other three registrations of the module contract arrived in Phase 7, and
// they are served BESIDE the actions because the editor needs all four to draw
// one step. Core declares no budgets and no leases of its own — its three
// actions cost nothing and hold nothing — so those are empty here, and that is
// the fact worth asserting: present and empty, not absent.
// one step. Core declares no budgets and no leases of its own — its four
// actions cost nothing, and `core.lease` BORROWS a lease rather than declaring
// one, which is the seam working the way round it is meant to: core owns the
// verb, a module owns the value. So both are empty here, and that is the fact
// worth asserting: present and empty, not absent.
assert.deepEqual(res.body.budgets, [])
assert.deepEqual(res.body.leases, [])
// One option source, and it is core's: `core.announce`'s leg param. It is here
// WITHOUT its resolver — the values are a request of their own.
// Two option sources, both core's: `core.announce`'s leg and `core.lease`'s
// lease. They are here WITHOUT their resolvers — the values are a request of
// their own.
assert.deepEqual(
res.body.optionSources.map((s) => s.id),
['core.options.legs'],
['core.options.legs', 'core.options.leases'],
)
for (const s of res.body.optionSources) assert.equal(s.resolve, undefined)
})
@@ -824,7 +839,12 @@ test('the board serves every registered action with its risk-class default, and
assert.equal(res.statusCode, 200)
const byId = Object.fromEntries(res.body.actions.map((a) => [a.id, a]))
assert.deepEqual(Object.keys(byId).sort(), ['core.announce', 'core.cue', 'core.wait'])
assert.deepEqual(Object.keys(byId).sort(), ['core.announce', 'core.cue', 'core.lease', 'core.wait'])
// And `core.lease` is the one core action the default-off rule bites: it is
// `change`, so a fresh deployment cannot borrow a value until an admin says so.
// §K's sentence, applied to core's own verb rather than only to a module's.
assert.equal(byId['core.lease'].enabled, false)
assert.equal(byId['core.lease'].changesWorld, true)
// core.wait is `inspect`, and it arrives ENABLED. Read §K's sentence literally
// and it would not, and every published event that waits would break on a fresh
// deployment (org lead, 2026-09-03).
@@ -843,6 +863,7 @@ test('the board never serves a callable', async () => {
for (const a of res.body.actions) {
assert.equal(a.perform, undefined)
assert.equal(a.revert, undefined)
assert.equal(a.reconcile, undefined)
assert.equal(a.cost, undefined)
}
})

18
server/test/eventsRoles.test.js
View File

@@ -162,6 +162,11 @@ const SURFACE = [
// Phase 6's switchboard — configuration that can break things.
['GET', '/events/actions', ['admin']],
['PUT', '/events/actions', ['admin']],
// Phase 8's cleanup, and it sits in the ADMIN column rather than with the live
// controls it is rendered beside. Re-running a teardown is not incident
// response — it asks core to write to the world again, which §K puts in the
// same row as the world-changing actions themselves.
['POST', '/events/runs/1/cleanup', ['admin']],
// Live control of a run in flight: admin and moderator, deliberately WIDER
// than start.
@@ -214,6 +219,19 @@ test('an editor may price an event but not publish or start it', async () => {
assert.equal(await forbidden('POST', '/events/1/runs', 'editor'), true)
})
test('a moderator may stop a run but not re-run its cleanup', async () => {
// The same shape as start-and-stop above, one row further on, and held as its
// own claim for the same reason: the two controls sit next to each other on the
// run console and a later tidying pass that gave them one gate would have to
// delete an assertion that says why they do not share one.
//
// Cancelling is the 2am incident. Cleanup asks core to delete things in a live
// world, which is the narrower decision even though it is the tidier-sounding
// button.
assert.equal(await forbidden('POST', '/events/runs/1/cancel', 'moderator'), false)
assert.equal(await forbidden('POST', '/events/runs/1/cleanup', 'moderator'), true)
})
test('the switchboard is admin only in both directions', async () => {
// Reading which actions are enabled is as much `admin` as writing it: the board
// is the deployment's posture, and §K puts it in the same row as the actions it