diff --git a/client/src/api/client.js b/client/src/api/client.js index 7ffba08..4aa0a9c 100644 --- a/client/src/api/client.js +++ b/client/src/api/client.js @@ -538,8 +538,12 @@ export const api = { pauseEventRun: (runId, reason) => req(`/admin/events/runs/${runId}/pause`, { method: 'POST', body: { reason } }), resumeEventRun: (runId) => req(`/admin/events/runs/${runId}/resume`, { method: 'POST' }), - cancelEventRun: (runId, reason) => - req(`/admin/events/runs/${runId}/cancel`, { method: 'POST', body: { reason } }), + // `cleanup` defaults to true server-side and has to be asked out of: EVENTS.md + // §L makes cancelling WITHOUT cleanup the separate, admin-only, logged action, + // so an absent flag means "give back what this run took". + cancelEventRun: (runId, reason, cleanup = true) => + req(`/admin/events/runs/${runId}/cancel`, { method: 'POST', body: { reason, cleanup } }), + cleanupEventRun: (runId) => req(`/admin/events/runs/${runId}/cleanup`, { method: 'POST' }), advanceEventRun: (runId, reason) => req(`/admin/events/runs/${runId}/advance`, { method: 'POST', body: { reason } }), confirmEventStep: (runId, stepId, note) => diff --git a/client/src/routes/admin/views/EventRun.jsx b/client/src/routes/admin/views/EventRun.jsx index 18aa3a6..e3ff02e 100644 --- a/client/src/routes/admin/views/EventRun.jsx +++ b/client/src/routes/admin/views/EventRun.jsx @@ -38,6 +38,15 @@ import { // SERVER'S. `gates[].where` arrives already rendered, because those labels are // defined in the condition grammar and a second renderer in the browser would // be a second opinion about what `gte` reads as. +// +// **Phase 8 gave it a third, and it is the one that outlives the event.** The +// resource ledger is what this run changed in the world and what became of it, +// and its unresolved rows are the reason a `completed` run can still need a +// person — EVENTS.md §L: a run reaches `completed` with `cleanup_status = +// 'incomplete'` rather than being held open, because a tidy `completed` row over +// a shard full of orphaned monsters is the failure that would end this feature's +// credibility on its first bad night. The panel is shown on finished runs for +// exactly that reason, and it is the only panel here whose empty state matters. const POLL_MS = 5000 @@ -50,6 +59,29 @@ const STATUS_COLOR = { completed: '#8fc79a', } +// The six ledger statuses, in the two groups that matter to a reader: green is +// resolved, amber wants a person. `orphaned` and `drifted` are amber rather than +// red because neither is a fault — one thing vanished, the other was taken by +// somebody with every right to take it — and red is reserved for "this did not +// come back and core kept asking". +const RESOURCE_COLOR = { + reverted: '#8fc79a', + confirmed: '#d9c184', + pending: '#d9c184', + reverting: '#d9c184', + drifted: '#d9c184', + orphaned: '#d9c184', +} + +const RESOURCE_WORD = { + pending: 'recorded, unconfirmed', + confirmed: 'still out there', + reverting: 'being given back', + reverted: 'given back', + orphaned: 'gone', + drifted: 'someone else moved it', +} + const STEP_COLOR = { done: '#8fc79a', failed: '#d98b84', @@ -148,6 +180,9 @@ export default function EventRun() { // into the run when it was created, so this is what THIS run is allowed rather // than what the switchboard says today. const [budget, setBudget] = useState([]) + // What this run created or borrowed, and what became of each (Phase 8). + const [resources, setResources] = useState([]) + const [unresolved, setUnresolved] = useState(0) const [lines, setLines] = useState([]) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) @@ -168,6 +203,8 @@ export default function EventRun() { setCounts(detail.counts || {}) setGates(detail.gates || []) setBudget(detail.budget || []) + setResources(detail.resources || []) + setUnresolved(detail.unresolvedResources || 0) setLines(log.log || []) }, [runId]) @@ -308,6 +345,18 @@ export default function EventRun() { onClick={() => act(() => api.admin.cancelEventRun(run.id, reason))}> Cancel run + {/* The separate, admin-only decision (§L). It is a second button rather + than a checkbox on the first because the two are not variants of one + action: one gives the world back, the other deliberately leaves it + changed. A checkbox next to Cancel is a thing an operator unticks by + accident at two in the morning. The server refuses this to a + moderator, and the refusal arrives as a sentence in `problem`. */} + {controls.cancel && ( + + )} {isTerminalRun(run.status) && ( @@ -400,6 +449,79 @@ export default function EventRun() { )} + {/* ── What this run changed in the world (Phase 8) ── + The WHOLE ledger, reverted rows included: "how much did last night's + invasion actually spawn, and did all of it come back" is one question + with two halves, and a list of only the failures answers neither. + Shown on finished runs for the same reason the caps meter is. */} + {(resources.length > 0 || run.cleanupStatus === 'incomplete') && ( +
0 ? '#d9c184' : 'var(--rule)'}`, + }} + > +
+

+ What this run changed +

+ {/* The manual retry. Offered only on a terminal run, because a run + still in flight has a ledger that is still growing and reverting a + resource the next step is about to use would be undoing an event + while it is happening. */} + {isTerminalRun(run.status) && unresolved > 0 && ( + + )} +
+

+ {unresolved > 0 ? ( + <> + {unresolved} of these {unresolved === 1 ? 'is' : 'are'} still unresolved. The runner + gives them back on its own and stops asking after a few tries;{' '} + Try cleanup again clears that count and asks once more. + + ) : ( + 'Everything this run created or borrowed has been given back.' + )} +

+ {resources.length === 0 ? ( +

+ Nothing named — a step changed the world and its answer never arrived, so core kept the + record it wrote beforehand and will ask the module to undo it by key. +

+ ) : ( + + + {resources.map((r) => ( + + + + + + + ))} + +
+ {r.kind}{' '} + {r.ref} + + {RESOURCE_WORD[r.status] || r.status} + + {r.module} + {r.leaseUntil ? ` · until ${clock(r.leaseUntil)}` : ''} + {r.revertAttempts > 0 ? ` · ${r.revertAttempts} attempt${r.revertAttempts === 1 ? '' : 's'}` : ''} + + {r.lastError || ''} +
+ )} +
+ )} + {/* ── Waiting on a person ── */} {parked.length > 0 && (
diff --git a/server/db/schema.sql b/server/db/schema.sql index fd86d56..76744bb 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -2090,7 +2090,7 @@ CREATE TABLE IF NOT EXISTS engagement_suppressions ( -- module contract. The rest arrive with the phases that give them a writer -- rather than as empty tables nothing reads -- `event_run_phase_gates` in P5, -- `event_action_settings` and `event_run_budget` in P6, `event_run_resources` in --- P8 and `event_run_participants` in P10. +-- P8 (below) and `event_run_participants` in P10. -- -- Core tables, so no module prefix, and no game vocabulary anywhere below: an -- action id, a scope, a resource kind and a budget dimension are all opaque @@ -2492,6 +2492,101 @@ CREATE TABLE IF NOT EXISTS event_run_budget ( UNIQUE KEY uq_evbud_dim (run_id, dimension) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- The cleanup ledger: everything one run created or leased, and what became of +-- it (EVENTS.md §D, §L "The ledger's two rules"; Phase 8). +-- +-- **It holds both kinds of thing an event owns.** An OBJECT it created is +-- `kind: 'creature'` with `ref` a serial, reverted by its own action's +-- `revert()`. A VALUE it leased is `kind: 'override'` with `ref` the lease id and +-- `payload` carrying the baseline and what was applied, restored by the lease's +-- own `restore()`. One table, because cleanup asks both the same question -- +-- what is still out there, and did putting it back work. +-- +-- **Rule 1: a resource is recorded BEFORE it is confirmed.** A spawn's serial +-- does not exist until the module answers, so what is written before the dispatch +-- is a PLACEHOLDER keyed by the step's idempotency key (`kind` = the reserved +-- '@step', `ref` = that key). On the answer the reported resources are inserted +-- `confirmed` and the placeholder is resolved. If the acknowledgement is lost the +-- placeholder survives, and cleanup calls `revert()` with the idempotency key and +-- no resources -- which is why §F's `revert({ runId, resources, idempotencyKey })` +-- takes the key at all. Recording afterwards instead would make every object +-- whose ack was lost invisible to cleanup for ever. +-- +-- **Rule 2: revert is idempotent, and its failure is loud and sticky.** A row +-- that never reverts stays visible -- the run reaches `completed` with +-- `cleanup_status = 'incomplete'` rather than being held `running`, because a +-- tidy `completed` over a shard full of orphaned monsters is the failure that +-- would end this feature's credibility on its first bad night. +-- +-- **The unique key is what stops two events leasing one target**, and it must +-- hold among LIVE rows only: last week's finished event must not keep this +-- week's from leasing the same rate. MariaDB has no partial index, so the +-- encoding is a STORED generated column that is NULL once the row is no longer +-- ours -- and multiple NULLs do not collide in a unique index. It is derived from +-- `status` ALONE and the opaque columns stay in the KEY, which is the shape +-- TEAMS.md §2.5 had to be corrected into: MariaDB refuses ON DELETE SET NULL on a +-- foreign key whose column is a base column of a stored generated column +-- (error 1901), so `step_id` must not appear in the expression. +-- +-- **The key is held by the three statuses that mean "core still believes this is +-- ours"** -- `pending`, `confirmed`, `reverting` -- and released by the three that +-- mean it is not. §D says "among non-reverted rows", which was written before the +-- six statuses had their meanings; taken literally it makes `drifted` and +-- `orphaned` hold a target for ever, so one bad night would disable a lease +-- permanently with no control able to clear it. `drifted` means somebody else has +-- hold of the value and this run has deliberately let go of it; `orphaned` means +-- it vanished. Neither is a claim on the target, and both stay LOUD by another +-- mechanism -- `cleanup_status = 'incomplete'` and a row on the run console -- +-- which is what §L's rule 2 actually asks for. Amended 2026-09-03. +CREATE TABLE IF NOT EXISTS event_run_resources ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + run_id BIGINT NOT NULL, + -- Which step made it. It is how cleanup finds the ACTION to call `revert()` on: + -- the row records the module and the opaque names, and the step records the + -- verb. SET NULL rather than CASCADE, for `engagement_sends`' reason -- a record + -- of what was changed in the world must outlive the row that scheduled it. + step_id BIGINT NULL, + -- The registering module, copied at record time rather than derived from the + -- action id, so an uninstalled module still names itself on the console. + owner_module VARCHAR(64) NOT NULL, + -- Both module-opaque, stored verbatim, never interpreted -- `ctx.teams.activity.push`'s + -- treatment. '@step' is the one reserved `kind` and core owns it. + kind VARCHAR(64) NOT NULL, + ref VARCHAR(190) NOT NULL, + payload JSON NULL, + -- A lease's deadline, and NULL for an owned object. It goes DOWN THE WIRE as + -- well: the game side restores baseline when it passes, without being asked + -- again, which is the fail-safe that makes an unattended world change + -- defensible. This column is core's copy of that promise, for the console and + -- for the boot-time check. + lease_until DATETIME NULL, + status ENUM('pending','confirmed','reverting','reverted','orphaned','drifted') + NOT NULL DEFAULT 'pending', + -- Bounded like a step's `attempts`, and for the same reason: a revert that can + -- never succeed must become visible rather than cycling for ever. Engagement + -- Phase 14's rule -- only a terminal row is ever retention-eligible -- is what + -- makes an unbounded counter a row nothing can ever sweep. + revert_attempts INT NOT NULL DEFAULT 0, + last_error VARCHAR(500) NULL, + -- Optional, and module-opaque like the rest: who received it, for a granted + -- reward that results should be able to name. `event_run_participants` joins on + -- the same key in Phase 10. + member_key VARCHAR(190) NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + -- 1 while core still believes this resource is this run's, NULL once it is not. + -- See the unique key below; derived from `status` alone, deliberately. + live_marker TINYINT AS (IF(status IN ('pending','confirmed','reverting'), 1, NULL)) STORED, + CONSTRAINT fk_evres_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE, + CONSTRAINT fk_evres_step FOREIGN KEY (step_id) REFERENCES event_run_steps(id) ON DELETE SET NULL, + -- "Two events cannot hold a lease on one target", among non-reverted rows. + UNIQUE KEY uq_evres_target (owner_module, kind, ref, live_marker), + -- The run console, and the cleanup sweep's read: one run's ledger in order. + INDEX idx_evres_run (run_id, status), + -- The cleanup leg's scan across runs, and the boot-time lease self-check. + INDEX idx_evres_live (status, lease_until) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- §K's last bound: "a scheduled definition that has never been verified is the -- case worth refusing to start". A version is immutable, so a dry run that passed -- against it stays true — which is what makes the pass a property of the VERSION diff --git a/server/routes.guards.json b/server/routes.guards.json index 63d1265..b2c5dd5 100644 --- a/server/routes.guards.json +++ b/server/routes.guards.json @@ -581,6 +581,15 @@ "requireAuth" ] }, + { + "method": "POST", + "path": "/api/v1/admin/events/runs/:runId/cleanup", + "handlers": 2, + "gates": [ + "noindex", + "requireAuth" + ] + }, { "method": "GET", "path": "/api/v1/admin/events/runs/:runId/log", diff --git a/server/routes.manifest.json b/server/routes.manifest.json index 5a85a9e..31caf15 100644 --- a/server/routes.manifest.json +++ b/server/routes.manifest.json @@ -257,6 +257,10 @@ "method": "POST", "path": "/api/v1/admin/events/runs/:runId/cancel" }, + { + "method": "POST", + "path": "/api/v1/admin/events/runs/:runId/cleanup" + }, { "method": "GET", "path": "/api/v1/admin/events/runs/:runId/log" diff --git a/server/src/config/coreEventActions.js b/server/src/config/coreEventActions.js index 1d171f0..e124ac7 100644 --- a/server/src/config/coreEventActions.js +++ b/server/src/config/coreEventActions.js @@ -13,6 +13,15 @@ // a human to go and do something. A deployment with no game module installed has // a working event system made of exactly these. // +// **Phase 8 added a fourth, and it is the odd one out on purpose.** `core.lease` +// names no game noun either — it borrows a value some module declared — but +// unlike the other three it genuinely changes the world, so it is `risk: 'change'` +// and therefore default-off, admin-only and cap-checked like any module verb. +// It is CORE's rather than each module's because §F puts the duration bound and +// the two-events-one-target conflict check on core's side of the seam: a lease +// verb per module would be that bound re-implemented once per module, advisory +// everywhere, and wrong in the first one that forgot it. +// // **Phase 2 gave all three real bodies**, and between them they exercise every // shape §F's envelope can take: `core.announce` does work and finishes, // `core.wait` finishes while deferring what follows it, and `core.cue` succeeds @@ -24,10 +33,42 @@ // which runs under `routeManifest.js` and `swagger.js` against a dead pool // (MODULE_API.md §2.2). Nothing below runs at require time; the announce leg is // looked up inside `perform()`, per call, which is also what makes a leg -// registered by a module that booted later reachable at all. +// registered by a module that booted later reachable at all. `core.lease` is the +// one action here that reaches a table, and it requires the model INSIDE +// `perform()` for the same reason — a top-level require would make this file +// build a pool during route-manifest generation. const registries = require('../modules/registries') + +/** + * Turn the `value` param's text into whatever the named lease says it holds. + * + * The range check is here too, and it is REQUIRED on the numeric types for the + * reason §F gives: unlike a cap, a bad lease value is in force the moment it is + * applied, so "0.5 to 5" is not advice. + */ +function coerceLeaseValue(lease, raw) { + const text = String(raw === undefined || raw === null ? '' : raw).trim() + if (lease.type === 'string') return { ok: true, value: text } + if (lease.type === 'bool') { + if (['true', '1', 'yes', 'on'].includes(text.toLowerCase())) return { ok: true, value: true } + if (['false', '0', 'no', 'off'].includes(text.toLowerCase())) return { ok: true, value: false } + return { ok: false, error: `"${raw}" is not a yes or no value for ${lease.label}` } + } + const n = Number(text) + if (text === '' || !Number.isFinite(n)) { + return { ok: false, error: `"${raw}" is not a number, and ${lease.label} holds one` } + } + if (lease.type === 'int' && !Number.isInteger(n)) { + return { ok: false, error: `${lease.label} holds a whole number, and "${raw}" is not one` } + } + if (n < lease.min || n > lease.max) { + return { ok: false, error: `${lease.label} accepts ${lease.min} to ${lease.max}, and "${raw}" is outside that` } + } + return { ok: true, value: n } +} + const ACTIONS = [ { id: 'core.announce', @@ -214,6 +255,164 @@ const ACTIONS = [ return { ok: true, await: 'human' } }, }, + + { + id: 'core.lease', + label: 'Borrow a value', + description: + 'Hold a module-declared value at a new setting for a bounded time, and put the old one back at teardown.', + + // The world changes and it changes back, so `change` rather than + // `irreversible` — and `change`'s default `on_failure` is `pause`, which is + // the right stop for a run that failed halfway through altering the world. + risk: 'change', + // The one action core ships in this class. `override` is what tells the + // cleanup sweep to restore through the LEASE registry rather than through an + // action's `revert()`, which is why this action needs no `revert()` of its own + // and why the registry refuses one on it. + reversible: 'override', + version: 1, + + params: [ + { + name: 'lease', + type: 'string', + required: true, + example: 'uo.rate.skillgain', + source: 'core.options.leases', + description: 'Which declared value to borrow.', + }, + { + // **A string, and the coercion is here rather than in the type system.** + // A param declares ONE type; a lease declares its own, and they are four + // different ones. Typing this `float` would make a boolean lease + // unauthorable and a string lease nonsense, so the field takes text and + // this action turns it into whatever the named lease said it holds — the + // one place that knows both halves. + name: 'value', + type: 'string', + required: true, + example: '3.0', + description: 'What to hold it at, in whatever type the lease declares.', + }, + { + name: 'minutes', + type: 'int', + required: true, + example: 120, + description: 'How long to hold it. Core refuses more than the lease allows.', + }, + ], + + // What a lease costs is the LEASE's business to bound, not a budget's: + // `maxDurationMs` and the numeric range are declared beside the callables and + // enforced below. A cap dimension here would be core inventing an accounting + // unit for something a module already bounds — and `registerEventBudgets` + // refuses a dimension nobody declared, which is exactly the rule that would + // then bite core's own action. + + /** + * Read the baseline, reserve the target, apply the value. + * + * **This is rule 1 in its strongest form.** Unlike a spawn, a lease's target + * is knowable before the dispatch — it is the lease id the step names — so + * the ledger row is written with its real `kind` and `ref` BEFORE anything + * touches the world, and the two-events-one-target refusal comes from the + * unique index at that moment rather than from a check that read and then + * wrote. A second run asking for a lease another run holds comes back + * `refused`, in the same words a cap breach uses and for the same reason: + * nothing is broken, the deployment already has that value spoken for. + * + * The order is read then reserve then apply, and a failure at each stage + * undoes the one before it: a reservation whose `apply` refuses is released + * here rather than left for the sweep, because there is nothing out there to + * give back and a shard that is merely down must not lock a lease out for the + * length of a retry cycle. + */ + async perform({ runId, stepId, params, verify }) { + // eslint-disable-next-line global-require + const resourcesDb = require('../model/events/eventRunResources.db') + const lease = registries.eventLease(params.lease) + if (!lease) { + return { ok: false, retry: false, error: `no module registers the lease "${params.lease}"` } + } + + const coerced = coerceLeaseValue(lease, params.value) + if (!coerced.ok) return { ok: false, retry: false, error: coerced.error } + + const minutes = Number(params.minutes) + if (!Number.isFinite(minutes) || minutes <= 0) { + return { ok: false, retry: false, error: `"${params.minutes}" is not a number of minutes` } + } + const ms = Math.round(minutes * 60_000) + if (ms > lease.maxDurationMs) { + return { + ok: false, + retry: false, + error: `${lease.label} may be held for at most ${Math.floor(lease.maxDurationMs / 60_000)} minutes, not ${minutes}`, + } + } + + // **The dry run stops here, and it has still checked everything worth + // checking**: the lease exists, the value is in range and the duration is + // allowed. What it deliberately does not do is reserve the target — a + // verify that took a lease would be a dry run that changed something, and + // it would then refuse the real run that followed it. + if (verify) return { ok: true } + + const baseline = await lease.read() + if (!baseline || baseline.ok !== true) { + return { ok: false, error: `could not read the current value of ${lease.label}` } + } + + const until = new Date(Date.now() + ms) + const reserved = await resourcesDb.reserve({ + runId, + stepId, + owner: lease.owner || 'core', + kind: 'override', + ref: lease.id, + payload: { target: lease.id, baseline: baseline.value, applied: coerced.value, until: until.toISOString() }, + leaseUntil: until, + }) + if (!reserved.ok) { + const heldBy = reserved.holder ? ` (run ${reserved.holder.run_id})` : '' + return { + ok: false, + retry: false, + error: `${lease.label} is already leased by another run${heldBy}`, + } + } + + // **`until` goes down the wire** (§F). The module passes it to its sidecar + // and the game side restores baseline when it passes, without being asked + // again — the fail-safe that makes an unattended, scheduled world change + // defensible, because the worst case is a world back at baseline early + // rather than one stuck changed indefinitely. + let applied + try { + applied = await lease.apply(coerced.value, until) + } catch (err) { + applied = { ok: false, error: err.message } + } + if (!applied || applied.ok !== true) { + await resourcesDb.markReverted(reserved.id) + return { ok: false, error: applied && applied.error ? String(applied.error) : `${lease.label} refused the new value` } + } + + await resourcesDb.confirm(reserved.id) + // **The run now owes the world something, and something has to say so.** + // The generic path marks a run dirty when it records a module's reported + // resources; this action reserves its own row and never goes through it, so + // a run whose only resource was a lease would have kept `cleanup_status = + // 'not_required'` and never been swept. Found by the live walk, and the + // cleanup leg's own scan was widened to make the class impossible rather + // than only this instance. + // eslint-disable-next-line global-require + await require('../events/ledger').markRunDirty(runId) + return { ok: true } + }, + }, ] // ── Core's own param option sources (§F, Phase 7) ────────────────── @@ -237,6 +436,16 @@ const OPTION_SOURCES = [ return registries.announceLegs().map((l) => ({ value: l.leg, label: l.label || l.leg })) }, }, + { + id: 'core.options.leases', + label: 'Borrowable values', + description: 'Every value a module has declared this deployment may lease.', + async resolve() { + return registries + .allEventLeases() + .map((l) => ({ value: l.id, label: l.label, group: l.id.split('.')[0] })) + }, + }, ] module.exports = { ACTIONS, OPTION_SOURCES } diff --git a/server/src/events/cleanup.js b/server/src/events/cleanup.js new file mode 100644 index 0000000..4fe2af5 --- /dev/null +++ b/server/src/events/cleanup.js @@ -0,0 +1,427 @@ +// ── Giving back what a run took ──────────────────────────────────────────── +// +// EVENTS.md §C ("Cleanup is generated, never authored"), §L and its two ledger +// rules, and Phase 8 of EVENTS_PLAN.md. `events/ledger.js` is the write half; +// this is the undo half, plus the reconcile that answers "is any of it still +// there?" after something outside core restarted. +// +// **Cleanup is derived from the ledger, never authored.** An operator cannot be +// relied on to write the undo, and an aborted run never reaches the phase they +// wrote it in — so there is no cleanup phase in a spec and no `on_teardown` on an +// action. There is one function, it reads rows, and it runs on EVERY terminal +// path: completion, cancellation and abort alike. +// +// **It is not built out of `event_run_steps` rows** (org lead, 2026-09-03). The +// plan's phrase is "cleanup steps are generated from the ledger", and the +// tempting reading is a synthetic phase of real step rows so the console's +// per-step retry comes free. It is the wrong shape here for one concrete reason: +// `event_run_resources` already carries `revert_attempts` and `last_error`, so +// synthetic steps would put a second retry counter beside the first and the two +// would disagree the first time a step reverted three of its four resources. +// The manual retry the API surface promises is a route over the ledger — +// `POST /admin/events/runs/:runId/cleanup` — rather than a step control. +// +// **Where it runs from.** One place: the runner's cleanup leg, which finds +// terminal runs that still owe the world something and works their rows. Hooking +// each terminal path instead would be four call sites, three of which are inside +// a request, and none of which would survive the process dying mid-cleanup. The +// leg is ordered AFTER advance in the tick, so a run that completes in one tick +// is cleaned in the same one. +// +// **The scan's WHERE clause cost two live-walk findings, in opposite +// directions.** A run whose only resource was a LEASE never went through +// `ledger.markRunDirty` — `core.lease` reserves its own row — so its +// `cleanup_status` stayed `not_required` and the lease was never given back at +// all. And a run whose first sweep failed was moved to `incomplete` by that very +// sweep, so it was never picked up again: `MAX_REVERT_ATTEMPTS` meant ONE attempt +// rather than three. The first is why `not_required` is in the scan; the second +// is why `incomplete` is written HERE only once nothing retryable is left. +// +// **Rule 2 is what the bounds are for.** A revert that never succeeds must stay +// visible rather than cycle: `MAX_REVERT_ATTEMPTS` stops the automatic retry, the +// run reaches `completed` with `cleanup_status = 'incomplete'`, and the rows stay +// on the console with their last error. Only a human's cleanup clears the +// counter — Engagement Phase 14's rule, whose defect was a sweep that reset every +// stale row and made the ceiling unreachable for ever. + +const resourcesDb = require('../model/events/eventRunResources.db') +const runsDb = require('../model/events/eventRuns.db') +const logDb = require('../model/events/eventRunLog.db') +const stepsDb = require('../model/events/eventRunSteps.db') +const registries = require('../modules/registries') +const { withDeadline } = require('./dispatch') +const log = require('../utils/logger')('events') + +// How many times the automatic sweep will ask before leaving a resource for a +// human. Three, like a step's, and for the same reason: a fourth attempt against +// a shard that has answered the same way three times is not new information. +const MAX_REVERT_ATTEMPTS = Number(process.env.EVENT_REVERT_MAX_ATTEMPTS) || 3 + +// The bound on one revert call, when the action that made the resource is gone +// and there is no `budgetMs` to read. A restore is a round trip like any other. +const DEFAULT_REVERT_BUDGET_MS = 10_000 + +// How many runs one cleanup leg looks at, and how many resource groups it works +// per run. Bounds rather than targets, exactly like `RUN_BATCH`: the tick runs +// again, and an unbounded teardown is how one run's bad night stalls every other. +const CLEANUP_RUN_BATCH = Number(process.env.EVENT_CLEANUP_RUN_BATCH) || 10 +const CLEANUP_GROUPS_PER_RUN = Number(process.env.EVENT_CLEANUP_GROUPS_PER_RUN) || 25 + +/** + * Classify one revert answer, with `dispatch.classify`'s posture: no shape a + * failure can take may read as success. + * + * The extra value here is `drifted`. It is NOT an error — the module did exactly + * what it was asked and found somebody else's value in place — so it is a third + * outcome rather than a failure with a flag, and the row it produces is the one + * §L wants surfaced beside the unreverted ones. + */ +function classifyRevert(raw, what) { + if (raw && raw.__timedOut) return { outcome: 'retry', error: raw.error } + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + return { outcome: 'retry', error: `${what} answered with no envelope` } + } + if (raw.ok === true) { + // §L, and the Rust wipe: "gone, and that is fine" is a successful revert. A + // module never has to distinguish "I deleted it" from "it was not there". + return { outcome: 'done', failed: Array.isArray(raw.failed) ? raw.failed.map(String) : [] } + } + if (raw.drifted === true) { + return { + outcome: 'drifted', + error: `the value is now ${JSON.stringify(raw.current)} rather than what this run applied, so it was left alone`, + } + } + return { + outcome: raw.retry === false ? 'terminal' : 'retry', + error: raw.error ? String(raw.error) : `${what} refused`, + } +} + +/** Call a lease's `restore`, under a deadline, never throwing. */ +async function restoreLease(row) { + const lease = registries.eventLease(row.ref) + if (!lease) { + // The module that owned it is uninstalled or failed to boot. Not a failure to + // retry away — nothing will change until an operator reinstalls it — and not + // an orphan either, because core has no idea whether the value is still + // applied. It stays unresolved with the reason on it, which is exactly what + // `cleanup_status = 'incomplete'` is for. + return { outcome: 'terminal', error: `no module registers the lease "${row.ref}"` } + } + const payload = row.payload || {} + let raw + try { + raw = await withDeadline( + () => lease.restore(payload.baseline, { expected: payload.applied, runId: row.run_id }), + DEFAULT_REVERT_BUDGET_MS, + row.ref, + ) + } catch (err) { + return { outcome: 'retry', error: err.message } + } + return classifyRevert(raw, row.ref) +} + +/** Call an action's `revert` over a group of its rows, under a deadline, never throwing. */ +async function revertGroup(actionId, rows, idempotencyKey) { + const action = registries.eventAction(actionId) + if (!action || typeof action.revert !== 'function') { + return { + outcome: 'terminal', + error: action + ? `${actionId} declares no revert()` + : `no module registers "${actionId}", so its resources cannot be given back`, + } + } + const payload = rows + .filter((r) => r.kind !== resourcesDb.STEP_KIND) + .map((r) => ({ kind: r.kind, ref: r.ref, payload: r.payload || null, memberKey: r.member_key || null })) + + let raw + try { + raw = await withDeadline( + () => action.revert({ runId: rows[0].run_id, resources: payload, idempotencyKey }), + action.budgetMs || DEFAULT_REVERT_BUDGET_MS, + actionId, + ) + } catch (err) { + // A module should not throw from `revert` any more than from `perform`, and + // one that does has produced a transient failure rather than a crashed sweep. + log.warn('event revert threw', { action: actionId, message: err.message }) + return { outcome: 'retry', error: err.message } + } + return classifyRevert(raw, actionId) +} + +/** + * Work one run's ledger once. + * + * Answers what it found and what it managed, and sets `cleanup_status` from the + * rows that are left rather than from what it did — the two differ whenever + * another writer touched the run, and the rows are the truth. + * + * `resetAttempts` is the human's flag. It is never set by the automatic leg. + */ +async function cleanupRun(run, { resetAttempts = false, actor = null } = {}) { + const summary = { attempted: 0, reverted: 0, drifted: 0, failed: 0, remaining: 0 } + + if (resetAttempts) { + const cleared = await resourcesDb.resetAttempts(run.id) + await runsDb.setCleanupStatus(run.id, 'pending') + if (cleared) { + await logDb.write({ + runId: run.id, + kind: 'cleanup.retry', + detail: { resources: cleared, by: actor }, + }) + } + } + + const rows = await resourcesDb.unresolvedForRun(run.id, { + maxAttempts: resetAttempts ? null : MAX_REVERT_ATTEMPTS, + }) + + // **Grouped by the step that made them**, because that is what names the verb: + // the resource row records the module and the opaque names, the step records + // the action, and `revert()` takes a LIST so one round trip can give back + // twelve creatures. A lease is its own group of one — core restores it through + // the lease registry rather than through any action, which is the split §F + // draws and the reason `core.lease` needs no `revert()` of its own. + const leases = rows.filter((r) => r.kind === 'override') + const byStep = new Map() + for (const row of rows) { + if (row.kind === 'override') continue + const key = row.step_id === null ? `orphan:${row.id}` : `step:${row.step_id}` + if (!byStep.has(key)) byStep.set(key, []) + byStep.get(key).push(row) + } + + const groups = [...leases.map((r) => ({ lease: r })), ...[...byStep.values()].map((rs) => ({ rows: rs }))] + + for (const group of groups.slice(0, CLEANUP_GROUPS_PER_RUN)) { + if (group.lease) { + const row = group.lease + if (!(await resourcesDb.claimRevert(row.id))) continue + summary.attempted += 1 + const verdict = await restoreLease(row) + await applyVerdict(run, [row], verdict, summary, row.ref) + continue + } + + const rs = group.rows + // The step is what names the action and carries the idempotency key. A row + // whose step was deleted keeps the action id in its own payload, which is why + // the placeholder writes one. + const step = rs[0].step_id === null ? null : await stepsDb.getById(rs[0].step_id) + const actionId = step?.action_id || rs[0].payload?.action || null + if (!actionId) { + await noteUnrevertable(run, rs, 'nothing records which action created this', summary) + continue + } + const claimed = [] + for (const row of rs) if (await resourcesDb.claimRevert(row.id)) claimed.push(row) + if (!claimed.length) continue + summary.attempted += claimed.length + const verdict = await revertGroup(actionId, claimed, step?.idempotency_key || rs[0].ref) + await applyVerdict(run, claimed, verdict, summary, actionId) + } + + summary.remaining = await resourcesDb.unresolvedCount(run.id) + // **`incomplete` means "finished with, and not finished"**, so it is written + // only once there is nothing left this sweep will try. Writing it after the + // FIRST failure — which is what the first draft did — took the run straight out + // of the leg's own scan, and `MAX_REVERT_ATTEMPTS` quietly meant one attempt + // rather than three. Found by the live walk, watching `revert_attempts` sit at + // 1 through half a minute of ticks. + const retryable = await resourcesDb.unresolvedForRun(run.id, { maxAttempts: MAX_REVERT_ATTEMPTS }) + const status = summary.remaining === 0 ? 'complete' : retryable.length ? 'pending' : 'incomplete' + await runsDb.setCleanupStatus(run.id, status) + + if (summary.attempted > 0) { + await logDb.write({ + runId: run.id, + kind: 'cleanup.swept', + detail: { ...summary, by: actor }, + }) + } + return summary +} + +/** Write one verdict across the rows it covers, and count it. */ +async function applyVerdict(run, rows, verdict, summary, what) { + for (const row of rows) { + if (verdict.outcome === 'done' && !(verdict.failed || []).includes(row.ref)) { + await resourcesDb.markReverted(row.id) + summary.reverted += 1 + continue + } + if (verdict.outcome === 'drifted') { + await resourcesDb.failRevert(row.id, verdict.error, 'drifted') + summary.drifted += 1 + continue + } + const error = + verdict.outcome === 'done' + ? `${what} could not give "${row.ref}" back` + : verdict.error + await resourcesDb.failRevert(row.id, error, 'confirmed') + summary.failed += 1 + } + await logDb.write({ + runId: run.id, + kind: verdict.outcome === 'done' ? 'cleanup.reverted' : 'cleanup.failed', + detail: { + what, + outcome: verdict.outcome, + resources: rows.map((r) => `${r.kind}:${r.ref}`), + ...(verdict.error ? { error: verdict.error } : {}), + }, + }) +} + +/** A group core cannot even name a verb for. Counted as failed, and said once. */ +async function noteUnrevertable(run, rows, reason, summary) { + for (const row of rows) { + if (!(await resourcesDb.claimRevert(row.id))) continue + await resourcesDb.failRevert(row.id, reason, 'confirmed') + summary.attempted += 1 + summary.failed += 1 + } + await logDb.write({ + runId: run.id, + kind: 'cleanup.failed', + detail: { what: null, outcome: 'terminal', resources: rows.map((r) => `${r.kind}:${r.ref}`), error: reason }, + }) +} + +/** + * The cleanup leg of the tick: every TERMINAL run with something left to give + * back. + * + * Terminal only. A run still in flight has a ledger that is still growing, and + * reverting a resource the next step is about to use would be core undoing an + * event while it is happening. + */ +async function sweep() { + // The ceiling goes INTO the query, so a run whose rows are all spent is not + // selected, worked over and found to have nothing to do on every tick for the + // rest of its life. It is also what excludes a run an admin cancelled without + // cleanup, whose counters were spent deliberately. + const candidates = await resourcesDb.runsNeedingCleanup(CLEANUP_RUN_BATCH, MAX_REVERT_ATTEMPTS) + let swept = 0 + for (const candidate of candidates) { + if (!runsDb.TERMINAL.includes(candidate.status)) continue + try { + await cleanupRun(candidate) + swept += 1 + } catch (err) { + log.error('event cleanup failed', { run: candidate.id, message: err.message }) + } + } + return swept +} + +/** + * Ask one module which of its ledgered resources the game still has (§L, and + * §N7's "the shard stays stateless about events"). + * + * **Core cannot know when to ask**, and that is not an omission: §F says core has + * no concept of the game being up, because a module with six sidecars cannot + * answer that question in the singular. So the module triggers this, through + * `ctx.events.reconcile()`, when it sees its own reconnect — module-uo already + * watches `bootId` for exactly that. Core also asks once at boot, for its own + * restart. + * + * **A resource the module no longer has becomes `orphaned`, never `reverted`.** + * Reverting it would be core recording that it put something back when what + * actually happened is that the thing vanished while nobody was looking, and the + * two are different sentences to the operator reading the console afterwards. + * + * A module with no `reconcile()` on the action is not broken: core keeps + * believing its own ledger, which is precisely the behaviour before this phase. + */ +async function reconcileModule(owner) { + const rows = await resourcesDb.liveForModule(owner) + const summary = { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 } + if (!rows.length) return summary + + const byStep = new Map() + for (const row of rows) { + if (row.kind === resourcesDb.STEP_KIND) continue // nothing to ask about yet + const key = row.step_id === null ? `orphan:${row.id}` : `step:${row.step_id}` + if (!byStep.has(key)) byStep.set(key, []) + byStep.get(key).push(row) + } + + for (const group of byStep.values()) { + const step = group[0].step_id === null ? null : await stepsDb.getById(group[0].step_id) + const actionId = step?.action_id || group[0].payload?.action || null + const action = actionId ? registries.eventAction(actionId) : null + if (!action || typeof action.reconcile !== 'function') { + summary.unanswered += group.length + continue + } + summary.asked += group.length + let raw + try { + raw = await withDeadline( + () => + action.reconcile({ + runId: group[0].run_id, + resources: group.map((r) => ({ kind: r.kind, ref: r.ref, payload: r.payload || null })), + }), + action.budgetMs || DEFAULT_REVERT_BUDGET_MS, + actionId, + ) + } catch (err) { + log.warn('event reconcile threw', { action: actionId, message: err.message }) + raw = null + } + // Same posture as everywhere else: nothing that is not an explicit answer + // counts as one. A module that could not answer leaves the ledger alone, + // because "I do not know" must never be read as "it is gone". + if (!raw || raw.__timedOut || raw.ok !== true || !Array.isArray(raw.inForce)) { + summary.unanswered += group.length + continue + } + const held = new Set(raw.inForce.map(String)) + for (const row of group) { + if (held.has(row.ref)) { + summary.inForce += 1 + continue + } + await resourcesDb.markOrphaned(row.id, 'the module reports this is no longer in force') + summary.orphaned += 1 + await logDb.write({ + runId: row.run_id, + kind: 'resource.orphaned', + detail: { module: owner, resource: `${row.kind}:${row.ref}`, action: actionId }, + }) + } + } + return summary +} + +/** Ask every module that owns a live row. Core's own boot-time sweep. */ +async function reconcileAll() { + const owners = await resourcesDb.modulesWithLiveRows() + const out = {} + for (const owner of owners) { + try { + out[owner] = await reconcileModule(owner) + } catch (err) { + log.error('event reconcile failed', { module: owner, message: err.message }) + } + } + return out +} + +module.exports = { + MAX_REVERT_ATTEMPTS, + classifyRevert, + cleanupRun, + sweep, + reconcileModule, + reconcileAll, +} diff --git a/server/src/events/ledger.js b/server/src/events/ledger.js new file mode 100644 index 0000000..7638619 --- /dev/null +++ b/server/src/events/ledger.js @@ -0,0 +1,221 @@ +// ── Recording what a run changed in the world ────────────────────────────── +// +// EVENTS.md §D and §L, and Phase 8 of EVENTS_PLAN.md. The write half of the +// resource ledger; `events/cleanup.js` is the read-and-undo half. +// +// **Rule 1 is the whole reason this file is not two lines inside `drainStep`.** +// A resource is recorded BEFORE it is confirmed. The obstacle is that a spawn's +// serial does not exist until the module answers, so there is nothing to write a +// row about yet — which is why what goes in before the dispatch is a PLACEHOLDER +// keyed by the step's idempotency key rather than by the object: +// +// pre-dispatch INSERT pending { kind: '@step', ref: } +// answer INSERT confirmed { kind: 'creature', ref: '0x40001234' } × n +// resolve the placeholder +// ack lost the placeholder is still `pending` +// cleanup revert({ idempotencyKey, resources: [] }) +// +// That last line is why §F's `revert({ runId, resources, idempotencyKey })` takes +// the key at all. A module that half-ran and never answered is reachable by its +// key and by nothing else, and Phase 11's plugin-side key ledger is what makes +// answering it exact. Until then the contract is still honest, because §L +// requires reverting something that does not exist to be a SUCCESS. +// +// **Recording is idempotent, and the database is what makes it so.** A retry +// re-dispatches the same idempotency key, and a module that answers with the same +// resources twice must not produce two rows. `uq_evres_target` refuses the second +// insert, and this file reads that refusal as "already recorded" rather than as an +// error — the same posture `materialisePhase`'s INSERT IGNORE takes. +// +// **A lease does not use the placeholder.** Its target is knowable before the +// dispatch — it is the lease id the step names — so `core.lease` reserves the +// real row first, which is both a stronger form of rule 1 and the only place the +// two-events-one-target refusal can happen before the world has been written to. + +const resourcesDb = require('../model/events/eventRunResources.db') +const runsDb = require('../model/events/eventRuns.db') +const registries = require('../modules/registries') +const log = require('../utils/logger')('events') + +// Which reversible classes get a pre-dispatch placeholder. `none` is gone once +// done and `self` undoes itself, so neither has anything core could come back +// for; `override` reserves its own target instead (see the header). That leaves +// `ledger` — the class that declares `revert()`, which is exactly the class whose +// refs core cannot know until the module speaks. +const PLACEHOLDER_CLASSES = ['ledger'] + +// A resource `kind` may be anything a module likes except core's own reserved +// one. Bounded to the column, and refused rather than truncated: a truncated ref +// is a cleanup call naming the wrong object. +const MAX_KIND = 64 +const MAX_REF = 190 +const MAX_MEMBER_KEY = 190 + +/** Does this action produce anything core will have to come back for? */ +const ledgers = (action) => action && (action.reversible === 'ledger' || action.reversible === 'override') + +/** + * Turn one entry of a module's `resources` array into a row, or say why not. + * + * Every failure here is the module's mistake rather than the world's, so none of + * them is a retry: a badly shaped resource will be just as badly shaped on the + * second attempt. They are logged and dropped, and the step still counts as done + * — because it IS done; something happened in the world, and refusing to record + * it would be the one outcome worse than recording it imperfectly. + */ +function normalise(entry, actionId) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + return { ok: false, reason: `${actionId} reported a resource that is not an object` } + } + const kind = String(entry.kind || '') + const ref = String(entry.ref === undefined || entry.ref === null ? '' : entry.ref) + if (!kind || kind.length > MAX_KIND) { + return { ok: false, reason: `${actionId} reported a resource with a bad kind "${entry.kind}"` } + } + if (kind === resourcesDb.STEP_KIND) { + // Core's own. A module that could write one would be a module that could make + // its own step's placeholder look resolved. + return { ok: false, reason: `${actionId} reported a resource of the reserved kind "${kind}"` } + } + if (!ref || ref.length > MAX_REF) { + return { ok: false, reason: `${actionId} reported a resource with a bad ref "${entry.ref}"` } + } + const memberKey = entry.memberKey === undefined || entry.memberKey === null ? null : String(entry.memberKey) + if (memberKey !== null && memberKey.length > MAX_MEMBER_KEY) { + return { ok: false, reason: `${actionId} reported a resource with an over-long memberKey` } + } + + let leaseUntil = null + if (entry.until !== undefined && entry.until !== null) { + const at = new Date(entry.until) + if (Number.isNaN(at.getTime())) { + return { ok: false, reason: `${actionId} reported a resource with a bad until "${entry.until}"` } + } + leaseUntil = at + } + + // A borrowed value must name a lease core knows how to give back. Core restores + // an `override` through the lease registry — that is the split §F draws — so a + // ref naming nothing registered is a resource core would be recording with no + // way to undo it, which is the promise rule 2 exists to stop core making. + if (kind === 'override' && !registries.eventLease(ref)) { + return { ok: false, reason: `${actionId} reported a lease "${ref}" no module registers` } + } + + return { + ok: true, + row: { + kind, + ref, + payload: entry.payload === undefined ? null : entry.payload, + leaseUntil, + memberKey, + }, + } +} + +/** + * Write the pre-dispatch placeholder for a step that is about to change the + * world. Answers the row id, or null when this action ledgers nothing. + * + * **A duplicate is not a failure.** A retry re-uses its step's idempotency key, so + * the second attempt's placeholder collides with the first's — and finding it + * already there is the correct answer, not an error. The existing row is reused. + */ +async function reserveStep(run, step, action) { + if (!ledgers(action) || !PLACEHOLDER_CLASSES.includes(action.reversible)) return null + + const owner = action.owner || 'core' + const reserved = await resourcesDb.reserve({ + runId: run.id, + stepId: step.id, + owner, + kind: resourcesDb.STEP_KIND, + ref: step.idempotency_key, + payload: { action: action.id, phase: step.phase, seq: step.seq }, + }) + if (reserved.ok) { + await markRunDirty(run.id) + return reserved.id + } + // The only way a '@step' row collides is with this step's own earlier attempt, + // because an idempotency key is minted once per step and never varies by + // attempt (§E). Reuse it. + const existing = await resourcesDb.findByTarget(owner, resourcesDb.STEP_KIND, step.idempotency_key) + return existing ? existing.id : null +} + +/** + * Record what a module said it made, and close out the placeholder. + * + * Answers `{ recorded, rejected }` — how many rows went in, and the reasons any + * entry was dropped. Never throws: a step that changed the world has changed it, + * and a ledger that threw would turn a bookkeeping problem into a failed step and + * then into a retry of a world write that already happened. + */ +async function recordAnswer({ run, step, action, placeholderId, resources }) { + const out = { recorded: 0, rejected: [] } + if (!ledgers(action)) return out + + const owner = action.owner || 'core' + const list = Array.isArray(resources) ? resources : [] + + for (const entry of list) { + const parsed = normalise(entry, action.id) + if (!parsed.ok) { + out.rejected.push(parsed.reason) + log.warn('event resource rejected', { run: run.id, step: step.id, reason: parsed.reason }) + continue + } + try { + const reserved = await resourcesDb.reserve({ + runId: run.id, + stepId: step.id, + owner, + kind: parsed.row.kind, + ref: parsed.row.ref, + payload: parsed.row.payload, + leaseUntil: parsed.row.leaseUntil, + memberKey: parsed.row.memberKey, + }) + if (!reserved.ok) { + // Already ledgered — by this step's own earlier attempt, or (a module bug + // rather than a race) by another run that still holds the same target. + // Either way there is a live row for it and a second would be the double + // cleanup the unique key exists to prevent. + if (reserved.holder && reserved.holder.run_id !== run.id) { + out.rejected.push(`${parsed.row.kind} "${parsed.row.ref}" is already held by run ${reserved.holder.run_id}`) + } + continue + } + await resourcesDb.confirm(reserved.id) + out.recorded += 1 + } catch (err) { + // Bookkeeping must not become the step's control flow. + out.rejected.push(err.message) + log.error('event resource insert failed', { run: run.id, step: step.id, message: err.message }) + } + } + + if (out.recorded > 0) await markRunDirty(run.id) + + // The placeholder's job is over the moment the real rows exist. It is resolved + // even when the module reported nothing at all — an action that ledgers and + // then answers `ok` with an empty list is saying "I made nothing", and holding + // its placeholder open would make cleanup call `revert()` for a step that has + // nothing to give back on every terminal path for ever. + if (placeholderId) await resourcesDb.resolvePlaceholder(placeholderId) + + return out +} + +/** + * There is now something to clean up. Idempotent and guarded, so it can never + * walk a run back from `complete` or `incomplete` to `pending` — only a human's + * cleanup does that, and it does it deliberately. + */ +async function markRunDirty(runId) { + await runsDb.setCleanupStatus(runId, 'pending', ['not_required']) +} + +module.exports = { ledgers, normalise, reserveStep, recordAnswer, markRunDirty, PLACEHOLDER_CLASSES } diff --git a/server/src/model/events/eventRunControls.model.js b/server/src/model/events/eventRunControls.model.js index d80a91a..a0ad08e 100644 --- a/server/src/model/events/eventRunControls.model.js +++ b/server/src/model/events/eventRunControls.model.js @@ -15,10 +15,13 @@ // diagnosis panel: a screen that explains why a phase has not started, beside a // control that does something about it. // -// **One of §I's controls is still not here.** `cleanup` needs Phase 8's resource -// ledger; there is nothing to revert, so cancel takes `{ reason }` and gains -// `cleanup` when there is something for it to do. Absent rather than inert, -// which is the posture Phase 1 set and every phase since has kept. +// **`cleanup` is the eighth, and Phase 8 is what gave it a ledger to work over.** +// It re-runs the teardown across every resource a run has not given back, and it +// is `admin` where the other seven are `admin` + `moderator`: it is not incident +// response, it is asking core to write to the world again. Its partner is +// cancel's new `cleanup: false`, which is §L's "cancelling WITHOUT cleanup is a +// separate, logged, admin-only action" — deliberately the flag that has to be +// asked for, because the safe default is to give back what the run took. // // **Every control is guarded on the status it may act from, and the guard is a // WHERE clause rather than a read-then-write.** A run console rendered thirty @@ -37,6 +40,7 @@ const runsDb = require('./eventRuns.db') const stepsDb = require('./eventRunSteps.db') const logDb = require('./eventRunLog.db') const gatesDb = require('./eventPhaseGates.db') +const resourcesDb = require('./eventRunResources.db') const gates = require('../../events/gates') const MAX_REASON = 500 @@ -152,11 +156,23 @@ async function resume(runId, options = {}, userId = null) { * and a second writer on that row would race the process that owns it. It * finishes into a cancelled run, which is honest. */ -async function cancel(runId, { reason } = {}, userId = null) { +async function cancel(runId, { reason, cleanup = true } = {}, userId = null, { isAdmin = true } = {}) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is already ${run.status}`) + // §L: cancelling WITHOUT cleanup is a separate, logged, ADMIN-only action. The + // route itself is `admin` + `moderator`, so the narrower gate cannot live in + // middleware — which of the two you have to be depends on what is in the body, + // exactly as the authoring role floor does (§K). + if (cleanup === false && !isAdmin) { + return { + ok: false, + status: 403, + errors: ['leaving a run\'s world changes in place is an administrator\'s decision'], + } + } + const note = clean(reason) const from = ['scheduled', 'starting', 'running', 'paused', 'ending'] if (!(await runsDb.transition(run.id, from, 'cancelled', { error: note || 'cancelled by staff' }))) { @@ -168,9 +184,83 @@ async function cancel(runId, { reason } = {}, userId = null) { runId: run.id, kind: 'run.status', phase: run.current_phase, - detail: { from: run.status, to: 'cancelled', control: 'cancel', by: userId, reason: note, cancelledSteps: closed }, + detail: { + from: run.status, + to: 'cancelled', + control: 'cancel', + by: userId, + reason: note, + cancelledSteps: closed, + cleanup: cleanup !== false, + }, }) - return { ok: true, run: await runsDb.getById(run.id), cancelledSteps: closed } + + // **The teardown is not done here, and the request does not wait for it.** + // Cleanup is one leg of the runner's tick over terminal runs (§L), which is + // what makes it survive a process that dies halfway through it — and a cancel + // pressed at two in the morning must answer at once rather than after a dozen + // round trips to a shard that may be the reason it is being cancelled. The run + // is terminal the moment this returns, so the very next tick picks its ledger + // up. + // + // `cleanup: false` is the operator saying leave it. The resources stay + // unresolved and the run carries `incomplete`, which is the truthful value: the + // world changes are still up, they are listed on the console, and the log line + // above records who decided that. + let cleanupStatus = run.cleanup_status + if (cleanup === false && (await resourcesDb.unresolvedCount(run.id)) > 0) { + // `incomplete` is what takes the run out of the cleanup leg's scan, and it is + // the truthful value: the world changes are still up, they are listed on the + // console, and the log line above records who decided that. + // + // **The first draft spent every row's `revert_attempts` instead**, to stop the + // sweep by the same mechanism a failed retry does. It worked and it made the + // console lie: the run page rendered "3 attempts" beside resources nothing had + // ever tried, which reads as "core tried three times and could not". Found by + // opening the page. A counter that means two things is a counter a screen + // cannot render. + await runsDb.setCleanupStatus(run.id, 'incomplete') + cleanupStatus = 'incomplete' + } + + return { + ok: true, + run: await runsDb.getById(run.id), + cancelledSteps: closed, + cleanup: cleanup !== false, + cleanupStatus, + } +} + +/** + * Re-run cleanup over everything a run has not given back. + * + * The manual retry §L promises, and the only thing that clears + * `revert_attempts`. That licence is the same one a human's step retry has, and + * it is deliberately not extended to the automatic sweep: Engagement Phase 14's + * defect was exactly a sweep that reset every stale row, which made the attempt + * ceiling unreachable and left the row cycling for ever. + * + * Legal on a TERMINAL run only. A run still in flight has a ledger that is still + * growing, and reverting a resource the next step is about to use would be core + * undoing an event while it is happening. + */ +async function cleanupRun(runId, userId = null) { + const run = await loadRun(runId) + if (!run) return { ok: false, status: 404, errors: ['no such run'] } + if (!runsDb.TERMINAL.includes(run.status)) { + return conflict(`this run is still ${run.status}; cancel it before cleaning up after it`) + } + if (run.cleanup_status === 'not_required') { + return conflict('this run recorded no resources, so there is nothing to give back') + } + + // eslint-disable-next-line global-require + const summary = await require('../../events/cleanup').cleanupRun(run, { + resetAttempts: true, + actor: userId, + }) + return { ok: true, run: await runsDb.getById(run.id), summary } } /** @@ -359,4 +449,4 @@ async function retryStep(runId, stepId, options = {}, userId = null) { } } -module.exports = { pause, resume, cancel, advancePhase, confirmStep, skipStep, retryStep } +module.exports = { pause, resume, cancel, cleanupRun, advancePhase, confirmStep, skipStep, retryStep } diff --git a/server/src/model/events/eventRunLog.db.js b/server/src/model/events/eventRunLog.db.js index 4cb291e..6abb204 100644 --- a/server/src/model/events/eventRunLog.db.js +++ b/server/src/model/events/eventRunLog.db.js @@ -47,6 +47,16 @@ const KINDS = [ 'run.budget', // the caps this run was seeded with, and which switch set each 'step.refused', // a step was not permitted: disabled, or over a cap 'version.verified', // a dry run passed against a version, unlocking scheduled starts + // Phase 8's six, and every one of them is an answer to "what did this event + // leave behind". `resource.recorded` is written at the ANSWER rather than at + // the placeholder, because a placeholder is a promise and the operator's + // question is about the world. + 'resource.recorded', // a step reported what it created or borrowed, and it is ledgered + 'resource.orphaned', // a module reports a ledgered resource is no longer in force + 'cleanup.reverted', // a group of resources came back + 'cleanup.failed', // a group did not, with the reason and how it was left + 'cleanup.swept', // one pass over a run's ledger, and what it found + 'cleanup.retry', // a human cleared the attempt counter and asked again ] const hydrate = (row) => row && { ...row, detail: parseJson(row.detail, null) } diff --git a/server/src/model/events/eventRunResources.db.js b/server/src/model/events/eventRunResources.db.js new file mode 100644 index 0000000..9f11af4 --- /dev/null +++ b/server/src/model/events/eventRunResources.db.js @@ -0,0 +1,353 @@ +// ── event_run_resources — SQL only ───────────────────────────────────────── +// +// EVENTS.md §D and §L ("The ledger's two rules"), and Phase 8 of EVENTS_PLAN.md. +// Everything one run created or leased, and what became of it. +// +// **Rule 1 lives in `reserve()`.** A resource is recorded BEFORE it is +// confirmed, so the placeholder this writes is the row that exists while the +// dispatch is in flight — and the row that SURVIVES when the acknowledgement is +// lost. Recording on the answer instead would make every object whose ack went +// missing invisible to cleanup for ever. +// +// **Rule 2 lives in the status column and in `failRevert()`.** A revert that +// never succeeds leaves its row unreverted, with the error on it, and the run +// completes with `cleanup_status = 'incomplete'` rather than being held open. +// Loud and sticky. +// +// **The unique key is enforced by the database, not by a read.** `reserve()` +// answers `{ ok: false, code: 'held' }` on a duplicate key rather than checking +// first and then inserting — two runs entering the same tick would both pass the +// check. It is the argument `event_run_budget.spend()` makes about the cap and +// `runsDb.transition` makes about a status, in the third place it applies. + +const { query } = require('../../utils/db') +const { parseJson } = require('./eventJson') + +// The one `kind` core owns. A module's kinds are opaque and stored verbatim; this +// one is core's own, and `registries` refuses a module resource that claims it. +const STEP_KIND = '@step' + +// The statuses that mean "core still believes this resource is this run's". They +// are exactly the ones the `live_marker` generated column keeps non-NULL, so the +// unique target key holds while a row is in one of them and releases when it +// leaves. Duplicated here as a JavaScript list because the sweeps read by it too, +// and a second copy that can drift is better than a query that cannot express it. +const HELD = ['pending', 'confirmed', 'reverting'] + +// Every status that still wants a human or a retry: `HELD` plus the two that mean +// "we let go, and not cleanly". This is what "unreverted" means everywhere in +// this feature — the console's list, `cleanup_status`, and the manual retry. +const UNRESOLVED = [...HELD, 'orphaned', 'drifted'] + +const COLUMNS = `id, run_id, step_id, owner_module, kind, ref, payload, lease_until, + status, revert_attempts, last_error, member_key, created_at, updated_at` + +// `payload` is opaque to core and stored verbatim, but it comes back as a string +// from the driver and every caller wants the object — the cleanup sweep reads a +// lease's baseline out of it, and the console renders it. Hydrated here for the +// same reason a step's params are: one place rather than at each read. +const hydrate = (row) => row && { ...row, payload: parseJson(row.payload, null) } + +/** + * Record a resource that does not exist yet. + * + * Answers `{ ok: true, id }`, or `{ ok: false, code: 'held', holder }` when the + * target is already someone's — which is the lease conflict, surfaced as a + * refusal rather than a failure because nothing is wrong with the system: another + * run has the thing. + * + * **`ER_DUP_ENTRY` is the check.** The holder is looked up only to name it in the + * refusal, and only after the insert has already lost the race. + */ +async function reserve({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) { + try { + const result = await query( + `INSERT INTO event_run_resources + (run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')`, + [runId, stepId, owner, kind, ref, payload === null ? null : JSON.stringify(payload), leaseUntil, memberKey], + ) + return { ok: true, id: Number(result.insertId) } + } catch (err) { + if (err && (err.code === 'ER_DUP_ENTRY' || err.errno === 1062)) { + const [holder] = await query( + `SELECT run_id, status FROM event_run_resources + WHERE owner_module = ? AND kind = ? AND ref = ? AND status IN (?, ?, ?) + LIMIT 1`, + [owner, kind, ref, ...HELD], + ) + return { ok: false, code: 'held', holder: holder || null } + } + throw err + } +} + +/** + * Promote a reserved row to `confirmed`, optionally attaching what the module + * finally said about it. + * + * Guarded on `pending` so a late answer cannot un-revert a row cleanup has + * already dealt with — the same reason every other write in this feature is a + * compare-and-set rather than a read-then-write. + */ +async function confirm(id, { payload, leaseUntil, memberKey } = {}) { + const sets = ["status = 'confirmed'"] + const params = [] + if (payload !== undefined) { + sets.push('payload = ?') + params.push(payload === null ? null : JSON.stringify(payload)) + } + if (leaseUntil !== undefined) { + sets.push('lease_until = ?') + params.push(leaseUntil) + } + if (memberKey !== undefined) { + sets.push('member_key = ?') + params.push(memberKey) + } + const result = await query( + `UPDATE event_run_resources SET ${sets.join(', ')} WHERE id = ? AND status = 'pending'`, + [...params, id], + ) + return (result.affectedRows || 0) > 0 +} + +/** + * Resolve a step placeholder once the module has named what it actually made. + * + * The placeholder's whole job is over at this point: the real rows exist, so the + * `@step` row must stop being one of the things cleanup will try to revert. + * `reverted` is the honest terminal state for it — there is nothing left to undo + * that the rows it stood in for do not now cover — and it releases the + * idempotency key for a later run, which matters because keys are per step and a + * re-materialised step reuses its own. + */ +async function resolvePlaceholder(id) { + const result = await query( + `UPDATE event_run_resources + SET status = 'reverted', last_error = NULL + WHERE id = ? AND kind = ? AND status IN ('pending', 'confirmed')`, + [id, STEP_KIND], + ) + return (result.affectedRows || 0) > 0 +} + +/** + * One row by its target, live or not — how a caller that lost the insert race + * finds the row it meant to write. Newest first, so a target that has been held + * and released several times answers with the current holder. + */ +async function findByTarget(owner, kind, ref) { + const [row] = await query( + `SELECT ${COLUMNS} FROM event_run_resources + WHERE owner_module = ? AND kind = ? AND ref = ? + ORDER BY id DESC LIMIT 1`, + [owner, kind, ref], + ) + return hydrate(row) || null +} + +/** One run's whole ledger, oldest first — the console's read. */ +async function forRun(runId) { + const rows = await query( + `SELECT ${COLUMNS} FROM event_run_resources WHERE run_id = ? ORDER BY id`, + [runId], + ) + return rows.map(hydrate) +} + +/** The rows of one run that still want something: the cleanup sweep's input. */ +async function unresolvedForRun(runId, { maxAttempts = null } = {}) { + const params = [runId, ...UNRESOLVED] + const attemptClause = maxAttempts === null ? '' : ' AND revert_attempts < ?' + if (maxAttempts !== null) params.push(maxAttempts) + const rows = await query( + `SELECT ${COLUMNS} FROM event_run_resources + WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)${attemptClause} + ORDER BY id`, + params, + ) + return rows.map(hydrate) +} + +/** How many of one run's rows are still unresolved — what `cleanup_status` is derived from. */ +async function unresolvedCount(runId) { + const [row] = await query( + `SELECT COUNT(*) AS n FROM event_run_resources + WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`, + [runId, ...UNRESOLVED], + ) + return Number(row?.n || 0) +} + +/** Unresolved counts for several runs at once, keyed by run id — the run LIST's read. */ +async function unresolvedCounts(runIds) { + const ids = [...new Set(runIds || [])].filter(Boolean) + if (!ids.length) return new Map() + const rows = await query( + `SELECT run_id, COUNT(*) AS n FROM event_run_resources + WHERE run_id IN (${ids.map(() => '?').join(',')}) AND status IN (?, ?, ?, ?, ?) + GROUP BY run_id`, + [...ids, ...UNRESOLVED], + ) + return new Map(rows.map((r) => [r.run_id, Number(r.n)])) +} + +/** + * Claim one row for a revert: `pending | confirmed | orphaned | drifted → reverting`. + * + * The compare-and-set that keeps the cleanup leg and the manual cleanup route off + * each other's rows. `reverting` is deliberately not claimable — a row another + * pass is mid-revert on is left alone, exactly as a step with a live claim is. + */ +async function claimRevert(id) { + const result = await query( + `UPDATE event_run_resources + SET status = 'reverting' + WHERE id = ? AND status IN ('pending', 'confirmed', 'orphaned', 'drifted')`, + [id], + ) + return (result.affectedRows || 0) > 0 +} + +/** The revert worked. `reverted` is terminal and releases the target. */ +async function markReverted(id) { + await query( + `UPDATE event_run_resources SET status = 'reverted', last_error = NULL WHERE id = ?`, + [id], + ) +} + +/** + * The revert did not work, and the row goes back to being unresolved. + * + * `revert_attempts` is incremented here and NOWHERE else, and it is never reset by + * a sweep — Engagement Phase 14's rule, whose defect was a reclaim that returned + * every stale row to its start state and made the attempt ceiling unreachable, so + * the row cycled for ever and was never eligible for any retention sweep. The one + * thing that may reset it is a human pressing cleanup, which is the same licence + * a human's step retry has. + * + * `restoreTo` is where the row lands: `drifted` when the module says somebody else + * moved the value, `orphaned` when it says the thing is gone, and `confirmed` + * otherwise — still ours, still out there, try again. + */ +async function failRevert(id, error, restoreTo = 'confirmed') { + await query( + `UPDATE event_run_resources + SET status = ?, revert_attempts = revert_attempts + 1, last_error = ? + WHERE id = ?`, + [restoreTo, String(error || 'the revert did not answer').slice(0, 500), id], + ) +} + +/** + * A human is trying again: clear the attempt counter on one run's unresolved rows. + * + * Only ever called from the cleanup route with an actor behind it. The automatic + * leg must never do this (see `failRevert`). + */ +async function resetAttempts(runId) { + const result = await query( + `UPDATE event_run_resources + SET revert_attempts = 0 + WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`, + [runId, ...UNRESOLVED], + ) + return result.affectedRows || 0 +} + +/** Every live row one module owns, for the reconcile sweep. */ +async function liveForModule(owner, { limit = 500 } = {}) { + const rows = await query( + `SELECT ${COLUMNS} FROM event_run_resources + WHERE owner_module = ? AND status IN ('pending', 'confirmed') + ORDER BY id LIMIT ?`, + [owner, Number(limit)], + ) + return rows.map(hydrate) +} + +/** Every module that currently owns a live row — who the reconcile sweep asks. */ +async function modulesWithLiveRows() { + const rows = await query( + `SELECT DISTINCT owner_module FROM event_run_resources + WHERE status IN ('pending', 'confirmed')`, + ) + return rows.map((r) => r.owner_module) +} + +/** + * The game no longer has it. Never reached by a revert — a revert that finds + * nothing there is a SUCCESS (§L, and it is what a Rust wipe needs) — only by + * reconcile, which is a different question: nobody asked for this to go. + */ +async function markOrphaned(id, detail = null) { + await query( + `UPDATE event_run_resources + SET status = 'orphaned', last_error = ? + WHERE id = ? AND status IN ('pending', 'confirmed', 'reverting')`, + [detail === null ? null : String(detail).slice(0, 500), id], + ) +} + +/** + * Terminal runs that still owe the world something — the cleanup leg's scan. + * + * **Both halves of the WHERE were live-walk findings, and they are opposite + * mistakes.** + * + * `cleanup_status = 'pending'` alone missed a run whose only resource was a + * LEASE: `core.lease` reserves its own row and never goes through the ledger's + * `markRunDirty`, so the flag stayed `not_required` and the lease was never given + * back at all. Hence `not_required` is in the list — a terminal run with an + * unresolved row has something to do whatever any summary column says, and + * treating that combination as work is the fail-safe direction. + * + * And the run status filter alone made `MAX_REVERT_ATTEMPTS` mean ONE attempt, + * because the first failing sweep set `incomplete` and nothing looked at the run + * again. That is fixed in `cleanupRun`, which now only writes `incomplete` once + * there is nothing left it will try — so `incomplete` genuinely means "finished + * with, and not finished", which is exactly what excludes both a run whose + * retries are spent and a run an admin cancelled without cleanup. + * + * The attempt bound is in the join for a different reason: without it a run whose + * rows are all spent would be selected, worked over and found to have nothing to + * do on every tick for the rest of its life. + */ +async function runsNeedingCleanup(limit = 25, maxAttempts = 3) { + return query( + `SELECT DISTINCT r.id, r.status, r.cleanup_status, r.version_id, r.definition_id, r.scope + FROM event_runs r + JOIN event_run_resources res ON res.run_id = r.id + WHERE r.status IN ('completed', 'cancelled', 'failed', 'missed') + AND r.cleanup_status IN ('pending', 'not_required') + AND res.status IN (?, ?, ?, ?, ?) + AND res.revert_attempts < ? + ORDER BY r.id + LIMIT ?`, + [...UNRESOLVED, Number(maxAttempts), Number(limit)], + ) +} + +module.exports = { + STEP_KIND, + HELD, + UNRESOLVED, + reserve, + confirm, + resolvePlaceholder, + findByTarget, + forRun, + unresolvedForRun, + unresolvedCount, + unresolvedCounts, + claimRevert, + markReverted, + failRevert, + resetAttempts, + liveForModule, + modulesWithLiveRows, + markOrphaned, + runsNeedingCleanup, +} diff --git a/server/src/model/events/eventRuns.db.js b/server/src/model/events/eventRuns.db.js index 5f09f13..5b224ff 100644 --- a/server/src/model/events/eventRuns.db.js +++ b/server/src/model/events/eventRuns.db.js @@ -386,6 +386,32 @@ async function setHealth(id, health) { return Number(result?.affectedRows || 0) === 1 } +/** + * Set `cleanup_status`, optionally guarded on where it is now (Phase 8). + * + * Four values and three writers, which is why the guard is a parameter rather + * than baked in. The ledger stamps `pending` the first time a run records + * anything, and it must do so only over `not_required` — a run already marked + * `complete` must not be walked back to `pending` by a late resource, and a + * `incomplete` one must not be silently tidied. The cleanup sweep sets `complete` + * or `incomplete` from what it found, unguarded, because the sweep IS the + * authority on that. A human's cleanup route re-opens `pending` deliberately, and + * says so in the log with the actor. + * + * **`pending` on a run that is still running is not a bug and reads correctly**: + * there is something to clean up and it has not happened yet. The alternative - + * a fifth value meaning "there will be something later" - is a state nothing + * would ever branch on. + */ +async function setCleanupStatus(id, to, from = null) { + const guard = from === null ? '' : ` AND cleanup_status IN (${from.map(() => '?').join(',')})` + const result = await query( + `UPDATE event_runs SET cleanup_status = ? WHERE id = ?${guard}`, + from === null ? [to, id] : [to, id, ...from], + ) + return Number(result?.affectedRows || 0) === 1 +} + /** * Runs whose start instant passed more than their own grace window ago (§E, §L). * @@ -488,6 +514,7 @@ module.exports = { statusOf, transition, setHealth, + setCleanupStatus, concurrencyHolder, reclaimStale, terminalBefore, diff --git a/server/src/model/events/eventRuns.model.js b/server/src/model/events/eventRuns.model.js index 5a41f57..dd00149 100644 --- a/server/src/model/events/eventRuns.model.js +++ b/server/src/model/events/eventRuns.model.js @@ -27,6 +27,7 @@ const definitionsDb = require('./eventDefinitions.db') const versionsDb = require('./eventVersions.db') const settingsDb = require('./eventActionSettings.db') const budgetDb = require('./eventRunBudget.db') +const resourcesDb = require('./eventRunResources.db') const authorize = require('../../events/authorize') const MAX_SCOPE = 190 @@ -204,11 +205,12 @@ async function create( async function detail(runId) { const run = await db.getById(runId) if (!run) return null - const [steps, counts, gateRows, budget] = await Promise.all([ + const [steps, counts, gateRows, budget, resources] = await Promise.all([ stepsDb.listForRun(runId), stepsDb.statusCounts(runId), gatesDb.listForRun(runId), budgetDb.forRun(runId), + resourcesDb.forRun(runId), ]) const now = new Date() return { @@ -226,6 +228,37 @@ async function detail(runId) { cap: b.cap, from: b.effective_from, })), + // What this run changed in the world, and what became of it (Phase 8). The + // WHOLE ledger, reverted rows included, because "what did last night's + // invasion actually spawn, and did all of it come back" is the question this + // panel exists for and a list of only the failures cannot answer the second + // half of it. + // + // **The `@step` placeholders are filtered out.** They are core's own + // bookkeeping — a row that says "a dispatch is in flight and may have made + // something" — and the console's list is of things in the world. One left in + // would read as a resource nobody can name, which is exactly the confusion it + // exists to prevent internally. + resources: resources + .filter((r) => r.kind !== resourcesDb.STEP_KIND) + .map((r) => ({ + id: r.id, + stepId: r.step_id, + module: r.owner_module, + kind: r.kind, + ref: r.ref, + payload: r.payload, + leaseUntil: r.lease_until, + status: r.status, + revertAttempts: r.revert_attempts, + lastError: r.last_error, + memberKey: r.member_key, + createdAt: r.created_at, + })), + // How many rows are still unresolved, counted over the WHOLE ledger rather + // than over the list above — a placeholder left standing by a lost + // acknowledgement is exactly the case `cleanup_status` must not call clean. + unresolvedResources: resources.filter((r) => resourcesDb.UNRESOLVED.includes(r.status)).length, } } diff --git a/server/src/modules/loader.js b/server/src/modules/loader.js index df0024d..7503186 100644 --- a/server/src/modules/loader.js +++ b/server/src/modules/loader.js @@ -228,6 +228,39 @@ function buildCtx(id, moduleRoot) { emit: (triggerId, envelope) => { engagementEmit.emit(id, triggerId, envelope) }, + // EVENTS.md §L, and the resource ledger (Phase 8). "On reconnect the runner + // asks each ledgered resource's module to reconcile" — and this is how the + // runner learns there has BEEN a reconnect. + // + // **Core cannot decide when to call this, and that is the contract rather + // than a gap.** §F: core has no concept of the game being up, because a + // module with six sidecars cannot answer that question in the singular. So + // the module says so, when it sees its own — module-uo already watches + // `bootId` to tell a shard restart from a sidecar reconnect, which is + // exactly the moment a ledger of live spawns has become a claim about a + // world that no longer exists. + // + // `id` is bound here and never taken from the arguments, like `emit` and + // `teams.activity.push` before it: a module reconciles its OWN ledger, and + // without the binding this would be a way to have core mark another + // module's resources orphaned. + // + // Fire-and-forget and returns undefined, for the third time and the same + // reason: this is called from inside a connection handler, and there is + // nothing a module could correctly do with a failure of core's bookkeeping. + reconcile: () => { + // eslint-disable-next-line global-require + require('../events/cleanup') + .reconcileModule(id) + .then( + (summary) => { + if (summary && summary.orphaned) { + log.warn('event resources orphaned on reconcile', { module: id, ...summary }) + } + }, + (err) => { log.error('ctx.events.reconcile failed', { module: id, message: err.message }) }, + ) + }, }, // The in-app sink (§5.1) — a module writing the inbox directly, without a // rule. Live from Phase 7; it threw until the `user_notifications` table diff --git a/server/src/modules/registries.js b/server/src/modules/registries.js index 2823c52..88d52e8 100644 --- a/server/src/modules/registries.js +++ b/server/src/modules/registries.js @@ -157,11 +157,11 @@ const eventBudgets = new Map() // lease id → { owner, id, label, type, min, max, maxDurationMs, description, // read, apply, restore } (§F "Leases: one more declaration", Phase 7). // -// **Phase 7 registers a lease and nothing acquires one.** Core owns the duration -// and the conflict check, the module owns reading the current value and writing a -// new one — and both halves of that live in the resource ledger, which is Phase -// 8's. What is here is the declaration, its validation and its catalog entry, so -// that the module contract is one version rather than two. +// **Core owns the duration and the conflict check; the module owns reading the +// current value and writing a new one.** Phase 7 registered a lease and nothing +// acquired one; Phase 8 gave it a verb — `core.lease`, a CORE action, so the +// bound and the two-events-one-target refusal are enforced in one place rather +// than re-implemented by every module that ships a lease. const eventLeases = new Map() // source id → { owner, id, label, description, resolve } (§F "Param option @@ -435,14 +435,14 @@ async function resolveAudience(id, params = {}) { /** * Every declaration WITHOUT its callables — what the admin catalog serves. * - * `perform`, `revert` and `cost` are stripped for the same reason `resolve` is + * `perform`, `revert`, `reconcile` and `cost` are stripped for the same reason `resolve` is * stripped from an audience and `handler` from a slash command: this is the * object that leaves the process, and the browser's whole relationship with an * action is naming one by id. §F's "a module registers actions server-side and * adds no routes for them" is only true if the functions never ride out. */ const allEventActions = () => - [...eventActions.values()].map(({ perform, revert, cost, ...rest }) => rest) + [...eventActions.values()].map(({ perform, revert, reconcile, cost, ...rest }) => rest) /** One declaration, callables included. The runner's lookup (Phase 2). */ const eventAction = (id) => eventActions.get(id) || null @@ -485,7 +485,7 @@ const isEventBudget = (id) => eventBudgets.has(id) const allEventLeases = () => [...eventLeases.values()].map(({ read, apply: applyValue, restore, ...rest }) => rest) -/** One lease, callables included. Phase 8's lookup; nothing calls it yet. */ +/** One lease, callables included. `core.lease` and the cleanup sweep read it. */ const eventLease = (id) => eventLeases.get(id) || null /** Every option source WITHOUT its resolver — the authoring form's list. */ @@ -985,7 +985,7 @@ function checkActionParam(actionId, entry, seen) { } /** - * `registerEventActions([{ id, label, risk, reversible, version, budgetMs, cost, params, perform, revert }])`. + * `registerEventActions([{ id, label, risk, reversible, version, budgetMs, cost, params, perform, revert, reconcile }])`. * * A typed verb core may ask a registrant to carry out. Everything decidable from * the argument alone is decided here, at the call; the collision — is this id @@ -1042,6 +1042,23 @@ function checkEventActionShape(entry) { `registerEventActions: ${a.id} declares revert() but is reversible: '${a.reversible}'`, ) } + // §L's reconnect row, and it is OPTIONAL where `revert` is required (Phase 8). + // `revert` is how a run gives a resource back; `reconcile` is how a module says + // which of them the game still has after something outside core restarted. A + // module that cannot answer that question is not broken -- core simply keeps + // believing its own ledger, which is the pre-Phase-8 behaviour -- whereas a + // module that created something and cannot undo it has made a promise core has + // no way to keep. Only meaningful for an action that ledgers anything. + if (a.reconcile !== undefined) { + if (typeof a.reconcile !== 'function') { + throw new Error(`registerEventActions: ${a.id} reconcile must be a function`) + } + if (a.reversible === 'none' || a.reversible === 'self') { + throw new Error( + `registerEventActions: ${a.id} declares reconcile() but is reversible: '${a.reversible}' and ledgers nothing`, + ) + } + } if (a.cost !== undefined && typeof a.cost !== 'function') { throw new Error(`registerEventActions: ${a.id} cost must be a function of its params`) } @@ -1076,6 +1093,7 @@ function checkEventActionShape(entry) { cost: a.cost || null, perform: a.perform, revert: a.revert || null, + reconcile: a.reconcile || null, } } @@ -1130,10 +1148,12 @@ function checkEventBudgetShape(entry) { * thing a module must not be allowed to skip. A lease whose restore writes blindly * is a lease that silently reverts an operator's manual fix. * - * **Nothing acquires a lease in Phase 7.** This registers, validates and serves - * one; the ledger that holds it, the deadline that goes down the wire and the - * drift answer are Phase 8's. Declaring it now is what keeps the module contract - * one version rather than two. + * **A lease is acquired by `core.lease` and by nothing else** (Phase 8). The step + * names a lease id, a value and a duration; core reads the baseline, reserves the + * target in `event_run_resources` — which is where the two-events-one-target + * refusal comes from — applies the value with the deadline, and restores it at + * teardown through the same `restore()` the drift check lives in. A module ships + * the three callables and never has to own any of that. */ function checkEventLeaseShape(entry) { const l = entry || {} diff --git a/server/src/router/v1/admin/events.controller.js b/server/src/router/v1/admin/events.controller.js index 009a46b..6442dce 100644 --- a/server/src/router/v1/admin/events.controller.js +++ b/server/src/router/v1/admin/events.controller.js @@ -9,13 +9,12 @@ // this screen does. // // **Phase 3 added the live run controls** at the bottom of this file: pause, -// resume, cancel, and a step's confirm, skip and retry. What is still absent is -// `advance`, `cleanup` and the action switchboard — `advance` has no honest -// meaning until Phase 5 gives a phase an advance condition, `cleanup` has no -// ledger to work over until Phase 8, and the switchboard is Phase 6's. Each of -// them is absent rather than stubbed, for the reason the whole set was in Phase -// 1: a control that returns 200 and does nothing is worse than one that is not -// there. +// resume, cancel, and a step's confirm, skip and retry. `advance` joined them in +// Phase 5, the action switchboard in Phase 6, and **`cleanup` in Phase 8** — +// each when the phase that gave it something to act on landed, and each absent +// rather than stubbed until then, for the reason the whole set was in Phase 1: a +// control that returns 200 and does nothing is worse than one that is not there. +// Nothing in the § API surface table is absent any more. const registries = require('../../../modules/registries') const spec = require('../../../events/spec') @@ -315,6 +314,12 @@ exports.getRun = async (req, res) => { // rather than "what is allowed now" — which is the question that survives // an admin moving a switch tomorrow. budget: found.budget, + // The resource ledger (Phase 8): everything this run created or borrowed, and + // what became of each. The WHOLE ledger, reverted rows included — "how much + // did last night's invasion spawn, and did all of it come back" is one + // question with two halves, and a list of only the failures answers neither. + resources: found.resources, + unresolvedResources: found.unresolvedResources, }) } @@ -678,14 +683,48 @@ exports.advanceRunPhase = async (req, res) => { exports.cancelRun = async (req, res) => { const runId = asId(req.params.runId) if (!runId) return res.status(400).json({ error: 'bad run id' }) - const result = await controls.cancel(runId, { reason: req.body?.reason }, req.user.id) + // **`cleanup` defaults to true and has to be asked out of.** §L makes cancelling + // WITHOUT cleanup the separate, admin-only, logged action, so an absent flag + // must mean "give back what this run took" — the safe direction, and the one a + // moderator's cancel at two in the morning takes without having to know the + // flag exists. + const withCleanup = req.body?.cleanup !== false + const result = await controls.cancel( + runId, + { reason: req.body?.reason, cleanup: withCleanup }, + req.user.id, + { isAdmin: req.user.role === 'admin' }, + ) if (!result.ok) return res.status(result.status || 409).json({ errors: result.errors }) await activity.log({ req, action: 'event.run.cancelled', - detail: { runId, reason: req.body?.reason || null, cancelledSteps: result.cancelledSteps }, + detail: { + runId, + reason: req.body?.reason || null, + cancelledSteps: result.cancelledSteps, + cleanup: result.cleanup, + }, }) - return res.json({ run: shapeRun(result.run), cancelledSteps: result.cancelledSteps }) + return res.json({ + run: shapeRun(result.run), + cancelledSteps: result.cancelledSteps, + cleanup: result.cleanup, + }) +} + +/** POST /api/v1/admin/events/runs/:runId/cleanup */ +exports.cleanupRun = async (req, res) => { + const runId = asId(req.params.runId) + if (!runId) return res.status(400).json({ error: 'bad run id' }) + const result = await controls.cleanupRun(runId, req.user.id) + if (!result.ok) return res.status(result.status || 409).json({ errors: result.errors }) + await activity.log({ req, action: 'event.run.cleaned', detail: { runId, ...result.summary } }) + // **A 200 whatever the sweep found.** The request succeeded; some resources may + // still be out there, and answering 4xx would make "the shard refused to delete + // three of these" indistinguishable from "you sent a bad run id" — the same + // argument the dry run's findings make. + return res.json({ run: shapeRun(result.run), summary: result.summary }) } /** POST /api/v1/admin/events/runs/:runId/steps/:stepId/confirm */ diff --git a/server/src/router/v1/admin/events.router.js b/server/src/router/v1/admin/events.router.js index f75fbc9..034f88b 100644 --- a/server/src/router/v1/admin/events.router.js +++ b/server/src/router/v1/admin/events.router.js @@ -17,8 +17,10 @@ // 5; **`verify` and the action switchboard arrived in Phase 6** — `verify` at // `admin, editor` because a dry run dispatches nothing, and both halves of // `/actions` at `admin`, because §K puts the switchboard in the same row as the -// world-changing actions it governs. `cleanup` is still absent rather than -// stubbed: there is no resource ledger until Phase 8. +// world-changing actions it governs. **`cleanup` completed the set in Phase 8**, +// and it is `admin` rather than admin+moderator for the same §K reason: it asks +// core to write to the world again, which is not incident response. There is no +// route in the § API surface table left absent. // // **Literal paths are declared before `/:id`**, so `/catalog`, `/series`, // `/calendar` and `/runs` are never read as an event id. @@ -207,9 +209,9 @@ eventsRouter.get( '/runs/:runId', // #swagger.tags = ['Admin · Events'] // #swagger.summary = 'One run: its status, health, cleanup state and every step with its params and idempotency key' - // #swagger.description = 'The run console. `counts` summarises the step list by status. Steps carry the idempotency key core minted at materialisation — stable across every attempt, which is what lets the game side recognise a repeat. `gates` is the diagnosis panel (Phase 5): one entry per phase that authored an advance condition, already rendered in the condition builder's own words — `gte` as "is at least", `present` as "is present" — with the tally, how long it has waited, and the last related firing whether or not it matched. A phase is waiting on its gate only once every one of its steps is terminal; `stalled` means an `on` gate has waited past EVENT_PHASE_STALL_MS, which is visibility and never a timeout — nothing advances a phase but its condition or a human.' + // #swagger.description = 'The run console. `counts` summarises the step list by status. Steps carry the idempotency key core minted at materialisation — stable across every attempt, which is what lets the game side recognise a repeat. `gates` is the diagnosis panel (Phase 5): one entry per phase that authored an advance condition, already rendered in the condition builder's own words — `gte` as "is at least", `present` as "is present" — with the tally, how long it has waited, and the last related firing whether or not it matched. A phase is waiting on its gate only once every one of its steps is terminal; `stalled` means an `on` gate has waited past EVENT_PHASE_STALL_MS, which is visibility and never a timeout — nothing advances a phase but its condition or a human. `budget` is the cap meter (Phase 6), and `resources` is the cleanup ledger (Phase 8): every object this run created and every value it borrowed, with what became of each — `confirmed` is still out there, `reverted` came back, `drifted` means somebody moved it and core left it alone, and `orphaned` means the module reports it is gone. `unresolvedResources` counts the ones still wanting something, including a placeholder left standing by a lost acknowledgement, which is why it can exceed the length of the list.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] - /* #swagger.responses[200] = { description: 'The run, its steps, the status counts and the phase gates', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, steps: { type: "array", items: { type: "object", additionalProperties: true } }, counts: { type: "object", additionalProperties: true }, gates: { type: "array", items: { type: "object", additionalProperties: true } } } } } } } */ + /* #swagger.responses[200] = { description: 'The run, its steps, the status counts, the phase gates, the cap meter and the resource ledger', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, steps: { type: "array", items: { type: "object", additionalProperties: true } }, counts: { type: "object", additionalProperties: true }, gates: { type: "array", items: { type: "object", additionalProperties: true } }, budget: { type: "array", items: { type: "object", additionalProperties: true } }, resources: { type: "array", items: { type: "object", additionalProperties: true } }, unresolvedResources: { type: "integer" } } } } } } */ /* #swagger.responses[404] = { description: 'No such run', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ controller.getRun, ) @@ -273,16 +275,29 @@ eventsRouter.post( '/runs/:runId/cancel', // #swagger.tags = ['Admin · Events'] // #swagger.summary = 'Cancel a run' - // #swagger.description = 'Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` is not a parameter yet — the resource ledger it would work over arrives in Phase 8, and a flag that changes nothing is worse than one that is not there.' + // #swagger.description = 'Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` arrived in Phase 8 and DEFAULTS TO TRUE: what the run created or borrowed is given back by the runner cleanup leg on its next tick, which is why this answers at once rather than after a round trip per resource. Sending `cleanup: false` deliberately leaves the world changes in place — that is admin-only even though the route is admin+moderator, because which of the two you have to be depends on what is in the body — and the run then carries `cleanup_status: incomplete` with every unreverted row listed on its console.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] - /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { reason: { type: "string", description: "Why. Recorded on the run and in its log, with the actor." } } } } } } */ - /* #swagger.responses[200] = { description: 'The cancelled run and how many steps were closed out with it', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, cancelledSteps: { type: "integer" } } } } } } */ + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { reason: { type: "string", description: "Why. Recorded on the run and in its log, with the actor." }, cleanup: { type: "boolean", description: "Default true. False leaves the world changes from this run in place, and is admin-only." } } } } } } */ + /* #swagger.responses[200] = { description: 'The cancelled run, how many steps were closed out with it, and whether cleanup was asked for', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, cancelledSteps: { type: "integer" }, cleanup: { type: "boolean" } } } } } } */ /* #swagger.responses[409] = { description: 'The run has already reached a terminal status', content: { "application/json": { schema: { type: "object", properties: { errors: { type: "array", items: { type: "string" } } } } } } } */ - /* #swagger.responses[403] = { description: 'Not an admin or moderator', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + /* #swagger.responses[403] = { description: 'Not an admin or moderator, or a moderator asking to skip cleanup', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ liveControl, controller.cancelRun, ) +eventsRouter.post( + '/runs/:runId/cleanup', + // #swagger.tags = ['Admin · Events'] + // #swagger.summary = 'Re-run cleanup over everything this run has not given back' + // #swagger.description = 'The manual retry EVENTS.md §L promises, and the only thing that clears a resource attempt counter — the automatic sweep never does, because a sweep that reset every stale row is what made an attempt ceiling unreachable in the engagement workstream. Legal on a TERMINAL run only: a run still in flight has a ledger that is still growing, and reverting a resource the next step is about to use would be core undoing an event while it is happening. `admin` rather than admin+moderator, unlike the seven live controls beside it, because this is not incident response — it asks core to write to the world again, which §K puts in the same row as the world-changing actions themselves. Answers 200 whatever it found: some resources may still be out there, and a 4xx would make that indistinguishable from a bad run id.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.responses[200] = { description: 'The run and what the sweep managed', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, summary: { type: "object", properties: { attempted: { type: "integer" }, reverted: { type: "integer" }, drifted: { type: "integer" }, failed: { type: "integer" }, remaining: { type: "integer" } } } } } } } } */ + /* #swagger.responses[409] = { description: 'The run is still in flight, or recorded no resources at all', content: { "application/json": { schema: { type: "object", properties: { errors: { type: "array", items: { type: "string" } } } } } } } */ + /* #swagger.responses[403] = { description: 'Not an admin', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + adminOnly, + controller.cleanupRun, +) + eventsRouter.post( '/runs/:runId/advance', // #swagger.tags = ['Admin · Events'] diff --git a/server/src/server.js b/server/src/server.js index be12168..a297cc1 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -16,6 +16,7 @@ const teamForumUploadSweep = require('./utils/teamForumUploadSweep') const teamDigestWorker = require('./utils/teamDigestWorker') const engagementWorker = require('./utils/engagementWorker') const eventRunner = require('./utils/eventRunner') +const eventCleanup = require('./events/cleanup') const { ensureSchema, close } = require('./utils/db') const { seedDefaults, createInitialAdminFromEnv } = require('../db/seed') const settings = require('./model/settings/settings.model') @@ -170,11 +171,25 @@ async function start() { // enables a rule: core seeds none and `enabled` defaults to 0. engagementWorker.start() - // Advance scheduled events (EVENTS.md §E). Materialise, advance, drain, and the - // run-log retention sweep. No-op until an admin publishes a definition and - // starts a run: core ships no event definitions. + // Advance scheduled events (EVENTS.md §E). Materialise, advance, drain, clean + // up and the run-log retention sweep. No-op until an admin publishes a + // definition and starts a run: core ships no event definitions. eventRunner.start() + // **Core's own restart is the one reconnect core can see** (EVENTS.md §L, + // Phase 8). Every other one belongs to a module, which reports it through + // `ctx.events.reconcile()`; this is the case where the thing that restarted was + // this process, and the ledger it wakes up holding may describe a world that + // moved on while it was down. Awaited by nobody and never fatal: a module that + // cannot answer leaves its rows alone, which is the pre-Phase-8 behaviour. + eventCleanup + .reconcileAll() + .then((summaries) => { + const orphaned = Object.values(summaries).reduce((n, x) => n + (x.orphaned || 0), 0) + if (orphaned) log.warn('event resources orphaned at boot', { orphaned, summaries }) + }) + .catch((err) => log.error('boot reconcile failed', { message: err.message })) + setupShutdown(server, internalServer) } diff --git a/server/src/utils/eventRunner.js b/server/src/utils/eventRunner.js index d4ef550..deb55ec 100644 --- a/server/src/utils/eventRunner.js +++ b/server/src/utils/eventRunner.js @@ -10,7 +10,16 @@ // 1. **reclaim** — release leases whose holder died, never touching `attempts` // 2. **materialise** — sweep occurrences past their grace window into `missed` // 3. **advance** — claim each due run and move it through its phases -// 4. **prune** — the `event_run_log` retention sweep, on its own long clock +// 4. **cleanup** — give back what a TERMINAL run still holds (Phase 8) +// 5. **prune** — the `event_run_log` retention sweep, on its own long clock +// +// **Cleanup is a leg rather than a limb of `advanceRun`**, and its position in +// that list is load-bearing: it runs after advance, so a run that completes in +// one tick is torn down in the same one, and it is one place rather than four, so +// a process that dies mid-teardown resumes on the next tick instead of leaving a +// world half-restored with nothing scheduled to finish it. §L's "cleanup steps +// are generated from the ledger and run" on cancellation and abort as well as on +// completion is one query here rather than a hook on each of the three. // // **What a phase advances on, as of Phase 5.** Every step terminal, and — if the // phase authored one — its GATE open as well. The gate is an ADDITIONAL @@ -69,6 +78,8 @@ const gates = require('../events/gates') const spec = require('../events/spec') const registries = require('../modules/registries') const { dispatchStep } = require('../events/dispatch') +const ledger = require('../events/ledger') +const cleanup = require('../events/cleanup') const authorize = require('../events/authorize') const log = require('./logger')('event-runner') @@ -261,6 +272,27 @@ async function drainStep(run, step, now, carry = {}) { } } + // ── Rule 1: record BEFORE the dispatch, not after ── + // + // §D. A step that ledgers gets a placeholder keyed by its idempotency key, + // written before anything reaches the module, so an answer that never comes + // back still leaves cleanup something to act on. Recording afterwards would + // make every object whose acknowledgement was lost invisible for ever, which is + // the one failure the whole world-write half cannot tolerate. + // + // It is deliberately AFTER the permission check: a refused step never reaches + // the module, so it has created nothing and must ledger nothing. + let placeholderId = null + try { + placeholderId = await ledger.reserveStep(run, step, action) + } catch (err) { + // The ledger is what makes a world write recoverable, so a step that cannot + // be recorded must not be dispatched. Transient — the next attempt tries the + // insert again — because the alternative is an unrecorded world change. + log.error('could not reserve the ledger row', { run: run.id, step: step.id, message: err.message }) + return applyFailure(run, step, `the resource ledger could not record this step: ${err.message}`) + } + const result = await dispatchStep(step, { run }) if (result.actionVersionDrift) { @@ -273,6 +305,36 @@ async function drainStep(run, step, now, carry = {}) { }) } + // ── …and promote it on the answer ── + // + // On both success shapes, because `await: 'human'` is a SUCCESS: the module did + // its part and something outside the system has to happen next, and a cue's + // confirm finishes the step without a second dispatch — so this is the only + // moment its resources can be recorded. A failure records nothing and leaves the + // placeholder standing, which is the whole point of writing one. + if (result.outcome === 'done' || result.outcome === 'parked') { + const recorded = await ledger.recordAnswer({ + run, + step, + action, + placeholderId, + resources: result.resources, + }) + if (recorded.recorded > 0 || recorded.rejected.length > 0) { + await logDb.write({ + runId: run.id, + stepId: step.id, + kind: 'resource.recorded', + phase: step.phase, + detail: { + action: step.action_id, + recorded: recorded.recorded, + ...(recorded.rejected.length ? { rejected: recorded.rejected } : {}), + }, + }) + } + } + if (result.outcome === 'parked') { // The GM cue. The step stays `running` with a NULL lease: genuinely in // flight, nothing holding it, so the stale reclaim passes it by and a cue @@ -459,9 +521,11 @@ async function advanceRun(run, now) { } if (run.status === 'ending') { - // A run that reached the wind-down and then lost its process. Phase 8 puts - // cleanup here; until then `ending` is a state a run passes through rather - // than one it does work in, and completing it is the whole recovery. + // A run that reached the wind-down and then lost its process. Completing it is + // still the whole recovery even with a ledger in the picture: cleanup is a leg + // of the tick over TERMINAL runs, so the row this leaves behind is exactly + // what that leg is looking for, and doing the teardown here as well would be + // the second call site the leg exists to avoid. await runsDb.transition(run.id, 'ending', 'completed') await logDb.write({ runId: run.id, kind: 'run.status', detail: { from: 'ending', to: 'completed' } }) return 'completed' @@ -536,8 +600,9 @@ async function advanceRun(run, now) { const next = phases[phaseIndex + 1] if (!next) { // §E's `ending` exists for the reason `sending` does in the outbox — it is - // what a claim sets — so the run passes through it even though Phase 2 has - // no cleanup to do there. Phase 8 is what gives it work. + // what a claim sets. The run still passes straight through it: the ledger's + // teardown is the tick's cleanup leg, which runs after this one in the same + // tick and takes the run as it now is. if (!(await runsDb.transition(run.id, 'running', 'ending'))) return 'taken' await logDb.write({ runId: run.id, kind: 'run.status', phase: phaseKey, detail: { from: 'running', to: 'ending' } }) await runsDb.transition(run.id, 'ending', 'completed') @@ -790,6 +855,17 @@ async function tick(now = new Date()) { } if (due && due.length) log.info('event runs swept', { due: due.length, ...counts }) + // **After advance, deliberately.** A run that reached `completed` two lines ago + // is torn down in this tick rather than the next, so §L's "a run reaches + // `completed` with `cleanup_status = 'incomplete'`" is what an operator sees + // instead of a completed run that briefly claims it has cleanup pending. + try { + const swept = await cleanup.sweep() + if (swept) log.info('event cleanup swept', { runs: swept }) + } catch (err) { + log.error('cleanup sweep failed', { message: err.message }) + } + try { await prune(now) } catch (err) { diff --git a/server/swagger/swagger-output.json b/server/swagger/swagger-output.json index 4210995..24c5fac 100644 --- a/server/swagger/swagger-output.json +++ b/server/swagger/swagger-output.json @@ -4137,7 +4137,7 @@ ], "responses": { "200": { - "description": "The run, its steps, the status counts and the phase gates", + "description": "The run, its steps, the status counts, the phase gates, the cap meter and the resource ledger", "content": { "application/json": { "schema": { @@ -4164,6 +4164,23 @@ "type": "object", "additionalProperties": true } + }, + "budget": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + }, + "unresolvedResources": { + "type": "integer" } } } @@ -4295,7 +4312,7 @@ "Admin · Events" ], "summary": "Cancel a run", - "description": "Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` is not a parameter yet — the resource ledger it would work over arrives in Phase 8, and a flag that changes nothing is worse than one that is not there.", + "description": "Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` arrived in Phase 8 and DEFAULTS TO TRUE: what the run created or borrowed is given back by the runner cleanup leg on its next tick, which is why this answers at once rather than after a round trip per resource. Sending `cleanup: false` deliberately leaves the world changes in place — that is admin-only even though the route is admin+moderator, because which of the two you have to be depends on what is in the body — and the run then carries `cleanup_status: incomplete` with every unreverted row listed on its console.", "parameters": [ { "name": "runId", @@ -4308,7 +4325,7 @@ ], "responses": { "200": { - "description": "The cancelled run and how many steps were closed out with it", + "description": "The cancelled run, how many steps were closed out with it, and whether cleanup was asked for", "content": { "application/json": { "schema": { @@ -4320,6 +4337,9 @@ }, "cancelledSteps": { "type": "integer" + }, + "cleanup": { + "type": "boolean" } } } @@ -4330,7 +4350,7 @@ "description": "Bad Request" }, "403": { - "description": "Not an admin or moderator", + "description": "Not an admin or moderator, or a moderator asking to skip cleanup", "content": { "application/json": { "schema": { @@ -4376,6 +4396,10 @@ "reason": { "type": "string", "description": "Why. Recorded on the run and in its log, with the actor." + }, + "cleanup": { + "type": "boolean", + "description": "Default true. False leaves the world changes from this run in place, and is admin-only." } } } @@ -4384,6 +4408,102 @@ } } }, + "/api/v1/admin/events/runs/{runId}/cleanup": { + "post": { + "tags": [ + "Admin · Events" + ], + "summary": "Re-run cleanup over everything this run has not given back", + "description": "The manual retry EVENTS.md §L promises, and the only thing that clears a resource attempt counter — the automatic sweep never does, because a sweep that reset every stale row is what made an attempt ceiling unreachable in the engagement workstream. Legal on a TERMINAL run only: a run still in flight has a ledger that is still growing, and reverting a resource the next step is about to use would be core undoing an event while it is happening. `admin` rather than admin+moderator, unlike the seven live controls beside it, because this is not incident response — it asks core to write to the world again, which §K puts in the same row as the world-changing actions themselves. Answers 200 whatever it found: some resources may still be out there, and a 4xx would make that indistinguishable from a bad run id.", + "parameters": [ + { + "name": "runId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The run and what the sweep managed", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "run": { + "type": "object", + "additionalProperties": true + }, + "summary": { + "type": "object", + "properties": { + "attempted": { + "type": "integer" + }, + "reverted": { + "type": "integer" + }, + "drifted": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "remaining": { + "type": "integer" + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad Request" + }, + "403": { + "description": "Not an admin", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "The run is still in flight, or recorded no resources at all", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ] + } + }, "/api/v1/admin/events/runs/{runId}/log": { "get": { "tags": [ diff --git a/server/test/eventActionRegistry.test.js b/server/test/eventActionRegistry.test.js index a2d4148..cbf9e16 100644 --- a/server/test/eventActionRegistry.test.js +++ b/server/test/eventActionRegistry.test.js @@ -44,10 +44,15 @@ const register = (owner, entries) => { registries.apply(api.staged) } -test('core registers its three actions on every boot', () => { +test('core registers its four actions on every boot', () => { registries.registerCore() const ids = registries.allEventActions().map((a) => a.id) - assert.deepEqual(ids, ['core.announce', 'core.wait', 'core.cue']) + // `core.lease` joined the three in Phase 8, and it is the only one of the four + // that genuinely changes the world — which is why it is core's rather than each + // module's: §F puts the duration bound and the two-events-one-target conflict + // check on core's side of the seam, and a lease verb per module would be that + // bound re-implemented once per module and advisory everywhere. + assert.deepEqual(ids, ['core.announce', 'core.wait', 'core.cue', 'core.lease']) assert.equal(ids.length, coreEventActions.ACTIONS.length) }) @@ -56,6 +61,7 @@ test('the catalog carries no callable', () => { for (const action of registries.allEventActions()) { assert.equal(action.perform, undefined, `${action.id} leaked perform`) assert.equal(action.revert, undefined, `${action.id} leaked revert`) + assert.equal(action.reconcile, undefined, `${action.id} leaked reconcile`) assert.equal(action.cost, undefined, `${action.id} leaked cost`) } // …and the runner's own lookup still has it, which is the half that makes the @@ -242,7 +248,7 @@ test('a whole batch is refused or taken, never half', () => { test('_reset() hands the process back', () => { registries.registerCore() - assert.equal(registries.allEventActions().length, 3) + assert.equal(registries.allEventActions().length, 4) registries._reset() assert.equal(registries.allEventActions().length, 0) assert.equal(registries.isEventAction('core.wait'), false) @@ -366,9 +372,80 @@ test('an option source needs a resolver, and core registers one of its own', () // Core through the same door (Phase 7): `core.announce`'s `leg` param names a // source, and the announce legs are already a registry with labels in them. registries.registerCore() - assert.deepEqual(registries.allEventOptionSources().map((s) => s.id), ['core.options.legs']) + assert.deepEqual(registries.allEventOptionSources().map((s) => s.id), [ + 'core.options.legs', + // Phase 8's, and it is the same argument one phase on: `core.lease`'s `lease` + // param would otherwise be a free-text box whose typo is caught at dispatch, + // mid-run — and the leases are already a registry with labels in them. + 'core.options.leases', + ]) const leg = registries.eventAction('core.announce').params.find((p) => p.name === 'leg') assert.equal(leg.source, 'core.options.legs') + const which = registries.eventAction('core.lease').params.find((p) => p.name === 'lease') + assert.equal(which.source, 'core.options.leases') +}) + +// ── `reconcile`, the one member Phase 8 added to the action shape ────────── +// +// Optional where `revert` is required, and the asymmetry is the design: a module +// that cannot say what the game still has is not broken — core keeps believing +// its own ledger, which is the behaviour before this phase — whereas a module +// that created something and cannot undo it has made a promise core has no way +// to keep. + +test('reconcile is optional, must be a function, and only on an action that ledgers', () => { + const ledgering = { + id: 'demo.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + perform: async () => ({ ok: true }), + revert: async () => ({ ok: true }), + } + + // Absent is legal, and it lands as an explicit null rather than as a missing + // key — the same shape `revert` and `cost` take, so the catalog's strip list + // and the sweep's `typeof` check both have something to look at. + register('demo', [ledgering]) + assert.equal(registries.eventAction('demo.spawn').reconcile, null) + registries._reset() + + assert.throws( + () => register('demo', [{ ...ledgering, reconcile: 'yes please' }]), + /reconcile must be a function/, + ) + + // The mirror check `revert` already has. An action that ledgers nothing has no + // rows for core to ask about, so a `reconcile` on one is an author who believes + // something is being tracked and a sweep that will never call it. + assert.throws( + () => + register('demo', [ + { + id: 'demo.shout', + label: 'Shout', + risk: 'notify', + reversible: 'none', + perform: async () => ({ ok: true }), + reconcile: async () => ({ ok: true, inForce: [] }), + }, + ]), + /declares reconcile\(\) but is reversible: 'none' and ledgers nothing/, + ) +}) + +test('an override action may reconcile, because a lease is ledgered too', () => { + register('demo', [ + { + id: 'demo.borrow', + label: 'Borrow', + risk: 'change', + reversible: 'override', + perform: async () => ({ ok: true }), + reconcile: async () => ({ ok: true, inForce: [] }), + }, + ]) + assert.equal(typeof registries.eventAction('demo.borrow').reconcile, 'function') }) test('_reset() hands back the three new registries too', () => { diff --git a/server/test/eventCleanup.test.js b/server/test/eventCleanup.test.js new file mode 100644 index 0000000..f70bc56 --- /dev/null +++ b/server/test/eventCleanup.test.js @@ -0,0 +1,500 @@ +// ── Giving back what a run took (EVENTS_PLAN.md Phase 8) ─────────────────── +// +// The phase's shipped claim: **cleanup is generated from the ledger and runs on +// every terminal path.** An operator cannot be relied on to write the undo, and +// an aborted run never reaches the phase they wrote it in — so there is no +// cleanup phase in a spec, no `on_teardown` on an action, and one function that +// reads rows. +// +// The properties around it are §L's, and two of them are ones this codebase has +// already paid for once: +// +// • **a revert that never succeeds stays visible rather than cycling** — rule 2, +// and `MAX_REVERT_ATTEMPTS` is what stops the automatic retry. Only a human +// clears the counter, which is Engagement Phase 14's rule stated a third time +// • **reverting something that does not exist is a SUCCESS** — §L, and what a +// Rust wipe needs +// • **drift is not an error.** The module did exactly what it was asked and +// found somebody else's value in place. A restore that wrote anyway would +// silently revert an operator's manual fix +// • **a resource the module no longer has becomes `orphaned`, never `reverted`** — +// reverting it would be core recording that it put something back when what +// happened is that the thing vanished +// • **"I do not know" is never read as "it is gone".** Every unanswerable +// reconcile leaves the ledger alone +// +// The sweep is driven against a stubbed db layer, exactly as `eventRunner.test.js` +// drives the runner: what a stub cannot prove is the SQL, and the unique key that +// makes two events unable to lease one target runs against a real MariaDB in +// `eventRunnerSql.test.js`. + +process.env.DB_HOST = '127.0.0.1' +process.env.DB_PORT = '59999' + +const { test, beforeEach, afterEach, after } = require('node:test') +const assert = require('node:assert/strict') + +const registries = require('../src/modules/registries') +const cleanup = require('../src/events/cleanup') +const resourcesDb = require('../src/model/events/eventRunResources.db') +const runsDb = require('../src/model/events/eventRuns.db') +const stepsDb = require('../src/model/events/eventRunSteps.db') +const logDb = require('../src/model/events/eventRunLog.db') +const db = require('../src/utils/db') + +after(() => db.close()) + +const UNRESOLVED = ['pending', 'confirmed', 'reverting', 'orphaned', 'drifted'] + +let store +const originals = { + resourcesDb: { ...resourcesDb }, + runsDb: { ...runsDb }, + stepsDb: { ...stepsDb }, + logDb: { ...logDb }, +} + +beforeEach(() => { + registries._reset() + store = { rows: new Map(), steps: new Map(), log: [], next: 1, cleanupStatus: 'pending' } + + resourcesDb.unresolvedForRun = async (runId, { maxAttempts = null } = {}) => + [...store.rows.values()] + .filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status)) + .filter((r) => maxAttempts === null || r.revert_attempts < maxAttempts) + .map((r) => ({ ...r })) + resourcesDb.unresolvedCount = async (runId) => + [...store.rows.values()].filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status)).length + resourcesDb.claimRevert = async (id) => { + const r = store.rows.get(id) + if (!r || !['pending', 'confirmed', 'orphaned', 'drifted'].includes(r.status)) return false + r.status = 'reverting' + return true + } + resourcesDb.markReverted = async (id) => { + const r = store.rows.get(id) + if (r) Object.assign(r, { status: 'reverted', last_error: null }) + } + resourcesDb.failRevert = async (id, error, restoreTo = 'confirmed') => { + const r = store.rows.get(id) + if (r) Object.assign(r, { status: restoreTo, revert_attempts: r.revert_attempts + 1, last_error: String(error) }) + } + resourcesDb.resetAttempts = async (runId) => { + let n = 0 + for (const r of store.rows.values()) { + if (r.run_id === runId && UNRESOLVED.includes(r.status)) { + r.revert_attempts = 0 + n += 1 + } + } + return n + } + resourcesDb.markOrphaned = async (id, detail = null) => { + const r = store.rows.get(id) + if (r && ['pending', 'confirmed', 'reverting'].includes(r.status)) { + Object.assign(r, { status: 'orphaned', last_error: detail }) + } + } + resourcesDb.liveForModule = async (owner) => + [...store.rows.values()].filter((r) => r.owner_module === owner && ['pending', 'confirmed'].includes(r.status)).map((r) => ({ ...r })) + resourcesDb.modulesWithLiveRows = async () => [ + ...new Set([...store.rows.values()].filter((r) => ['pending', 'confirmed'].includes(r.status)).map((r) => r.owner_module)), + ] + resourcesDb.runsNeedingCleanup = async () => store.candidates || [] + + runsDb.setCleanupStatus = async (id, to, from = null) => { + if (from && !from.includes(store.cleanupStatus)) return false + store.cleanupStatus = to + return true + } + stepsDb.getById = async (id) => store.steps.get(id) || null + logDb.write = async (entry) => { + store.log.push(entry) + } +}) + +afterEach(() => { + Object.assign(resourcesDb, originals.resourcesDb) + Object.assign(runsDb, originals.runsDb) + Object.assign(stepsDb, originals.stepsDb) + Object.assign(logDb, originals.logDb) + registries._reset() +}) + +const RUN = { id: 7, status: 'completed', cleanup_status: 'pending' } + +function addStep(id, actionId, key = 'k'.repeat(40)) { + store.steps.set(id, { id, run_id: RUN.id, action_id: actionId, idempotency_key: key, phase: 'p', seq: 0 }) +} + +function addResource(over = {}) { + const id = store.next++ + const row = { + id, + run_id: RUN.id, + step_id: 1, + owner_module: 'demo', + kind: 'creature', + ref: `0x${id}`, + payload: null, + lease_until: null, + status: 'confirmed', + revert_attempts: 0, + last_error: null, + member_key: null, + ...over, + } + store.rows.set(id, row) + return row +} + +function registerAction(over = {}) { + const api = registries.stage('demo') + api.registerEventActions([ + { + id: 'demo.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + budgetMs: 500, + perform: async () => ({ ok: true }), + revert: async () => ({ ok: true }), + ...over, + }, + ]) + registries.apply(api.staged) +} + +function registerLease(over = {}) { + const api = registries.stage('demo') + api.registerEventLeases([ + { + id: 'demo.rate', + label: 'Gather rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3_600_000, + read: async () => ({ ok: true, value: 1 }), + apply: async () => ({ ok: true }), + restore: async () => ({ ok: true }), + ...over, + }, + ]) + registries.apply(api.staged) +} + +const kinds = () => store.log.map((e) => e.kind) + +// ── The shipped claim ────────────────────────────────────────────────────── + +test('a run\'s ledger is given back in one call per step, and the run goes complete', async () => { + // `revert` takes a LIST because that is what makes twelve creatures one round + // trip rather than twelve. The grouping key is the STEP, because the resource + // row records the module and the opaque names while the step records the verb. + let seen = null + registerAction({ revert: async (arg) => { seen = arg; return { ok: true } } }) + addStep(1, 'demo.spawn', 'key-one') + addResource({ ref: '0xA' }) + addResource({ ref: '0xB' }) + + const summary = await cleanup.cleanupRun(RUN) + assert.deepEqual(summary, { attempted: 2, reverted: 2, drifted: 0, failed: 0, remaining: 0 }) + assert.equal(seen.runId, RUN.id) + assert.equal(seen.idempotencyKey, 'key-one') + assert.deepEqual(seen.resources.map((r) => r.ref), ['0xA', '0xB']) + assert.equal(store.cleanupStatus, 'complete') +}) + +test('two steps are two calls, and one failing does not take the other down', async () => { + const calls = [] + registerAction({ + revert: async ({ idempotencyKey, resources }) => { + calls.push(idempotencyKey) + return idempotencyKey === 'bad' ? { ok: false, error: 'the shard did not answer' } : { ok: true, resources } + }, + }) + addStep(1, 'demo.spawn', 'good') + addStep(2, 'demo.spawn', 'bad') + addResource({ step_id: 1 }) + addResource({ step_id: 2 }) + + const summary = await cleanup.cleanupRun(RUN) + assert.equal(calls.length, 2) + assert.equal(summary.reverted, 1) + assert.equal(summary.failed, 1) + assert.equal(summary.remaining, 1) + // **Still `pending`, because the failed row has retries left.** `incomplete` + // means "finished with, and not finished" — it is what takes a run out of the + // sweep's own scan, so writing it after the FIRST failure made + // `MAX_REVERT_ATTEMPTS` quietly mean one attempt. Found by watching + // `revert_attempts` sit at 1 through half a minute of live ticks. + assert.equal(store.cleanupStatus, 'pending') +}) + +test('reverting something that does not exist is a success', async () => { + // §L, and the Rust wipe: "gone, and that is fine". The module never has to + // distinguish "I deleted it" from "it was not there" — which is also what makes + // a placeholder for an object that may never have existed safe to write. + registerAction({ revert: async () => ({ ok: true, detail: 'resource no longer exists' }) }) + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.reverted, 1) + assert.equal(store.cleanupStatus, 'complete') +}) + +test('a module may name the ones that did not come back', async () => { + // Partial cleanup is the ordinary case — eleven of twelve creatures deleted — + // and it is why the ledger is a row per object rather than a row per step. + registerAction({ revert: async () => ({ ok: true, failed: ['0x2'] }) }) + addStep(1, 'demo.spawn') + addResource({ ref: '0x1' }) + addResource({ ref: '0x2' }) + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.reverted, 1) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()].find((r) => r.ref === '0x2').last_error, /could not give "0x2" back/) +}) + +// ── Rule 2: loud and sticky ──────────────────────────────────────────────── + +test('a revert that never works stops retrying and stays visible', async () => { + let calls = 0 + registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } }) + addStep(1, 'demo.spawn') + addResource() + + for (let i = 0; i < 6; i++) await cleanup.cleanupRun(RUN) + + // Bounded at MAX_REVERT_ATTEMPTS, exactly like a step's attempts. A fourth ask + // of a shard that has answered the same way three times is not new information, + // and an unbounded counter is a row nothing can ever sweep. + assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS) + const row = [...store.rows.values()][0] + assert.equal(row.revert_attempts, cleanup.MAX_REVERT_ATTEMPTS) + assert.equal(row.status, 'confirmed') + assert.equal(row.last_error, 'nope') + // And ONLY now, with nothing left to try, does the run stop being the sweep's + // business. §L: it does not stay `running` — an event whose world changes are + // still up is a real state, and pretending the event is in progress hides it. + assert.equal(store.cleanupStatus, 'incomplete') +}) + +test('the run goes back to pending each time it still has an attempt left', async () => { + // The other half of the same rule, watched one pass at a time rather than at + // the end. Each of the first two sweeps leaves the run in the scan; the third + // takes it out. A test that only looked at the end state would pass against the + // defect this replaced. + registerAction({ revert: async () => ({ ok: false, error: 'nope' }) }) + addStep(1, 'demo.spawn') + addResource() + + const seen = [] + for (let i = 0; i < 3; i++) { + await cleanup.cleanupRun(RUN) + seen.push(store.cleanupStatus) + } + assert.deepEqual(seen, ['pending', 'pending', 'incomplete']) +}) + +test('only a human clears the attempt counter', async () => { + // Engagement Phase 14's defect, stated a third time: a SWEEP that returned every + // stale row to its start state made the attempt ceiling unreachable, so the row + // cycled for ever and was never eligible for any retention sweep. The automatic + // leg must never do this; the cleanup route may, because a person asked. + let calls = 0 + registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } }) + addStep(1, 'demo.spawn') + addResource() + + for (let i = 0; i < 5; i++) await cleanup.cleanupRun(RUN) + assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS) + + await cleanup.cleanupRun(RUN, { resetAttempts: true, actor: 9 }) + assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS + 1) + assert.ok(store.log.some((e) => e.kind === 'cleanup.retry' && e.detail.by === 9)) +}) + +test('a module that throws from revert is a transient failure, not a crashed sweep', async () => { + registerAction({ revert: async () => { throw new Error('socket hung up') } }) + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()][0].last_error, /socket hung up/) +}) + +test('no shape a revert failure can take reads as success', () => { + // `dispatch.classify`'s rule, applied to the other direction of the contract. + // The expensive mistake here is the mirror of the one there: recording that a + // world change was UNDONE when it was not. + for (const raw of [null, undefined, 'ok', [], {}, { ok: 'yes' }, { ok: 1 }]) { + assert.notEqual(cleanup.classifyRevert(raw, 'demo.spawn').outcome, 'done', JSON.stringify(raw)) + } + assert.equal(cleanup.classifyRevert({ __timedOut: true, error: 'slow' }, 'x').outcome, 'retry') + assert.equal(cleanup.classifyRevert({ ok: false, retry: false, error: 'never' }, 'x').outcome, 'terminal') + assert.equal(cleanup.classifyRevert({ ok: true }, 'x').outcome, 'done') +}) + +test('an action whose module is gone leaves its rows unresolved with the reason', async () => { + // Not a retry — nothing will change until an operator reinstalls it — and not + // an orphan either, because core has no idea whether the thing is still there. + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()][0].last_error, /no module registers "demo.spawn"/) + // Retried like any other failure rather than given up on at once, and that is + // the right uniformity here: "nothing registers this" stops being true the + // moment an operator reinstalls the module, and three registry lookups cost + // nothing. So it is `pending` until the attempts are spent. + assert.equal(store.cleanupStatus, 'pending') +}) + +// ── Leases ───────────────────────────────────────────────────────────────── + +test('a lease is restored through the LEASE registry, not through any action', async () => { + // The split §F draws: core owns the duration and the conflict check, the module + // owns reading and writing. It is why `core.lease` needs no `revert()` of its + // own, and why an `override` row routes here rather than to its step's action. + let seen = null + registerLease({ restore: async (baseline, opts) => { seen = { baseline, opts }; return { ok: true } } }) + addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } }) + + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.reverted, 1) + assert.equal(seen.baseline, 1) + // The drift check's input. `restore` MUST verify current === expected before + // writing, and a lease whose restore wrote blindly would silently revert an + // operator's manual fix. + assert.equal(seen.opts.expected, 3) +}) + +test('drift is not an error: the world is left alone and the row says so', async () => { + registerLease({ restore: async () => ({ ok: false, drifted: true, current: 4.5 }) }) + addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } }) + + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.drifted, 1) + assert.equal(summary.failed, 0) + const row = [...store.rows.values()][0] + assert.equal(row.status, 'drifted') + assert.match(row.last_error, /now 4\.5 rather than what this run applied/) + // Still surfaced. §L: "surfaced beside the unreverted ones" — the run does not + // get to call itself clean because somebody else took the value. It is `pending` + // rather than `incomplete` for one more reason worth keeping: drift is retried + // like any other failure, because a GM who puts the value back between two ticks + // should have the lease close cleanly. + assert.equal(store.cleanupStatus, 'pending') +}) + +test('a lease whose module is uninstalled is unresolved, never assumed restored', async () => { + addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } }) + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()][0].last_error, /no module registers the lease "demo.rate"/) +}) + +// ── The sweep ────────────────────────────────────────────────────────────── + +test('the sweep only touches TERMINAL runs', async () => { + // A run still in flight has a ledger that is still growing, and reverting a + // resource the next step is about to use would be core undoing an event while + // it is happening. + registerAction() + addStep(1, 'demo.spawn') + addResource() + store.candidates = [{ id: RUN.id, status: 'running', cleanup_status: 'pending' }] + assert.equal(await cleanup.sweep(), 0) + assert.equal([...store.rows.values()][0].status, 'confirmed') + + store.candidates = [{ id: RUN.id, status: 'cancelled', cleanup_status: 'pending' }] + assert.equal(await cleanup.sweep(), 1) + assert.equal([...store.rows.values()][0].status, 'reverted') +}) + +// ── Reconcile ────────────────────────────────────────────────────────────── + +test('a resource the module no longer has becomes orphaned, never reverted', async () => { + // §L, and the distinction matters to the operator reading the console + // afterwards: `reverted` says core put something back, `orphaned` says the + // thing vanished while nobody was looking. Recording the second as the first + // would be core claiming credit for a shard restart. + registerAction({ reconcile: async () => ({ ok: true, inForce: ['0x1'] }) }) + addStep(1, 'demo.spawn') + addResource({ ref: '0x1' }) + addResource({ ref: '0x2' }) + + const summary = await cleanup.reconcileModule('demo') + assert.deepEqual(summary, { asked: 2, inForce: 1, orphaned: 1, unanswered: 0 }) + assert.equal([...store.rows.values()].find((r) => r.ref === '0x2').status, 'orphaned') + assert.equal([...store.rows.values()].find((r) => r.ref === '0x1').status, 'confirmed') + assert.ok(store.log.some((e) => e.kind === 'resource.orphaned')) +}) + +test('"I do not know" is never read as "it is gone"', async () => { + // Every unanswerable shape leaves the ledger exactly as it was. A reconcile + // that read silence as absence would orphan a whole shard's worth of live + // spawns the first time a sidecar was slow. + for (const answer of [null, undefined, { ok: false }, { ok: true }, { ok: true, inForce: 'all' }, 'yes']) { + store.rows.clear() + store.log.length = 0 + registries._reset() + registerAction({ reconcile: async () => answer }) + addStep(1, 'demo.spawn') + addResource({ ref: '0x1' }) + const summary = await cleanup.reconcileModule('demo') + assert.equal(summary.orphaned, 0, JSON.stringify(answer)) + assert.equal(summary.unanswered, 1, JSON.stringify(answer)) + assert.equal([...store.rows.values()][0].status, 'confirmed') + } +}) + +test('a module with no reconcile is not broken; core keeps believing its ledger', async () => { + // Optional where `revert` is required. A module that cannot answer leaves core + // exactly where it was before this phase, which is a capability its deployment + // does without rather than a boot it fails. + registerAction() + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.reconcileModule('demo') + assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 1 }) + assert.equal([...store.rows.values()][0].status, 'confirmed') +}) + +test('a reconcile that throws orphans nothing', async () => { + registerAction({ reconcile: async () => { throw new Error('sidecar gone') } }) + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.reconcileModule('demo') + assert.equal(summary.unanswered, 1) + assert.equal([...store.rows.values()][0].status, 'confirmed') +}) + +test('placeholders are not asked about, because there is nothing to ask yet', async () => { + // A `@step` row names no object — it says "a dispatch was in flight and may + // have made something". Asking a module whether it is in force is a question + // with no answer, and reading a shrug as absence would resolve the one row whose + // survival is the safety property. + registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) }) + addStep(1, 'demo.spawn') + addResource({ kind: resourcesDb.STEP_KIND, ref: 'a'.repeat(40), status: 'pending' }) + const summary = await cleanup.reconcileModule('demo') + assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 }) + assert.equal([...store.rows.values()][0].status, 'pending') +}) + +test('reconcileAll asks every module that owns a live row', async () => { + registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) }) + addStep(1, 'demo.spawn') + addStep(2, 'other.thing') + addResource({ owner_module: 'demo' }) + addResource({ owner_module: 'other', step_id: 2 }) + const out = await cleanup.reconcileAll() + assert.deepEqual(Object.keys(out).sort(), ['demo', 'other']) + assert.equal(out.demo.orphaned, 1) + // The other module registers nothing, so its row is left alone rather than + // orphaned by a module that is not there to be asked. + assert.equal(out.other.unanswered, 1) +}) diff --git a/server/test/eventLedger.test.js b/server/test/eventLedger.test.js new file mode 100644 index 0000000..2f41ce8 --- /dev/null +++ b/server/test/eventLedger.test.js @@ -0,0 +1,337 @@ +// ── The resource ledger's write half (EVENTS_PLAN.md Phase 8) ────────────── +// +// §D's two rules, and rule 1 is the one this file exists for: **a resource is +// recorded BEFORE it is confirmed.** The obstacle it works around is that a +// spawn's serial does not exist until the module answers, so what goes in before +// the dispatch is a placeholder keyed by the step's idempotency key — and the +// property worth a test is that the placeholder SURVIVES an answer that never +// comes, because that is the case where recording afterwards would have lost the +// object for ever. +// +// The other rules here are about what core will and will not write down on a +// module's say-so. Every one of them is fail-closed in a specific direction: +// +// • the reserved `@step` kind is core's and a module may not claim it +// • an `override` must name a lease core knows how to give back, or core would +// be recording something it has no way to restore +// • a duplicate is "already recorded", not an error — a retry re-sends the same +// idempotency key and a module may honestly report the same resources twice +// • a badly shaped resource is dropped and LOGGED, never a failed step: the +// step changed the world, and turning bookkeeping into a retry would re-run +// a world write that already happened +// +// The db layer is stubbed with a store that enforces `uq_evres_target`, because +// that refusal is behaviour the callers branch on rather than an implementation +// detail. The SQL itself is `eventRunnerSql.test.js`'s, against a real MariaDB. + +process.env.DB_HOST = '127.0.0.1' +process.env.DB_PORT = '59999' + +const { test, beforeEach, afterEach, after } = require('node:test') +const assert = require('node:assert/strict') + +const registries = require('../src/modules/registries') +const ledger = require('../src/events/ledger') +const resourcesDb = require('../src/model/events/eventRunResources.db') +const runsDb = require('../src/model/events/eventRuns.db') +const db = require('../src/utils/db') + +after(() => db.close()) + +const HELD = ['pending', 'confirmed', 'reverting'] + +let store +const originals = { resourcesDb: { ...resourcesDb }, runsDb: { ...runsDb } } + +beforeEach(() => { + registries._reset() + store = { rows: new Map(), next: 1, cleanupStatus: 'not_required' } + + resourcesDb.reserve = async ({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) => { + const holder = [...store.rows.values()].find( + (r) => r.owner_module === owner && r.kind === kind && r.ref === ref && HELD.includes(r.status), + ) + if (holder) return { ok: false, code: 'held', holder: { run_id: holder.run_id, status: holder.status } } + const id = store.next++ + store.rows.set(id, { + id, + run_id: runId, + step_id: stepId, + owner_module: owner, + kind, + ref, + payload, + lease_until: leaseUntil, + status: 'pending', + revert_attempts: 0, + last_error: null, + member_key: memberKey, + }) + return { ok: true, id } + } + resourcesDb.confirm = async (id) => { + const r = store.rows.get(id) + if (!r || r.status !== 'pending') return false + r.status = 'confirmed' + return true + } + resourcesDb.resolvePlaceholder = async (id) => { + const r = store.rows.get(id) + if (!r || r.kind !== resourcesDb.STEP_KIND) return false + r.status = 'reverted' + return true + } + resourcesDb.findByTarget = async (owner, kind, ref) => + [...store.rows.values()].reverse().find((r) => r.owner_module === owner && r.kind === kind && r.ref === ref) || null + + runsDb.setCleanupStatus = async (id, to, from = null) => { + if (from && !from.includes(store.cleanupStatus)) return false + store.cleanupStatus = to + return true + } +}) + +afterEach(() => { + Object.assign(resourcesDb, originals.resourcesDb) + Object.assign(runsDb, originals.runsDb) + registries._reset() +}) + +const RUN = { id: 7 } +const STEP = { id: 42, phase: 'invasion', seq: 0, idempotency_key: 'a'.repeat(40) } + +const action = (over = {}) => ({ + id: 'demo.spawn', + owner: 'demo', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + budgetMs: 1000, + ...over, +}) + +const rows = () => [...store.rows.values()] + +// ── Which actions ledger at all ──────────────────────────────────────────── + +test('only the two reversible classes core has to come back for are ledgered', () => { + // `none` is gone once done, `self` undoes itself. Neither has anything core + // could revert, and giving one a placeholder would put a row in the ledger that + // teardown could never resolve — the exact reason `core.announce` is declared + // `none` rather than `ledger`. + assert.equal(ledger.ledgers(action({ reversible: 'ledger' })), true) + assert.equal(ledger.ledgers(action({ reversible: 'override' })), true) + assert.equal(ledger.ledgers(action({ reversible: 'none' })), false) + assert.equal(ledger.ledgers(action({ reversible: 'self' })), false) +}) + +test('only a ledger action gets a placeholder; an override reserves its own target', async () => { + // The asymmetry is the design. A spawn's ref is unknown until the module + // answers, so the placeholder stands in for it; a lease's target is the lease + // id the step already names, so `core.lease` writes the real row before it + // touches the world — which is rule 1 in a stronger form, and the only place + // the two-events-one-target refusal can happen before the world has changed. + assert.equal(typeof (await ledger.reserveStep(RUN, STEP, action())), 'number') + assert.equal(await ledger.reserveStep(RUN, STEP, action({ id: 'demo.b', reversible: 'override' })), null) + assert.equal(await ledger.reserveStep(RUN, STEP, action({ id: 'demo.c', reversible: 'none' })), null) + assert.equal(rows().length, 1) + assert.equal(rows()[0].kind, '@step') + assert.equal(rows()[0].ref, STEP.idempotency_key) +}) + +test('the first ledger row is what makes a run dirty, and only from not_required', async () => { + assert.equal(store.cleanupStatus, 'not_required') + await ledger.reserveStep(RUN, STEP, action()) + assert.equal(store.cleanupStatus, 'pending') + + // A run whose sweep has already finished must not be walked back to `pending` + // by a late row: only a human's cleanup re-opens it, and it does so + // deliberately and with an actor on the log line. + store.cleanupStatus = 'complete' + await ledger.recordAnswer({ + run: RUN, + step: { ...STEP, id: 43, idempotency_key: 'b'.repeat(40) }, + action: action(), + placeholderId: null, + resources: [{ kind: 'creature', ref: '0x1' }], + }) + assert.equal(store.cleanupStatus, 'complete') +}) + +// ── Rule 1 ───────────────────────────────────────────────────────────────── + +test('a lost acknowledgement leaves the placeholder standing, which is the whole point', async () => { + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + // The dispatch timed out: no answer, so `recordAnswer` is never reached. This + // is the case rule 1 exists for — record afterwards and the object the module + // may well have created is invisible to cleanup for ever. + assert.equal(rows()[0].status, 'pending') + assert.equal(rows()[0].payload.action, 'demo.spawn') + assert.ok(placeholderId) +}) + +test('a retry reuses its own placeholder rather than writing a second', async () => { + // An idempotency key is minted once per step and does not vary by attempt (§E), + // so the second attempt's insert collides with the first attempt's row. Finding + // it already there is the correct answer, and a second row would be a second + // thing for cleanup to revert. + const first = await ledger.reserveStep(RUN, STEP, action()) + const second = await ledger.reserveStep(RUN, STEP, action()) + assert.equal(first, second) + assert.equal(rows().length, 1) +}) + +test('the placeholder is resolved once the real rows exist', async () => { + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + const out = await ledger.recordAnswer({ + run: RUN, + step: STEP, + action: action(), + placeholderId, + resources: [ + { kind: 'creature', ref: '0x40001234' }, + { kind: 'creature', ref: '0x40001235' }, + ], + }) + assert.equal(out.recorded, 2) + assert.deepEqual(out.rejected, []) + assert.equal(store.rows.get(placeholderId).status, 'reverted') + assert.deepEqual( + rows().filter((r) => r.kind === 'creature').map((r) => [r.ref, r.status]), + [['0x40001234', 'confirmed'], ['0x40001235', 'confirmed']], + ) +}) + +test('an action that ledgers and reports nothing still resolves its placeholder', async () => { + // "I made nothing" is a real answer. Holding the placeholder open for it would + // make cleanup call `revert()` on every terminal path, for ever, for a step that + // has nothing to give back. + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + const out = await ledger.recordAnswer({ run: RUN, step: STEP, action: action(), placeholderId, resources: [] }) + assert.equal(out.recorded, 0) + assert.equal(store.rows.get(placeholderId).status, 'reverted') +}) + +test('a module reporting the same resources twice produces one row', async () => { + // The database is what makes recording idempotent: `uq_evres_target` refuses + // the second insert and this file reads that as "already recorded". Without it + // a retry against a module that honestly re-reports its work would double every + // row cleanup then has to revert. + const args = { run: RUN, step: STEP, action: action(), placeholderId: null, resources: [{ kind: 'creature', ref: '0x1' }] } + await ledger.recordAnswer(args) + const again = await ledger.recordAnswer(args) + assert.equal(again.recorded, 0) + assert.deepEqual(again.rejected, []) + assert.equal(rows().filter((r) => r.kind === 'creature').length, 1) +}) + +test('a target another RUN holds is rejected by name rather than silently skipped', async () => { + await ledger.recordAnswer({ + run: { id: 1 }, + step: STEP, + action: action(), + placeholderId: null, + resources: [{ kind: 'creature', ref: '0x1' }], + }) + const out = await ledger.recordAnswer({ + run: { id: 2 }, + step: { ...STEP, id: 99 }, + action: action(), + placeholderId: null, + resources: [{ kind: 'creature', ref: '0x1' }], + }) + assert.equal(out.recorded, 0) + assert.match(out.rejected.join('\n'), /already held by run 1/) +}) + +// ── What core will not write down ────────────────────────────────────────── + +test('a module may not claim core\'s reserved kind', () => { + // A module that could write a `@step` row could make its own step's placeholder + // look resolved — which is the one row whose survival is the safety property. + const bad = ledger.normalise({ kind: '@step', ref: 'x' }, 'demo.spawn') + assert.equal(bad.ok, false) + assert.match(bad.reason, /reserved kind/) +}) + +test('an override must name a lease core knows how to give back', () => { + // Core restores an `override` through the LEASE registry — that is the split §F + // draws — so a ref naming nothing registered is a resource core would be + // recording with no way to undo it. Refusing to record it is the fail-closed + // direction: rule 2 is a promise core must not make and then break. + assert.equal(ledger.normalise({ kind: 'override', ref: 'demo.rate' }, 'demo.x').ok, false) + + const api = registries.stage('demo') + api.registerEventLeases([ + { + id: 'demo.rate', + label: 'Rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3_600_000, + read: async () => ({ ok: true, value: 1 }), + apply: async () => ({ ok: true }), + restore: async () => ({ ok: true }), + }, + ]) + registries.apply(api.staged) + assert.equal(ledger.normalise({ kind: 'override', ref: 'demo.rate' }, 'demo.x').ok, true) +}) + +test('every bad shape is refused, and none of them is a retry', () => { + // A badly shaped resource is the module's mistake rather than the world's, and + // it will be just as badly shaped on the second attempt. They are dropped and + // reported; the STEP still counts as done, because it is — something happened + // in the world, and refusing to record it would be the one outcome worse than + // recording it imperfectly. + const bad = [ + null, + 'a string', + ['an array'], + { ref: 'x' }, // no kind + { kind: 'creature' }, // no ref + { kind: 'creature', ref: 'x'.repeat(200) }, + { kind: 'k'.repeat(80), ref: 'x' }, + { kind: 'creature', ref: 'x', memberKey: 'm'.repeat(200) }, + { kind: 'creature', ref: 'x', until: 'not a date' }, + ] + for (const entry of bad) { + assert.equal(ledger.normalise(entry, 'demo.spawn').ok, false, JSON.stringify(entry)) + } +}) + +test('a bad resource never fails the step it came from', async () => { + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + const out = await ledger.recordAnswer({ + run: RUN, + step: STEP, + action: action(), + placeholderId, + resources: [{ kind: 'creature', ref: '0x1' }, { nonsense: true }], + }) + assert.equal(out.recorded, 1) + assert.equal(out.rejected.length, 1) + // And the placeholder is still resolved: the good row exists, and leaving the + // placeholder open would ask the module to undo the step a second time. + assert.equal(store.rows.get(placeholderId).status, 'reverted') +}) + +test('a lease deadline and a member key ride through verbatim', async () => { + const until = new Date('2026-09-04T00:00:00Z') + await ledger.recordAnswer({ + run: RUN, + step: STEP, + action: action(), + placeholderId: null, + resources: [{ kind: 'reward', ref: 'item-1', memberKey: 'Darrow', until, payload: { cliloc: 1234 } }], + }) + const row = rows()[0] + assert.equal(row.member_key, 'Darrow') + assert.equal(row.lease_until.getTime(), until.getTime()) + assert.deepEqual(row.payload, { cliloc: 1234 }) + // Opaque: core stores what the module said and never interprets it, which is + // `ctx.teams.activity.push`'s exact treatment one registry along. + assert.equal(row.kind, 'reward') + assert.equal(row.owner_module, 'demo') +}) diff --git a/server/test/eventModuleContract.test.js b/server/test/eventModuleContract.test.js index 1823400..c9d6520 100644 --- a/server/test/eventModuleContract.test.js +++ b/server/test/eventModuleContract.test.js @@ -513,3 +513,242 @@ test('an action whose module is gone goes dormant, and a step naming it fails te assert.equal(result.dormant, true) assert.match(result.error, /no module registers "demo\.summon"/) }) + +// ── Phase 8: the ledger's two callables, from a module ───────────────────── +// +// `revert` was already required at registration for `reversible: 'ledger'` — +// Phase 1 put that check in. What Phase 8 added is a caller for it, and +// `reconcile` beside it. Both are proved here through the REAL loader for the +// same reason the four registrations are: a `revert` a test called directly is a +// `revert` core might still have no way to reach. + +test('a module\'s revert is reached by the cleanup sweep, resources and key in hand', async () => { + const record = loadModule('demo', ` + let seen = null + module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'demo.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true, resources: [{ kind: 'creature', ref: '0xA' }] } }, + async revert(arg) { seen = arg; return { ok: true } }, + async reconcile() { return { ok: true, inForce: [] } }, + }]) + api.registerEventOptionSources([ + { id: 'demo.options.seen', label: 'seen', async resolve() { return [{ value: JSON.stringify(seen), label: 'seen' }] } }, + ]) + } + `) + assertRegistered(record) + + const action = registries.eventAction('demo.spawn') + // Both callables survived the registration copy — which is not a given: that + // copy is explicit rather than a spread, precisely so nothing rides along, and + // a member added to the contract without being added to it is a member that + // silently does not exist. + assert.equal(typeof action.revert, 'function') + assert.equal(typeof action.reconcile, 'function') + assert.equal(action.owner, 'demo') + + const answer = await action.revert({ + runId: 3, + resources: [{ kind: 'creature', ref: '0xA', payload: null, memberKey: null }], + idempotencyKey: 'k-1', + }) + assert.deepEqual(answer, { ok: true }) + + // Read back through the module's own option source rather than out of a + // closure this file holds: the point is that what core PASSED is what the + // module SAW, across the seam. + const seen = JSON.parse((await registries.resolveOptionSource('demo.options.seen')).options[0].value) + assert.equal(seen.runId, 3) + assert.equal(seen.idempotencyKey, 'k-1') + assert.deepEqual(seen.resources, [{ kind: 'creature', ref: '0xA', payload: null, memberKey: null }]) +}) + +test('reconcile is optional, and a module without one still registers', () => { + // The asymmetry with `revert`, from the loader's side. A module that cannot say + // what the game still has is not broken — core keeps believing its own ledger, + // which is the behaviour before this phase — whereas one that creates something + // and cannot undo it has made a promise core has no way to keep. + const withNone = loadModule('quiet', `module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'quiet.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true } }, + async revert() { return { ok: true } }, + }]) + }`) + assertRegistered(withNone) + assert.equal(registries.eventAction('quiet.spawn').reconcile, null) + + const withoutRevert = loadModule('broken', `module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'broken.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true } }, + }]) + }`) + assert.equal(withoutRevert.state, 'startup_failed') + assert.match(withoutRevert.reason, /reversible: 'ledger' but has no revert\(\)/) +}) + +test('a module cannot claim core\'s reserved resource kind', async () => { + // `@step` is the placeholder's kind, and the placeholder is the row whose + // survival is the safety property: a module able to write one could make its + // own step look already accounted for. Refused at recording, and the STEP still + // succeeds — because it did. + const record = loadModule('sneaky', `module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'sneaky.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true, resources: [{ kind: '@step', ref: 'anything' }] } }, + async revert() { return { ok: true } }, + }]) + }`) + assertRegistered(record) + + // eslint-disable-next-line global-require + const ledger = require('../src/events/ledger') + const result = await dispatch.dispatchStep(step('sneaky.spawn'), { run: RUN }) + assert.equal(result.outcome, 'done') + const parsed = ledger.normalise(result.resources[0], 'sneaky.spawn') + assert.equal(parsed.ok, false) + assert.match(parsed.reason, /reserved kind/) +}) + +test('core registers the lease VERB and a module registers the lease', async () => { + // The seam working the way round it is meant to (Phase 8). A module ships the + // three callables; the verb an author puts in a step is `core.lease`, so the + // duration bound and the two-events-one-target conflict check live in one place + // rather than being re-implemented once per module and advisory everywhere. + const record = loadModule('demo', `module.exports = (ctx, api) => { + api.registerEventLeases([{ + id: 'demo.rate.gain', + label: 'Gain rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3600000, + async read() { return { ok: true, value: 1 } }, + async apply() { return { ok: true } }, + async restore() { return { ok: true } }, + }]) + }`) + assertRegistered(record) + registries.registerCore() + + // The module registers no ACTION at all, and its lease is still reachable. + assert.equal(registries.eventAction('demo.lease'), null) + assert.equal(registries.eventAction('core.lease').reversible, 'override') + + // And the dropdown behind `core.lease`'s first param is answered by what the + // module declared — resolved per request, so a module that booted later is + // still in the list. + const options = await registries.resolveOptionSource('core.options.leases') + assert.equal(options.ok, true) + assert.deepEqual(options.options, [{ value: 'demo.rate.gain', label: 'Gain rate', group: 'demo' }]) +}) + +test('core refuses a lease held longer than the module allows', async () => { + // The bound is the MODULE's number and the enforcement is CORE's, which is the + // §F split stated as one assertion. `retry: false` because a duration that is + // too long will still be too long in sixty seconds: it is an authoring error, + // not an outage. + const record = loadModule('demo', `module.exports = (ctx, api) => { + api.registerEventLeases([{ + id: 'demo.rate.gain', + label: 'Gain rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3600000, + async read() { return { ok: true, value: 1 } }, + async apply() { return { ok: true } }, + async restore() { return { ok: true } }, + }]) + }`) + assertRegistered(record) + registries.registerCore() + + const tooLong = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 120 }), + { run: RUN }, + ) + assert.equal(tooLong.outcome, 'terminal') + assert.match(tooLong.error, /at most 60 minutes, not 120/) + + // The same for a value outside the declared range. Unlike a cap, a bad lease + // value is in force the moment it is applied, which is why min/max are required + // on the numeric types rather than advisory. + const tooBig = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '9', minutes: 10 }), + { run: RUN }, + ) + assert.equal(tooBig.outcome, 'terminal') + assert.match(tooBig.error, /accepts 0\.5 to 5/) + + // And a lease nobody registers, which is the dormancy rule one registry along. + const missing = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.nope', value: '3', minutes: 10 }), + { run: RUN }, + ) + assert.equal(missing.outcome, 'terminal') + assert.match(missing.error, /no module registers the lease "demo\.nope"/) +}) + +test('a dry run of core.lease checks everything and takes nothing', async () => { + // `verify: true` must change nothing and must answer honestly (§F). A verify + // that reserved the target would be a dry run that changed something — and it + // would then refuse the real run that followed it, which is the worst of both. + let applied = 0 + const record = loadModule('demo', ` + let applied = 0 + module.exports = (ctx, api) => { + api.registerEventLeases([{ + id: 'demo.rate.gain', + label: 'Gain rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3600000, + async read() { return { ok: true, value: 1 } }, + async apply() { applied += 1; return { ok: true } }, + async restore() { return { ok: true } }, + }]) + api.registerEventOptionSources([ + { id: 'demo.options.applied', label: 'applied', async resolve() { return [{ value: String(applied), label: 'n' }] } }, + ]) + } + `) + assertRegistered(record) + registries.registerCore() + void applied + + const ok = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 10 }), + { run: RUN, verify: true }, + ) + assert.equal(ok.outcome, 'done') + assert.equal((await registries.resolveOptionSource('demo.options.applied')).options[0].value, '0') + + // A dry run that is still a real check: the bad duration is caught with + // `verify: true` as well, which is the whole value of the switchboard's + // "find out before you schedule it". + const bad = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 999 }), + { run: RUN, verify: true }, + ) + assert.equal(bad.outcome, 'terminal') +}) diff --git a/server/test/eventRunControls.test.js b/server/test/eventRunControls.test.js index 1cefb6c..2ae7bb0 100644 --- a/server/test/eventRunControls.test.js +++ b/server/test/eventRunControls.test.js @@ -35,6 +35,11 @@ const runsDb = require('../src/model/events/eventRuns.db') const stepsDb = require('../src/model/events/eventRunSteps.db') const logDb = require('../src/model/events/eventRunLog.db') const gatesDb = require('../src/model/events/eventPhaseGates.db') +// Phase 8: cancel now decides what happens to the run's world changes, and +// `cleanupRun` is the eighth control. Same rule as every phase since the fourth — +// a new leg under a model needs a stub in every file that stubs that layer. +const resourcesDb = require('../src/model/events/eventRunResources.db') +const eventCleanup = require('../src/events/cleanup') const db = require('../src/utils/db') after(() => db.close()) @@ -48,12 +53,35 @@ const originals = [ ['steps', stepsDb, { ...stepsDb }], ['log', logDb, { ...logDb }], ['gates', gatesDb, { ...gatesDb }], + ['resources', resourcesDb, { ...resourcesDb }], + ['cleanup', eventCleanup, { ...eventCleanup }], ] function installStubs() { - store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), nextStepId: 1, nextGateId: 1 } + store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), sweeps: [], unresolved: {}, nextStepId: 1, nextGateId: 1 } const snap = (o) => ({ ...o }) + runsDb.setCleanupStatus = async (id, to, from = null) => { + const r = store.runs.get(Number(id)) + if (!r) return false + if (from && !from.includes(r.cleanup_status)) return false + r.cleanup_status = to + return true + } + + // The sweep itself is `eventCleanup.test.js`'s subject. What this file is + // about is which control calls it, with what, and whether it is allowed to. + eventCleanup.cleanupRun = async (run, opts = {}) => { + store.sweeps.push({ runId: run.id, ...opts }) + return { attempted: 1, reverted: 1, drifted: 0, failed: 0, remaining: 0 } + } + + // Cancel asks the LEDGER whether the run owes the world anything, so that + // `cleanup: false` on a run with nothing recorded does not stamp `incomplete` + // over `not_required`. Unstubbed this is the ten-second dead-port wait, for the + // fifth time in this feature. + resourcesDb.unresolvedCount = async (runId) => store.unresolved[runId] ?? 0 + runsDb.getById = async (id) => { const r = store.runs.get(Number(id)) return r ? snap(r) : null @@ -184,7 +212,7 @@ function seedGate(runId, phase, { kind = 'on', trigger = 'test.trigger', needed return g } -function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) { +function seedRun({ status = 'running', phase = 'main', steps = [], cleanupStatus = 'not_required' } = {}) { const id = nextRunId++ store.runs.set(id, { id, @@ -192,6 +220,7 @@ function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) { version_id: id, status, health: 'ok', + cleanup_status: cleanupStatus, current_phase: phase, claimed_by: null, claim_expires_at: null, @@ -559,3 +588,115 @@ test('an empty reason is stored as NULL rather than as an empty string', async ( await controls.pause(id, { reason: ' ' }, ACTOR) assert.equal(lastLog().detail.reason, null) }) + +// ── cancel decides what happens to the world (Phase 8) ───────────────────── + +test('cancel gives back what the run took, by default and without waiting for it', async () => { + // The teardown is the runner cleanup leg over TERMINAL runs, not this request. + // Two reasons, and both are why the control answers at once: a cancel pressed + // at two in the morning must not block on a dozen round trips to the shard + // that may BE the reason it is being cancelled, and a process that dies + // halfway through a teardown has to resume rather than leave a world half + // restored with nothing scheduled to finish it. + const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 3 + + const result = await controls.cancel(id, { reason: 'called off' }, ACTOR) + + assert.equal(result.ok, true) + assert.equal(result.cleanup, true) + assert.deepEqual(store.sweeps, [], 'the request must not do the teardown itself') + // Still `pending`, which is what the leg looks for. The run is terminal the + // moment this returns, so the very next tick picks its ledger up. + assert.equal(runRow(id).cleanup_status, 'pending') + assert.equal(store.log.at(-1).detail.cleanup, true) +}) + +test('cancel WITHOUT cleanup is admin-only, even though the route is wider', async () => { + // §L: "cancelling without cleanup is a separate, logged, admin-only action." + // The route is `admin` + `moderator`, so the narrower gate cannot live in + // middleware — WHICH of the two you have to be depends on what is in the body, + // exactly as the authoring role floor does (§K). + const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 3 + + const refused = await controls.cancel(id, { cleanup: false }, ACTOR, { isAdmin: false }) + assert.equal(refused.ok, false) + assert.equal(refused.status, 403) + assert.equal(runRow(id).status, 'running', 'and the run is not cancelled either') + + // A moderator asking for the ordinary cancel is fine: the safe direction is + // the default, so the widest gate keeps the button it exists for. + const allowed = await controls.cancel(id, {}, ACTOR, { isAdmin: false }) + assert.equal(allowed.ok, true) + assert.equal(allowed.cleanup, true) +}) + +test('cancel without cleanup leaves the world changes up, and says so on the run', async () => { + // `incomplete` is the truthful value rather than a tidy one: the changes are + // still up, they are listed on the console, and the log line records who + // decided that. A `complete` here would be the "tidy completed row over a shard + // full of orphaned monsters" §L names as the failure that ends this feature's + // credibility. + const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 3 + + const result = await controls.cancel(id, { cleanup: false, reason: 'leave it up' }, ACTOR) + + assert.equal(result.ok, true) + assert.equal(result.cleanup, false) + assert.equal(runRow(id).cleanup_status, 'incomplete') + assert.equal(store.log.at(-1).detail.cleanup, false) + assert.equal(store.log.at(-1).detail.by, ACTOR) +}) + +test('a run that recorded nothing is unaffected by either flag', async () => { + // `not_required` is not walked to `incomplete` by a cancel that skipped a + // teardown there was nothing to do — and it is the LEDGER that says so, not the + // status column, because `not_required` is also what a run holding only a lease + // wrongly carried before the live walk found it. + const id = seedRun({ status: 'running', cleanupStatus: 'not_required', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 0 + await controls.cancel(id, { cleanup: false }, ACTOR) + assert.equal(runRow(id).cleanup_status, 'not_required') +}) + +// ── cleanup, the eighth control ──────────────────────────────────────────── + +test('cleanup re-runs the teardown and clears the attempt counter', async () => { + // The manual retry §L promises. `resetAttempts` is the licence a human has and + // the automatic sweep does not — Engagement Phase 14's rule, whose defect was + // a sweep that reset every stale row and made the attempt ceiling unreachable. + const id = seedRun({ status: 'completed', cleanupStatus: 'incomplete' }) + + const result = await controls.cleanupRun(id, ACTOR) + + assert.equal(result.ok, true) + assert.deepEqual(store.sweeps, [{ runId: id, resetAttempts: true, actor: ACTOR }]) + assert.equal(result.summary.reverted, 1) +}) + +test('cleanup refuses a run that is still in flight', async () => { + // A run still going has a ledger that is still growing, and reverting a + // resource the next step is about to use would be core undoing an event while + // it is happening. Cancel is the control for a run that should stop. + for (const status of ['scheduled', 'starting', 'running', 'paused', 'ending']) { + const id = seedRun({ status, cleanupStatus: 'pending' }) + const result = await controls.cleanupRun(id, ACTOR) + assert.equal(result.ok, false, status) + assert.match(result.errors[0], /cancel it before cleaning up after it/) + } + assert.deepEqual(store.sweeps, []) +}) + +test('cleanup refuses a run that recorded no resources', async () => { + const id = seedRun({ status: 'completed', cleanupStatus: 'not_required' }) + const result = await controls.cleanupRun(id, ACTOR) + assert.equal(result.ok, false) + assert.match(result.errors[0], /nothing to give back/) +}) + +test('cleanup on an unknown run is a 404, not a 409', async () => { + const result = await controls.cleanupRun(9999, ACTOR) + assert.equal(result.status, 404) +}) diff --git a/server/test/eventRunner.test.js b/server/test/eventRunner.test.js index d441766..e14efa4 100644 --- a/server/test/eventRunner.test.js +++ b/server/test/eventRunner.test.js @@ -45,6 +45,11 @@ const gatesDb = require('../src/model/events/eventPhaseGates.db') // every file that stubs the layer under it needs the stub. const settingsDb = require('../src/model/events/eventActionSettings.db') const budgetDb = require('../src/model/events/eventRunBudget.db') +// Phase 8 put a ledger write in front of every world-changing dispatch and a +// cleanup leg at the end of the tick. **Fourth time, same rule, and this file's +// own note is what caught it**: unstubbed, one of these is not a wrong answer, +// it is a ten-second wait on the dead port. +const resourcesDb = require('../src/model/events/eventRunResources.db') const gates = require('../src/events/gates') const db = require('../src/utils/db') @@ -58,7 +63,7 @@ const later = (ms) => new Date(T0.getTime() + ms) let store const originals = {} -for (const [name, mod] of [['runsDb', runsDb], ['stepsDb', stepsDb], ['logDb', logDb], ['versionsDb', versionsDb], ['definitionsDb', definitionsDb], ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb]]) { +for (const [name, mod] of [['runsDb', runsDb], ['stepsDb', stepsDb], ['logDb', logDb], ['versionsDb', versionsDb], ['definitionsDb', definitionsDb], ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb], ['resourcesDb', resourcesDb]]) { originals[name] = { mod, fns: { ...mod } } } @@ -79,8 +84,10 @@ function installStubs() { gates: new Map(), settings: new Map(), budget: new Map(), + resources: new Map(), nextStepId: 1, nextGateId: 1, + nextResourceId: 1, } // Phase 4 put a schedule-expansion leg in front of the tick. This file is @@ -91,6 +98,69 @@ function installStubs() { // connection timeout. Object.assign(definitionsDb, { findSchedulable: async () => [] }) + // ── The resource ledger (Phase 8) ── + // + // `reserve` enforces `uq_evres_target` in the stub, because the refusal it + // produces is BEHAVIOUR the runner branches on rather than an implementation + // detail: a placeholder that collides is this step's own earlier attempt and is + // reused, and a lease that collides is another run holding the target. A stub + // that let both inserts through would make the retry path grow a second row and + // the conflict test pass for no reason. + const HELD_STATUSES = ['pending', 'confirmed', 'reverting'] + resourcesDb.reserve = async ({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) => { + const holder = [...store.resources.values()].find( + (r) => r.owner_module === owner && r.kind === kind && r.ref === ref && HELD_STATUSES.includes(r.status), + ) + if (holder) return { ok: false, code: 'held', holder: { run_id: holder.run_id, status: holder.status } } + const id = store.nextResourceId++ + store.resources.set(id, { + id, + run_id: runId, + step_id: stepId, + owner_module: owner, + kind, + ref, + payload, + lease_until: leaseUntil, + status: 'pending', + revert_attempts: 0, + last_error: null, + member_key: memberKey, + }) + return { ok: true, id } + } + resourcesDb.confirm = async (id) => { + const r = store.resources.get(id) + if (!r || r.status !== 'pending') return false + r.status = 'confirmed' + return true + } + resourcesDb.resolvePlaceholder = async (id) => { + const r = store.resources.get(id) + if (!r || r.kind !== resourcesDb.STEP_KIND || !['pending', 'confirmed'].includes(r.status)) return false + r.status = 'reverted' + return true + } + resourcesDb.findByTarget = async (owner, kind, ref) => + [...store.resources.values()].reverse().find((r) => r.owner_module === owner && r.kind === kind && r.ref === ref) || null + resourcesDb.forRun = async (runId) => [...store.resources.values()].filter((r) => r.run_id === runId) + resourcesDb.markReverted = async (id) => { + const r = store.resources.get(id) + if (r) r.status = 'reverted' + } + // The cleanup leg's scan. This file is about the runner's own legs, and the + // sweep has its own file — answering with nothing is what keeps every `tick()` + // here measuring the runner rather than a teardown. + resourcesDb.runsNeedingCleanup = async () => [] + + runsDb.setCleanupStatus = async (id, to, from = null) => { + const r = store.runs.get(id) + if (!r) return false + if (from && !from.includes(r.cleanup_status)) return false + r.cleanup_status = to + return true + } + // Snapshots, not live references. A SQL SELECT hands back a copy, and the // runner reads `step.attempts` as the value BEFORE its own claim incremented // it — returning references here would make the retry budget off by one in the @@ -1319,3 +1389,163 @@ test('the runner never re-checks the role of whoever started the run', async () assert.equal(stepsOf(id)[0].status, 'done') }) + +// ── The resource ledger, from the runner's side (Phase 8) ────────────────── +// +// `eventLedger.test.js` owns the recording RULES and `eventCleanup.test.js` owns +// the undo. What belongs here is the ORDER — that the placeholder is written +// before the module is reached and after the permission check, and that a step +// which never answers leaves it standing. Those are properties of `drainStep`, +// and nothing below the runner can observe them. + +const ledgerRows = (id) => [...store.resources.values()].filter((r) => r.run_id === id) + +test('a world-changing step is recorded BEFORE it is dispatched', async () => { + // §D rule 1. The assertion is made from INSIDE `perform()`, which is the only + // place that can tell "recorded first" from "recorded at all" — and the + // difference between the two is every object whose acknowledgement is lost. + let seenDuringDispatch = null + register([scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) })]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn')] }]) + scripted['test.spawn'] = { + calls: [], + answer: () => { + seenDuringDispatch = ledgerRows(id).map((r) => [r.kind, r.status]) + return { ok: true, resources: [{ kind: 'creature', ref: '0x40001234' }] } + }, + } + + await runner.tick(T0) + + assert.deepEqual(seenDuringDispatch, [['@step', 'pending']]) + // And on the answer the real row exists and the placeholder is done with. + assert.deepEqual( + ledgerRows(id).map((r) => [r.kind, r.ref, r.status]), + [ + ['@step', stepsOf(id)[0].idempotency_key, 'reverted'], + ['creature', '0x40001234', 'confirmed'], + ], + ) + assert.equal(run(id).cleanup_status, 'pending') + assert.ok(kinds(id).includes('resource.recorded')) +}) + +test('a step that never answers leaves its placeholder pending', async () => { + // The whole reason the placeholder exists. The module timed out, so core does + // not know whether anything was created — and the row that survives is what + // lets cleanup ask it by idempotency key later. Record on the answer instead + // and this run finishes looking spotless over a shard full of orphans. + register([ + scriptedAction('test.spawn', { + risk: 'change', + reversible: 'ledger', + budgetMs: 5, + revert: async () => ({ ok: true }), + perform: () => new Promise(() => {}), + }), + ]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + + await runner.tick(T0) + + assert.deepEqual(ledgerRows(id).map((r) => [r.kind, r.status]), [['@step', 'pending']]) +}) + +test('a refused step ledgers nothing, because it never reached the module', async () => { + // The placeholder is written AFTER the permission check, deliberately. A step + // refused by a cap or by a switch created nothing, and a ledger row for it + // would be core asking a module to undo something it was never asked to do. + register([scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) })]) + // No `setSwitch`, so it is default-off: §K's world-changing default. + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + + await runner.tick(T0) + + assert.equal(stepsOf(id)[0].status, 'refused') + assert.deepEqual(ledgerRows(id), []) + assert.equal(run(id).cleanup_status, 'not_required') +}) + +test('an announce step ledgers nothing at all', async () => { + // `reversible: 'none'`, so there is nothing core could come back for. A + // placeholder here would be a row teardown could never resolve — which is the + // reason `core.announce` is declared `none` rather than `ledger` even though a + // sent message cannot be unsent. + register([scriptedAction('test.say')]) + const id = seedRun([{ key: 'main', steps: [step('test.say')] }]) + await runner.tick(T0) + assert.equal(run(id).status, 'completed') + assert.deepEqual(ledgerRows(id), []) + assert.equal(run(id).cleanup_status, 'not_required') +}) + +test('a parked step records what it made, because its confirm never dispatches again', async () => { + // `await: 'human'` is a SUCCESS: the module did its part and something outside + // the system has to happen next. The cue's confirm finishes the step without a + // second dispatch, so this is the only moment its resources can be recorded. + register([ + scriptedAction('test.stage', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }), + ]) + setSwitch('test.stage', true) + const id = seedRun([{ key: 'main', steps: [step('test.stage')] }]) + scripted['test.stage'] = { + calls: [], + answer: { ok: true, await: 'human', resources: [{ kind: 'prop', ref: 'gate-1' }] }, + } + + await runner.tick(T0) + + assert.equal(stepsOf(id)[0].status, 'running') + assert.deepEqual( + ledgerRows(id).map((r) => [r.kind, r.status]), + [['@step', 'reverted'], ['prop', 'confirmed']], + ) +}) + +test('a retry re-uses its placeholder and records the resources once', async () => { + // An idempotency key does not vary by attempt (§E), so the second attempt's + // placeholder insert collides with the first attempt's row — and a module that + // honestly re-reports the same creature must not produce a second thing for + // cleanup to revert. + register([ + scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }), + ]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + scripted['test.spawn'] = { + calls: [], + answers: [ + { ok: false, error: 'the shard did not answer' }, + { ok: true, resources: [{ kind: 'creature', ref: '0xFEED' }] }, + ], + } + + await runner.tick(T0) + await runner.tick(later(runner.RETRY_MS + 1000)) + + assert.equal(stepsOf(id)[0].status, 'done') + assert.equal(ledgerRows(id).filter((r) => r.kind === '@step').length, 1) + assert.equal(ledgerRows(id).filter((r) => r.kind === 'creature').length, 1) +}) + +test('a ledger write that fails stops the dispatch rather than losing the record', async () => { + // The ledger is what makes a world write recoverable, so a step that cannot be + // recorded must not be sent. Transient, because the alternative is an + // unrecorded world change — the one outcome §D rule 1 exists to make impossible. + register([ + scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }), + ]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + scripted['test.spawn'] = { calls: [], answer: { ok: true } } + resourcesDb.reserve = async () => { + throw new Error('the ledger is unreachable') + } + + await runner.tick(T0) + + assert.equal(scripted['test.spawn'].calls.length, 0, 'the module must not have been reached') + assert.match(stepsOf(id)[0].last_error, /the resource ledger could not record this step/) +}) diff --git a/server/test/eventRunnerSql.test.js b/server/test/eventRunnerSql.test.js index ac66978..ce69605 100644 --- a/server/test/eventRunnerSql.test.js +++ b/server/test/eventRunnerSql.test.js @@ -80,6 +80,18 @@ // that died between entering a phase and opening its gate opens no second // one on the next tick. // +// **Phase 8 added the resource ledger**, and its unique key is the single most +// server-dependent thing in this feature. "Two events cannot hold a lease on one +// target" has to hold among LIVE rows only — last week's finished event must not +// keep this week's from leasing the same rate — and MariaDB has no partial index, +// so the encoding is a STORED generated column that goes NULL once the row is no +// longer ours. Whether multiple NULLs collide in a unique index is a property of +// the server and of nothing else, and TEAMS.md §2.5 already had to be corrected +// once on this exact shape: MariaDB refuses ON DELETE SET NULL on a foreign key +// whose column is a base column of a stored generated column (error 1901), which +// is why the expression reads `status` alone and `step_id` stays a SET NULL FK. +// Both halves of that are proved below rather than believed. +// // Plus the two unique indexes that are load-bearing rather than tidy: // `uq_evrun_occurrence` (which, not the claim, is what stops two runs of one // occurrence existing) and `uq_evstep_slot` (which is what makes re-materialising @@ -208,6 +220,29 @@ CREATE TABLE event_run_budget ( CONSTRAINT fk_evbud_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE, UNIQUE KEY uq_evbud_dim (run_id, dimension) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE event_run_resources ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + run_id BIGINT NOT NULL, + step_id BIGINT NULL, + owner_module VARCHAR(64) NOT NULL, + kind VARCHAR(64) NOT NULL, + ref VARCHAR(190) NOT NULL, + payload JSON NULL, + lease_until DATETIME NULL, + status ENUM('pending','confirmed','reverting','reverted','orphaned','drifted') + NOT NULL DEFAULT 'pending', + revert_attempts INT NOT NULL DEFAULT 0, + last_error VARCHAR(500) NULL, + member_key VARCHAR(190) NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + live_marker TINYINT AS (IF(status IN ('pending','confirmed','reverting'), 1, NULL)) STORED, + CONSTRAINT fk_evres_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE, + CONSTRAINT fk_evres_step FOREIGN KEY (step_id) REFERENCES event_run_steps(id) ON DELETE SET NULL, + UNIQUE KEY uq_evres_target (owner_module, kind, ref, live_marker), + INDEX idx_evres_run (run_id, status), + INDEX idx_evres_live (status, lease_until) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE event_action_settings ( action_id VARCHAR(96) NOT NULL PRIMARY KEY, enabled TINYINT(1) NOT NULL DEFAULT 0, @@ -438,6 +473,7 @@ const rows = (r) => Number(r.affectedRows) beforeEach(async () => { if (!available) return await pool.query('DELETE FROM event_run_phase_gates') + await pool.query('DELETE FROM event_run_resources') await pool.query('DELETE FROM event_run_steps') await pool.query('DELETE FROM event_runs') await pool.query('DELETE FROM event_definitions') @@ -1139,7 +1175,21 @@ test('a gate goes with its run', async (t) => { // live server, so the pool holds open connections and the process never exits — // 49 green tests and a file that hangs until the harness kills it. Every other // event test file already closes it in an `after`; this one now has a reason to. +// +// **And that second pool has to be aimed at the throwaway database**, which it +// was not before Phase 8 noticed. `utils/db` builds its pool at REQUIRE time from +// `DB_NAME`, and its own `dotenv.config()` reads `server/.env` — so a model test +// run on a developer's machine was reaching that developer's real schema while +// the fixtures it was asserting against were being written next door. It passed +// only because the two tables happened to exist in both. `dotenv` does not +// overwrite a variable that already exists, so setting it here, before the +// require below, is what beats the file. This file drops its database in `after`, +// so aiming the model pool at it is also what keeps the whole run disposable. +process.env.DB_NAME = DB + const budgetDb = require('../src/model/events/eventRunBudget.db') +const resourcesDb = require('../src/model/events/eventRunResources.db') +const runsDb = require('../src/model/events/eventRuns.db') const appDb = require('../src/utils/db') after(() => appDb.close()) @@ -1303,3 +1353,324 @@ test('a non-positive spend never reaches the database', async (t) => { assert.equal(await budgetDb.spend(runId, 'uo.creatures', 0), true) assert.equal(await consumedOf(runId), 0) }) + +// ── The resource ledger's unique key (Phase 8) ───────────────────────────── +// +// Every test here is about a property of the SERVER. A stub can enforce whatever +// rule its author had in mind; only MariaDB can say whether this encoding of +// "unique among live rows" actually is one. + +const insertResource = async (runId, over = {}) => + ( + await pool.query( + `INSERT INTO event_run_resources (run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + runId, + over.stepId ?? null, + over.owner ?? 'demo', + over.kind ?? 'creature', + over.ref ?? '0xA', + over.payload ?? null, + over.leaseUntil ?? null, + over.memberKey ?? null, + over.status ?? 'pending', + ], + ) + ).insertId + +const dup = async (fn) => { + try { + await fn() + return null + } catch (err) { + return err.code || String(err.errno) + } +} + +test('two runs cannot hold one target, and the refusal comes from the server', async (t) => { + if (needDb(t)) return + // §D: "the unique index is what stops two events leasing one target." Not a + // read-then-insert — two runs entering the same tick would both pass the check + // — so the whole conflict story is this one constraint answering. + const a = await seedRun() + const b = await seedRun() + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' }) + const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })) + assert.equal(code, 'ER_DUP_ENTRY') +}) + +test('the key is released by the three statuses that mean it is no longer ours', async (t) => { + if (needDb(t)) return + // Amended 2026-09-03. §D says "among non-reverted rows", which was written + // before the six statuses had their meanings; taken literally it makes + // `drifted` and `orphaned` hold a target for ever, so one bad night would + // disable a lease permanently with no control able to clear it. `drifted` means + // somebody else has hold of the value and this run has let go; `orphaned` means + // it vanished. Neither is a claim on the target. + for (const status of ['reverted', 'drifted', 'orphaned']) { + await pool.query('DELETE FROM event_run_resources') + const a = await seedRun() + const b = await seedRun() + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status }) + const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })) + assert.equal(code, null, `a ${status} row must not hold the target`) + } +}) + +test('the key is HELD by the three that mean core still believes it is ours', async (t) => { + if (needDb(t)) return + for (const status of ['pending', 'confirmed', 'reverting']) { + await pool.query('DELETE FROM event_run_resources') + const a = await seedRun() + const b = await seedRun() + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status }) + const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })) + assert.equal(code, 'ER_DUP_ENTRY', `a ${status} row must hold the target`) + } +}) + +test('many released rows on one target coexist, which is the whole encoding', async (t) => { + if (needDb(t)) return + // The property the NULL depends on: multiple NULLs do not collide in a unique + // index. A weekly event that leases the same rate every Saturday accumulates one + // released row per week, and the fiftieth must not fail to insert. + const a = await seedRun() + for (let i = 0; i < 5; i++) { + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'reverted' }) + } + const held = await pool.query( + 'SELECT COUNT(*) AS n FROM event_run_resources WHERE ref = ? AND live_marker IS NULL', + ['demo.rate'], + ) + assert.equal(Number(held[0].n), 5) + // And a live one still goes on top of them. + assert.equal(await dup(() => insertResource(a.runId, { kind: 'override', ref: 'demo.rate' })), null) +}) + +test('an UPDATE that releases a row frees the target at once', async (t) => { + if (needDb(t)) return + // The generated column is STORED, so this is really asking whether MariaDB + // recomputes it on UPDATE and re-indexes. Cleanup depends on it entirely: the + // moment a lease is restored, the next event may take it. + const a = await seedRun() + const b = await seedRun() + const id = await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' }) + assert.equal(await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })), 'ER_DUP_ENTRY') + await pool.query("UPDATE event_run_resources SET status = 'reverted' WHERE id = ?", [id]) + assert.equal(await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })), null) +}) + +test('the key is per owner and per kind, not per ref', async (t) => { + if (needDb(t)) return + // `kind` and `ref` are module-opaque strings core stores verbatim, so two + // modules using the same word for different things must not collide — and one + // module's `creature:0xA` and `item:0xA` are two objects. + const a = await seedRun() + await insertResource(a.runId, { owner: 'demo', kind: 'creature', ref: '0xA', status: 'confirmed' }) + assert.equal(await dup(() => insertResource(a.runId, { owner: 'other', kind: 'creature', ref: '0xA' })), null) + assert.equal(await dup(() => insertResource(a.runId, { owner: 'demo', kind: 'item', ref: '0xA' })), null) + assert.equal(await dup(() => insertResource(a.runId, { owner: 'demo', kind: 'creature', ref: '0xB' })), null) +}) + +test('a deleted step leaves its resources behind, and a deleted run does not', async (t) => { + if (needDb(t)) return + // TEAMS.md §2.5's correction, held as a test: `step_id` is SET NULL and it only + // works because the generated column reads `status` alone. If `step_id` were in + // that expression MariaDB would refuse the constraint outright (error 1901), and + // the migration would fail on a fresh install rather than here. + // + // The two directions are different on purpose. A record of what was changed in + // the WORLD must outlive the row that scheduled it — `engagement_sends`' + // argument — while a deleted RUN takes its ledger with it, because the ledger + // exists to answer questions about a run. + const a = await seedRun() + const stepId = await seedStep(a.runId) + const id = await insertResource(a.runId, { stepId, status: 'confirmed' }) + + await pool.query('DELETE FROM event_run_steps WHERE id = ?', [stepId]) + const orphaned = (await pool.query('SELECT step_id, status FROM event_run_resources WHERE id = ?', [id]))[0] + assert.equal(orphaned.step_id, null) + assert.equal(orphaned.status, 'confirmed') + + await pool.query('DELETE FROM event_runs WHERE id = ?', [a.runId]) + const gone = await pool.query('SELECT id FROM event_run_resources WHERE id = ?', [id]) + assert.equal(gone.length, 0) +}) + +test('the ledger model reserves, confirms, reverts and refuses for real', async (t) => { + if (needDb(t)) return + // Through the shipping module rather than a copy of its statements, like the + // budget tests above: `reserve` reads ER_DUP_ENTRY as a refusal and looks the + // holder up to name it, and both halves of that are the connector's behaviour + // rather than this file's. + const a = await seedRun() + const b = await seedRun() + + const first = await resourcesDb.reserve({ runId: a.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' }) + assert.equal(first.ok, true) + await resourcesDb.confirm(first.id) + + const second = await resourcesDb.reserve({ runId: b.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' }) + assert.equal(second.ok, false) + assert.equal(second.code, 'held') + assert.equal(Number(second.holder.run_id), Number(a.runId)) + assert.equal(second.holder.status, 'confirmed') + + // And once it is given back the second run gets it. + await resourcesDb.markReverted(first.id) + const third = await resourcesDb.reserve({ runId: b.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' }) + assert.equal(third.ok, true) +}) + +test('failRevert increments and NEVER resets, and only a human clears it', async (t) => { + if (needDb(t)) return + // Engagement Phase 14's rule at the statement level: `revert_attempts = + // revert_attempts + 1` is written in one place, and the reset is a separate + // statement with an actor behind it. A `SET revert_attempts = ?` anywhere in + // the sweep would make the ceiling unreachable and the row cycle for ever. + const a = await seedRun() + const id = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' }) + .then((r) => r.id) + await resourcesDb.confirm(id) + + await resourcesDb.failRevert(id, 'the shard did not answer') + await resourcesDb.failRevert(id, 'still nothing') + let row = (await pool.query('SELECT * FROM event_run_resources WHERE id = ?', [id]))[0] + assert.equal(Number(row.revert_attempts), 2) + assert.equal(row.status, 'confirmed') + assert.equal(row.last_error, 'still nothing') + + assert.equal(await resourcesDb.resetAttempts(a.runId), 1) + row = (await pool.query('SELECT * FROM event_run_resources WHERE id = ?', [id]))[0] + assert.equal(Number(row.revert_attempts), 0) +}) + +test('claimRevert is a compare-and-set, and reverting is not re-claimable', async (t) => { + if (needDb(t)) return + // The cleanup leg and the manual cleanup route can both be working one run at + // once. `reverting` is deliberately not claimable — a row another pass is + // mid-revert on is left alone, exactly as a step with a live claim is. + const a = await seedRun() + const id = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' }) + .then((r) => r.id) + await resourcesDb.confirm(id) + + assert.equal(await resourcesDb.claimRevert(id), true) + assert.equal(await resourcesDb.claimRevert(id), false, 'a second pass must not take a row mid-revert') + + await resourcesDb.markReverted(id) + assert.equal(await resourcesDb.claimRevert(id), false, 'and a reverted row is finished') +}) + +test('the placeholder is resolvable exactly once, and only if it is a placeholder', async (t) => { + if (needDb(t)) return + // The `kind = '@step'` guard in the statement, not in JavaScript. It is what + // stops a bug elsewhere resolving a real resource — which would be core marking + // a live creature as given back without asking anyone. + const a = await seedRun() + const placeholder = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: resourcesDb.STEP_KIND, ref: 'k'.repeat(40) }) + .then((r) => r.id) + const real = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' }) + .then((r) => r.id) + + assert.equal(await resourcesDb.resolvePlaceholder(placeholder), true) + assert.equal(await resourcesDb.resolvePlaceholder(placeholder), false) + assert.equal(await resourcesDb.resolvePlaceholder(real), false) + const stillThere = (await pool.query('SELECT status FROM event_run_resources WHERE id = ?', [real]))[0] + assert.equal(stillThere.status, 'pending') +}) + +test('the unresolved reads see the five statuses that still want something', async (t) => { + if (needDb(t)) return + // `cleanup_status` is derived from this count, so what it includes IS the + // definition of "clean". A `drifted` row left out of it would let a run whose + // lease somebody else took call itself complete. + const a = await seedRun() + for (const status of ['pending', 'confirmed', 'reverting', 'reverted', 'orphaned', 'drifted']) { + await insertResource(a.runId, { ref: `ref-${status}`, status }) + } + assert.equal(await resourcesDb.unresolvedCount(a.runId), 5) + const rows = await resourcesDb.unresolvedForRun(a.runId) + assert.deepEqual( + rows.map((r) => r.status).sort(), + ['confirmed', 'drifted', 'orphaned', 'pending', 'reverting'], + ) + const counts = await resourcesDb.unresolvedCounts([a.runId]) + assert.equal(counts.get(a.runId) ?? counts.get(String(a.runId)), 5) +}) + +test('payload comes back as an object rather than as a string', async (t) => { + if (needDb(t)) return + // A lease's baseline lives in here and the cleanup sweep reads it out to pass + // to `restore`. The connector hands JSON back as text, so a missing hydration + // is a `restore(undefined)` — a lease put back to nothing, silently. + const a = await seedRun() + const id = await resourcesDb + .reserve({ + runId: a.runId, + owner: 'demo', + kind: 'override', + ref: 'demo.rate', + payload: { baseline: 1, applied: 3 }, + }) + .then((r) => r.id) + void id + const [row] = await resourcesDb.forRun(a.runId) + assert.deepEqual(row.payload, { baseline: 1, applied: 3 }) +}) + +test('the cleanup scan finds a run that owes something, whatever its status column says', async (t) => { + if (needDb(t)) return + await pool.query('ALTER TABLE event_runs ADD COLUMN IF NOT EXISTS cleanup_status ' + + "ENUM('not_required','pending','complete','incomplete') NOT NULL DEFAULT 'not_required'") + + // **`not_required` is in the scan because of a live-walk defect**, not for + // tidiness. A run whose only resource was a LEASE never went through the + // ledger's `markRunDirty` — `core.lease` reserves its own row — so its column + // stayed `not_required` and the lease was never given back at all. A terminal + // run with an unresolved row has work to do whatever any summary column says. + const lease = await seedRun({ status: 'completed' }) + await insertResource(lease.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' }) + const marked = await seedRun({ status: 'cancelled' }) + await insertResource(marked.runId, { ref: '0xB', status: 'pending' }) + await pool.query("UPDATE event_runs SET cleanup_status = 'pending' WHERE id = ?", [marked.runId]) + + // Three that must NOT be selected, one per reason. + const running = await seedRun({ status: 'running' }) + await insertResource(running.runId, { ref: '0xC', status: 'confirmed' }) + const done = await seedRun({ status: 'completed' }) + await insertResource(done.runId, { ref: '0xD', status: 'reverted' }) + const givenUp = await seedRun({ status: 'completed' }) + await insertResource(givenUp.runId, { ref: '0xE', status: 'confirmed' }) + await pool.query("UPDATE event_runs SET cleanup_status = 'incomplete' WHERE id = ?", [givenUp.runId]) + + const found = (await resourcesDb.runsNeedingCleanup(10, 3)).map((r) => Number(r.id)) + assert.deepEqual(found.sort(), [Number(lease.runId), Number(marked.runId)].sort()) +}) + +test('the scan stops selecting a run whose attempts are spent', async (t) => { + if (needDb(t)) return + // Without the bound in the join, a run whose rows are all spent would be + // selected, worked over and found to have nothing to do on every tick for the + // rest of its life. + const a = await seedRun({ status: 'completed' }) + const id = await insertResource(a.runId, { status: 'confirmed' }) + await pool.query("UPDATE event_runs SET cleanup_status = 'pending' WHERE id = ?", [a.runId]) + assert.equal((await resourcesDb.runsNeedingCleanup(10, 3)).length, 1) + + await pool.query('UPDATE event_run_resources SET revert_attempts = 3 WHERE id = ?', [id]) + assert.equal((await resourcesDb.runsNeedingCleanup(10, 3)).length, 0) +}) + +test('setCleanupStatus is guarded, which is what stops a late row re-opening a swept run', async (t) => { + if (needDb(t)) return + const a = await seedRun({ status: 'completed' }) + assert.equal(await runsDb.setCleanupStatus(a.runId, 'complete'), true) + assert.equal(await runsDb.setCleanupStatus(a.runId, 'pending', ['not_required']), false) + assert.equal(await runsDb.setCleanupStatus(a.runId, 'pending'), true) +}) diff --git a/server/test/eventsAdmin.test.js b/server/test/eventsAdmin.test.js index ff3179d..0b77599 100644 --- a/server/test/eventsAdmin.test.js +++ b/server/test/eventsAdmin.test.js @@ -42,6 +42,13 @@ const logDb = require('../src/model/events/eventRunLog.db') // model needs a stub in every file that stubs that layer. const settingsDb = require('../src/model/events/eventActionSettings.db') const budgetDb = require('../src/model/events/eventRunBudget.db') +// Phase 8: the run console reads the resource ledger. The SAME rule, for the +// fourth time in this feature -- Phase 4's expansion leg, Phase 5's gate read, +// Phase 6's settings read and now this one. **Unstubbed it is not a wrong +// answer, it is a ten-second ECONNREFUSED against the dead port**, which is why +// one missing stub here cost the run detail test ten seconds and said nothing +// about the route it was testing. +const resourcesDb = require('../src/model/events/eventRunResources.db') const seriesDb = require('../src/model/events/eventSeries.db') const gatesDb = require('../src/model/events/eventPhaseGates.db') const activity = require('../src/model/activity/activity.model') @@ -63,6 +70,7 @@ for (const [name, mod] of [ ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb], + ['resourcesDb', resourcesDb], ['activity', activity], ]) { originals[name] = { mod, fns: { ...mod } } @@ -88,6 +96,7 @@ function installStubs() { gates: [], settings: new Map(), budget: new Map(), + resources: [], occurrences: new Set(), nextDefinition: 1, nextVersion: 1, @@ -341,6 +350,9 @@ function installStubs() { [...store.budget.values()] .filter((b) => Number(b.run_id) === Number(runId)) .sort((a, b) => a.dimension.localeCompare(b.dimension)) + + resourcesDb.forRun = async (runId) => + store.resources.filter((r) => Number(r.run_id) === Number(runId)) } // ── Fixtures ─────────────────────────────────────────────────────────────── @@ -415,23 +427,26 @@ test('the catalog serves the registry, callables stripped, with its vocabularies assert.equal(res.statusCode, 200) assert.deepEqual( res.body.actions.map((a) => a.id), - ['core.announce', 'core.wait', 'core.cue'], + ['core.announce', 'core.wait', 'core.cue', 'core.lease'], ) for (const action of res.body.actions) assert.equal(action.perform, undefined) assert.deepEqual(res.body.risks, ['notify', 'inspect', 'change', 'irreversible']) assert.deepEqual(res.body.onFailure, ['skip', 'pause', 'abort_run']) // The other three registrations of the module contract arrived in Phase 7, and // they are served BESIDE the actions because the editor needs all four to draw - // one step. Core declares no budgets and no leases of its own — its three - // actions cost nothing and hold nothing — so those are empty here, and that is - // the fact worth asserting: present and empty, not absent. + // one step. Core declares no budgets and no leases of its own — its four + // actions cost nothing, and `core.lease` BORROWS a lease rather than declaring + // one, which is the seam working the way round it is meant to: core owns the + // verb, a module owns the value. So both are empty here, and that is the fact + // worth asserting: present and empty, not absent. assert.deepEqual(res.body.budgets, []) assert.deepEqual(res.body.leases, []) - // One option source, and it is core's: `core.announce`'s leg param. It is here - // WITHOUT its resolver — the values are a request of their own. + // Two option sources, both core's: `core.announce`'s leg and `core.lease`'s + // lease. They are here WITHOUT their resolvers — the values are a request of + // their own. assert.deepEqual( res.body.optionSources.map((s) => s.id), - ['core.options.legs'], + ['core.options.legs', 'core.options.leases'], ) for (const s of res.body.optionSources) assert.equal(s.resolve, undefined) }) @@ -824,7 +839,12 @@ test('the board serves every registered action with its risk-class default, and assert.equal(res.statusCode, 200) const byId = Object.fromEntries(res.body.actions.map((a) => [a.id, a])) - assert.deepEqual(Object.keys(byId).sort(), ['core.announce', 'core.cue', 'core.wait']) + assert.deepEqual(Object.keys(byId).sort(), ['core.announce', 'core.cue', 'core.lease', 'core.wait']) + // And `core.lease` is the one core action the default-off rule bites: it is + // `change`, so a fresh deployment cannot borrow a value until an admin says so. + // §K's sentence, applied to core's own verb rather than only to a module's. + assert.equal(byId['core.lease'].enabled, false) + assert.equal(byId['core.lease'].changesWorld, true) // core.wait is `inspect`, and it arrives ENABLED. Read §K's sentence literally // and it would not, and every published event that waits would break on a fresh // deployment (org lead, 2026-09-03). @@ -843,6 +863,7 @@ test('the board never serves a callable', async () => { for (const a of res.body.actions) { assert.equal(a.perform, undefined) assert.equal(a.revert, undefined) + assert.equal(a.reconcile, undefined) assert.equal(a.cost, undefined) } }) diff --git a/server/test/eventsRoles.test.js b/server/test/eventsRoles.test.js index 9abd2dc..d0d64f5 100644 --- a/server/test/eventsRoles.test.js +++ b/server/test/eventsRoles.test.js @@ -162,6 +162,11 @@ const SURFACE = [ // Phase 6's switchboard — configuration that can break things. ['GET', '/events/actions', ['admin']], ['PUT', '/events/actions', ['admin']], + // Phase 8's cleanup, and it sits in the ADMIN column rather than with the live + // controls it is rendered beside. Re-running a teardown is not incident + // response — it asks core to write to the world again, which §K puts in the + // same row as the world-changing actions themselves. + ['POST', '/events/runs/1/cleanup', ['admin']], // Live control of a run in flight: admin and moderator, deliberately WIDER // than start. @@ -214,6 +219,19 @@ test('an editor may price an event but not publish or start it', async () => { assert.equal(await forbidden('POST', '/events/1/runs', 'editor'), true) }) +test('a moderator may stop a run but not re-run its cleanup', async () => { + // The same shape as start-and-stop above, one row further on, and held as its + // own claim for the same reason: the two controls sit next to each other on the + // run console and a later tidying pass that gave them one gate would have to + // delete an assertion that says why they do not share one. + // + // Cancelling is the 2am incident. Cleanup asks core to delete things in a live + // world, which is the narrower decision even though it is the tidier-sounding + // button. + assert.equal(await forbidden('POST', '/events/runs/1/cancel', 'moderator'), false) + assert.equal(await forbidden('POST', '/events/runs/1/cleanup', 'moderator'), true) +}) + test('the switchboard is admin only in both directions', async () => { // Reading which actions are enabled is as much `admin` as writing it: the board // is the deployment's posture, and §K puts it in the same row as the actions it