From fdc118166c9ef01498288d206cebe9d5b40da8ff Mon Sep 17 00:00:00 2001 From: wtclaude Date: Thu, 3 Sep 2026 21:19:27 -0500 Subject: [PATCH] feat(events): the resource ledger, leases and cleanup (Phase 8) Event System Phase 8 (EVENTS_PLAN.md). Docs half: RunicGateway/docs#NNN. One table, one core action, one route, one body field, and two members added to MODULE_API 1.10.0 in place. The safety property the whole world-write half depends on: core now remembers what a run changed in the world, and gives it back on every terminal path. Four decisions settled by the org lead on 2026-09-03, all as recommended: - A lease is acquired by a new CORE action, `core.lease`. Section F puts the duration bound and the two-events-one-target conflict check on core's side of the seam, and a lease verb per module would be both re-implemented once per module, advisory everywhere. - Record-before-confirm is a PLACEHOLDER keyed by the step's idempotency key. A spawn's ref does not exist until the module answers, so what core writes before the dispatch is `kind: '@step'`, `ref` = that key. If the answer never comes it stands, and cleanup calls revert() with the key and no resources -- which is why section F's revert takes the key at all. - Cleanup is one sweep over the ledger, not synthetic step rows. The step-shaped version costs a second retry counter beside `revert_attempts`. - `reconcile` is declared here and TRIGGERED BY THE MODULE, through `ctx.events.reconcile()`. Core has no concept of the game being up, so it cannot decide when to ask; it asks once at its own boot. MODULE_API stays 1.10.0. A protocol owes a bump once it has landed on `main`; while it is on `edge` it is amended in place, so the whole module contract reaches an author as one version they read once. Verify - `npm test` -- 2025 tests, 1935 pass, 89 skipped, 1 fail. That one is the pre-existing engagementManifest CRLF failure, in a file this branch does not touch (`edge` before: 1950/1876/73/1). +75 tests. - The unique key was proved against a REAL MariaDB, because nothing else can prove it: whether multiple NULLs collide in a unique index, whether a STORED generated column is recomputed on UPDATE, and whether the SET NULL foreign key survives beside it are properties of the server. eventRunnerSql.test.js gained 16 tests; 65 pass against the container. The real schema.sql was applied to a fresh database and to an existing one. - Client: 362 pass, and it builds. routes:manifest and swagger -- one route added, none moved. The live walk found three defects, and two of them are the phase's real finding Driven by a throwaway `rig` module in website/modules/, deleted before commit. 1. A lease was never given back at all. `core.lease` reserves its own ledger row, so it never went through the ledger's dirty-marking, so a run holding only a lease kept `cleanup_status = 'not_required'` and the cleanup leg -- which selected on `pending` -- never looked at it. 2. EVENT_REVERT_MAX_ATTEMPTS meant one attempt, not three. The first failing sweep moved the run to `incomplete`, which took it out of the leg's own scan for ever. The test covering the bound asserted `<= 3` and was satisfied by 1: a bound has two halves, and a test that only asserts the ceiling passes against a floor. 3. The first fix for (2) made the console lie. Spending every row's `revert_attempts` was a tidy way to take a `cleanup: false` run out of a counter-bounded scan, and the run page then rendered "3 attempts" beside resources nothing had ever tried. Found by opening the page. Both (1) and (2) are the same mistake: deriving "is there anything to do" from a summary column instead of from the rows. Neither was visible to a unit test, because a test that calls the sweep directly never asks what would have selected the run. The two properties that need the process to die were walked as the plan asks. With the module's perform() hanging, the placeholder existed while the dispatch was in flight and nothing was named; after taskkill and a restart the reclaim re-dispatched the same idempotency key, the retry re-used its own placeholder, and everything was given back. Then, with the module reporting one of two resources as no longer in force, the boot-time reconcile marked the other `orphaned` -- never `reverted`. This branch does NOT bump MODULE_API_VERSION, so the integration kit stays as Phase 7 left it: red until the Phase 16 cutover re-pins ci/core-ref.json. Co-Authored-By: Claude --- client/src/api/client.js | 8 +- client/src/routes/admin/views/EventRun.jsx | 122 +++++ server/db/schema.sql | 97 +++- server/routes.guards.json | 9 + server/routes.manifest.json | 4 + server/src/config/coreEventActions.js | 211 +++++++- server/src/events/cleanup.js | 427 +++++++++++++++ server/src/events/ledger.js | 221 ++++++++ .../model/events/eventRunControls.model.js | 106 +++- server/src/model/events/eventRunLog.db.js | 10 + .../src/model/events/eventRunResources.db.js | 353 +++++++++++++ server/src/model/events/eventRuns.db.js | 27 + server/src/model/events/eventRuns.model.js | 35 +- server/src/modules/loader.js | 33 ++ server/src/modules/registries.js | 46 +- .../src/router/v1/admin/events.controller.js | 59 ++- server/src/router/v1/admin/events.router.js | 31 +- server/src/server.js | 21 +- server/src/utils/eventRunner.js | 88 ++- server/swagger/swagger-output.json | 128 ++++- server/test/eventActionRegistry.test.js | 85 ++- server/test/eventCleanup.test.js | 500 ++++++++++++++++++ server/test/eventLedger.test.js | 337 ++++++++++++ server/test/eventModuleContract.test.js | 239 +++++++++ server/test/eventRunControls.test.js | 145 ++++- server/test/eventRunner.test.js | 232 +++++++- server/test/eventRunnerSql.test.js | 371 +++++++++++++ server/test/eventsAdmin.test.js | 37 +- server/test/eventsRoles.test.js | 18 + 29 files changed, 3928 insertions(+), 72 deletions(-) create mode 100644 server/src/events/cleanup.js create mode 100644 server/src/events/ledger.js create mode 100644 server/src/model/events/eventRunResources.db.js create mode 100644 server/test/eventCleanup.test.js create mode 100644 server/test/eventLedger.test.js diff --git a/client/src/api/client.js b/client/src/api/client.js index 7ffba08..4aa0a9c 100644 --- a/client/src/api/client.js +++ b/client/src/api/client.js @@ -538,8 +538,12 @@ export const api = { pauseEventRun: (runId, reason) => req(`/admin/events/runs/${runId}/pause`, { method: 'POST', body: { reason } }), resumeEventRun: (runId) => req(`/admin/events/runs/${runId}/resume`, { method: 'POST' }), - cancelEventRun: (runId, reason) => - req(`/admin/events/runs/${runId}/cancel`, { method: 'POST', body: { reason } }), + // `cleanup` defaults to true server-side and has to be asked out of: EVENTS.md + // §L makes cancelling WITHOUT cleanup the separate, admin-only, logged action, + // so an absent flag means "give back what this run took". + cancelEventRun: (runId, reason, cleanup = true) => + req(`/admin/events/runs/${runId}/cancel`, { method: 'POST', body: { reason, cleanup } }), + cleanupEventRun: (runId) => req(`/admin/events/runs/${runId}/cleanup`, { method: 'POST' }), advanceEventRun: (runId, reason) => req(`/admin/events/runs/${runId}/advance`, { method: 'POST', body: { reason } }), confirmEventStep: (runId, stepId, note) => diff --git a/client/src/routes/admin/views/EventRun.jsx b/client/src/routes/admin/views/EventRun.jsx index 18aa3a6..e3ff02e 100644 --- a/client/src/routes/admin/views/EventRun.jsx +++ b/client/src/routes/admin/views/EventRun.jsx @@ -38,6 +38,15 @@ import { // SERVER'S. `gates[].where` arrives already rendered, because those labels are // defined in the condition grammar and a second renderer in the browser would // be a second opinion about what `gte` reads as. +// +// **Phase 8 gave it a third, and it is the one that outlives the event.** The +// resource ledger is what this run changed in the world and what became of it, +// and its unresolved rows are the reason a `completed` run can still need a +// person — EVENTS.md §L: a run reaches `completed` with `cleanup_status = +// 'incomplete'` rather than being held open, because a tidy `completed` row over +// a shard full of orphaned monsters is the failure that would end this feature's +// credibility on its first bad night. The panel is shown on finished runs for +// exactly that reason, and it is the only panel here whose empty state matters. const POLL_MS = 5000 @@ -50,6 +59,29 @@ const STATUS_COLOR = { completed: '#8fc79a', } +// The six ledger statuses, in the two groups that matter to a reader: green is +// resolved, amber wants a person. `orphaned` and `drifted` are amber rather than +// red because neither is a fault — one thing vanished, the other was taken by +// somebody with every right to take it — and red is reserved for "this did not +// come back and core kept asking". +const RESOURCE_COLOR = { + reverted: '#8fc79a', + confirmed: '#d9c184', + pending: '#d9c184', + reverting: '#d9c184', + drifted: '#d9c184', + orphaned: '#d9c184', +} + +const RESOURCE_WORD = { + pending: 'recorded, unconfirmed', + confirmed: 'still out there', + reverting: 'being given back', + reverted: 'given back', + orphaned: 'gone', + drifted: 'someone else moved it', +} + const STEP_COLOR = { done: '#8fc79a', failed: '#d98b84', @@ -148,6 +180,9 @@ export default function EventRun() { // into the run when it was created, so this is what THIS run is allowed rather // than what the switchboard says today. const [budget, setBudget] = useState([]) + // What this run created or borrowed, and what became of each (Phase 8). + const [resources, setResources] = useState([]) + const [unresolved, setUnresolved] = useState(0) const [lines, setLines] = useState([]) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) @@ -168,6 +203,8 @@ export default function EventRun() { setCounts(detail.counts || {}) setGates(detail.gates || []) setBudget(detail.budget || []) + setResources(detail.resources || []) + setUnresolved(detail.unresolvedResources || 0) setLines(log.log || []) }, [runId]) @@ -308,6 +345,18 @@ export default function EventRun() { onClick={() => act(() => api.admin.cancelEventRun(run.id, reason))}> Cancel run + {/* The separate, admin-only decision (§L). It is a second button rather + than a checkbox on the first because the two are not variants of one + action: one gives the world back, the other deliberately leaves it + changed. A checkbox next to Cancel is a thing an operator unticks by + accident at two in the morning. The server refuses this to a + moderator, and the refusal arrives as a sentence in `problem`. */} + {controls.cancel && ( + + )} {isTerminalRun(run.status) && ( @@ -400,6 +449,79 @@ export default function EventRun() { )} + {/* ── What this run changed in the world (Phase 8) ── + The WHOLE ledger, reverted rows included: "how much did last night's + invasion actually spawn, and did all of it come back" is one question + with two halves, and a list of only the failures answers neither. + Shown on finished runs for the same reason the caps meter is. */} + {(resources.length > 0 || run.cleanupStatus === 'incomplete') && ( +
0 ? '#d9c184' : 'var(--rule)'}`, + }} + > +
+

+ What this run changed +

+ {/* The manual retry. Offered only on a terminal run, because a run + still in flight has a ledger that is still growing and reverting a + resource the next step is about to use would be undoing an event + while it is happening. */} + {isTerminalRun(run.status) && unresolved > 0 && ( + + )} +
+

+ {unresolved > 0 ? ( + <> + {unresolved} of these {unresolved === 1 ? 'is' : 'are'} still unresolved. The runner + gives them back on its own and stops asking after a few tries;{' '} + Try cleanup again clears that count and asks once more. + + ) : ( + 'Everything this run created or borrowed has been given back.' + )} +

+ {resources.length === 0 ? ( +

+ Nothing named — a step changed the world and its answer never arrived, so core kept the + record it wrote beforehand and will ask the module to undo it by key. +

+ ) : ( + + + {resources.map((r) => ( + + + + + + + ))} + +
+ {r.kind}{' '} + {r.ref} + + {RESOURCE_WORD[r.status] || r.status} + + {r.module} + {r.leaseUntil ? ` · until ${clock(r.leaseUntil)}` : ''} + {r.revertAttempts > 0 ? ` · ${r.revertAttempts} attempt${r.revertAttempts === 1 ? '' : 's'}` : ''} + + {r.lastError || ''} +
+ )} +
+ )} + {/* ── Waiting on a person ── */} {parked.length > 0 && (
diff --git a/server/db/schema.sql b/server/db/schema.sql index fd86d56..76744bb 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -2090,7 +2090,7 @@ CREATE TABLE IF NOT EXISTS engagement_suppressions ( -- module contract. The rest arrive with the phases that give them a writer -- rather than as empty tables nothing reads -- `event_run_phase_gates` in P5, -- `event_action_settings` and `event_run_budget` in P6, `event_run_resources` in --- P8 and `event_run_participants` in P10. +-- P8 (below) and `event_run_participants` in P10. -- -- Core tables, so no module prefix, and no game vocabulary anywhere below: an -- action id, a scope, a resource kind and a budget dimension are all opaque @@ -2492,6 +2492,101 @@ CREATE TABLE IF NOT EXISTS event_run_budget ( UNIQUE KEY uq_evbud_dim (run_id, dimension) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- The cleanup ledger: everything one run created or leased, and what became of +-- it (EVENTS.md §D, §L "The ledger's two rules"; Phase 8). +-- +-- **It holds both kinds of thing an event owns.** An OBJECT it created is +-- `kind: 'creature'` with `ref` a serial, reverted by its own action's +-- `revert()`. A VALUE it leased is `kind: 'override'` with `ref` the lease id and +-- `payload` carrying the baseline and what was applied, restored by the lease's +-- own `restore()`. One table, because cleanup asks both the same question -- +-- what is still out there, and did putting it back work. +-- +-- **Rule 1: a resource is recorded BEFORE it is confirmed.** A spawn's serial +-- does not exist until the module answers, so what is written before the dispatch +-- is a PLACEHOLDER keyed by the step's idempotency key (`kind` = the reserved +-- '@step', `ref` = that key). On the answer the reported resources are inserted +-- `confirmed` and the placeholder is resolved. If the acknowledgement is lost the +-- placeholder survives, and cleanup calls `revert()` with the idempotency key and +-- no resources -- which is why §F's `revert({ runId, resources, idempotencyKey })` +-- takes the key at all. Recording afterwards instead would make every object +-- whose ack was lost invisible to cleanup for ever. +-- +-- **Rule 2: revert is idempotent, and its failure is loud and sticky.** A row +-- that never reverts stays visible -- the run reaches `completed` with +-- `cleanup_status = 'incomplete'` rather than being held `running`, because a +-- tidy `completed` over a shard full of orphaned monsters is the failure that +-- would end this feature's credibility on its first bad night. +-- +-- **The unique key is what stops two events leasing one target**, and it must +-- hold among LIVE rows only: last week's finished event must not keep this +-- week's from leasing the same rate. MariaDB has no partial index, so the +-- encoding is a STORED generated column that is NULL once the row is no longer +-- ours -- and multiple NULLs do not collide in a unique index. It is derived from +-- `status` ALONE and the opaque columns stay in the KEY, which is the shape +-- TEAMS.md §2.5 had to be corrected into: MariaDB refuses ON DELETE SET NULL on a +-- foreign key whose column is a base column of a stored generated column +-- (error 1901), so `step_id` must not appear in the expression. +-- +-- **The key is held by the three statuses that mean "core still believes this is +-- ours"** -- `pending`, `confirmed`, `reverting` -- and released by the three that +-- mean it is not. §D says "among non-reverted rows", which was written before the +-- six statuses had their meanings; taken literally it makes `drifted` and +-- `orphaned` hold a target for ever, so one bad night would disable a lease +-- permanently with no control able to clear it. `drifted` means somebody else has +-- hold of the value and this run has deliberately let go of it; `orphaned` means +-- it vanished. Neither is a claim on the target, and both stay LOUD by another +-- mechanism -- `cleanup_status = 'incomplete'` and a row on the run console -- +-- which is what §L's rule 2 actually asks for. Amended 2026-09-03. +CREATE TABLE IF NOT EXISTS event_run_resources ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + run_id BIGINT NOT NULL, + -- Which step made it. It is how cleanup finds the ACTION to call `revert()` on: + -- the row records the module and the opaque names, and the step records the + -- verb. SET NULL rather than CASCADE, for `engagement_sends`' reason -- a record + -- of what was changed in the world must outlive the row that scheduled it. + step_id BIGINT NULL, + -- The registering module, copied at record time rather than derived from the + -- action id, so an uninstalled module still names itself on the console. + owner_module VARCHAR(64) NOT NULL, + -- Both module-opaque, stored verbatim, never interpreted -- `ctx.teams.activity.push`'s + -- treatment. '@step' is the one reserved `kind` and core owns it. + kind VARCHAR(64) NOT NULL, + ref VARCHAR(190) NOT NULL, + payload JSON NULL, + -- A lease's deadline, and NULL for an owned object. It goes DOWN THE WIRE as + -- well: the game side restores baseline when it passes, without being asked + -- again, which is the fail-safe that makes an unattended world change + -- defensible. This column is core's copy of that promise, for the console and + -- for the boot-time check. + lease_until DATETIME NULL, + status ENUM('pending','confirmed','reverting','reverted','orphaned','drifted') + NOT NULL DEFAULT 'pending', + -- Bounded like a step's `attempts`, and for the same reason: a revert that can + -- never succeed must become visible rather than cycling for ever. Engagement + -- Phase 14's rule -- only a terminal row is ever retention-eligible -- is what + -- makes an unbounded counter a row nothing can ever sweep. + revert_attempts INT NOT NULL DEFAULT 0, + last_error VARCHAR(500) NULL, + -- Optional, and module-opaque like the rest: who received it, for a granted + -- reward that results should be able to name. `event_run_participants` joins on + -- the same key in Phase 10. + member_key VARCHAR(190) NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + -- 1 while core still believes this resource is this run's, NULL once it is not. + -- See the unique key below; derived from `status` alone, deliberately. + live_marker TINYINT AS (IF(status IN ('pending','confirmed','reverting'), 1, NULL)) STORED, + CONSTRAINT fk_evres_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE, + CONSTRAINT fk_evres_step FOREIGN KEY (step_id) REFERENCES event_run_steps(id) ON DELETE SET NULL, + -- "Two events cannot hold a lease on one target", among non-reverted rows. + UNIQUE KEY uq_evres_target (owner_module, kind, ref, live_marker), + -- The run console, and the cleanup sweep's read: one run's ledger in order. + INDEX idx_evres_run (run_id, status), + -- The cleanup leg's scan across runs, and the boot-time lease self-check. + INDEX idx_evres_live (status, lease_until) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- §K's last bound: "a scheduled definition that has never been verified is the -- case worth refusing to start". A version is immutable, so a dry run that passed -- against it stays true — which is what makes the pass a property of the VERSION diff --git a/server/routes.guards.json b/server/routes.guards.json index 63d1265..b2c5dd5 100644 --- a/server/routes.guards.json +++ b/server/routes.guards.json @@ -581,6 +581,15 @@ "requireAuth" ] }, + { + "method": "POST", + "path": "/api/v1/admin/events/runs/:runId/cleanup", + "handlers": 2, + "gates": [ + "noindex", + "requireAuth" + ] + }, { "method": "GET", "path": "/api/v1/admin/events/runs/:runId/log", diff --git a/server/routes.manifest.json b/server/routes.manifest.json index 5a85a9e..31caf15 100644 --- a/server/routes.manifest.json +++ b/server/routes.manifest.json @@ -257,6 +257,10 @@ "method": "POST", "path": "/api/v1/admin/events/runs/:runId/cancel" }, + { + "method": "POST", + "path": "/api/v1/admin/events/runs/:runId/cleanup" + }, { "method": "GET", "path": "/api/v1/admin/events/runs/:runId/log" diff --git a/server/src/config/coreEventActions.js b/server/src/config/coreEventActions.js index 1d171f0..e124ac7 100644 --- a/server/src/config/coreEventActions.js +++ b/server/src/config/coreEventActions.js @@ -13,6 +13,15 @@ // a human to go and do something. A deployment with no game module installed has // a working event system made of exactly these. // +// **Phase 8 added a fourth, and it is the odd one out on purpose.** `core.lease` +// names no game noun either — it borrows a value some module declared — but +// unlike the other three it genuinely changes the world, so it is `risk: 'change'` +// and therefore default-off, admin-only and cap-checked like any module verb. +// It is CORE's rather than each module's because §F puts the duration bound and +// the two-events-one-target conflict check on core's side of the seam: a lease +// verb per module would be that bound re-implemented once per module, advisory +// everywhere, and wrong in the first one that forgot it. +// // **Phase 2 gave all three real bodies**, and between them they exercise every // shape §F's envelope can take: `core.announce` does work and finishes, // `core.wait` finishes while deferring what follows it, and `core.cue` succeeds @@ -24,10 +33,42 @@ // which runs under `routeManifest.js` and `swagger.js` against a dead pool // (MODULE_API.md §2.2). Nothing below runs at require time; the announce leg is // looked up inside `perform()`, per call, which is also what makes a leg -// registered by a module that booted later reachable at all. +// registered by a module that booted later reachable at all. `core.lease` is the +// one action here that reaches a table, and it requires the model INSIDE +// `perform()` for the same reason — a top-level require would make this file +// build a pool during route-manifest generation. const registries = require('../modules/registries') + +/** + * Turn the `value` param's text into whatever the named lease says it holds. + * + * The range check is here too, and it is REQUIRED on the numeric types for the + * reason §F gives: unlike a cap, a bad lease value is in force the moment it is + * applied, so "0.5 to 5" is not advice. + */ +function coerceLeaseValue(lease, raw) { + const text = String(raw === undefined || raw === null ? '' : raw).trim() + if (lease.type === 'string') return { ok: true, value: text } + if (lease.type === 'bool') { + if (['true', '1', 'yes', 'on'].includes(text.toLowerCase())) return { ok: true, value: true } + if (['false', '0', 'no', 'off'].includes(text.toLowerCase())) return { ok: true, value: false } + return { ok: false, error: `"${raw}" is not a yes or no value for ${lease.label}` } + } + const n = Number(text) + if (text === '' || !Number.isFinite(n)) { + return { ok: false, error: `"${raw}" is not a number, and ${lease.label} holds one` } + } + if (lease.type === 'int' && !Number.isInteger(n)) { + return { ok: false, error: `${lease.label} holds a whole number, and "${raw}" is not one` } + } + if (n < lease.min || n > lease.max) { + return { ok: false, error: `${lease.label} accepts ${lease.min} to ${lease.max}, and "${raw}" is outside that` } + } + return { ok: true, value: n } +} + const ACTIONS = [ { id: 'core.announce', @@ -214,6 +255,164 @@ const ACTIONS = [ return { ok: true, await: 'human' } }, }, + + { + id: 'core.lease', + label: 'Borrow a value', + description: + 'Hold a module-declared value at a new setting for a bounded time, and put the old one back at teardown.', + + // The world changes and it changes back, so `change` rather than + // `irreversible` — and `change`'s default `on_failure` is `pause`, which is + // the right stop for a run that failed halfway through altering the world. + risk: 'change', + // The one action core ships in this class. `override` is what tells the + // cleanup sweep to restore through the LEASE registry rather than through an + // action's `revert()`, which is why this action needs no `revert()` of its own + // and why the registry refuses one on it. + reversible: 'override', + version: 1, + + params: [ + { + name: 'lease', + type: 'string', + required: true, + example: 'uo.rate.skillgain', + source: 'core.options.leases', + description: 'Which declared value to borrow.', + }, + { + // **A string, and the coercion is here rather than in the type system.** + // A param declares ONE type; a lease declares its own, and they are four + // different ones. Typing this `float` would make a boolean lease + // unauthorable and a string lease nonsense, so the field takes text and + // this action turns it into whatever the named lease said it holds — the + // one place that knows both halves. + name: 'value', + type: 'string', + required: true, + example: '3.0', + description: 'What to hold it at, in whatever type the lease declares.', + }, + { + name: 'minutes', + type: 'int', + required: true, + example: 120, + description: 'How long to hold it. Core refuses more than the lease allows.', + }, + ], + + // What a lease costs is the LEASE's business to bound, not a budget's: + // `maxDurationMs` and the numeric range are declared beside the callables and + // enforced below. A cap dimension here would be core inventing an accounting + // unit for something a module already bounds — and `registerEventBudgets` + // refuses a dimension nobody declared, which is exactly the rule that would + // then bite core's own action. + + /** + * Read the baseline, reserve the target, apply the value. + * + * **This is rule 1 in its strongest form.** Unlike a spawn, a lease's target + * is knowable before the dispatch — it is the lease id the step names — so + * the ledger row is written with its real `kind` and `ref` BEFORE anything + * touches the world, and the two-events-one-target refusal comes from the + * unique index at that moment rather than from a check that read and then + * wrote. A second run asking for a lease another run holds comes back + * `refused`, in the same words a cap breach uses and for the same reason: + * nothing is broken, the deployment already has that value spoken for. + * + * The order is read then reserve then apply, and a failure at each stage + * undoes the one before it: a reservation whose `apply` refuses is released + * here rather than left for the sweep, because there is nothing out there to + * give back and a shard that is merely down must not lock a lease out for the + * length of a retry cycle. + */ + async perform({ runId, stepId, params, verify }) { + // eslint-disable-next-line global-require + const resourcesDb = require('../model/events/eventRunResources.db') + const lease = registries.eventLease(params.lease) + if (!lease) { + return { ok: false, retry: false, error: `no module registers the lease "${params.lease}"` } + } + + const coerced = coerceLeaseValue(lease, params.value) + if (!coerced.ok) return { ok: false, retry: false, error: coerced.error } + + const minutes = Number(params.minutes) + if (!Number.isFinite(minutes) || minutes <= 0) { + return { ok: false, retry: false, error: `"${params.minutes}" is not a number of minutes` } + } + const ms = Math.round(minutes * 60_000) + if (ms > lease.maxDurationMs) { + return { + ok: false, + retry: false, + error: `${lease.label} may be held for at most ${Math.floor(lease.maxDurationMs / 60_000)} minutes, not ${minutes}`, + } + } + + // **The dry run stops here, and it has still checked everything worth + // checking**: the lease exists, the value is in range and the duration is + // allowed. What it deliberately does not do is reserve the target — a + // verify that took a lease would be a dry run that changed something, and + // it would then refuse the real run that followed it. + if (verify) return { ok: true } + + const baseline = await lease.read() + if (!baseline || baseline.ok !== true) { + return { ok: false, error: `could not read the current value of ${lease.label}` } + } + + const until = new Date(Date.now() + ms) + const reserved = await resourcesDb.reserve({ + runId, + stepId, + owner: lease.owner || 'core', + kind: 'override', + ref: lease.id, + payload: { target: lease.id, baseline: baseline.value, applied: coerced.value, until: until.toISOString() }, + leaseUntil: until, + }) + if (!reserved.ok) { + const heldBy = reserved.holder ? ` (run ${reserved.holder.run_id})` : '' + return { + ok: false, + retry: false, + error: `${lease.label} is already leased by another run${heldBy}`, + } + } + + // **`until` goes down the wire** (§F). The module passes it to its sidecar + // and the game side restores baseline when it passes, without being asked + // again — the fail-safe that makes an unattended, scheduled world change + // defensible, because the worst case is a world back at baseline early + // rather than one stuck changed indefinitely. + let applied + try { + applied = await lease.apply(coerced.value, until) + } catch (err) { + applied = { ok: false, error: err.message } + } + if (!applied || applied.ok !== true) { + await resourcesDb.markReverted(reserved.id) + return { ok: false, error: applied && applied.error ? String(applied.error) : `${lease.label} refused the new value` } + } + + await resourcesDb.confirm(reserved.id) + // **The run now owes the world something, and something has to say so.** + // The generic path marks a run dirty when it records a module's reported + // resources; this action reserves its own row and never goes through it, so + // a run whose only resource was a lease would have kept `cleanup_status = + // 'not_required'` and never been swept. Found by the live walk, and the + // cleanup leg's own scan was widened to make the class impossible rather + // than only this instance. + // eslint-disable-next-line global-require + await require('../events/ledger').markRunDirty(runId) + return { ok: true } + }, + }, ] // ── Core's own param option sources (§F, Phase 7) ────────────────── @@ -237,6 +436,16 @@ const OPTION_SOURCES = [ return registries.announceLegs().map((l) => ({ value: l.leg, label: l.label || l.leg })) }, }, + { + id: 'core.options.leases', + label: 'Borrowable values', + description: 'Every value a module has declared this deployment may lease.', + async resolve() { + return registries + .allEventLeases() + .map((l) => ({ value: l.id, label: l.label, group: l.id.split('.')[0] })) + }, + }, ] module.exports = { ACTIONS, OPTION_SOURCES } diff --git a/server/src/events/cleanup.js b/server/src/events/cleanup.js new file mode 100644 index 0000000..4fe2af5 --- /dev/null +++ b/server/src/events/cleanup.js @@ -0,0 +1,427 @@ +// ── Giving back what a run took ──────────────────────────────────────────── +// +// EVENTS.md §C ("Cleanup is generated, never authored"), §L and its two ledger +// rules, and Phase 8 of EVENTS_PLAN.md. `events/ledger.js` is the write half; +// this is the undo half, plus the reconcile that answers "is any of it still +// there?" after something outside core restarted. +// +// **Cleanup is derived from the ledger, never authored.** An operator cannot be +// relied on to write the undo, and an aborted run never reaches the phase they +// wrote it in — so there is no cleanup phase in a spec and no `on_teardown` on an +// action. There is one function, it reads rows, and it runs on EVERY terminal +// path: completion, cancellation and abort alike. +// +// **It is not built out of `event_run_steps` rows** (org lead, 2026-09-03). The +// plan's phrase is "cleanup steps are generated from the ledger", and the +// tempting reading is a synthetic phase of real step rows so the console's +// per-step retry comes free. It is the wrong shape here for one concrete reason: +// `event_run_resources` already carries `revert_attempts` and `last_error`, so +// synthetic steps would put a second retry counter beside the first and the two +// would disagree the first time a step reverted three of its four resources. +// The manual retry the API surface promises is a route over the ledger — +// `POST /admin/events/runs/:runId/cleanup` — rather than a step control. +// +// **Where it runs from.** One place: the runner's cleanup leg, which finds +// terminal runs that still owe the world something and works their rows. Hooking +// each terminal path instead would be four call sites, three of which are inside +// a request, and none of which would survive the process dying mid-cleanup. The +// leg is ordered AFTER advance in the tick, so a run that completes in one tick +// is cleaned in the same one. +// +// **The scan's WHERE clause cost two live-walk findings, in opposite +// directions.** A run whose only resource was a LEASE never went through +// `ledger.markRunDirty` — `core.lease` reserves its own row — so its +// `cleanup_status` stayed `not_required` and the lease was never given back at +// all. And a run whose first sweep failed was moved to `incomplete` by that very +// sweep, so it was never picked up again: `MAX_REVERT_ATTEMPTS` meant ONE attempt +// rather than three. The first is why `not_required` is in the scan; the second +// is why `incomplete` is written HERE only once nothing retryable is left. +// +// **Rule 2 is what the bounds are for.** A revert that never succeeds must stay +// visible rather than cycle: `MAX_REVERT_ATTEMPTS` stops the automatic retry, the +// run reaches `completed` with `cleanup_status = 'incomplete'`, and the rows stay +// on the console with their last error. Only a human's cleanup clears the +// counter — Engagement Phase 14's rule, whose defect was a sweep that reset every +// stale row and made the ceiling unreachable for ever. + +const resourcesDb = require('../model/events/eventRunResources.db') +const runsDb = require('../model/events/eventRuns.db') +const logDb = require('../model/events/eventRunLog.db') +const stepsDb = require('../model/events/eventRunSteps.db') +const registries = require('../modules/registries') +const { withDeadline } = require('./dispatch') +const log = require('../utils/logger')('events') + +// How many times the automatic sweep will ask before leaving a resource for a +// human. Three, like a step's, and for the same reason: a fourth attempt against +// a shard that has answered the same way three times is not new information. +const MAX_REVERT_ATTEMPTS = Number(process.env.EVENT_REVERT_MAX_ATTEMPTS) || 3 + +// The bound on one revert call, when the action that made the resource is gone +// and there is no `budgetMs` to read. A restore is a round trip like any other. +const DEFAULT_REVERT_BUDGET_MS = 10_000 + +// How many runs one cleanup leg looks at, and how many resource groups it works +// per run. Bounds rather than targets, exactly like `RUN_BATCH`: the tick runs +// again, and an unbounded teardown is how one run's bad night stalls every other. +const CLEANUP_RUN_BATCH = Number(process.env.EVENT_CLEANUP_RUN_BATCH) || 10 +const CLEANUP_GROUPS_PER_RUN = Number(process.env.EVENT_CLEANUP_GROUPS_PER_RUN) || 25 + +/** + * Classify one revert answer, with `dispatch.classify`'s posture: no shape a + * failure can take may read as success. + * + * The extra value here is `drifted`. It is NOT an error — the module did exactly + * what it was asked and found somebody else's value in place — so it is a third + * outcome rather than a failure with a flag, and the row it produces is the one + * §L wants surfaced beside the unreverted ones. + */ +function classifyRevert(raw, what) { + if (raw && raw.__timedOut) return { outcome: 'retry', error: raw.error } + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + return { outcome: 'retry', error: `${what} answered with no envelope` } + } + if (raw.ok === true) { + // §L, and the Rust wipe: "gone, and that is fine" is a successful revert. A + // module never has to distinguish "I deleted it" from "it was not there". + return { outcome: 'done', failed: Array.isArray(raw.failed) ? raw.failed.map(String) : [] } + } + if (raw.drifted === true) { + return { + outcome: 'drifted', + error: `the value is now ${JSON.stringify(raw.current)} rather than what this run applied, so it was left alone`, + } + } + return { + outcome: raw.retry === false ? 'terminal' : 'retry', + error: raw.error ? String(raw.error) : `${what} refused`, + } +} + +/** Call a lease's `restore`, under a deadline, never throwing. */ +async function restoreLease(row) { + const lease = registries.eventLease(row.ref) + if (!lease) { + // The module that owned it is uninstalled or failed to boot. Not a failure to + // retry away — nothing will change until an operator reinstalls it — and not + // an orphan either, because core has no idea whether the value is still + // applied. It stays unresolved with the reason on it, which is exactly what + // `cleanup_status = 'incomplete'` is for. + return { outcome: 'terminal', error: `no module registers the lease "${row.ref}"` } + } + const payload = row.payload || {} + let raw + try { + raw = await withDeadline( + () => lease.restore(payload.baseline, { expected: payload.applied, runId: row.run_id }), + DEFAULT_REVERT_BUDGET_MS, + row.ref, + ) + } catch (err) { + return { outcome: 'retry', error: err.message } + } + return classifyRevert(raw, row.ref) +} + +/** Call an action's `revert` over a group of its rows, under a deadline, never throwing. */ +async function revertGroup(actionId, rows, idempotencyKey) { + const action = registries.eventAction(actionId) + if (!action || typeof action.revert !== 'function') { + return { + outcome: 'terminal', + error: action + ? `${actionId} declares no revert()` + : `no module registers "${actionId}", so its resources cannot be given back`, + } + } + const payload = rows + .filter((r) => r.kind !== resourcesDb.STEP_KIND) + .map((r) => ({ kind: r.kind, ref: r.ref, payload: r.payload || null, memberKey: r.member_key || null })) + + let raw + try { + raw = await withDeadline( + () => action.revert({ runId: rows[0].run_id, resources: payload, idempotencyKey }), + action.budgetMs || DEFAULT_REVERT_BUDGET_MS, + actionId, + ) + } catch (err) { + // A module should not throw from `revert` any more than from `perform`, and + // one that does has produced a transient failure rather than a crashed sweep. + log.warn('event revert threw', { action: actionId, message: err.message }) + return { outcome: 'retry', error: err.message } + } + return classifyRevert(raw, actionId) +} + +/** + * Work one run's ledger once. + * + * Answers what it found and what it managed, and sets `cleanup_status` from the + * rows that are left rather than from what it did — the two differ whenever + * another writer touched the run, and the rows are the truth. + * + * `resetAttempts` is the human's flag. It is never set by the automatic leg. + */ +async function cleanupRun(run, { resetAttempts = false, actor = null } = {}) { + const summary = { attempted: 0, reverted: 0, drifted: 0, failed: 0, remaining: 0 } + + if (resetAttempts) { + const cleared = await resourcesDb.resetAttempts(run.id) + await runsDb.setCleanupStatus(run.id, 'pending') + if (cleared) { + await logDb.write({ + runId: run.id, + kind: 'cleanup.retry', + detail: { resources: cleared, by: actor }, + }) + } + } + + const rows = await resourcesDb.unresolvedForRun(run.id, { + maxAttempts: resetAttempts ? null : MAX_REVERT_ATTEMPTS, + }) + + // **Grouped by the step that made them**, because that is what names the verb: + // the resource row records the module and the opaque names, the step records + // the action, and `revert()` takes a LIST so one round trip can give back + // twelve creatures. A lease is its own group of one — core restores it through + // the lease registry rather than through any action, which is the split §F + // draws and the reason `core.lease` needs no `revert()` of its own. + const leases = rows.filter((r) => r.kind === 'override') + const byStep = new Map() + for (const row of rows) { + if (row.kind === 'override') continue + const key = row.step_id === null ? `orphan:${row.id}` : `step:${row.step_id}` + if (!byStep.has(key)) byStep.set(key, []) + byStep.get(key).push(row) + } + + const groups = [...leases.map((r) => ({ lease: r })), ...[...byStep.values()].map((rs) => ({ rows: rs }))] + + for (const group of groups.slice(0, CLEANUP_GROUPS_PER_RUN)) { + if (group.lease) { + const row = group.lease + if (!(await resourcesDb.claimRevert(row.id))) continue + summary.attempted += 1 + const verdict = await restoreLease(row) + await applyVerdict(run, [row], verdict, summary, row.ref) + continue + } + + const rs = group.rows + // The step is what names the action and carries the idempotency key. A row + // whose step was deleted keeps the action id in its own payload, which is why + // the placeholder writes one. + const step = rs[0].step_id === null ? null : await stepsDb.getById(rs[0].step_id) + const actionId = step?.action_id || rs[0].payload?.action || null + if (!actionId) { + await noteUnrevertable(run, rs, 'nothing records which action created this', summary) + continue + } + const claimed = [] + for (const row of rs) if (await resourcesDb.claimRevert(row.id)) claimed.push(row) + if (!claimed.length) continue + summary.attempted += claimed.length + const verdict = await revertGroup(actionId, claimed, step?.idempotency_key || rs[0].ref) + await applyVerdict(run, claimed, verdict, summary, actionId) + } + + summary.remaining = await resourcesDb.unresolvedCount(run.id) + // **`incomplete` means "finished with, and not finished"**, so it is written + // only once there is nothing left this sweep will try. Writing it after the + // FIRST failure — which is what the first draft did — took the run straight out + // of the leg's own scan, and `MAX_REVERT_ATTEMPTS` quietly meant one attempt + // rather than three. Found by the live walk, watching `revert_attempts` sit at + // 1 through half a minute of ticks. + const retryable = await resourcesDb.unresolvedForRun(run.id, { maxAttempts: MAX_REVERT_ATTEMPTS }) + const status = summary.remaining === 0 ? 'complete' : retryable.length ? 'pending' : 'incomplete' + await runsDb.setCleanupStatus(run.id, status) + + if (summary.attempted > 0) { + await logDb.write({ + runId: run.id, + kind: 'cleanup.swept', + detail: { ...summary, by: actor }, + }) + } + return summary +} + +/** Write one verdict across the rows it covers, and count it. */ +async function applyVerdict(run, rows, verdict, summary, what) { + for (const row of rows) { + if (verdict.outcome === 'done' && !(verdict.failed || []).includes(row.ref)) { + await resourcesDb.markReverted(row.id) + summary.reverted += 1 + continue + } + if (verdict.outcome === 'drifted') { + await resourcesDb.failRevert(row.id, verdict.error, 'drifted') + summary.drifted += 1 + continue + } + const error = + verdict.outcome === 'done' + ? `${what} could not give "${row.ref}" back` + : verdict.error + await resourcesDb.failRevert(row.id, error, 'confirmed') + summary.failed += 1 + } + await logDb.write({ + runId: run.id, + kind: verdict.outcome === 'done' ? 'cleanup.reverted' : 'cleanup.failed', + detail: { + what, + outcome: verdict.outcome, + resources: rows.map((r) => `${r.kind}:${r.ref}`), + ...(verdict.error ? { error: verdict.error } : {}), + }, + }) +} + +/** A group core cannot even name a verb for. Counted as failed, and said once. */ +async function noteUnrevertable(run, rows, reason, summary) { + for (const row of rows) { + if (!(await resourcesDb.claimRevert(row.id))) continue + await resourcesDb.failRevert(row.id, reason, 'confirmed') + summary.attempted += 1 + summary.failed += 1 + } + await logDb.write({ + runId: run.id, + kind: 'cleanup.failed', + detail: { what: null, outcome: 'terminal', resources: rows.map((r) => `${r.kind}:${r.ref}`), error: reason }, + }) +} + +/** + * The cleanup leg of the tick: every TERMINAL run with something left to give + * back. + * + * Terminal only. A run still in flight has a ledger that is still growing, and + * reverting a resource the next step is about to use would be core undoing an + * event while it is happening. + */ +async function sweep() { + // The ceiling goes INTO the query, so a run whose rows are all spent is not + // selected, worked over and found to have nothing to do on every tick for the + // rest of its life. It is also what excludes a run an admin cancelled without + // cleanup, whose counters were spent deliberately. + const candidates = await resourcesDb.runsNeedingCleanup(CLEANUP_RUN_BATCH, MAX_REVERT_ATTEMPTS) + let swept = 0 + for (const candidate of candidates) { + if (!runsDb.TERMINAL.includes(candidate.status)) continue + try { + await cleanupRun(candidate) + swept += 1 + } catch (err) { + log.error('event cleanup failed', { run: candidate.id, message: err.message }) + } + } + return swept +} + +/** + * Ask one module which of its ledgered resources the game still has (§L, and + * §N7's "the shard stays stateless about events"). + * + * **Core cannot know when to ask**, and that is not an omission: §F says core has + * no concept of the game being up, because a module with six sidecars cannot + * answer that question in the singular. So the module triggers this, through + * `ctx.events.reconcile()`, when it sees its own reconnect — module-uo already + * watches `bootId` for exactly that. Core also asks once at boot, for its own + * restart. + * + * **A resource the module no longer has becomes `orphaned`, never `reverted`.** + * Reverting it would be core recording that it put something back when what + * actually happened is that the thing vanished while nobody was looking, and the + * two are different sentences to the operator reading the console afterwards. + * + * A module with no `reconcile()` on the action is not broken: core keeps + * believing its own ledger, which is precisely the behaviour before this phase. + */ +async function reconcileModule(owner) { + const rows = await resourcesDb.liveForModule(owner) + const summary = { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 } + if (!rows.length) return summary + + const byStep = new Map() + for (const row of rows) { + if (row.kind === resourcesDb.STEP_KIND) continue // nothing to ask about yet + const key = row.step_id === null ? `orphan:${row.id}` : `step:${row.step_id}` + if (!byStep.has(key)) byStep.set(key, []) + byStep.get(key).push(row) + } + + for (const group of byStep.values()) { + const step = group[0].step_id === null ? null : await stepsDb.getById(group[0].step_id) + const actionId = step?.action_id || group[0].payload?.action || null + const action = actionId ? registries.eventAction(actionId) : null + if (!action || typeof action.reconcile !== 'function') { + summary.unanswered += group.length + continue + } + summary.asked += group.length + let raw + try { + raw = await withDeadline( + () => + action.reconcile({ + runId: group[0].run_id, + resources: group.map((r) => ({ kind: r.kind, ref: r.ref, payload: r.payload || null })), + }), + action.budgetMs || DEFAULT_REVERT_BUDGET_MS, + actionId, + ) + } catch (err) { + log.warn('event reconcile threw', { action: actionId, message: err.message }) + raw = null + } + // Same posture as everywhere else: nothing that is not an explicit answer + // counts as one. A module that could not answer leaves the ledger alone, + // because "I do not know" must never be read as "it is gone". + if (!raw || raw.__timedOut || raw.ok !== true || !Array.isArray(raw.inForce)) { + summary.unanswered += group.length + continue + } + const held = new Set(raw.inForce.map(String)) + for (const row of group) { + if (held.has(row.ref)) { + summary.inForce += 1 + continue + } + await resourcesDb.markOrphaned(row.id, 'the module reports this is no longer in force') + summary.orphaned += 1 + await logDb.write({ + runId: row.run_id, + kind: 'resource.orphaned', + detail: { module: owner, resource: `${row.kind}:${row.ref}`, action: actionId }, + }) + } + } + return summary +} + +/** Ask every module that owns a live row. Core's own boot-time sweep. */ +async function reconcileAll() { + const owners = await resourcesDb.modulesWithLiveRows() + const out = {} + for (const owner of owners) { + try { + out[owner] = await reconcileModule(owner) + } catch (err) { + log.error('event reconcile failed', { module: owner, message: err.message }) + } + } + return out +} + +module.exports = { + MAX_REVERT_ATTEMPTS, + classifyRevert, + cleanupRun, + sweep, + reconcileModule, + reconcileAll, +} diff --git a/server/src/events/ledger.js b/server/src/events/ledger.js new file mode 100644 index 0000000..7638619 --- /dev/null +++ b/server/src/events/ledger.js @@ -0,0 +1,221 @@ +// ── Recording what a run changed in the world ────────────────────────────── +// +// EVENTS.md §D and §L, and Phase 8 of EVENTS_PLAN.md. The write half of the +// resource ledger; `events/cleanup.js` is the read-and-undo half. +// +// **Rule 1 is the whole reason this file is not two lines inside `drainStep`.** +// A resource is recorded BEFORE it is confirmed. The obstacle is that a spawn's +// serial does not exist until the module answers, so there is nothing to write a +// row about yet — which is why what goes in before the dispatch is a PLACEHOLDER +// keyed by the step's idempotency key rather than by the object: +// +// pre-dispatch INSERT pending { kind: '@step', ref: } +// answer INSERT confirmed { kind: 'creature', ref: '0x40001234' } × n +// resolve the placeholder +// ack lost the placeholder is still `pending` +// cleanup revert({ idempotencyKey, resources: [] }) +// +// That last line is why §F's `revert({ runId, resources, idempotencyKey })` takes +// the key at all. A module that half-ran and never answered is reachable by its +// key and by nothing else, and Phase 11's plugin-side key ledger is what makes +// answering it exact. Until then the contract is still honest, because §L +// requires reverting something that does not exist to be a SUCCESS. +// +// **Recording is idempotent, and the database is what makes it so.** A retry +// re-dispatches the same idempotency key, and a module that answers with the same +// resources twice must not produce two rows. `uq_evres_target` refuses the second +// insert, and this file reads that refusal as "already recorded" rather than as an +// error — the same posture `materialisePhase`'s INSERT IGNORE takes. +// +// **A lease does not use the placeholder.** Its target is knowable before the +// dispatch — it is the lease id the step names — so `core.lease` reserves the +// real row first, which is both a stronger form of rule 1 and the only place the +// two-events-one-target refusal can happen before the world has been written to. + +const resourcesDb = require('../model/events/eventRunResources.db') +const runsDb = require('../model/events/eventRuns.db') +const registries = require('../modules/registries') +const log = require('../utils/logger')('events') + +// Which reversible classes get a pre-dispatch placeholder. `none` is gone once +// done and `self` undoes itself, so neither has anything core could come back +// for; `override` reserves its own target instead (see the header). That leaves +// `ledger` — the class that declares `revert()`, which is exactly the class whose +// refs core cannot know until the module speaks. +const PLACEHOLDER_CLASSES = ['ledger'] + +// A resource `kind` may be anything a module likes except core's own reserved +// one. Bounded to the column, and refused rather than truncated: a truncated ref +// is a cleanup call naming the wrong object. +const MAX_KIND = 64 +const MAX_REF = 190 +const MAX_MEMBER_KEY = 190 + +/** Does this action produce anything core will have to come back for? */ +const ledgers = (action) => action && (action.reversible === 'ledger' || action.reversible === 'override') + +/** + * Turn one entry of a module's `resources` array into a row, or say why not. + * + * Every failure here is the module's mistake rather than the world's, so none of + * them is a retry: a badly shaped resource will be just as badly shaped on the + * second attempt. They are logged and dropped, and the step still counts as done + * — because it IS done; something happened in the world, and refusing to record + * it would be the one outcome worse than recording it imperfectly. + */ +function normalise(entry, actionId) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + return { ok: false, reason: `${actionId} reported a resource that is not an object` } + } + const kind = String(entry.kind || '') + const ref = String(entry.ref === undefined || entry.ref === null ? '' : entry.ref) + if (!kind || kind.length > MAX_KIND) { + return { ok: false, reason: `${actionId} reported a resource with a bad kind "${entry.kind}"` } + } + if (kind === resourcesDb.STEP_KIND) { + // Core's own. A module that could write one would be a module that could make + // its own step's placeholder look resolved. + return { ok: false, reason: `${actionId} reported a resource of the reserved kind "${kind}"` } + } + if (!ref || ref.length > MAX_REF) { + return { ok: false, reason: `${actionId} reported a resource with a bad ref "${entry.ref}"` } + } + const memberKey = entry.memberKey === undefined || entry.memberKey === null ? null : String(entry.memberKey) + if (memberKey !== null && memberKey.length > MAX_MEMBER_KEY) { + return { ok: false, reason: `${actionId} reported a resource with an over-long memberKey` } + } + + let leaseUntil = null + if (entry.until !== undefined && entry.until !== null) { + const at = new Date(entry.until) + if (Number.isNaN(at.getTime())) { + return { ok: false, reason: `${actionId} reported a resource with a bad until "${entry.until}"` } + } + leaseUntil = at + } + + // A borrowed value must name a lease core knows how to give back. Core restores + // an `override` through the lease registry — that is the split §F draws — so a + // ref naming nothing registered is a resource core would be recording with no + // way to undo it, which is the promise rule 2 exists to stop core making. + if (kind === 'override' && !registries.eventLease(ref)) { + return { ok: false, reason: `${actionId} reported a lease "${ref}" no module registers` } + } + + return { + ok: true, + row: { + kind, + ref, + payload: entry.payload === undefined ? null : entry.payload, + leaseUntil, + memberKey, + }, + } +} + +/** + * Write the pre-dispatch placeholder for a step that is about to change the + * world. Answers the row id, or null when this action ledgers nothing. + * + * **A duplicate is not a failure.** A retry re-uses its step's idempotency key, so + * the second attempt's placeholder collides with the first's — and finding it + * already there is the correct answer, not an error. The existing row is reused. + */ +async function reserveStep(run, step, action) { + if (!ledgers(action) || !PLACEHOLDER_CLASSES.includes(action.reversible)) return null + + const owner = action.owner || 'core' + const reserved = await resourcesDb.reserve({ + runId: run.id, + stepId: step.id, + owner, + kind: resourcesDb.STEP_KIND, + ref: step.idempotency_key, + payload: { action: action.id, phase: step.phase, seq: step.seq }, + }) + if (reserved.ok) { + await markRunDirty(run.id) + return reserved.id + } + // The only way a '@step' row collides is with this step's own earlier attempt, + // because an idempotency key is minted once per step and never varies by + // attempt (§E). Reuse it. + const existing = await resourcesDb.findByTarget(owner, resourcesDb.STEP_KIND, step.idempotency_key) + return existing ? existing.id : null +} + +/** + * Record what a module said it made, and close out the placeholder. + * + * Answers `{ recorded, rejected }` — how many rows went in, and the reasons any + * entry was dropped. Never throws: a step that changed the world has changed it, + * and a ledger that threw would turn a bookkeeping problem into a failed step and + * then into a retry of a world write that already happened. + */ +async function recordAnswer({ run, step, action, placeholderId, resources }) { + const out = { recorded: 0, rejected: [] } + if (!ledgers(action)) return out + + const owner = action.owner || 'core' + const list = Array.isArray(resources) ? resources : [] + + for (const entry of list) { + const parsed = normalise(entry, action.id) + if (!parsed.ok) { + out.rejected.push(parsed.reason) + log.warn('event resource rejected', { run: run.id, step: step.id, reason: parsed.reason }) + continue + } + try { + const reserved = await resourcesDb.reserve({ + runId: run.id, + stepId: step.id, + owner, + kind: parsed.row.kind, + ref: parsed.row.ref, + payload: parsed.row.payload, + leaseUntil: parsed.row.leaseUntil, + memberKey: parsed.row.memberKey, + }) + if (!reserved.ok) { + // Already ledgered — by this step's own earlier attempt, or (a module bug + // rather than a race) by another run that still holds the same target. + // Either way there is a live row for it and a second would be the double + // cleanup the unique key exists to prevent. + if (reserved.holder && reserved.holder.run_id !== run.id) { + out.rejected.push(`${parsed.row.kind} "${parsed.row.ref}" is already held by run ${reserved.holder.run_id}`) + } + continue + } + await resourcesDb.confirm(reserved.id) + out.recorded += 1 + } catch (err) { + // Bookkeeping must not become the step's control flow. + out.rejected.push(err.message) + log.error('event resource insert failed', { run: run.id, step: step.id, message: err.message }) + } + } + + if (out.recorded > 0) await markRunDirty(run.id) + + // The placeholder's job is over the moment the real rows exist. It is resolved + // even when the module reported nothing at all — an action that ledgers and + // then answers `ok` with an empty list is saying "I made nothing", and holding + // its placeholder open would make cleanup call `revert()` for a step that has + // nothing to give back on every terminal path for ever. + if (placeholderId) await resourcesDb.resolvePlaceholder(placeholderId) + + return out +} + +/** + * There is now something to clean up. Idempotent and guarded, so it can never + * walk a run back from `complete` or `incomplete` to `pending` — only a human's + * cleanup does that, and it does it deliberately. + */ +async function markRunDirty(runId) { + await runsDb.setCleanupStatus(runId, 'pending', ['not_required']) +} + +module.exports = { ledgers, normalise, reserveStep, recordAnswer, markRunDirty, PLACEHOLDER_CLASSES } diff --git a/server/src/model/events/eventRunControls.model.js b/server/src/model/events/eventRunControls.model.js index d80a91a..a0ad08e 100644 --- a/server/src/model/events/eventRunControls.model.js +++ b/server/src/model/events/eventRunControls.model.js @@ -15,10 +15,13 @@ // diagnosis panel: a screen that explains why a phase has not started, beside a // control that does something about it. // -// **One of §I's controls is still not here.** `cleanup` needs Phase 8's resource -// ledger; there is nothing to revert, so cancel takes `{ reason }` and gains -// `cleanup` when there is something for it to do. Absent rather than inert, -// which is the posture Phase 1 set and every phase since has kept. +// **`cleanup` is the eighth, and Phase 8 is what gave it a ledger to work over.** +// It re-runs the teardown across every resource a run has not given back, and it +// is `admin` where the other seven are `admin` + `moderator`: it is not incident +// response, it is asking core to write to the world again. Its partner is +// cancel's new `cleanup: false`, which is §L's "cancelling WITHOUT cleanup is a +// separate, logged, admin-only action" — deliberately the flag that has to be +// asked for, because the safe default is to give back what the run took. // // **Every control is guarded on the status it may act from, and the guard is a // WHERE clause rather than a read-then-write.** A run console rendered thirty @@ -37,6 +40,7 @@ const runsDb = require('./eventRuns.db') const stepsDb = require('./eventRunSteps.db') const logDb = require('./eventRunLog.db') const gatesDb = require('./eventPhaseGates.db') +const resourcesDb = require('./eventRunResources.db') const gates = require('../../events/gates') const MAX_REASON = 500 @@ -152,11 +156,23 @@ async function resume(runId, options = {}, userId = null) { * and a second writer on that row would race the process that owns it. It * finishes into a cancelled run, which is honest. */ -async function cancel(runId, { reason } = {}, userId = null) { +async function cancel(runId, { reason, cleanup = true } = {}, userId = null, { isAdmin = true } = {}) { const run = await loadRun(runId) if (!run) return { ok: false, status: 404, errors: ['no such run'] } if (runsDb.TERMINAL.includes(run.status)) return conflict(`this run is already ${run.status}`) + // §L: cancelling WITHOUT cleanup is a separate, logged, ADMIN-only action. The + // route itself is `admin` + `moderator`, so the narrower gate cannot live in + // middleware — which of the two you have to be depends on what is in the body, + // exactly as the authoring role floor does (§K). + if (cleanup === false && !isAdmin) { + return { + ok: false, + status: 403, + errors: ['leaving a run\'s world changes in place is an administrator\'s decision'], + } + } + const note = clean(reason) const from = ['scheduled', 'starting', 'running', 'paused', 'ending'] if (!(await runsDb.transition(run.id, from, 'cancelled', { error: note || 'cancelled by staff' }))) { @@ -168,9 +184,83 @@ async function cancel(runId, { reason } = {}, userId = null) { runId: run.id, kind: 'run.status', phase: run.current_phase, - detail: { from: run.status, to: 'cancelled', control: 'cancel', by: userId, reason: note, cancelledSteps: closed }, + detail: { + from: run.status, + to: 'cancelled', + control: 'cancel', + by: userId, + reason: note, + cancelledSteps: closed, + cleanup: cleanup !== false, + }, }) - return { ok: true, run: await runsDb.getById(run.id), cancelledSteps: closed } + + // **The teardown is not done here, and the request does not wait for it.** + // Cleanup is one leg of the runner's tick over terminal runs (§L), which is + // what makes it survive a process that dies halfway through it — and a cancel + // pressed at two in the morning must answer at once rather than after a dozen + // round trips to a shard that may be the reason it is being cancelled. The run + // is terminal the moment this returns, so the very next tick picks its ledger + // up. + // + // `cleanup: false` is the operator saying leave it. The resources stay + // unresolved and the run carries `incomplete`, which is the truthful value: the + // world changes are still up, they are listed on the console, and the log line + // above records who decided that. + let cleanupStatus = run.cleanup_status + if (cleanup === false && (await resourcesDb.unresolvedCount(run.id)) > 0) { + // `incomplete` is what takes the run out of the cleanup leg's scan, and it is + // the truthful value: the world changes are still up, they are listed on the + // console, and the log line above records who decided that. + // + // **The first draft spent every row's `revert_attempts` instead**, to stop the + // sweep by the same mechanism a failed retry does. It worked and it made the + // console lie: the run page rendered "3 attempts" beside resources nothing had + // ever tried, which reads as "core tried three times and could not". Found by + // opening the page. A counter that means two things is a counter a screen + // cannot render. + await runsDb.setCleanupStatus(run.id, 'incomplete') + cleanupStatus = 'incomplete' + } + + return { + ok: true, + run: await runsDb.getById(run.id), + cancelledSteps: closed, + cleanup: cleanup !== false, + cleanupStatus, + } +} + +/** + * Re-run cleanup over everything a run has not given back. + * + * The manual retry §L promises, and the only thing that clears + * `revert_attempts`. That licence is the same one a human's step retry has, and + * it is deliberately not extended to the automatic sweep: Engagement Phase 14's + * defect was exactly a sweep that reset every stale row, which made the attempt + * ceiling unreachable and left the row cycling for ever. + * + * Legal on a TERMINAL run only. A run still in flight has a ledger that is still + * growing, and reverting a resource the next step is about to use would be core + * undoing an event while it is happening. + */ +async function cleanupRun(runId, userId = null) { + const run = await loadRun(runId) + if (!run) return { ok: false, status: 404, errors: ['no such run'] } + if (!runsDb.TERMINAL.includes(run.status)) { + return conflict(`this run is still ${run.status}; cancel it before cleaning up after it`) + } + if (run.cleanup_status === 'not_required') { + return conflict('this run recorded no resources, so there is nothing to give back') + } + + // eslint-disable-next-line global-require + const summary = await require('../../events/cleanup').cleanupRun(run, { + resetAttempts: true, + actor: userId, + }) + return { ok: true, run: await runsDb.getById(run.id), summary } } /** @@ -359,4 +449,4 @@ async function retryStep(runId, stepId, options = {}, userId = null) { } } -module.exports = { pause, resume, cancel, advancePhase, confirmStep, skipStep, retryStep } +module.exports = { pause, resume, cancel, cleanupRun, advancePhase, confirmStep, skipStep, retryStep } diff --git a/server/src/model/events/eventRunLog.db.js b/server/src/model/events/eventRunLog.db.js index 4cb291e..6abb204 100644 --- a/server/src/model/events/eventRunLog.db.js +++ b/server/src/model/events/eventRunLog.db.js @@ -47,6 +47,16 @@ const KINDS = [ 'run.budget', // the caps this run was seeded with, and which switch set each 'step.refused', // a step was not permitted: disabled, or over a cap 'version.verified', // a dry run passed against a version, unlocking scheduled starts + // Phase 8's six, and every one of them is an answer to "what did this event + // leave behind". `resource.recorded` is written at the ANSWER rather than at + // the placeholder, because a placeholder is a promise and the operator's + // question is about the world. + 'resource.recorded', // a step reported what it created or borrowed, and it is ledgered + 'resource.orphaned', // a module reports a ledgered resource is no longer in force + 'cleanup.reverted', // a group of resources came back + 'cleanup.failed', // a group did not, with the reason and how it was left + 'cleanup.swept', // one pass over a run's ledger, and what it found + 'cleanup.retry', // a human cleared the attempt counter and asked again ] const hydrate = (row) => row && { ...row, detail: parseJson(row.detail, null) } diff --git a/server/src/model/events/eventRunResources.db.js b/server/src/model/events/eventRunResources.db.js new file mode 100644 index 0000000..9f11af4 --- /dev/null +++ b/server/src/model/events/eventRunResources.db.js @@ -0,0 +1,353 @@ +// ── event_run_resources — SQL only ───────────────────────────────────────── +// +// EVENTS.md §D and §L ("The ledger's two rules"), and Phase 8 of EVENTS_PLAN.md. +// Everything one run created or leased, and what became of it. +// +// **Rule 1 lives in `reserve()`.** A resource is recorded BEFORE it is +// confirmed, so the placeholder this writes is the row that exists while the +// dispatch is in flight — and the row that SURVIVES when the acknowledgement is +// lost. Recording on the answer instead would make every object whose ack went +// missing invisible to cleanup for ever. +// +// **Rule 2 lives in the status column and in `failRevert()`.** A revert that +// never succeeds leaves its row unreverted, with the error on it, and the run +// completes with `cleanup_status = 'incomplete'` rather than being held open. +// Loud and sticky. +// +// **The unique key is enforced by the database, not by a read.** `reserve()` +// answers `{ ok: false, code: 'held' }` on a duplicate key rather than checking +// first and then inserting — two runs entering the same tick would both pass the +// check. It is the argument `event_run_budget.spend()` makes about the cap and +// `runsDb.transition` makes about a status, in the third place it applies. + +const { query } = require('../../utils/db') +const { parseJson } = require('./eventJson') + +// The one `kind` core owns. A module's kinds are opaque and stored verbatim; this +// one is core's own, and `registries` refuses a module resource that claims it. +const STEP_KIND = '@step' + +// The statuses that mean "core still believes this resource is this run's". They +// are exactly the ones the `live_marker` generated column keeps non-NULL, so the +// unique target key holds while a row is in one of them and releases when it +// leaves. Duplicated here as a JavaScript list because the sweeps read by it too, +// and a second copy that can drift is better than a query that cannot express it. +const HELD = ['pending', 'confirmed', 'reverting'] + +// Every status that still wants a human or a retry: `HELD` plus the two that mean +// "we let go, and not cleanly". This is what "unreverted" means everywhere in +// this feature — the console's list, `cleanup_status`, and the manual retry. +const UNRESOLVED = [...HELD, 'orphaned', 'drifted'] + +const COLUMNS = `id, run_id, step_id, owner_module, kind, ref, payload, lease_until, + status, revert_attempts, last_error, member_key, created_at, updated_at` + +// `payload` is opaque to core and stored verbatim, but it comes back as a string +// from the driver and every caller wants the object — the cleanup sweep reads a +// lease's baseline out of it, and the console renders it. Hydrated here for the +// same reason a step's params are: one place rather than at each read. +const hydrate = (row) => row && { ...row, payload: parseJson(row.payload, null) } + +/** + * Record a resource that does not exist yet. + * + * Answers `{ ok: true, id }`, or `{ ok: false, code: 'held', holder }` when the + * target is already someone's — which is the lease conflict, surfaced as a + * refusal rather than a failure because nothing is wrong with the system: another + * run has the thing. + * + * **`ER_DUP_ENTRY` is the check.** The holder is looked up only to name it in the + * refusal, and only after the insert has already lost the race. + */ +async function reserve({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) { + try { + const result = await query( + `INSERT INTO event_run_resources + (run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')`, + [runId, stepId, owner, kind, ref, payload === null ? null : JSON.stringify(payload), leaseUntil, memberKey], + ) + return { ok: true, id: Number(result.insertId) } + } catch (err) { + if (err && (err.code === 'ER_DUP_ENTRY' || err.errno === 1062)) { + const [holder] = await query( + `SELECT run_id, status FROM event_run_resources + WHERE owner_module = ? AND kind = ? AND ref = ? AND status IN (?, ?, ?) + LIMIT 1`, + [owner, kind, ref, ...HELD], + ) + return { ok: false, code: 'held', holder: holder || null } + } + throw err + } +} + +/** + * Promote a reserved row to `confirmed`, optionally attaching what the module + * finally said about it. + * + * Guarded on `pending` so a late answer cannot un-revert a row cleanup has + * already dealt with — the same reason every other write in this feature is a + * compare-and-set rather than a read-then-write. + */ +async function confirm(id, { payload, leaseUntil, memberKey } = {}) { + const sets = ["status = 'confirmed'"] + const params = [] + if (payload !== undefined) { + sets.push('payload = ?') + params.push(payload === null ? null : JSON.stringify(payload)) + } + if (leaseUntil !== undefined) { + sets.push('lease_until = ?') + params.push(leaseUntil) + } + if (memberKey !== undefined) { + sets.push('member_key = ?') + params.push(memberKey) + } + const result = await query( + `UPDATE event_run_resources SET ${sets.join(', ')} WHERE id = ? AND status = 'pending'`, + [...params, id], + ) + return (result.affectedRows || 0) > 0 +} + +/** + * Resolve a step placeholder once the module has named what it actually made. + * + * The placeholder's whole job is over at this point: the real rows exist, so the + * `@step` row must stop being one of the things cleanup will try to revert. + * `reverted` is the honest terminal state for it — there is nothing left to undo + * that the rows it stood in for do not now cover — and it releases the + * idempotency key for a later run, which matters because keys are per step and a + * re-materialised step reuses its own. + */ +async function resolvePlaceholder(id) { + const result = await query( + `UPDATE event_run_resources + SET status = 'reverted', last_error = NULL + WHERE id = ? AND kind = ? AND status IN ('pending', 'confirmed')`, + [id, STEP_KIND], + ) + return (result.affectedRows || 0) > 0 +} + +/** + * One row by its target, live or not — how a caller that lost the insert race + * finds the row it meant to write. Newest first, so a target that has been held + * and released several times answers with the current holder. + */ +async function findByTarget(owner, kind, ref) { + const [row] = await query( + `SELECT ${COLUMNS} FROM event_run_resources + WHERE owner_module = ? AND kind = ? AND ref = ? + ORDER BY id DESC LIMIT 1`, + [owner, kind, ref], + ) + return hydrate(row) || null +} + +/** One run's whole ledger, oldest first — the console's read. */ +async function forRun(runId) { + const rows = await query( + `SELECT ${COLUMNS} FROM event_run_resources WHERE run_id = ? ORDER BY id`, + [runId], + ) + return rows.map(hydrate) +} + +/** The rows of one run that still want something: the cleanup sweep's input. */ +async function unresolvedForRun(runId, { maxAttempts = null } = {}) { + const params = [runId, ...UNRESOLVED] + const attemptClause = maxAttempts === null ? '' : ' AND revert_attempts < ?' + if (maxAttempts !== null) params.push(maxAttempts) + const rows = await query( + `SELECT ${COLUMNS} FROM event_run_resources + WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)${attemptClause} + ORDER BY id`, + params, + ) + return rows.map(hydrate) +} + +/** How many of one run's rows are still unresolved — what `cleanup_status` is derived from. */ +async function unresolvedCount(runId) { + const [row] = await query( + `SELECT COUNT(*) AS n FROM event_run_resources + WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`, + [runId, ...UNRESOLVED], + ) + return Number(row?.n || 0) +} + +/** Unresolved counts for several runs at once, keyed by run id — the run LIST's read. */ +async function unresolvedCounts(runIds) { + const ids = [...new Set(runIds || [])].filter(Boolean) + if (!ids.length) return new Map() + const rows = await query( + `SELECT run_id, COUNT(*) AS n FROM event_run_resources + WHERE run_id IN (${ids.map(() => '?').join(',')}) AND status IN (?, ?, ?, ?, ?) + GROUP BY run_id`, + [...ids, ...UNRESOLVED], + ) + return new Map(rows.map((r) => [r.run_id, Number(r.n)])) +} + +/** + * Claim one row for a revert: `pending | confirmed | orphaned | drifted → reverting`. + * + * The compare-and-set that keeps the cleanup leg and the manual cleanup route off + * each other's rows. `reverting` is deliberately not claimable — a row another + * pass is mid-revert on is left alone, exactly as a step with a live claim is. + */ +async function claimRevert(id) { + const result = await query( + `UPDATE event_run_resources + SET status = 'reverting' + WHERE id = ? AND status IN ('pending', 'confirmed', 'orphaned', 'drifted')`, + [id], + ) + return (result.affectedRows || 0) > 0 +} + +/** The revert worked. `reverted` is terminal and releases the target. */ +async function markReverted(id) { + await query( + `UPDATE event_run_resources SET status = 'reverted', last_error = NULL WHERE id = ?`, + [id], + ) +} + +/** + * The revert did not work, and the row goes back to being unresolved. + * + * `revert_attempts` is incremented here and NOWHERE else, and it is never reset by + * a sweep — Engagement Phase 14's rule, whose defect was a reclaim that returned + * every stale row to its start state and made the attempt ceiling unreachable, so + * the row cycled for ever and was never eligible for any retention sweep. The one + * thing that may reset it is a human pressing cleanup, which is the same licence + * a human's step retry has. + * + * `restoreTo` is where the row lands: `drifted` when the module says somebody else + * moved the value, `orphaned` when it says the thing is gone, and `confirmed` + * otherwise — still ours, still out there, try again. + */ +async function failRevert(id, error, restoreTo = 'confirmed') { + await query( + `UPDATE event_run_resources + SET status = ?, revert_attempts = revert_attempts + 1, last_error = ? + WHERE id = ?`, + [restoreTo, String(error || 'the revert did not answer').slice(0, 500), id], + ) +} + +/** + * A human is trying again: clear the attempt counter on one run's unresolved rows. + * + * Only ever called from the cleanup route with an actor behind it. The automatic + * leg must never do this (see `failRevert`). + */ +async function resetAttempts(runId) { + const result = await query( + `UPDATE event_run_resources + SET revert_attempts = 0 + WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`, + [runId, ...UNRESOLVED], + ) + return result.affectedRows || 0 +} + +/** Every live row one module owns, for the reconcile sweep. */ +async function liveForModule(owner, { limit = 500 } = {}) { + const rows = await query( + `SELECT ${COLUMNS} FROM event_run_resources + WHERE owner_module = ? AND status IN ('pending', 'confirmed') + ORDER BY id LIMIT ?`, + [owner, Number(limit)], + ) + return rows.map(hydrate) +} + +/** Every module that currently owns a live row — who the reconcile sweep asks. */ +async function modulesWithLiveRows() { + const rows = await query( + `SELECT DISTINCT owner_module FROM event_run_resources + WHERE status IN ('pending', 'confirmed')`, + ) + return rows.map((r) => r.owner_module) +} + +/** + * The game no longer has it. Never reached by a revert — a revert that finds + * nothing there is a SUCCESS (§L, and it is what a Rust wipe needs) — only by + * reconcile, which is a different question: nobody asked for this to go. + */ +async function markOrphaned(id, detail = null) { + await query( + `UPDATE event_run_resources + SET status = 'orphaned', last_error = ? + WHERE id = ? AND status IN ('pending', 'confirmed', 'reverting')`, + [detail === null ? null : String(detail).slice(0, 500), id], + ) +} + +/** + * Terminal runs that still owe the world something — the cleanup leg's scan. + * + * **Both halves of the WHERE were live-walk findings, and they are opposite + * mistakes.** + * + * `cleanup_status = 'pending'` alone missed a run whose only resource was a + * LEASE: `core.lease` reserves its own row and never goes through the ledger's + * `markRunDirty`, so the flag stayed `not_required` and the lease was never given + * back at all. Hence `not_required` is in the list — a terminal run with an + * unresolved row has something to do whatever any summary column says, and + * treating that combination as work is the fail-safe direction. + * + * And the run status filter alone made `MAX_REVERT_ATTEMPTS` mean ONE attempt, + * because the first failing sweep set `incomplete` and nothing looked at the run + * again. That is fixed in `cleanupRun`, which now only writes `incomplete` once + * there is nothing left it will try — so `incomplete` genuinely means "finished + * with, and not finished", which is exactly what excludes both a run whose + * retries are spent and a run an admin cancelled without cleanup. + * + * The attempt bound is in the join for a different reason: without it a run whose + * rows are all spent would be selected, worked over and found to have nothing to + * do on every tick for the rest of its life. + */ +async function runsNeedingCleanup(limit = 25, maxAttempts = 3) { + return query( + `SELECT DISTINCT r.id, r.status, r.cleanup_status, r.version_id, r.definition_id, r.scope + FROM event_runs r + JOIN event_run_resources res ON res.run_id = r.id + WHERE r.status IN ('completed', 'cancelled', 'failed', 'missed') + AND r.cleanup_status IN ('pending', 'not_required') + AND res.status IN (?, ?, ?, ?, ?) + AND res.revert_attempts < ? + ORDER BY r.id + LIMIT ?`, + [...UNRESOLVED, Number(maxAttempts), Number(limit)], + ) +} + +module.exports = { + STEP_KIND, + HELD, + UNRESOLVED, + reserve, + confirm, + resolvePlaceholder, + findByTarget, + forRun, + unresolvedForRun, + unresolvedCount, + unresolvedCounts, + claimRevert, + markReverted, + failRevert, + resetAttempts, + liveForModule, + modulesWithLiveRows, + markOrphaned, + runsNeedingCleanup, +} diff --git a/server/src/model/events/eventRuns.db.js b/server/src/model/events/eventRuns.db.js index 5f09f13..5b224ff 100644 --- a/server/src/model/events/eventRuns.db.js +++ b/server/src/model/events/eventRuns.db.js @@ -386,6 +386,32 @@ async function setHealth(id, health) { return Number(result?.affectedRows || 0) === 1 } +/** + * Set `cleanup_status`, optionally guarded on where it is now (Phase 8). + * + * Four values and three writers, which is why the guard is a parameter rather + * than baked in. The ledger stamps `pending` the first time a run records + * anything, and it must do so only over `not_required` — a run already marked + * `complete` must not be walked back to `pending` by a late resource, and a + * `incomplete` one must not be silently tidied. The cleanup sweep sets `complete` + * or `incomplete` from what it found, unguarded, because the sweep IS the + * authority on that. A human's cleanup route re-opens `pending` deliberately, and + * says so in the log with the actor. + * + * **`pending` on a run that is still running is not a bug and reads correctly**: + * there is something to clean up and it has not happened yet. The alternative - + * a fifth value meaning "there will be something later" - is a state nothing + * would ever branch on. + */ +async function setCleanupStatus(id, to, from = null) { + const guard = from === null ? '' : ` AND cleanup_status IN (${from.map(() => '?').join(',')})` + const result = await query( + `UPDATE event_runs SET cleanup_status = ? WHERE id = ?${guard}`, + from === null ? [to, id] : [to, id, ...from], + ) + return Number(result?.affectedRows || 0) === 1 +} + /** * Runs whose start instant passed more than their own grace window ago (§E, §L). * @@ -488,6 +514,7 @@ module.exports = { statusOf, transition, setHealth, + setCleanupStatus, concurrencyHolder, reclaimStale, terminalBefore, diff --git a/server/src/model/events/eventRuns.model.js b/server/src/model/events/eventRuns.model.js index 5a41f57..dd00149 100644 --- a/server/src/model/events/eventRuns.model.js +++ b/server/src/model/events/eventRuns.model.js @@ -27,6 +27,7 @@ const definitionsDb = require('./eventDefinitions.db') const versionsDb = require('./eventVersions.db') const settingsDb = require('./eventActionSettings.db') const budgetDb = require('./eventRunBudget.db') +const resourcesDb = require('./eventRunResources.db') const authorize = require('../../events/authorize') const MAX_SCOPE = 190 @@ -204,11 +205,12 @@ async function create( async function detail(runId) { const run = await db.getById(runId) if (!run) return null - const [steps, counts, gateRows, budget] = await Promise.all([ + const [steps, counts, gateRows, budget, resources] = await Promise.all([ stepsDb.listForRun(runId), stepsDb.statusCounts(runId), gatesDb.listForRun(runId), budgetDb.forRun(runId), + resourcesDb.forRun(runId), ]) const now = new Date() return { @@ -226,6 +228,37 @@ async function detail(runId) { cap: b.cap, from: b.effective_from, })), + // What this run changed in the world, and what became of it (Phase 8). The + // WHOLE ledger, reverted rows included, because "what did last night's + // invasion actually spawn, and did all of it come back" is the question this + // panel exists for and a list of only the failures cannot answer the second + // half of it. + // + // **The `@step` placeholders are filtered out.** They are core's own + // bookkeeping — a row that says "a dispatch is in flight and may have made + // something" — and the console's list is of things in the world. One left in + // would read as a resource nobody can name, which is exactly the confusion it + // exists to prevent internally. + resources: resources + .filter((r) => r.kind !== resourcesDb.STEP_KIND) + .map((r) => ({ + id: r.id, + stepId: r.step_id, + module: r.owner_module, + kind: r.kind, + ref: r.ref, + payload: r.payload, + leaseUntil: r.lease_until, + status: r.status, + revertAttempts: r.revert_attempts, + lastError: r.last_error, + memberKey: r.member_key, + createdAt: r.created_at, + })), + // How many rows are still unresolved, counted over the WHOLE ledger rather + // than over the list above — a placeholder left standing by a lost + // acknowledgement is exactly the case `cleanup_status` must not call clean. + unresolvedResources: resources.filter((r) => resourcesDb.UNRESOLVED.includes(r.status)).length, } } diff --git a/server/src/modules/loader.js b/server/src/modules/loader.js index df0024d..7503186 100644 --- a/server/src/modules/loader.js +++ b/server/src/modules/loader.js @@ -228,6 +228,39 @@ function buildCtx(id, moduleRoot) { emit: (triggerId, envelope) => { engagementEmit.emit(id, triggerId, envelope) }, + // EVENTS.md §L, and the resource ledger (Phase 8). "On reconnect the runner + // asks each ledgered resource's module to reconcile" — and this is how the + // runner learns there has BEEN a reconnect. + // + // **Core cannot decide when to call this, and that is the contract rather + // than a gap.** §F: core has no concept of the game being up, because a + // module with six sidecars cannot answer that question in the singular. So + // the module says so, when it sees its own — module-uo already watches + // `bootId` to tell a shard restart from a sidecar reconnect, which is + // exactly the moment a ledger of live spawns has become a claim about a + // world that no longer exists. + // + // `id` is bound here and never taken from the arguments, like `emit` and + // `teams.activity.push` before it: a module reconciles its OWN ledger, and + // without the binding this would be a way to have core mark another + // module's resources orphaned. + // + // Fire-and-forget and returns undefined, for the third time and the same + // reason: this is called from inside a connection handler, and there is + // nothing a module could correctly do with a failure of core's bookkeeping. + reconcile: () => { + // eslint-disable-next-line global-require + require('../events/cleanup') + .reconcileModule(id) + .then( + (summary) => { + if (summary && summary.orphaned) { + log.warn('event resources orphaned on reconcile', { module: id, ...summary }) + } + }, + (err) => { log.error('ctx.events.reconcile failed', { module: id, message: err.message }) }, + ) + }, }, // The in-app sink (§5.1) — a module writing the inbox directly, without a // rule. Live from Phase 7; it threw until the `user_notifications` table diff --git a/server/src/modules/registries.js b/server/src/modules/registries.js index 2823c52..88d52e8 100644 --- a/server/src/modules/registries.js +++ b/server/src/modules/registries.js @@ -157,11 +157,11 @@ const eventBudgets = new Map() // lease id → { owner, id, label, type, min, max, maxDurationMs, description, // read, apply, restore } (§F "Leases: one more declaration", Phase 7). // -// **Phase 7 registers a lease and nothing acquires one.** Core owns the duration -// and the conflict check, the module owns reading the current value and writing a -// new one — and both halves of that live in the resource ledger, which is Phase -// 8's. What is here is the declaration, its validation and its catalog entry, so -// that the module contract is one version rather than two. +// **Core owns the duration and the conflict check; the module owns reading the +// current value and writing a new one.** Phase 7 registered a lease and nothing +// acquired one; Phase 8 gave it a verb — `core.lease`, a CORE action, so the +// bound and the two-events-one-target refusal are enforced in one place rather +// than re-implemented by every module that ships a lease. const eventLeases = new Map() // source id → { owner, id, label, description, resolve } (§F "Param option @@ -435,14 +435,14 @@ async function resolveAudience(id, params = {}) { /** * Every declaration WITHOUT its callables — what the admin catalog serves. * - * `perform`, `revert` and `cost` are stripped for the same reason `resolve` is + * `perform`, `revert`, `reconcile` and `cost` are stripped for the same reason `resolve` is * stripped from an audience and `handler` from a slash command: this is the * object that leaves the process, and the browser's whole relationship with an * action is naming one by id. §F's "a module registers actions server-side and * adds no routes for them" is only true if the functions never ride out. */ const allEventActions = () => - [...eventActions.values()].map(({ perform, revert, cost, ...rest }) => rest) + [...eventActions.values()].map(({ perform, revert, reconcile, cost, ...rest }) => rest) /** One declaration, callables included. The runner's lookup (Phase 2). */ const eventAction = (id) => eventActions.get(id) || null @@ -485,7 +485,7 @@ const isEventBudget = (id) => eventBudgets.has(id) const allEventLeases = () => [...eventLeases.values()].map(({ read, apply: applyValue, restore, ...rest }) => rest) -/** One lease, callables included. Phase 8's lookup; nothing calls it yet. */ +/** One lease, callables included. `core.lease` and the cleanup sweep read it. */ const eventLease = (id) => eventLeases.get(id) || null /** Every option source WITHOUT its resolver — the authoring form's list. */ @@ -985,7 +985,7 @@ function checkActionParam(actionId, entry, seen) { } /** - * `registerEventActions([{ id, label, risk, reversible, version, budgetMs, cost, params, perform, revert }])`. + * `registerEventActions([{ id, label, risk, reversible, version, budgetMs, cost, params, perform, revert, reconcile }])`. * * A typed verb core may ask a registrant to carry out. Everything decidable from * the argument alone is decided here, at the call; the collision — is this id @@ -1042,6 +1042,23 @@ function checkEventActionShape(entry) { `registerEventActions: ${a.id} declares revert() but is reversible: '${a.reversible}'`, ) } + // §L's reconnect row, and it is OPTIONAL where `revert` is required (Phase 8). + // `revert` is how a run gives a resource back; `reconcile` is how a module says + // which of them the game still has after something outside core restarted. A + // module that cannot answer that question is not broken -- core simply keeps + // believing its own ledger, which is the pre-Phase-8 behaviour -- whereas a + // module that created something and cannot undo it has made a promise core has + // no way to keep. Only meaningful for an action that ledgers anything. + if (a.reconcile !== undefined) { + if (typeof a.reconcile !== 'function') { + throw new Error(`registerEventActions: ${a.id} reconcile must be a function`) + } + if (a.reversible === 'none' || a.reversible === 'self') { + throw new Error( + `registerEventActions: ${a.id} declares reconcile() but is reversible: '${a.reversible}' and ledgers nothing`, + ) + } + } if (a.cost !== undefined && typeof a.cost !== 'function') { throw new Error(`registerEventActions: ${a.id} cost must be a function of its params`) } @@ -1076,6 +1093,7 @@ function checkEventActionShape(entry) { cost: a.cost || null, perform: a.perform, revert: a.revert || null, + reconcile: a.reconcile || null, } } @@ -1130,10 +1148,12 @@ function checkEventBudgetShape(entry) { * thing a module must not be allowed to skip. A lease whose restore writes blindly * is a lease that silently reverts an operator's manual fix. * - * **Nothing acquires a lease in Phase 7.** This registers, validates and serves - * one; the ledger that holds it, the deadline that goes down the wire and the - * drift answer are Phase 8's. Declaring it now is what keeps the module contract - * one version rather than two. + * **A lease is acquired by `core.lease` and by nothing else** (Phase 8). The step + * names a lease id, a value and a duration; core reads the baseline, reserves the + * target in `event_run_resources` — which is where the two-events-one-target + * refusal comes from — applies the value with the deadline, and restores it at + * teardown through the same `restore()` the drift check lives in. A module ships + * the three callables and never has to own any of that. */ function checkEventLeaseShape(entry) { const l = entry || {} diff --git a/server/src/router/v1/admin/events.controller.js b/server/src/router/v1/admin/events.controller.js index 009a46b..6442dce 100644 --- a/server/src/router/v1/admin/events.controller.js +++ b/server/src/router/v1/admin/events.controller.js @@ -9,13 +9,12 @@ // this screen does. // // **Phase 3 added the live run controls** at the bottom of this file: pause, -// resume, cancel, and a step's confirm, skip and retry. What is still absent is -// `advance`, `cleanup` and the action switchboard — `advance` has no honest -// meaning until Phase 5 gives a phase an advance condition, `cleanup` has no -// ledger to work over until Phase 8, and the switchboard is Phase 6's. Each of -// them is absent rather than stubbed, for the reason the whole set was in Phase -// 1: a control that returns 200 and does nothing is worse than one that is not -// there. +// resume, cancel, and a step's confirm, skip and retry. `advance` joined them in +// Phase 5, the action switchboard in Phase 6, and **`cleanup` in Phase 8** — +// each when the phase that gave it something to act on landed, and each absent +// rather than stubbed until then, for the reason the whole set was in Phase 1: a +// control that returns 200 and does nothing is worse than one that is not there. +// Nothing in the § API surface table is absent any more. const registries = require('../../../modules/registries') const spec = require('../../../events/spec') @@ -315,6 +314,12 @@ exports.getRun = async (req, res) => { // rather than "what is allowed now" — which is the question that survives // an admin moving a switch tomorrow. budget: found.budget, + // The resource ledger (Phase 8): everything this run created or borrowed, and + // what became of each. The WHOLE ledger, reverted rows included — "how much + // did last night's invasion spawn, and did all of it come back" is one + // question with two halves, and a list of only the failures answers neither. + resources: found.resources, + unresolvedResources: found.unresolvedResources, }) } @@ -678,14 +683,48 @@ exports.advanceRunPhase = async (req, res) => { exports.cancelRun = async (req, res) => { const runId = asId(req.params.runId) if (!runId) return res.status(400).json({ error: 'bad run id' }) - const result = await controls.cancel(runId, { reason: req.body?.reason }, req.user.id) + // **`cleanup` defaults to true and has to be asked out of.** §L makes cancelling + // WITHOUT cleanup the separate, admin-only, logged action, so an absent flag + // must mean "give back what this run took" — the safe direction, and the one a + // moderator's cancel at two in the morning takes without having to know the + // flag exists. + const withCleanup = req.body?.cleanup !== false + const result = await controls.cancel( + runId, + { reason: req.body?.reason, cleanup: withCleanup }, + req.user.id, + { isAdmin: req.user.role === 'admin' }, + ) if (!result.ok) return res.status(result.status || 409).json({ errors: result.errors }) await activity.log({ req, action: 'event.run.cancelled', - detail: { runId, reason: req.body?.reason || null, cancelledSteps: result.cancelledSteps }, + detail: { + runId, + reason: req.body?.reason || null, + cancelledSteps: result.cancelledSteps, + cleanup: result.cleanup, + }, }) - return res.json({ run: shapeRun(result.run), cancelledSteps: result.cancelledSteps }) + return res.json({ + run: shapeRun(result.run), + cancelledSteps: result.cancelledSteps, + cleanup: result.cleanup, + }) +} + +/** POST /api/v1/admin/events/runs/:runId/cleanup */ +exports.cleanupRun = async (req, res) => { + const runId = asId(req.params.runId) + if (!runId) return res.status(400).json({ error: 'bad run id' }) + const result = await controls.cleanupRun(runId, req.user.id) + if (!result.ok) return res.status(result.status || 409).json({ errors: result.errors }) + await activity.log({ req, action: 'event.run.cleaned', detail: { runId, ...result.summary } }) + // **A 200 whatever the sweep found.** The request succeeded; some resources may + // still be out there, and answering 4xx would make "the shard refused to delete + // three of these" indistinguishable from "you sent a bad run id" — the same + // argument the dry run's findings make. + return res.json({ run: shapeRun(result.run), summary: result.summary }) } /** POST /api/v1/admin/events/runs/:runId/steps/:stepId/confirm */ diff --git a/server/src/router/v1/admin/events.router.js b/server/src/router/v1/admin/events.router.js index f75fbc9..034f88b 100644 --- a/server/src/router/v1/admin/events.router.js +++ b/server/src/router/v1/admin/events.router.js @@ -17,8 +17,10 @@ // 5; **`verify` and the action switchboard arrived in Phase 6** — `verify` at // `admin, editor` because a dry run dispatches nothing, and both halves of // `/actions` at `admin`, because §K puts the switchboard in the same row as the -// world-changing actions it governs. `cleanup` is still absent rather than -// stubbed: there is no resource ledger until Phase 8. +// world-changing actions it governs. **`cleanup` completed the set in Phase 8**, +// and it is `admin` rather than admin+moderator for the same §K reason: it asks +// core to write to the world again, which is not incident response. There is no +// route in the § API surface table left absent. // // **Literal paths are declared before `/:id`**, so `/catalog`, `/series`, // `/calendar` and `/runs` are never read as an event id. @@ -207,9 +209,9 @@ eventsRouter.get( '/runs/:runId', // #swagger.tags = ['Admin · Events'] // #swagger.summary = 'One run: its status, health, cleanup state and every step with its params and idempotency key' - // #swagger.description = 'The run console. `counts` summarises the step list by status. Steps carry the idempotency key core minted at materialisation — stable across every attempt, which is what lets the game side recognise a repeat. `gates` is the diagnosis panel (Phase 5): one entry per phase that authored an advance condition, already rendered in the condition builder's own words — `gte` as "is at least", `present` as "is present" — with the tally, how long it has waited, and the last related firing whether or not it matched. A phase is waiting on its gate only once every one of its steps is terminal; `stalled` means an `on` gate has waited past EVENT_PHASE_STALL_MS, which is visibility and never a timeout — nothing advances a phase but its condition or a human.' + // #swagger.description = 'The run console. `counts` summarises the step list by status. Steps carry the idempotency key core minted at materialisation — stable across every attempt, which is what lets the game side recognise a repeat. `gates` is the diagnosis panel (Phase 5): one entry per phase that authored an advance condition, already rendered in the condition builder's own words — `gte` as "is at least", `present` as "is present" — with the tally, how long it has waited, and the last related firing whether or not it matched. A phase is waiting on its gate only once every one of its steps is terminal; `stalled` means an `on` gate has waited past EVENT_PHASE_STALL_MS, which is visibility and never a timeout — nothing advances a phase but its condition or a human. `budget` is the cap meter (Phase 6), and `resources` is the cleanup ledger (Phase 8): every object this run created and every value it borrowed, with what became of each — `confirmed` is still out there, `reverted` came back, `drifted` means somebody moved it and core left it alone, and `orphaned` means the module reports it is gone. `unresolvedResources` counts the ones still wanting something, including a placeholder left standing by a lost acknowledgement, which is why it can exceed the length of the list.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] - /* #swagger.responses[200] = { description: 'The run, its steps, the status counts and the phase gates', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, steps: { type: "array", items: { type: "object", additionalProperties: true } }, counts: { type: "object", additionalProperties: true }, gates: { type: "array", items: { type: "object", additionalProperties: true } } } } } } } */ + /* #swagger.responses[200] = { description: 'The run, its steps, the status counts, the phase gates, the cap meter and the resource ledger', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, steps: { type: "array", items: { type: "object", additionalProperties: true } }, counts: { type: "object", additionalProperties: true }, gates: { type: "array", items: { type: "object", additionalProperties: true } }, budget: { type: "array", items: { type: "object", additionalProperties: true } }, resources: { type: "array", items: { type: "object", additionalProperties: true } }, unresolvedResources: { type: "integer" } } } } } } */ /* #swagger.responses[404] = { description: 'No such run', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ controller.getRun, ) @@ -273,16 +275,29 @@ eventsRouter.post( '/runs/:runId/cancel', // #swagger.tags = ['Admin · Events'] // #swagger.summary = 'Cancel a run' - // #swagger.description = 'Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` is not a parameter yet — the resource ledger it would work over arrives in Phase 8, and a flag that changes nothing is worse than one that is not there.' + // #swagger.description = 'Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` arrived in Phase 8 and DEFAULTS TO TRUE: what the run created or borrowed is given back by the runner cleanup leg on its next tick, which is why this answers at once rather than after a round trip per resource. Sending `cleanup: false` deliberately leaves the world changes in place — that is admin-only even though the route is admin+moderator, because which of the two you have to be depends on what is in the body — and the run then carries `cleanup_status: incomplete` with every unreverted row listed on its console.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] - /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { reason: { type: "string", description: "Why. Recorded on the run and in its log, with the actor." } } } } } } */ - /* #swagger.responses[200] = { description: 'The cancelled run and how many steps were closed out with it', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, cancelledSteps: { type: "integer" } } } } } } */ + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { reason: { type: "string", description: "Why. Recorded on the run and in its log, with the actor." }, cleanup: { type: "boolean", description: "Default true. False leaves the world changes from this run in place, and is admin-only." } } } } } } */ + /* #swagger.responses[200] = { description: 'The cancelled run, how many steps were closed out with it, and whether cleanup was asked for', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, cancelledSteps: { type: "integer" }, cleanup: { type: "boolean" } } } } } } */ /* #swagger.responses[409] = { description: 'The run has already reached a terminal status', content: { "application/json": { schema: { type: "object", properties: { errors: { type: "array", items: { type: "string" } } } } } } } */ - /* #swagger.responses[403] = { description: 'Not an admin or moderator', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + /* #swagger.responses[403] = { description: 'Not an admin or moderator, or a moderator asking to skip cleanup', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ liveControl, controller.cancelRun, ) +eventsRouter.post( + '/runs/:runId/cleanup', + // #swagger.tags = ['Admin · Events'] + // #swagger.summary = 'Re-run cleanup over everything this run has not given back' + // #swagger.description = 'The manual retry EVENTS.md §L promises, and the only thing that clears a resource attempt counter — the automatic sweep never does, because a sweep that reset every stale row is what made an attempt ceiling unreachable in the engagement workstream. Legal on a TERMINAL run only: a run still in flight has a ledger that is still growing, and reverting a resource the next step is about to use would be core undoing an event while it is happening. `admin` rather than admin+moderator, unlike the seven live controls beside it, because this is not incident response — it asks core to write to the world again, which §K puts in the same row as the world-changing actions themselves. Answers 200 whatever it found: some resources may still be out there, and a 4xx would make that indistinguishable from a bad run id.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.responses[200] = { description: 'The run and what the sweep managed', content: { "application/json": { schema: { type: "object", properties: { run: { type: "object", additionalProperties: true }, summary: { type: "object", properties: { attempted: { type: "integer" }, reverted: { type: "integer" }, drifted: { type: "integer" }, failed: { type: "integer" }, remaining: { type: "integer" } } } } } } } } */ + /* #swagger.responses[409] = { description: 'The run is still in flight, or recorded no resources at all', content: { "application/json": { schema: { type: "object", properties: { errors: { type: "array", items: { type: "string" } } } } } } } */ + /* #swagger.responses[403] = { description: 'Not an admin', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + adminOnly, + controller.cleanupRun, +) + eventsRouter.post( '/runs/:runId/advance', // #swagger.tags = ['Admin · Events'] diff --git a/server/src/server.js b/server/src/server.js index be12168..a297cc1 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -16,6 +16,7 @@ const teamForumUploadSweep = require('./utils/teamForumUploadSweep') const teamDigestWorker = require('./utils/teamDigestWorker') const engagementWorker = require('./utils/engagementWorker') const eventRunner = require('./utils/eventRunner') +const eventCleanup = require('./events/cleanup') const { ensureSchema, close } = require('./utils/db') const { seedDefaults, createInitialAdminFromEnv } = require('../db/seed') const settings = require('./model/settings/settings.model') @@ -170,11 +171,25 @@ async function start() { // enables a rule: core seeds none and `enabled` defaults to 0. engagementWorker.start() - // Advance scheduled events (EVENTS.md §E). Materialise, advance, drain, and the - // run-log retention sweep. No-op until an admin publishes a definition and - // starts a run: core ships no event definitions. + // Advance scheduled events (EVENTS.md §E). Materialise, advance, drain, clean + // up and the run-log retention sweep. No-op until an admin publishes a + // definition and starts a run: core ships no event definitions. eventRunner.start() + // **Core's own restart is the one reconnect core can see** (EVENTS.md §L, + // Phase 8). Every other one belongs to a module, which reports it through + // `ctx.events.reconcile()`; this is the case where the thing that restarted was + // this process, and the ledger it wakes up holding may describe a world that + // moved on while it was down. Awaited by nobody and never fatal: a module that + // cannot answer leaves its rows alone, which is the pre-Phase-8 behaviour. + eventCleanup + .reconcileAll() + .then((summaries) => { + const orphaned = Object.values(summaries).reduce((n, x) => n + (x.orphaned || 0), 0) + if (orphaned) log.warn('event resources orphaned at boot', { orphaned, summaries }) + }) + .catch((err) => log.error('boot reconcile failed', { message: err.message })) + setupShutdown(server, internalServer) } diff --git a/server/src/utils/eventRunner.js b/server/src/utils/eventRunner.js index d4ef550..deb55ec 100644 --- a/server/src/utils/eventRunner.js +++ b/server/src/utils/eventRunner.js @@ -10,7 +10,16 @@ // 1. **reclaim** — release leases whose holder died, never touching `attempts` // 2. **materialise** — sweep occurrences past their grace window into `missed` // 3. **advance** — claim each due run and move it through its phases -// 4. **prune** — the `event_run_log` retention sweep, on its own long clock +// 4. **cleanup** — give back what a TERMINAL run still holds (Phase 8) +// 5. **prune** — the `event_run_log` retention sweep, on its own long clock +// +// **Cleanup is a leg rather than a limb of `advanceRun`**, and its position in +// that list is load-bearing: it runs after advance, so a run that completes in +// one tick is torn down in the same one, and it is one place rather than four, so +// a process that dies mid-teardown resumes on the next tick instead of leaving a +// world half-restored with nothing scheduled to finish it. §L's "cleanup steps +// are generated from the ledger and run" on cancellation and abort as well as on +// completion is one query here rather than a hook on each of the three. // // **What a phase advances on, as of Phase 5.** Every step terminal, and — if the // phase authored one — its GATE open as well. The gate is an ADDITIONAL @@ -69,6 +78,8 @@ const gates = require('../events/gates') const spec = require('../events/spec') const registries = require('../modules/registries') const { dispatchStep } = require('../events/dispatch') +const ledger = require('../events/ledger') +const cleanup = require('../events/cleanup') const authorize = require('../events/authorize') const log = require('./logger')('event-runner') @@ -261,6 +272,27 @@ async function drainStep(run, step, now, carry = {}) { } } + // ── Rule 1: record BEFORE the dispatch, not after ── + // + // §D. A step that ledgers gets a placeholder keyed by its idempotency key, + // written before anything reaches the module, so an answer that never comes + // back still leaves cleanup something to act on. Recording afterwards would + // make every object whose acknowledgement was lost invisible for ever, which is + // the one failure the whole world-write half cannot tolerate. + // + // It is deliberately AFTER the permission check: a refused step never reaches + // the module, so it has created nothing and must ledger nothing. + let placeholderId = null + try { + placeholderId = await ledger.reserveStep(run, step, action) + } catch (err) { + // The ledger is what makes a world write recoverable, so a step that cannot + // be recorded must not be dispatched. Transient — the next attempt tries the + // insert again — because the alternative is an unrecorded world change. + log.error('could not reserve the ledger row', { run: run.id, step: step.id, message: err.message }) + return applyFailure(run, step, `the resource ledger could not record this step: ${err.message}`) + } + const result = await dispatchStep(step, { run }) if (result.actionVersionDrift) { @@ -273,6 +305,36 @@ async function drainStep(run, step, now, carry = {}) { }) } + // ── …and promote it on the answer ── + // + // On both success shapes, because `await: 'human'` is a SUCCESS: the module did + // its part and something outside the system has to happen next, and a cue's + // confirm finishes the step without a second dispatch — so this is the only + // moment its resources can be recorded. A failure records nothing and leaves the + // placeholder standing, which is the whole point of writing one. + if (result.outcome === 'done' || result.outcome === 'parked') { + const recorded = await ledger.recordAnswer({ + run, + step, + action, + placeholderId, + resources: result.resources, + }) + if (recorded.recorded > 0 || recorded.rejected.length > 0) { + await logDb.write({ + runId: run.id, + stepId: step.id, + kind: 'resource.recorded', + phase: step.phase, + detail: { + action: step.action_id, + recorded: recorded.recorded, + ...(recorded.rejected.length ? { rejected: recorded.rejected } : {}), + }, + }) + } + } + if (result.outcome === 'parked') { // The GM cue. The step stays `running` with a NULL lease: genuinely in // flight, nothing holding it, so the stale reclaim passes it by and a cue @@ -459,9 +521,11 @@ async function advanceRun(run, now) { } if (run.status === 'ending') { - // A run that reached the wind-down and then lost its process. Phase 8 puts - // cleanup here; until then `ending` is a state a run passes through rather - // than one it does work in, and completing it is the whole recovery. + // A run that reached the wind-down and then lost its process. Completing it is + // still the whole recovery even with a ledger in the picture: cleanup is a leg + // of the tick over TERMINAL runs, so the row this leaves behind is exactly + // what that leg is looking for, and doing the teardown here as well would be + // the second call site the leg exists to avoid. await runsDb.transition(run.id, 'ending', 'completed') await logDb.write({ runId: run.id, kind: 'run.status', detail: { from: 'ending', to: 'completed' } }) return 'completed' @@ -536,8 +600,9 @@ async function advanceRun(run, now) { const next = phases[phaseIndex + 1] if (!next) { // §E's `ending` exists for the reason `sending` does in the outbox — it is - // what a claim sets — so the run passes through it even though Phase 2 has - // no cleanup to do there. Phase 8 is what gives it work. + // what a claim sets. The run still passes straight through it: the ledger's + // teardown is the tick's cleanup leg, which runs after this one in the same + // tick and takes the run as it now is. if (!(await runsDb.transition(run.id, 'running', 'ending'))) return 'taken' await logDb.write({ runId: run.id, kind: 'run.status', phase: phaseKey, detail: { from: 'running', to: 'ending' } }) await runsDb.transition(run.id, 'ending', 'completed') @@ -790,6 +855,17 @@ async function tick(now = new Date()) { } if (due && due.length) log.info('event runs swept', { due: due.length, ...counts }) + // **After advance, deliberately.** A run that reached `completed` two lines ago + // is torn down in this tick rather than the next, so §L's "a run reaches + // `completed` with `cleanup_status = 'incomplete'`" is what an operator sees + // instead of a completed run that briefly claims it has cleanup pending. + try { + const swept = await cleanup.sweep() + if (swept) log.info('event cleanup swept', { runs: swept }) + } catch (err) { + log.error('cleanup sweep failed', { message: err.message }) + } + try { await prune(now) } catch (err) { diff --git a/server/swagger/swagger-output.json b/server/swagger/swagger-output.json index 4210995..24c5fac 100644 --- a/server/swagger/swagger-output.json +++ b/server/swagger/swagger-output.json @@ -4137,7 +4137,7 @@ ], "responses": { "200": { - "description": "The run, its steps, the status counts and the phase gates", + "description": "The run, its steps, the status counts, the phase gates, the cap meter and the resource ledger", "content": { "application/json": { "schema": { @@ -4164,6 +4164,23 @@ "type": "object", "additionalProperties": true } + }, + "budget": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + }, + "unresolvedResources": { + "type": "integer" } } } @@ -4295,7 +4312,7 @@ "Admin · Events" ], "summary": "Cancel a run", - "description": "Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` is not a parameter yet — the resource ledger it would work over arrives in Phase 8, and a flag that changes nothing is worse than one that is not there.", + "description": "Legal from every non-terminal status, `scheduled` included. Pending steps and any parked cue are cancelled with it; a step with a live lease is left alone, because nothing can recall a command already sent and a second writer on that row would race the process dispatching it. `cleanup` arrived in Phase 8 and DEFAULTS TO TRUE: what the run created or borrowed is given back by the runner cleanup leg on its next tick, which is why this answers at once rather than after a round trip per resource. Sending `cleanup: false` deliberately leaves the world changes in place — that is admin-only even though the route is admin+moderator, because which of the two you have to be depends on what is in the body — and the run then carries `cleanup_status: incomplete` with every unreverted row listed on its console.", "parameters": [ { "name": "runId", @@ -4308,7 +4325,7 @@ ], "responses": { "200": { - "description": "The cancelled run and how many steps were closed out with it", + "description": "The cancelled run, how many steps were closed out with it, and whether cleanup was asked for", "content": { "application/json": { "schema": { @@ -4320,6 +4337,9 @@ }, "cancelledSteps": { "type": "integer" + }, + "cleanup": { + "type": "boolean" } } } @@ -4330,7 +4350,7 @@ "description": "Bad Request" }, "403": { - "description": "Not an admin or moderator", + "description": "Not an admin or moderator, or a moderator asking to skip cleanup", "content": { "application/json": { "schema": { @@ -4376,6 +4396,10 @@ "reason": { "type": "string", "description": "Why. Recorded on the run and in its log, with the actor." + }, + "cleanup": { + "type": "boolean", + "description": "Default true. False leaves the world changes from this run in place, and is admin-only." } } } @@ -4384,6 +4408,102 @@ } } }, + "/api/v1/admin/events/runs/{runId}/cleanup": { + "post": { + "tags": [ + "Admin · Events" + ], + "summary": "Re-run cleanup over everything this run has not given back", + "description": "The manual retry EVENTS.md §L promises, and the only thing that clears a resource attempt counter — the automatic sweep never does, because a sweep that reset every stale row is what made an attempt ceiling unreachable in the engagement workstream. Legal on a TERMINAL run only: a run still in flight has a ledger that is still growing, and reverting a resource the next step is about to use would be core undoing an event while it is happening. `admin` rather than admin+moderator, unlike the seven live controls beside it, because this is not incident response — it asks core to write to the world again, which §K puts in the same row as the world-changing actions themselves. Answers 200 whatever it found: some resources may still be out there, and a 4xx would make that indistinguishable from a bad run id.", + "parameters": [ + { + "name": "runId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The run and what the sweep managed", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "run": { + "type": "object", + "additionalProperties": true + }, + "summary": { + "type": "object", + "properties": { + "attempted": { + "type": "integer" + }, + "reverted": { + "type": "integer" + }, + "drifted": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "remaining": { + "type": "integer" + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad Request" + }, + "403": { + "description": "Not an admin", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "The run is still in flight, or recorded no resources at all", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ] + } + }, "/api/v1/admin/events/runs/{runId}/log": { "get": { "tags": [ diff --git a/server/test/eventActionRegistry.test.js b/server/test/eventActionRegistry.test.js index a2d4148..cbf9e16 100644 --- a/server/test/eventActionRegistry.test.js +++ b/server/test/eventActionRegistry.test.js @@ -44,10 +44,15 @@ const register = (owner, entries) => { registries.apply(api.staged) } -test('core registers its three actions on every boot', () => { +test('core registers its four actions on every boot', () => { registries.registerCore() const ids = registries.allEventActions().map((a) => a.id) - assert.deepEqual(ids, ['core.announce', 'core.wait', 'core.cue']) + // `core.lease` joined the three in Phase 8, and it is the only one of the four + // that genuinely changes the world — which is why it is core's rather than each + // module's: §F puts the duration bound and the two-events-one-target conflict + // check on core's side of the seam, and a lease verb per module would be that + // bound re-implemented once per module and advisory everywhere. + assert.deepEqual(ids, ['core.announce', 'core.wait', 'core.cue', 'core.lease']) assert.equal(ids.length, coreEventActions.ACTIONS.length) }) @@ -56,6 +61,7 @@ test('the catalog carries no callable', () => { for (const action of registries.allEventActions()) { assert.equal(action.perform, undefined, `${action.id} leaked perform`) assert.equal(action.revert, undefined, `${action.id} leaked revert`) + assert.equal(action.reconcile, undefined, `${action.id} leaked reconcile`) assert.equal(action.cost, undefined, `${action.id} leaked cost`) } // …and the runner's own lookup still has it, which is the half that makes the @@ -242,7 +248,7 @@ test('a whole batch is refused or taken, never half', () => { test('_reset() hands the process back', () => { registries.registerCore() - assert.equal(registries.allEventActions().length, 3) + assert.equal(registries.allEventActions().length, 4) registries._reset() assert.equal(registries.allEventActions().length, 0) assert.equal(registries.isEventAction('core.wait'), false) @@ -366,9 +372,80 @@ test('an option source needs a resolver, and core registers one of its own', () // Core through the same door (Phase 7): `core.announce`'s `leg` param names a // source, and the announce legs are already a registry with labels in them. registries.registerCore() - assert.deepEqual(registries.allEventOptionSources().map((s) => s.id), ['core.options.legs']) + assert.deepEqual(registries.allEventOptionSources().map((s) => s.id), [ + 'core.options.legs', + // Phase 8's, and it is the same argument one phase on: `core.lease`'s `lease` + // param would otherwise be a free-text box whose typo is caught at dispatch, + // mid-run — and the leases are already a registry with labels in them. + 'core.options.leases', + ]) const leg = registries.eventAction('core.announce').params.find((p) => p.name === 'leg') assert.equal(leg.source, 'core.options.legs') + const which = registries.eventAction('core.lease').params.find((p) => p.name === 'lease') + assert.equal(which.source, 'core.options.leases') +}) + +// ── `reconcile`, the one member Phase 8 added to the action shape ────────── +// +// Optional where `revert` is required, and the asymmetry is the design: a module +// that cannot say what the game still has is not broken — core keeps believing +// its own ledger, which is the behaviour before this phase — whereas a module +// that created something and cannot undo it has made a promise core has no way +// to keep. + +test('reconcile is optional, must be a function, and only on an action that ledgers', () => { + const ledgering = { + id: 'demo.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + perform: async () => ({ ok: true }), + revert: async () => ({ ok: true }), + } + + // Absent is legal, and it lands as an explicit null rather than as a missing + // key — the same shape `revert` and `cost` take, so the catalog's strip list + // and the sweep's `typeof` check both have something to look at. + register('demo', [ledgering]) + assert.equal(registries.eventAction('demo.spawn').reconcile, null) + registries._reset() + + assert.throws( + () => register('demo', [{ ...ledgering, reconcile: 'yes please' }]), + /reconcile must be a function/, + ) + + // The mirror check `revert` already has. An action that ledgers nothing has no + // rows for core to ask about, so a `reconcile` on one is an author who believes + // something is being tracked and a sweep that will never call it. + assert.throws( + () => + register('demo', [ + { + id: 'demo.shout', + label: 'Shout', + risk: 'notify', + reversible: 'none', + perform: async () => ({ ok: true }), + reconcile: async () => ({ ok: true, inForce: [] }), + }, + ]), + /declares reconcile\(\) but is reversible: 'none' and ledgers nothing/, + ) +}) + +test('an override action may reconcile, because a lease is ledgered too', () => { + register('demo', [ + { + id: 'demo.borrow', + label: 'Borrow', + risk: 'change', + reversible: 'override', + perform: async () => ({ ok: true }), + reconcile: async () => ({ ok: true, inForce: [] }), + }, + ]) + assert.equal(typeof registries.eventAction('demo.borrow').reconcile, 'function') }) test('_reset() hands back the three new registries too', () => { diff --git a/server/test/eventCleanup.test.js b/server/test/eventCleanup.test.js new file mode 100644 index 0000000..f70bc56 --- /dev/null +++ b/server/test/eventCleanup.test.js @@ -0,0 +1,500 @@ +// ── Giving back what a run took (EVENTS_PLAN.md Phase 8) ─────────────────── +// +// The phase's shipped claim: **cleanup is generated from the ledger and runs on +// every terminal path.** An operator cannot be relied on to write the undo, and +// an aborted run never reaches the phase they wrote it in — so there is no +// cleanup phase in a spec, no `on_teardown` on an action, and one function that +// reads rows. +// +// The properties around it are §L's, and two of them are ones this codebase has +// already paid for once: +// +// • **a revert that never succeeds stays visible rather than cycling** — rule 2, +// and `MAX_REVERT_ATTEMPTS` is what stops the automatic retry. Only a human +// clears the counter, which is Engagement Phase 14's rule stated a third time +// • **reverting something that does not exist is a SUCCESS** — §L, and what a +// Rust wipe needs +// • **drift is not an error.** The module did exactly what it was asked and +// found somebody else's value in place. A restore that wrote anyway would +// silently revert an operator's manual fix +// • **a resource the module no longer has becomes `orphaned`, never `reverted`** — +// reverting it would be core recording that it put something back when what +// happened is that the thing vanished +// • **"I do not know" is never read as "it is gone".** Every unanswerable +// reconcile leaves the ledger alone +// +// The sweep is driven against a stubbed db layer, exactly as `eventRunner.test.js` +// drives the runner: what a stub cannot prove is the SQL, and the unique key that +// makes two events unable to lease one target runs against a real MariaDB in +// `eventRunnerSql.test.js`. + +process.env.DB_HOST = '127.0.0.1' +process.env.DB_PORT = '59999' + +const { test, beforeEach, afterEach, after } = require('node:test') +const assert = require('node:assert/strict') + +const registries = require('../src/modules/registries') +const cleanup = require('../src/events/cleanup') +const resourcesDb = require('../src/model/events/eventRunResources.db') +const runsDb = require('../src/model/events/eventRuns.db') +const stepsDb = require('../src/model/events/eventRunSteps.db') +const logDb = require('../src/model/events/eventRunLog.db') +const db = require('../src/utils/db') + +after(() => db.close()) + +const UNRESOLVED = ['pending', 'confirmed', 'reverting', 'orphaned', 'drifted'] + +let store +const originals = { + resourcesDb: { ...resourcesDb }, + runsDb: { ...runsDb }, + stepsDb: { ...stepsDb }, + logDb: { ...logDb }, +} + +beforeEach(() => { + registries._reset() + store = { rows: new Map(), steps: new Map(), log: [], next: 1, cleanupStatus: 'pending' } + + resourcesDb.unresolvedForRun = async (runId, { maxAttempts = null } = {}) => + [...store.rows.values()] + .filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status)) + .filter((r) => maxAttempts === null || r.revert_attempts < maxAttempts) + .map((r) => ({ ...r })) + resourcesDb.unresolvedCount = async (runId) => + [...store.rows.values()].filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status)).length + resourcesDb.claimRevert = async (id) => { + const r = store.rows.get(id) + if (!r || !['pending', 'confirmed', 'orphaned', 'drifted'].includes(r.status)) return false + r.status = 'reverting' + return true + } + resourcesDb.markReverted = async (id) => { + const r = store.rows.get(id) + if (r) Object.assign(r, { status: 'reverted', last_error: null }) + } + resourcesDb.failRevert = async (id, error, restoreTo = 'confirmed') => { + const r = store.rows.get(id) + if (r) Object.assign(r, { status: restoreTo, revert_attempts: r.revert_attempts + 1, last_error: String(error) }) + } + resourcesDb.resetAttempts = async (runId) => { + let n = 0 + for (const r of store.rows.values()) { + if (r.run_id === runId && UNRESOLVED.includes(r.status)) { + r.revert_attempts = 0 + n += 1 + } + } + return n + } + resourcesDb.markOrphaned = async (id, detail = null) => { + const r = store.rows.get(id) + if (r && ['pending', 'confirmed', 'reverting'].includes(r.status)) { + Object.assign(r, { status: 'orphaned', last_error: detail }) + } + } + resourcesDb.liveForModule = async (owner) => + [...store.rows.values()].filter((r) => r.owner_module === owner && ['pending', 'confirmed'].includes(r.status)).map((r) => ({ ...r })) + resourcesDb.modulesWithLiveRows = async () => [ + ...new Set([...store.rows.values()].filter((r) => ['pending', 'confirmed'].includes(r.status)).map((r) => r.owner_module)), + ] + resourcesDb.runsNeedingCleanup = async () => store.candidates || [] + + runsDb.setCleanupStatus = async (id, to, from = null) => { + if (from && !from.includes(store.cleanupStatus)) return false + store.cleanupStatus = to + return true + } + stepsDb.getById = async (id) => store.steps.get(id) || null + logDb.write = async (entry) => { + store.log.push(entry) + } +}) + +afterEach(() => { + Object.assign(resourcesDb, originals.resourcesDb) + Object.assign(runsDb, originals.runsDb) + Object.assign(stepsDb, originals.stepsDb) + Object.assign(logDb, originals.logDb) + registries._reset() +}) + +const RUN = { id: 7, status: 'completed', cleanup_status: 'pending' } + +function addStep(id, actionId, key = 'k'.repeat(40)) { + store.steps.set(id, { id, run_id: RUN.id, action_id: actionId, idempotency_key: key, phase: 'p', seq: 0 }) +} + +function addResource(over = {}) { + const id = store.next++ + const row = { + id, + run_id: RUN.id, + step_id: 1, + owner_module: 'demo', + kind: 'creature', + ref: `0x${id}`, + payload: null, + lease_until: null, + status: 'confirmed', + revert_attempts: 0, + last_error: null, + member_key: null, + ...over, + } + store.rows.set(id, row) + return row +} + +function registerAction(over = {}) { + const api = registries.stage('demo') + api.registerEventActions([ + { + id: 'demo.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + budgetMs: 500, + perform: async () => ({ ok: true }), + revert: async () => ({ ok: true }), + ...over, + }, + ]) + registries.apply(api.staged) +} + +function registerLease(over = {}) { + const api = registries.stage('demo') + api.registerEventLeases([ + { + id: 'demo.rate', + label: 'Gather rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3_600_000, + read: async () => ({ ok: true, value: 1 }), + apply: async () => ({ ok: true }), + restore: async () => ({ ok: true }), + ...over, + }, + ]) + registries.apply(api.staged) +} + +const kinds = () => store.log.map((e) => e.kind) + +// ── The shipped claim ────────────────────────────────────────────────────── + +test('a run\'s ledger is given back in one call per step, and the run goes complete', async () => { + // `revert` takes a LIST because that is what makes twelve creatures one round + // trip rather than twelve. The grouping key is the STEP, because the resource + // row records the module and the opaque names while the step records the verb. + let seen = null + registerAction({ revert: async (arg) => { seen = arg; return { ok: true } } }) + addStep(1, 'demo.spawn', 'key-one') + addResource({ ref: '0xA' }) + addResource({ ref: '0xB' }) + + const summary = await cleanup.cleanupRun(RUN) + assert.deepEqual(summary, { attempted: 2, reverted: 2, drifted: 0, failed: 0, remaining: 0 }) + assert.equal(seen.runId, RUN.id) + assert.equal(seen.idempotencyKey, 'key-one') + assert.deepEqual(seen.resources.map((r) => r.ref), ['0xA', '0xB']) + assert.equal(store.cleanupStatus, 'complete') +}) + +test('two steps are two calls, and one failing does not take the other down', async () => { + const calls = [] + registerAction({ + revert: async ({ idempotencyKey, resources }) => { + calls.push(idempotencyKey) + return idempotencyKey === 'bad' ? { ok: false, error: 'the shard did not answer' } : { ok: true, resources } + }, + }) + addStep(1, 'demo.spawn', 'good') + addStep(2, 'demo.spawn', 'bad') + addResource({ step_id: 1 }) + addResource({ step_id: 2 }) + + const summary = await cleanup.cleanupRun(RUN) + assert.equal(calls.length, 2) + assert.equal(summary.reverted, 1) + assert.equal(summary.failed, 1) + assert.equal(summary.remaining, 1) + // **Still `pending`, because the failed row has retries left.** `incomplete` + // means "finished with, and not finished" — it is what takes a run out of the + // sweep's own scan, so writing it after the FIRST failure made + // `MAX_REVERT_ATTEMPTS` quietly mean one attempt. Found by watching + // `revert_attempts` sit at 1 through half a minute of live ticks. + assert.equal(store.cleanupStatus, 'pending') +}) + +test('reverting something that does not exist is a success', async () => { + // §L, and the Rust wipe: "gone, and that is fine". The module never has to + // distinguish "I deleted it" from "it was not there" — which is also what makes + // a placeholder for an object that may never have existed safe to write. + registerAction({ revert: async () => ({ ok: true, detail: 'resource no longer exists' }) }) + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.reverted, 1) + assert.equal(store.cleanupStatus, 'complete') +}) + +test('a module may name the ones that did not come back', async () => { + // Partial cleanup is the ordinary case — eleven of twelve creatures deleted — + // and it is why the ledger is a row per object rather than a row per step. + registerAction({ revert: async () => ({ ok: true, failed: ['0x2'] }) }) + addStep(1, 'demo.spawn') + addResource({ ref: '0x1' }) + addResource({ ref: '0x2' }) + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.reverted, 1) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()].find((r) => r.ref === '0x2').last_error, /could not give "0x2" back/) +}) + +// ── Rule 2: loud and sticky ──────────────────────────────────────────────── + +test('a revert that never works stops retrying and stays visible', async () => { + let calls = 0 + registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } }) + addStep(1, 'demo.spawn') + addResource() + + for (let i = 0; i < 6; i++) await cleanup.cleanupRun(RUN) + + // Bounded at MAX_REVERT_ATTEMPTS, exactly like a step's attempts. A fourth ask + // of a shard that has answered the same way three times is not new information, + // and an unbounded counter is a row nothing can ever sweep. + assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS) + const row = [...store.rows.values()][0] + assert.equal(row.revert_attempts, cleanup.MAX_REVERT_ATTEMPTS) + assert.equal(row.status, 'confirmed') + assert.equal(row.last_error, 'nope') + // And ONLY now, with nothing left to try, does the run stop being the sweep's + // business. §L: it does not stay `running` — an event whose world changes are + // still up is a real state, and pretending the event is in progress hides it. + assert.equal(store.cleanupStatus, 'incomplete') +}) + +test('the run goes back to pending each time it still has an attempt left', async () => { + // The other half of the same rule, watched one pass at a time rather than at + // the end. Each of the first two sweeps leaves the run in the scan; the third + // takes it out. A test that only looked at the end state would pass against the + // defect this replaced. + registerAction({ revert: async () => ({ ok: false, error: 'nope' }) }) + addStep(1, 'demo.spawn') + addResource() + + const seen = [] + for (let i = 0; i < 3; i++) { + await cleanup.cleanupRun(RUN) + seen.push(store.cleanupStatus) + } + assert.deepEqual(seen, ['pending', 'pending', 'incomplete']) +}) + +test('only a human clears the attempt counter', async () => { + // Engagement Phase 14's defect, stated a third time: a SWEEP that returned every + // stale row to its start state made the attempt ceiling unreachable, so the row + // cycled for ever and was never eligible for any retention sweep. The automatic + // leg must never do this; the cleanup route may, because a person asked. + let calls = 0 + registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } }) + addStep(1, 'demo.spawn') + addResource() + + for (let i = 0; i < 5; i++) await cleanup.cleanupRun(RUN) + assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS) + + await cleanup.cleanupRun(RUN, { resetAttempts: true, actor: 9 }) + assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS + 1) + assert.ok(store.log.some((e) => e.kind === 'cleanup.retry' && e.detail.by === 9)) +}) + +test('a module that throws from revert is a transient failure, not a crashed sweep', async () => { + registerAction({ revert: async () => { throw new Error('socket hung up') } }) + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()][0].last_error, /socket hung up/) +}) + +test('no shape a revert failure can take reads as success', () => { + // `dispatch.classify`'s rule, applied to the other direction of the contract. + // The expensive mistake here is the mirror of the one there: recording that a + // world change was UNDONE when it was not. + for (const raw of [null, undefined, 'ok', [], {}, { ok: 'yes' }, { ok: 1 }]) { + assert.notEqual(cleanup.classifyRevert(raw, 'demo.spawn').outcome, 'done', JSON.stringify(raw)) + } + assert.equal(cleanup.classifyRevert({ __timedOut: true, error: 'slow' }, 'x').outcome, 'retry') + assert.equal(cleanup.classifyRevert({ ok: false, retry: false, error: 'never' }, 'x').outcome, 'terminal') + assert.equal(cleanup.classifyRevert({ ok: true }, 'x').outcome, 'done') +}) + +test('an action whose module is gone leaves its rows unresolved with the reason', async () => { + // Not a retry — nothing will change until an operator reinstalls it — and not + // an orphan either, because core has no idea whether the thing is still there. + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()][0].last_error, /no module registers "demo.spawn"/) + // Retried like any other failure rather than given up on at once, and that is + // the right uniformity here: "nothing registers this" stops being true the + // moment an operator reinstalls the module, and three registry lookups cost + // nothing. So it is `pending` until the attempts are spent. + assert.equal(store.cleanupStatus, 'pending') +}) + +// ── Leases ───────────────────────────────────────────────────────────────── + +test('a lease is restored through the LEASE registry, not through any action', async () => { + // The split §F draws: core owns the duration and the conflict check, the module + // owns reading and writing. It is why `core.lease` needs no `revert()` of its + // own, and why an `override` row routes here rather than to its step's action. + let seen = null + registerLease({ restore: async (baseline, opts) => { seen = { baseline, opts }; return { ok: true } } }) + addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } }) + + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.reverted, 1) + assert.equal(seen.baseline, 1) + // The drift check's input. `restore` MUST verify current === expected before + // writing, and a lease whose restore wrote blindly would silently revert an + // operator's manual fix. + assert.equal(seen.opts.expected, 3) +}) + +test('drift is not an error: the world is left alone and the row says so', async () => { + registerLease({ restore: async () => ({ ok: false, drifted: true, current: 4.5 }) }) + addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } }) + + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.drifted, 1) + assert.equal(summary.failed, 0) + const row = [...store.rows.values()][0] + assert.equal(row.status, 'drifted') + assert.match(row.last_error, /now 4\.5 rather than what this run applied/) + // Still surfaced. §L: "surfaced beside the unreverted ones" — the run does not + // get to call itself clean because somebody else took the value. It is `pending` + // rather than `incomplete` for one more reason worth keeping: drift is retried + // like any other failure, because a GM who puts the value back between two ticks + // should have the lease close cleanly. + assert.equal(store.cleanupStatus, 'pending') +}) + +test('a lease whose module is uninstalled is unresolved, never assumed restored', async () => { + addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } }) + const summary = await cleanup.cleanupRun(RUN) + assert.equal(summary.failed, 1) + assert.match([...store.rows.values()][0].last_error, /no module registers the lease "demo.rate"/) +}) + +// ── The sweep ────────────────────────────────────────────────────────────── + +test('the sweep only touches TERMINAL runs', async () => { + // A run still in flight has a ledger that is still growing, and reverting a + // resource the next step is about to use would be core undoing an event while + // it is happening. + registerAction() + addStep(1, 'demo.spawn') + addResource() + store.candidates = [{ id: RUN.id, status: 'running', cleanup_status: 'pending' }] + assert.equal(await cleanup.sweep(), 0) + assert.equal([...store.rows.values()][0].status, 'confirmed') + + store.candidates = [{ id: RUN.id, status: 'cancelled', cleanup_status: 'pending' }] + assert.equal(await cleanup.sweep(), 1) + assert.equal([...store.rows.values()][0].status, 'reverted') +}) + +// ── Reconcile ────────────────────────────────────────────────────────────── + +test('a resource the module no longer has becomes orphaned, never reverted', async () => { + // §L, and the distinction matters to the operator reading the console + // afterwards: `reverted` says core put something back, `orphaned` says the + // thing vanished while nobody was looking. Recording the second as the first + // would be core claiming credit for a shard restart. + registerAction({ reconcile: async () => ({ ok: true, inForce: ['0x1'] }) }) + addStep(1, 'demo.spawn') + addResource({ ref: '0x1' }) + addResource({ ref: '0x2' }) + + const summary = await cleanup.reconcileModule('demo') + assert.deepEqual(summary, { asked: 2, inForce: 1, orphaned: 1, unanswered: 0 }) + assert.equal([...store.rows.values()].find((r) => r.ref === '0x2').status, 'orphaned') + assert.equal([...store.rows.values()].find((r) => r.ref === '0x1').status, 'confirmed') + assert.ok(store.log.some((e) => e.kind === 'resource.orphaned')) +}) + +test('"I do not know" is never read as "it is gone"', async () => { + // Every unanswerable shape leaves the ledger exactly as it was. A reconcile + // that read silence as absence would orphan a whole shard's worth of live + // spawns the first time a sidecar was slow. + for (const answer of [null, undefined, { ok: false }, { ok: true }, { ok: true, inForce: 'all' }, 'yes']) { + store.rows.clear() + store.log.length = 0 + registries._reset() + registerAction({ reconcile: async () => answer }) + addStep(1, 'demo.spawn') + addResource({ ref: '0x1' }) + const summary = await cleanup.reconcileModule('demo') + assert.equal(summary.orphaned, 0, JSON.stringify(answer)) + assert.equal(summary.unanswered, 1, JSON.stringify(answer)) + assert.equal([...store.rows.values()][0].status, 'confirmed') + } +}) + +test('a module with no reconcile is not broken; core keeps believing its ledger', async () => { + // Optional where `revert` is required. A module that cannot answer leaves core + // exactly where it was before this phase, which is a capability its deployment + // does without rather than a boot it fails. + registerAction() + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.reconcileModule('demo') + assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 1 }) + assert.equal([...store.rows.values()][0].status, 'confirmed') +}) + +test('a reconcile that throws orphans nothing', async () => { + registerAction({ reconcile: async () => { throw new Error('sidecar gone') } }) + addStep(1, 'demo.spawn') + addResource() + const summary = await cleanup.reconcileModule('demo') + assert.equal(summary.unanswered, 1) + assert.equal([...store.rows.values()][0].status, 'confirmed') +}) + +test('placeholders are not asked about, because there is nothing to ask yet', async () => { + // A `@step` row names no object — it says "a dispatch was in flight and may + // have made something". Asking a module whether it is in force is a question + // with no answer, and reading a shrug as absence would resolve the one row whose + // survival is the safety property. + registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) }) + addStep(1, 'demo.spawn') + addResource({ kind: resourcesDb.STEP_KIND, ref: 'a'.repeat(40), status: 'pending' }) + const summary = await cleanup.reconcileModule('demo') + assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 }) + assert.equal([...store.rows.values()][0].status, 'pending') +}) + +test('reconcileAll asks every module that owns a live row', async () => { + registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) }) + addStep(1, 'demo.spawn') + addStep(2, 'other.thing') + addResource({ owner_module: 'demo' }) + addResource({ owner_module: 'other', step_id: 2 }) + const out = await cleanup.reconcileAll() + assert.deepEqual(Object.keys(out).sort(), ['demo', 'other']) + assert.equal(out.demo.orphaned, 1) + // The other module registers nothing, so its row is left alone rather than + // orphaned by a module that is not there to be asked. + assert.equal(out.other.unanswered, 1) +}) diff --git a/server/test/eventLedger.test.js b/server/test/eventLedger.test.js new file mode 100644 index 0000000..2f41ce8 --- /dev/null +++ b/server/test/eventLedger.test.js @@ -0,0 +1,337 @@ +// ── The resource ledger's write half (EVENTS_PLAN.md Phase 8) ────────────── +// +// §D's two rules, and rule 1 is the one this file exists for: **a resource is +// recorded BEFORE it is confirmed.** The obstacle it works around is that a +// spawn's serial does not exist until the module answers, so what goes in before +// the dispatch is a placeholder keyed by the step's idempotency key — and the +// property worth a test is that the placeholder SURVIVES an answer that never +// comes, because that is the case where recording afterwards would have lost the +// object for ever. +// +// The other rules here are about what core will and will not write down on a +// module's say-so. Every one of them is fail-closed in a specific direction: +// +// • the reserved `@step` kind is core's and a module may not claim it +// • an `override` must name a lease core knows how to give back, or core would +// be recording something it has no way to restore +// • a duplicate is "already recorded", not an error — a retry re-sends the same +// idempotency key and a module may honestly report the same resources twice +// • a badly shaped resource is dropped and LOGGED, never a failed step: the +// step changed the world, and turning bookkeeping into a retry would re-run +// a world write that already happened +// +// The db layer is stubbed with a store that enforces `uq_evres_target`, because +// that refusal is behaviour the callers branch on rather than an implementation +// detail. The SQL itself is `eventRunnerSql.test.js`'s, against a real MariaDB. + +process.env.DB_HOST = '127.0.0.1' +process.env.DB_PORT = '59999' + +const { test, beforeEach, afterEach, after } = require('node:test') +const assert = require('node:assert/strict') + +const registries = require('../src/modules/registries') +const ledger = require('../src/events/ledger') +const resourcesDb = require('../src/model/events/eventRunResources.db') +const runsDb = require('../src/model/events/eventRuns.db') +const db = require('../src/utils/db') + +after(() => db.close()) + +const HELD = ['pending', 'confirmed', 'reverting'] + +let store +const originals = { resourcesDb: { ...resourcesDb }, runsDb: { ...runsDb } } + +beforeEach(() => { + registries._reset() + store = { rows: new Map(), next: 1, cleanupStatus: 'not_required' } + + resourcesDb.reserve = async ({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) => { + const holder = [...store.rows.values()].find( + (r) => r.owner_module === owner && r.kind === kind && r.ref === ref && HELD.includes(r.status), + ) + if (holder) return { ok: false, code: 'held', holder: { run_id: holder.run_id, status: holder.status } } + const id = store.next++ + store.rows.set(id, { + id, + run_id: runId, + step_id: stepId, + owner_module: owner, + kind, + ref, + payload, + lease_until: leaseUntil, + status: 'pending', + revert_attempts: 0, + last_error: null, + member_key: memberKey, + }) + return { ok: true, id } + } + resourcesDb.confirm = async (id) => { + const r = store.rows.get(id) + if (!r || r.status !== 'pending') return false + r.status = 'confirmed' + return true + } + resourcesDb.resolvePlaceholder = async (id) => { + const r = store.rows.get(id) + if (!r || r.kind !== resourcesDb.STEP_KIND) return false + r.status = 'reverted' + return true + } + resourcesDb.findByTarget = async (owner, kind, ref) => + [...store.rows.values()].reverse().find((r) => r.owner_module === owner && r.kind === kind && r.ref === ref) || null + + runsDb.setCleanupStatus = async (id, to, from = null) => { + if (from && !from.includes(store.cleanupStatus)) return false + store.cleanupStatus = to + return true + } +}) + +afterEach(() => { + Object.assign(resourcesDb, originals.resourcesDb) + Object.assign(runsDb, originals.runsDb) + registries._reset() +}) + +const RUN = { id: 7 } +const STEP = { id: 42, phase: 'invasion', seq: 0, idempotency_key: 'a'.repeat(40) } + +const action = (over = {}) => ({ + id: 'demo.spawn', + owner: 'demo', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + budgetMs: 1000, + ...over, +}) + +const rows = () => [...store.rows.values()] + +// ── Which actions ledger at all ──────────────────────────────────────────── + +test('only the two reversible classes core has to come back for are ledgered', () => { + // `none` is gone once done, `self` undoes itself. Neither has anything core + // could revert, and giving one a placeholder would put a row in the ledger that + // teardown could never resolve — the exact reason `core.announce` is declared + // `none` rather than `ledger`. + assert.equal(ledger.ledgers(action({ reversible: 'ledger' })), true) + assert.equal(ledger.ledgers(action({ reversible: 'override' })), true) + assert.equal(ledger.ledgers(action({ reversible: 'none' })), false) + assert.equal(ledger.ledgers(action({ reversible: 'self' })), false) +}) + +test('only a ledger action gets a placeholder; an override reserves its own target', async () => { + // The asymmetry is the design. A spawn's ref is unknown until the module + // answers, so the placeholder stands in for it; a lease's target is the lease + // id the step already names, so `core.lease` writes the real row before it + // touches the world — which is rule 1 in a stronger form, and the only place + // the two-events-one-target refusal can happen before the world has changed. + assert.equal(typeof (await ledger.reserveStep(RUN, STEP, action())), 'number') + assert.equal(await ledger.reserveStep(RUN, STEP, action({ id: 'demo.b', reversible: 'override' })), null) + assert.equal(await ledger.reserveStep(RUN, STEP, action({ id: 'demo.c', reversible: 'none' })), null) + assert.equal(rows().length, 1) + assert.equal(rows()[0].kind, '@step') + assert.equal(rows()[0].ref, STEP.idempotency_key) +}) + +test('the first ledger row is what makes a run dirty, and only from not_required', async () => { + assert.equal(store.cleanupStatus, 'not_required') + await ledger.reserveStep(RUN, STEP, action()) + assert.equal(store.cleanupStatus, 'pending') + + // A run whose sweep has already finished must not be walked back to `pending` + // by a late row: only a human's cleanup re-opens it, and it does so + // deliberately and with an actor on the log line. + store.cleanupStatus = 'complete' + await ledger.recordAnswer({ + run: RUN, + step: { ...STEP, id: 43, idempotency_key: 'b'.repeat(40) }, + action: action(), + placeholderId: null, + resources: [{ kind: 'creature', ref: '0x1' }], + }) + assert.equal(store.cleanupStatus, 'complete') +}) + +// ── Rule 1 ───────────────────────────────────────────────────────────────── + +test('a lost acknowledgement leaves the placeholder standing, which is the whole point', async () => { + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + // The dispatch timed out: no answer, so `recordAnswer` is never reached. This + // is the case rule 1 exists for — record afterwards and the object the module + // may well have created is invisible to cleanup for ever. + assert.equal(rows()[0].status, 'pending') + assert.equal(rows()[0].payload.action, 'demo.spawn') + assert.ok(placeholderId) +}) + +test('a retry reuses its own placeholder rather than writing a second', async () => { + // An idempotency key is minted once per step and does not vary by attempt (§E), + // so the second attempt's insert collides with the first attempt's row. Finding + // it already there is the correct answer, and a second row would be a second + // thing for cleanup to revert. + const first = await ledger.reserveStep(RUN, STEP, action()) + const second = await ledger.reserveStep(RUN, STEP, action()) + assert.equal(first, second) + assert.equal(rows().length, 1) +}) + +test('the placeholder is resolved once the real rows exist', async () => { + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + const out = await ledger.recordAnswer({ + run: RUN, + step: STEP, + action: action(), + placeholderId, + resources: [ + { kind: 'creature', ref: '0x40001234' }, + { kind: 'creature', ref: '0x40001235' }, + ], + }) + assert.equal(out.recorded, 2) + assert.deepEqual(out.rejected, []) + assert.equal(store.rows.get(placeholderId).status, 'reverted') + assert.deepEqual( + rows().filter((r) => r.kind === 'creature').map((r) => [r.ref, r.status]), + [['0x40001234', 'confirmed'], ['0x40001235', 'confirmed']], + ) +}) + +test('an action that ledgers and reports nothing still resolves its placeholder', async () => { + // "I made nothing" is a real answer. Holding the placeholder open for it would + // make cleanup call `revert()` on every terminal path, for ever, for a step that + // has nothing to give back. + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + const out = await ledger.recordAnswer({ run: RUN, step: STEP, action: action(), placeholderId, resources: [] }) + assert.equal(out.recorded, 0) + assert.equal(store.rows.get(placeholderId).status, 'reverted') +}) + +test('a module reporting the same resources twice produces one row', async () => { + // The database is what makes recording idempotent: `uq_evres_target` refuses + // the second insert and this file reads that as "already recorded". Without it + // a retry against a module that honestly re-reports its work would double every + // row cleanup then has to revert. + const args = { run: RUN, step: STEP, action: action(), placeholderId: null, resources: [{ kind: 'creature', ref: '0x1' }] } + await ledger.recordAnswer(args) + const again = await ledger.recordAnswer(args) + assert.equal(again.recorded, 0) + assert.deepEqual(again.rejected, []) + assert.equal(rows().filter((r) => r.kind === 'creature').length, 1) +}) + +test('a target another RUN holds is rejected by name rather than silently skipped', async () => { + await ledger.recordAnswer({ + run: { id: 1 }, + step: STEP, + action: action(), + placeholderId: null, + resources: [{ kind: 'creature', ref: '0x1' }], + }) + const out = await ledger.recordAnswer({ + run: { id: 2 }, + step: { ...STEP, id: 99 }, + action: action(), + placeholderId: null, + resources: [{ kind: 'creature', ref: '0x1' }], + }) + assert.equal(out.recorded, 0) + assert.match(out.rejected.join('\n'), /already held by run 1/) +}) + +// ── What core will not write down ────────────────────────────────────────── + +test('a module may not claim core\'s reserved kind', () => { + // A module that could write a `@step` row could make its own step's placeholder + // look resolved — which is the one row whose survival is the safety property. + const bad = ledger.normalise({ kind: '@step', ref: 'x' }, 'demo.spawn') + assert.equal(bad.ok, false) + assert.match(bad.reason, /reserved kind/) +}) + +test('an override must name a lease core knows how to give back', () => { + // Core restores an `override` through the LEASE registry — that is the split §F + // draws — so a ref naming nothing registered is a resource core would be + // recording with no way to undo it. Refusing to record it is the fail-closed + // direction: rule 2 is a promise core must not make and then break. + assert.equal(ledger.normalise({ kind: 'override', ref: 'demo.rate' }, 'demo.x').ok, false) + + const api = registries.stage('demo') + api.registerEventLeases([ + { + id: 'demo.rate', + label: 'Rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3_600_000, + read: async () => ({ ok: true, value: 1 }), + apply: async () => ({ ok: true }), + restore: async () => ({ ok: true }), + }, + ]) + registries.apply(api.staged) + assert.equal(ledger.normalise({ kind: 'override', ref: 'demo.rate' }, 'demo.x').ok, true) +}) + +test('every bad shape is refused, and none of them is a retry', () => { + // A badly shaped resource is the module's mistake rather than the world's, and + // it will be just as badly shaped on the second attempt. They are dropped and + // reported; the STEP still counts as done, because it is — something happened + // in the world, and refusing to record it would be the one outcome worse than + // recording it imperfectly. + const bad = [ + null, + 'a string', + ['an array'], + { ref: 'x' }, // no kind + { kind: 'creature' }, // no ref + { kind: 'creature', ref: 'x'.repeat(200) }, + { kind: 'k'.repeat(80), ref: 'x' }, + { kind: 'creature', ref: 'x', memberKey: 'm'.repeat(200) }, + { kind: 'creature', ref: 'x', until: 'not a date' }, + ] + for (const entry of bad) { + assert.equal(ledger.normalise(entry, 'demo.spawn').ok, false, JSON.stringify(entry)) + } +}) + +test('a bad resource never fails the step it came from', async () => { + const placeholderId = await ledger.reserveStep(RUN, STEP, action()) + const out = await ledger.recordAnswer({ + run: RUN, + step: STEP, + action: action(), + placeholderId, + resources: [{ kind: 'creature', ref: '0x1' }, { nonsense: true }], + }) + assert.equal(out.recorded, 1) + assert.equal(out.rejected.length, 1) + // And the placeholder is still resolved: the good row exists, and leaving the + // placeholder open would ask the module to undo the step a second time. + assert.equal(store.rows.get(placeholderId).status, 'reverted') +}) + +test('a lease deadline and a member key ride through verbatim', async () => { + const until = new Date('2026-09-04T00:00:00Z') + await ledger.recordAnswer({ + run: RUN, + step: STEP, + action: action(), + placeholderId: null, + resources: [{ kind: 'reward', ref: 'item-1', memberKey: 'Darrow', until, payload: { cliloc: 1234 } }], + }) + const row = rows()[0] + assert.equal(row.member_key, 'Darrow') + assert.equal(row.lease_until.getTime(), until.getTime()) + assert.deepEqual(row.payload, { cliloc: 1234 }) + // Opaque: core stores what the module said and never interprets it, which is + // `ctx.teams.activity.push`'s exact treatment one registry along. + assert.equal(row.kind, 'reward') + assert.equal(row.owner_module, 'demo') +}) diff --git a/server/test/eventModuleContract.test.js b/server/test/eventModuleContract.test.js index 1823400..c9d6520 100644 --- a/server/test/eventModuleContract.test.js +++ b/server/test/eventModuleContract.test.js @@ -513,3 +513,242 @@ test('an action whose module is gone goes dormant, and a step naming it fails te assert.equal(result.dormant, true) assert.match(result.error, /no module registers "demo\.summon"/) }) + +// ── Phase 8: the ledger's two callables, from a module ───────────────────── +// +// `revert` was already required at registration for `reversible: 'ledger'` — +// Phase 1 put that check in. What Phase 8 added is a caller for it, and +// `reconcile` beside it. Both are proved here through the REAL loader for the +// same reason the four registrations are: a `revert` a test called directly is a +// `revert` core might still have no way to reach. + +test('a module\'s revert is reached by the cleanup sweep, resources and key in hand', async () => { + const record = loadModule('demo', ` + let seen = null + module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'demo.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true, resources: [{ kind: 'creature', ref: '0xA' }] } }, + async revert(arg) { seen = arg; return { ok: true } }, + async reconcile() { return { ok: true, inForce: [] } }, + }]) + api.registerEventOptionSources([ + { id: 'demo.options.seen', label: 'seen', async resolve() { return [{ value: JSON.stringify(seen), label: 'seen' }] } }, + ]) + } + `) + assertRegistered(record) + + const action = registries.eventAction('demo.spawn') + // Both callables survived the registration copy — which is not a given: that + // copy is explicit rather than a spread, precisely so nothing rides along, and + // a member added to the contract without being added to it is a member that + // silently does not exist. + assert.equal(typeof action.revert, 'function') + assert.equal(typeof action.reconcile, 'function') + assert.equal(action.owner, 'demo') + + const answer = await action.revert({ + runId: 3, + resources: [{ kind: 'creature', ref: '0xA', payload: null, memberKey: null }], + idempotencyKey: 'k-1', + }) + assert.deepEqual(answer, { ok: true }) + + // Read back through the module's own option source rather than out of a + // closure this file holds: the point is that what core PASSED is what the + // module SAW, across the seam. + const seen = JSON.parse((await registries.resolveOptionSource('demo.options.seen')).options[0].value) + assert.equal(seen.runId, 3) + assert.equal(seen.idempotencyKey, 'k-1') + assert.deepEqual(seen.resources, [{ kind: 'creature', ref: '0xA', payload: null, memberKey: null }]) +}) + +test('reconcile is optional, and a module without one still registers', () => { + // The asymmetry with `revert`, from the loader's side. A module that cannot say + // what the game still has is not broken — core keeps believing its own ledger, + // which is the behaviour before this phase — whereas one that creates something + // and cannot undo it has made a promise core has no way to keep. + const withNone = loadModule('quiet', `module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'quiet.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true } }, + async revert() { return { ok: true } }, + }]) + }`) + assertRegistered(withNone) + assert.equal(registries.eventAction('quiet.spawn').reconcile, null) + + const withoutRevert = loadModule('broken', `module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'broken.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true } }, + }]) + }`) + assert.equal(withoutRevert.state, 'startup_failed') + assert.match(withoutRevert.reason, /reversible: 'ledger' but has no revert\(\)/) +}) + +test('a module cannot claim core\'s reserved resource kind', async () => { + // `@step` is the placeholder's kind, and the placeholder is the row whose + // survival is the safety property: a module able to write one could make its + // own step look already accounted for. Refused at recording, and the STEP still + // succeeds — because it did. + const record = loadModule('sneaky', `module.exports = (ctx, api) => { + api.registerEventActions([{ + id: 'sneaky.spawn', + label: 'Spawn', + risk: 'change', + reversible: 'ledger', + params: [], + async perform() { return { ok: true, resources: [{ kind: '@step', ref: 'anything' }] } }, + async revert() { return { ok: true } }, + }]) + }`) + assertRegistered(record) + + // eslint-disable-next-line global-require + const ledger = require('../src/events/ledger') + const result = await dispatch.dispatchStep(step('sneaky.spawn'), { run: RUN }) + assert.equal(result.outcome, 'done') + const parsed = ledger.normalise(result.resources[0], 'sneaky.spawn') + assert.equal(parsed.ok, false) + assert.match(parsed.reason, /reserved kind/) +}) + +test('core registers the lease VERB and a module registers the lease', async () => { + // The seam working the way round it is meant to (Phase 8). A module ships the + // three callables; the verb an author puts in a step is `core.lease`, so the + // duration bound and the two-events-one-target conflict check live in one place + // rather than being re-implemented once per module and advisory everywhere. + const record = loadModule('demo', `module.exports = (ctx, api) => { + api.registerEventLeases([{ + id: 'demo.rate.gain', + label: 'Gain rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3600000, + async read() { return { ok: true, value: 1 } }, + async apply() { return { ok: true } }, + async restore() { return { ok: true } }, + }]) + }`) + assertRegistered(record) + registries.registerCore() + + // The module registers no ACTION at all, and its lease is still reachable. + assert.equal(registries.eventAction('demo.lease'), null) + assert.equal(registries.eventAction('core.lease').reversible, 'override') + + // And the dropdown behind `core.lease`'s first param is answered by what the + // module declared — resolved per request, so a module that booted later is + // still in the list. + const options = await registries.resolveOptionSource('core.options.leases') + assert.equal(options.ok, true) + assert.deepEqual(options.options, [{ value: 'demo.rate.gain', label: 'Gain rate', group: 'demo' }]) +}) + +test('core refuses a lease held longer than the module allows', async () => { + // The bound is the MODULE's number and the enforcement is CORE's, which is the + // §F split stated as one assertion. `retry: false` because a duration that is + // too long will still be too long in sixty seconds: it is an authoring error, + // not an outage. + const record = loadModule('demo', `module.exports = (ctx, api) => { + api.registerEventLeases([{ + id: 'demo.rate.gain', + label: 'Gain rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3600000, + async read() { return { ok: true, value: 1 } }, + async apply() { return { ok: true } }, + async restore() { return { ok: true } }, + }]) + }`) + assertRegistered(record) + registries.registerCore() + + const tooLong = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 120 }), + { run: RUN }, + ) + assert.equal(tooLong.outcome, 'terminal') + assert.match(tooLong.error, /at most 60 minutes, not 120/) + + // The same for a value outside the declared range. Unlike a cap, a bad lease + // value is in force the moment it is applied, which is why min/max are required + // on the numeric types rather than advisory. + const tooBig = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '9', minutes: 10 }), + { run: RUN }, + ) + assert.equal(tooBig.outcome, 'terminal') + assert.match(tooBig.error, /accepts 0\.5 to 5/) + + // And a lease nobody registers, which is the dormancy rule one registry along. + const missing = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.nope', value: '3', minutes: 10 }), + { run: RUN }, + ) + assert.equal(missing.outcome, 'terminal') + assert.match(missing.error, /no module registers the lease "demo\.nope"/) +}) + +test('a dry run of core.lease checks everything and takes nothing', async () => { + // `verify: true` must change nothing and must answer honestly (§F). A verify + // that reserved the target would be a dry run that changed something — and it + // would then refuse the real run that followed it, which is the worst of both. + let applied = 0 + const record = loadModule('demo', ` + let applied = 0 + module.exports = (ctx, api) => { + api.registerEventLeases([{ + id: 'demo.rate.gain', + label: 'Gain rate', + type: 'float', + min: 0.5, + max: 5, + maxDurationMs: 3600000, + async read() { return { ok: true, value: 1 } }, + async apply() { applied += 1; return { ok: true } }, + async restore() { return { ok: true } }, + }]) + api.registerEventOptionSources([ + { id: 'demo.options.applied', label: 'applied', async resolve() { return [{ value: String(applied), label: 'n' }] } }, + ]) + } + `) + assertRegistered(record) + registries.registerCore() + void applied + + const ok = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 10 }), + { run: RUN, verify: true }, + ) + assert.equal(ok.outcome, 'done') + assert.equal((await registries.resolveOptionSource('demo.options.applied')).options[0].value, '0') + + // A dry run that is still a real check: the bad duration is caught with + // `verify: true` as well, which is the whole value of the switchboard's + // "find out before you schedule it". + const bad = await dispatch.dispatchStep( + step('core.lease', { lease: 'demo.rate.gain', value: '3', minutes: 999 }), + { run: RUN, verify: true }, + ) + assert.equal(bad.outcome, 'terminal') +}) diff --git a/server/test/eventRunControls.test.js b/server/test/eventRunControls.test.js index 1cefb6c..2ae7bb0 100644 --- a/server/test/eventRunControls.test.js +++ b/server/test/eventRunControls.test.js @@ -35,6 +35,11 @@ const runsDb = require('../src/model/events/eventRuns.db') const stepsDb = require('../src/model/events/eventRunSteps.db') const logDb = require('../src/model/events/eventRunLog.db') const gatesDb = require('../src/model/events/eventPhaseGates.db') +// Phase 8: cancel now decides what happens to the run's world changes, and +// `cleanupRun` is the eighth control. Same rule as every phase since the fourth — +// a new leg under a model needs a stub in every file that stubs that layer. +const resourcesDb = require('../src/model/events/eventRunResources.db') +const eventCleanup = require('../src/events/cleanup') const db = require('../src/utils/db') after(() => db.close()) @@ -48,12 +53,35 @@ const originals = [ ['steps', stepsDb, { ...stepsDb }], ['log', logDb, { ...logDb }], ['gates', gatesDb, { ...gatesDb }], + ['resources', resourcesDb, { ...resourcesDb }], + ['cleanup', eventCleanup, { ...eventCleanup }], ] function installStubs() { - store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), nextStepId: 1, nextGateId: 1 } + store = { runs: new Map(), steps: new Map(), log: [], gates: new Map(), sweeps: [], unresolved: {}, nextStepId: 1, nextGateId: 1 } const snap = (o) => ({ ...o }) + runsDb.setCleanupStatus = async (id, to, from = null) => { + const r = store.runs.get(Number(id)) + if (!r) return false + if (from && !from.includes(r.cleanup_status)) return false + r.cleanup_status = to + return true + } + + // The sweep itself is `eventCleanup.test.js`'s subject. What this file is + // about is which control calls it, with what, and whether it is allowed to. + eventCleanup.cleanupRun = async (run, opts = {}) => { + store.sweeps.push({ runId: run.id, ...opts }) + return { attempted: 1, reverted: 1, drifted: 0, failed: 0, remaining: 0 } + } + + // Cancel asks the LEDGER whether the run owes the world anything, so that + // `cleanup: false` on a run with nothing recorded does not stamp `incomplete` + // over `not_required`. Unstubbed this is the ten-second dead-port wait, for the + // fifth time in this feature. + resourcesDb.unresolvedCount = async (runId) => store.unresolved[runId] ?? 0 + runsDb.getById = async (id) => { const r = store.runs.get(Number(id)) return r ? snap(r) : null @@ -184,7 +212,7 @@ function seedGate(runId, phase, { kind = 'on', trigger = 'test.trigger', needed return g } -function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) { +function seedRun({ status = 'running', phase = 'main', steps = [], cleanupStatus = 'not_required' } = {}) { const id = nextRunId++ store.runs.set(id, { id, @@ -192,6 +220,7 @@ function seedRun({ status = 'running', phase = 'main', steps = [] } = {}) { version_id: id, status, health: 'ok', + cleanup_status: cleanupStatus, current_phase: phase, claimed_by: null, claim_expires_at: null, @@ -559,3 +588,115 @@ test('an empty reason is stored as NULL rather than as an empty string', async ( await controls.pause(id, { reason: ' ' }, ACTOR) assert.equal(lastLog().detail.reason, null) }) + +// ── cancel decides what happens to the world (Phase 8) ───────────────────── + +test('cancel gives back what the run took, by default and without waiting for it', async () => { + // The teardown is the runner cleanup leg over TERMINAL runs, not this request. + // Two reasons, and both are why the control answers at once: a cancel pressed + // at two in the morning must not block on a dozen round trips to the shard + // that may BE the reason it is being cancelled, and a process that dies + // halfway through a teardown has to resume rather than leave a world half + // restored with nothing scheduled to finish it. + const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 3 + + const result = await controls.cancel(id, { reason: 'called off' }, ACTOR) + + assert.equal(result.ok, true) + assert.equal(result.cleanup, true) + assert.deepEqual(store.sweeps, [], 'the request must not do the teardown itself') + // Still `pending`, which is what the leg looks for. The run is terminal the + // moment this returns, so the very next tick picks its ledger up. + assert.equal(runRow(id).cleanup_status, 'pending') + assert.equal(store.log.at(-1).detail.cleanup, true) +}) + +test('cancel WITHOUT cleanup is admin-only, even though the route is wider', async () => { + // §L: "cancelling without cleanup is a separate, logged, admin-only action." + // The route is `admin` + `moderator`, so the narrower gate cannot live in + // middleware — WHICH of the two you have to be depends on what is in the body, + // exactly as the authoring role floor does (§K). + const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 3 + + const refused = await controls.cancel(id, { cleanup: false }, ACTOR, { isAdmin: false }) + assert.equal(refused.ok, false) + assert.equal(refused.status, 403) + assert.equal(runRow(id).status, 'running', 'and the run is not cancelled either') + + // A moderator asking for the ordinary cancel is fine: the safe direction is + // the default, so the widest gate keeps the button it exists for. + const allowed = await controls.cancel(id, {}, ACTOR, { isAdmin: false }) + assert.equal(allowed.ok, true) + assert.equal(allowed.cleanup, true) +}) + +test('cancel without cleanup leaves the world changes up, and says so on the run', async () => { + // `incomplete` is the truthful value rather than a tidy one: the changes are + // still up, they are listed on the console, and the log line records who + // decided that. A `complete` here would be the "tidy completed row over a shard + // full of orphaned monsters" §L names as the failure that ends this feature's + // credibility. + const id = seedRun({ status: 'running', cleanupStatus: 'pending', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 3 + + const result = await controls.cancel(id, { cleanup: false, reason: 'leave it up' }, ACTOR) + + assert.equal(result.ok, true) + assert.equal(result.cleanup, false) + assert.equal(runRow(id).cleanup_status, 'incomplete') + assert.equal(store.log.at(-1).detail.cleanup, false) + assert.equal(store.log.at(-1).detail.by, ACTOR) +}) + +test('a run that recorded nothing is unaffected by either flag', async () => { + // `not_required` is not walked to `incomplete` by a cancel that skipped a + // teardown there was nothing to do — and it is the LEDGER that says so, not the + // status column, because `not_required` is also what a run holding only a lease + // wrongly carried before the live walk found it. + const id = seedRun({ status: 'running', cleanupStatus: 'not_required', steps: [{ status: 'pending' }] }) + store.unresolved[id] = 0 + await controls.cancel(id, { cleanup: false }, ACTOR) + assert.equal(runRow(id).cleanup_status, 'not_required') +}) + +// ── cleanup, the eighth control ──────────────────────────────────────────── + +test('cleanup re-runs the teardown and clears the attempt counter', async () => { + // The manual retry §L promises. `resetAttempts` is the licence a human has and + // the automatic sweep does not — Engagement Phase 14's rule, whose defect was + // a sweep that reset every stale row and made the attempt ceiling unreachable. + const id = seedRun({ status: 'completed', cleanupStatus: 'incomplete' }) + + const result = await controls.cleanupRun(id, ACTOR) + + assert.equal(result.ok, true) + assert.deepEqual(store.sweeps, [{ runId: id, resetAttempts: true, actor: ACTOR }]) + assert.equal(result.summary.reverted, 1) +}) + +test('cleanup refuses a run that is still in flight', async () => { + // A run still going has a ledger that is still growing, and reverting a + // resource the next step is about to use would be core undoing an event while + // it is happening. Cancel is the control for a run that should stop. + for (const status of ['scheduled', 'starting', 'running', 'paused', 'ending']) { + const id = seedRun({ status, cleanupStatus: 'pending' }) + const result = await controls.cleanupRun(id, ACTOR) + assert.equal(result.ok, false, status) + assert.match(result.errors[0], /cancel it before cleaning up after it/) + } + assert.deepEqual(store.sweeps, []) +}) + +test('cleanup refuses a run that recorded no resources', async () => { + const id = seedRun({ status: 'completed', cleanupStatus: 'not_required' }) + const result = await controls.cleanupRun(id, ACTOR) + assert.equal(result.ok, false) + assert.match(result.errors[0], /nothing to give back/) +}) + +test('cleanup on an unknown run is a 404, not a 409', async () => { + const result = await controls.cleanupRun(9999, ACTOR) + assert.equal(result.status, 404) +}) diff --git a/server/test/eventRunner.test.js b/server/test/eventRunner.test.js index d441766..e14efa4 100644 --- a/server/test/eventRunner.test.js +++ b/server/test/eventRunner.test.js @@ -45,6 +45,11 @@ const gatesDb = require('../src/model/events/eventPhaseGates.db') // every file that stubs the layer under it needs the stub. const settingsDb = require('../src/model/events/eventActionSettings.db') const budgetDb = require('../src/model/events/eventRunBudget.db') +// Phase 8 put a ledger write in front of every world-changing dispatch and a +// cleanup leg at the end of the tick. **Fourth time, same rule, and this file's +// own note is what caught it**: unstubbed, one of these is not a wrong answer, +// it is a ten-second wait on the dead port. +const resourcesDb = require('../src/model/events/eventRunResources.db') const gates = require('../src/events/gates') const db = require('../src/utils/db') @@ -58,7 +63,7 @@ const later = (ms) => new Date(T0.getTime() + ms) let store const originals = {} -for (const [name, mod] of [['runsDb', runsDb], ['stepsDb', stepsDb], ['logDb', logDb], ['versionsDb', versionsDb], ['definitionsDb', definitionsDb], ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb]]) { +for (const [name, mod] of [['runsDb', runsDb], ['stepsDb', stepsDb], ['logDb', logDb], ['versionsDb', versionsDb], ['definitionsDb', definitionsDb], ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb], ['resourcesDb', resourcesDb]]) { originals[name] = { mod, fns: { ...mod } } } @@ -79,8 +84,10 @@ function installStubs() { gates: new Map(), settings: new Map(), budget: new Map(), + resources: new Map(), nextStepId: 1, nextGateId: 1, + nextResourceId: 1, } // Phase 4 put a schedule-expansion leg in front of the tick. This file is @@ -91,6 +98,69 @@ function installStubs() { // connection timeout. Object.assign(definitionsDb, { findSchedulable: async () => [] }) + // ── The resource ledger (Phase 8) ── + // + // `reserve` enforces `uq_evres_target` in the stub, because the refusal it + // produces is BEHAVIOUR the runner branches on rather than an implementation + // detail: a placeholder that collides is this step's own earlier attempt and is + // reused, and a lease that collides is another run holding the target. A stub + // that let both inserts through would make the retry path grow a second row and + // the conflict test pass for no reason. + const HELD_STATUSES = ['pending', 'confirmed', 'reverting'] + resourcesDb.reserve = async ({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) => { + const holder = [...store.resources.values()].find( + (r) => r.owner_module === owner && r.kind === kind && r.ref === ref && HELD_STATUSES.includes(r.status), + ) + if (holder) return { ok: false, code: 'held', holder: { run_id: holder.run_id, status: holder.status } } + const id = store.nextResourceId++ + store.resources.set(id, { + id, + run_id: runId, + step_id: stepId, + owner_module: owner, + kind, + ref, + payload, + lease_until: leaseUntil, + status: 'pending', + revert_attempts: 0, + last_error: null, + member_key: memberKey, + }) + return { ok: true, id } + } + resourcesDb.confirm = async (id) => { + const r = store.resources.get(id) + if (!r || r.status !== 'pending') return false + r.status = 'confirmed' + return true + } + resourcesDb.resolvePlaceholder = async (id) => { + const r = store.resources.get(id) + if (!r || r.kind !== resourcesDb.STEP_KIND || !['pending', 'confirmed'].includes(r.status)) return false + r.status = 'reverted' + return true + } + resourcesDb.findByTarget = async (owner, kind, ref) => + [...store.resources.values()].reverse().find((r) => r.owner_module === owner && r.kind === kind && r.ref === ref) || null + resourcesDb.forRun = async (runId) => [...store.resources.values()].filter((r) => r.run_id === runId) + resourcesDb.markReverted = async (id) => { + const r = store.resources.get(id) + if (r) r.status = 'reverted' + } + // The cleanup leg's scan. This file is about the runner's own legs, and the + // sweep has its own file — answering with nothing is what keeps every `tick()` + // here measuring the runner rather than a teardown. + resourcesDb.runsNeedingCleanup = async () => [] + + runsDb.setCleanupStatus = async (id, to, from = null) => { + const r = store.runs.get(id) + if (!r) return false + if (from && !from.includes(r.cleanup_status)) return false + r.cleanup_status = to + return true + } + // Snapshots, not live references. A SQL SELECT hands back a copy, and the // runner reads `step.attempts` as the value BEFORE its own claim incremented // it — returning references here would make the retry budget off by one in the @@ -1319,3 +1389,163 @@ test('the runner never re-checks the role of whoever started the run', async () assert.equal(stepsOf(id)[0].status, 'done') }) + +// ── The resource ledger, from the runner's side (Phase 8) ────────────────── +// +// `eventLedger.test.js` owns the recording RULES and `eventCleanup.test.js` owns +// the undo. What belongs here is the ORDER — that the placeholder is written +// before the module is reached and after the permission check, and that a step +// which never answers leaves it standing. Those are properties of `drainStep`, +// and nothing below the runner can observe them. + +const ledgerRows = (id) => [...store.resources.values()].filter((r) => r.run_id === id) + +test('a world-changing step is recorded BEFORE it is dispatched', async () => { + // §D rule 1. The assertion is made from INSIDE `perform()`, which is the only + // place that can tell "recorded first" from "recorded at all" — and the + // difference between the two is every object whose acknowledgement is lost. + let seenDuringDispatch = null + register([scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) })]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn')] }]) + scripted['test.spawn'] = { + calls: [], + answer: () => { + seenDuringDispatch = ledgerRows(id).map((r) => [r.kind, r.status]) + return { ok: true, resources: [{ kind: 'creature', ref: '0x40001234' }] } + }, + } + + await runner.tick(T0) + + assert.deepEqual(seenDuringDispatch, [['@step', 'pending']]) + // And on the answer the real row exists and the placeholder is done with. + assert.deepEqual( + ledgerRows(id).map((r) => [r.kind, r.ref, r.status]), + [ + ['@step', stepsOf(id)[0].idempotency_key, 'reverted'], + ['creature', '0x40001234', 'confirmed'], + ], + ) + assert.equal(run(id).cleanup_status, 'pending') + assert.ok(kinds(id).includes('resource.recorded')) +}) + +test('a step that never answers leaves its placeholder pending', async () => { + // The whole reason the placeholder exists. The module timed out, so core does + // not know whether anything was created — and the row that survives is what + // lets cleanup ask it by idempotency key later. Record on the answer instead + // and this run finishes looking spotless over a shard full of orphans. + register([ + scriptedAction('test.spawn', { + risk: 'change', + reversible: 'ledger', + budgetMs: 5, + revert: async () => ({ ok: true }), + perform: () => new Promise(() => {}), + }), + ]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + + await runner.tick(T0) + + assert.deepEqual(ledgerRows(id).map((r) => [r.kind, r.status]), [['@step', 'pending']]) +}) + +test('a refused step ledgers nothing, because it never reached the module', async () => { + // The placeholder is written AFTER the permission check, deliberately. A step + // refused by a cap or by a switch created nothing, and a ledger row for it + // would be core asking a module to undo something it was never asked to do. + register([scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) })]) + // No `setSwitch`, so it is default-off: §K's world-changing default. + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + + await runner.tick(T0) + + assert.equal(stepsOf(id)[0].status, 'refused') + assert.deepEqual(ledgerRows(id), []) + assert.equal(run(id).cleanup_status, 'not_required') +}) + +test('an announce step ledgers nothing at all', async () => { + // `reversible: 'none'`, so there is nothing core could come back for. A + // placeholder here would be a row teardown could never resolve — which is the + // reason `core.announce` is declared `none` rather than `ledger` even though a + // sent message cannot be unsent. + register([scriptedAction('test.say')]) + const id = seedRun([{ key: 'main', steps: [step('test.say')] }]) + await runner.tick(T0) + assert.equal(run(id).status, 'completed') + assert.deepEqual(ledgerRows(id), []) + assert.equal(run(id).cleanup_status, 'not_required') +}) + +test('a parked step records what it made, because its confirm never dispatches again', async () => { + // `await: 'human'` is a SUCCESS: the module did its part and something outside + // the system has to happen next. The cue's confirm finishes the step without a + // second dispatch, so this is the only moment its resources can be recorded. + register([ + scriptedAction('test.stage', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }), + ]) + setSwitch('test.stage', true) + const id = seedRun([{ key: 'main', steps: [step('test.stage')] }]) + scripted['test.stage'] = { + calls: [], + answer: { ok: true, await: 'human', resources: [{ kind: 'prop', ref: 'gate-1' }] }, + } + + await runner.tick(T0) + + assert.equal(stepsOf(id)[0].status, 'running') + assert.deepEqual( + ledgerRows(id).map((r) => [r.kind, r.status]), + [['@step', 'reverted'], ['prop', 'confirmed']], + ) +}) + +test('a retry re-uses its placeholder and records the resources once', async () => { + // An idempotency key does not vary by attempt (§E), so the second attempt's + // placeholder insert collides with the first attempt's row — and a module that + // honestly re-reports the same creature must not produce a second thing for + // cleanup to revert. + register([ + scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }), + ]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + scripted['test.spawn'] = { + calls: [], + answers: [ + { ok: false, error: 'the shard did not answer' }, + { ok: true, resources: [{ kind: 'creature', ref: '0xFEED' }] }, + ], + } + + await runner.tick(T0) + await runner.tick(later(runner.RETRY_MS + 1000)) + + assert.equal(stepsOf(id)[0].status, 'done') + assert.equal(ledgerRows(id).filter((r) => r.kind === '@step').length, 1) + assert.equal(ledgerRows(id).filter((r) => r.kind === 'creature').length, 1) +}) + +test('a ledger write that fails stops the dispatch rather than losing the record', async () => { + // The ledger is what makes a world write recoverable, so a step that cannot be + // recorded must not be sent. Transient, because the alternative is an + // unrecorded world change — the one outcome §D rule 1 exists to make impossible. + register([ + scriptedAction('test.spawn', { risk: 'change', reversible: 'ledger', revert: async () => ({ ok: true }) }), + ]) + setSwitch('test.spawn', true) + const id = seedRun([{ key: 'main', steps: [step('test.spawn', {}, 'skip')] }]) + scripted['test.spawn'] = { calls: [], answer: { ok: true } } + resourcesDb.reserve = async () => { + throw new Error('the ledger is unreachable') + } + + await runner.tick(T0) + + assert.equal(scripted['test.spawn'].calls.length, 0, 'the module must not have been reached') + assert.match(stepsOf(id)[0].last_error, /the resource ledger could not record this step/) +}) diff --git a/server/test/eventRunnerSql.test.js b/server/test/eventRunnerSql.test.js index ac66978..ce69605 100644 --- a/server/test/eventRunnerSql.test.js +++ b/server/test/eventRunnerSql.test.js @@ -80,6 +80,18 @@ // that died between entering a phase and opening its gate opens no second // one on the next tick. // +// **Phase 8 added the resource ledger**, and its unique key is the single most +// server-dependent thing in this feature. "Two events cannot hold a lease on one +// target" has to hold among LIVE rows only — last week's finished event must not +// keep this week's from leasing the same rate — and MariaDB has no partial index, +// so the encoding is a STORED generated column that goes NULL once the row is no +// longer ours. Whether multiple NULLs collide in a unique index is a property of +// the server and of nothing else, and TEAMS.md §2.5 already had to be corrected +// once on this exact shape: MariaDB refuses ON DELETE SET NULL on a foreign key +// whose column is a base column of a stored generated column (error 1901), which +// is why the expression reads `status` alone and `step_id` stays a SET NULL FK. +// Both halves of that are proved below rather than believed. +// // Plus the two unique indexes that are load-bearing rather than tidy: // `uq_evrun_occurrence` (which, not the claim, is what stops two runs of one // occurrence existing) and `uq_evstep_slot` (which is what makes re-materialising @@ -208,6 +220,29 @@ CREATE TABLE event_run_budget ( CONSTRAINT fk_evbud_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE, UNIQUE KEY uq_evbud_dim (run_id, dimension) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE event_run_resources ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + run_id BIGINT NOT NULL, + step_id BIGINT NULL, + owner_module VARCHAR(64) NOT NULL, + kind VARCHAR(64) NOT NULL, + ref VARCHAR(190) NOT NULL, + payload JSON NULL, + lease_until DATETIME NULL, + status ENUM('pending','confirmed','reverting','reverted','orphaned','drifted') + NOT NULL DEFAULT 'pending', + revert_attempts INT NOT NULL DEFAULT 0, + last_error VARCHAR(500) NULL, + member_key VARCHAR(190) NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + live_marker TINYINT AS (IF(status IN ('pending','confirmed','reverting'), 1, NULL)) STORED, + CONSTRAINT fk_evres_run FOREIGN KEY (run_id) REFERENCES event_runs(id) ON DELETE CASCADE, + CONSTRAINT fk_evres_step FOREIGN KEY (step_id) REFERENCES event_run_steps(id) ON DELETE SET NULL, + UNIQUE KEY uq_evres_target (owner_module, kind, ref, live_marker), + INDEX idx_evres_run (run_id, status), + INDEX idx_evres_live (status, lease_until) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE event_action_settings ( action_id VARCHAR(96) NOT NULL PRIMARY KEY, enabled TINYINT(1) NOT NULL DEFAULT 0, @@ -438,6 +473,7 @@ const rows = (r) => Number(r.affectedRows) beforeEach(async () => { if (!available) return await pool.query('DELETE FROM event_run_phase_gates') + await pool.query('DELETE FROM event_run_resources') await pool.query('DELETE FROM event_run_steps') await pool.query('DELETE FROM event_runs') await pool.query('DELETE FROM event_definitions') @@ -1139,7 +1175,21 @@ test('a gate goes with its run', async (t) => { // live server, so the pool holds open connections and the process never exits — // 49 green tests and a file that hangs until the harness kills it. Every other // event test file already closes it in an `after`; this one now has a reason to. +// +// **And that second pool has to be aimed at the throwaway database**, which it +// was not before Phase 8 noticed. `utils/db` builds its pool at REQUIRE time from +// `DB_NAME`, and its own `dotenv.config()` reads `server/.env` — so a model test +// run on a developer's machine was reaching that developer's real schema while +// the fixtures it was asserting against were being written next door. It passed +// only because the two tables happened to exist in both. `dotenv` does not +// overwrite a variable that already exists, so setting it here, before the +// require below, is what beats the file. This file drops its database in `after`, +// so aiming the model pool at it is also what keeps the whole run disposable. +process.env.DB_NAME = DB + const budgetDb = require('../src/model/events/eventRunBudget.db') +const resourcesDb = require('../src/model/events/eventRunResources.db') +const runsDb = require('../src/model/events/eventRuns.db') const appDb = require('../src/utils/db') after(() => appDb.close()) @@ -1303,3 +1353,324 @@ test('a non-positive spend never reaches the database', async (t) => { assert.equal(await budgetDb.spend(runId, 'uo.creatures', 0), true) assert.equal(await consumedOf(runId), 0) }) + +// ── The resource ledger's unique key (Phase 8) ───────────────────────────── +// +// Every test here is about a property of the SERVER. A stub can enforce whatever +// rule its author had in mind; only MariaDB can say whether this encoding of +// "unique among live rows" actually is one. + +const insertResource = async (runId, over = {}) => + ( + await pool.query( + `INSERT INTO event_run_resources (run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + runId, + over.stepId ?? null, + over.owner ?? 'demo', + over.kind ?? 'creature', + over.ref ?? '0xA', + over.payload ?? null, + over.leaseUntil ?? null, + over.memberKey ?? null, + over.status ?? 'pending', + ], + ) + ).insertId + +const dup = async (fn) => { + try { + await fn() + return null + } catch (err) { + return err.code || String(err.errno) + } +} + +test('two runs cannot hold one target, and the refusal comes from the server', async (t) => { + if (needDb(t)) return + // §D: "the unique index is what stops two events leasing one target." Not a + // read-then-insert — two runs entering the same tick would both pass the check + // — so the whole conflict story is this one constraint answering. + const a = await seedRun() + const b = await seedRun() + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' }) + const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })) + assert.equal(code, 'ER_DUP_ENTRY') +}) + +test('the key is released by the three statuses that mean it is no longer ours', async (t) => { + if (needDb(t)) return + // Amended 2026-09-03. §D says "among non-reverted rows", which was written + // before the six statuses had their meanings; taken literally it makes + // `drifted` and `orphaned` hold a target for ever, so one bad night would + // disable a lease permanently with no control able to clear it. `drifted` means + // somebody else has hold of the value and this run has let go; `orphaned` means + // it vanished. Neither is a claim on the target. + for (const status of ['reverted', 'drifted', 'orphaned']) { + await pool.query('DELETE FROM event_run_resources') + const a = await seedRun() + const b = await seedRun() + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status }) + const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })) + assert.equal(code, null, `a ${status} row must not hold the target`) + } +}) + +test('the key is HELD by the three that mean core still believes it is ours', async (t) => { + if (needDb(t)) return + for (const status of ['pending', 'confirmed', 'reverting']) { + await pool.query('DELETE FROM event_run_resources') + const a = await seedRun() + const b = await seedRun() + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status }) + const code = await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })) + assert.equal(code, 'ER_DUP_ENTRY', `a ${status} row must hold the target`) + } +}) + +test('many released rows on one target coexist, which is the whole encoding', async (t) => { + if (needDb(t)) return + // The property the NULL depends on: multiple NULLs do not collide in a unique + // index. A weekly event that leases the same rate every Saturday accumulates one + // released row per week, and the fiftieth must not fail to insert. + const a = await seedRun() + for (let i = 0; i < 5; i++) { + await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'reverted' }) + } + const held = await pool.query( + 'SELECT COUNT(*) AS n FROM event_run_resources WHERE ref = ? AND live_marker IS NULL', + ['demo.rate'], + ) + assert.equal(Number(held[0].n), 5) + // And a live one still goes on top of them. + assert.equal(await dup(() => insertResource(a.runId, { kind: 'override', ref: 'demo.rate' })), null) +}) + +test('an UPDATE that releases a row frees the target at once', async (t) => { + if (needDb(t)) return + // The generated column is STORED, so this is really asking whether MariaDB + // recomputes it on UPDATE and re-indexes. Cleanup depends on it entirely: the + // moment a lease is restored, the next event may take it. + const a = await seedRun() + const b = await seedRun() + const id = await insertResource(a.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' }) + assert.equal(await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })), 'ER_DUP_ENTRY') + await pool.query("UPDATE event_run_resources SET status = 'reverted' WHERE id = ?", [id]) + assert.equal(await dup(() => insertResource(b.runId, { kind: 'override', ref: 'demo.rate' })), null) +}) + +test('the key is per owner and per kind, not per ref', async (t) => { + if (needDb(t)) return + // `kind` and `ref` are module-opaque strings core stores verbatim, so two + // modules using the same word for different things must not collide — and one + // module's `creature:0xA` and `item:0xA` are two objects. + const a = await seedRun() + await insertResource(a.runId, { owner: 'demo', kind: 'creature', ref: '0xA', status: 'confirmed' }) + assert.equal(await dup(() => insertResource(a.runId, { owner: 'other', kind: 'creature', ref: '0xA' })), null) + assert.equal(await dup(() => insertResource(a.runId, { owner: 'demo', kind: 'item', ref: '0xA' })), null) + assert.equal(await dup(() => insertResource(a.runId, { owner: 'demo', kind: 'creature', ref: '0xB' })), null) +}) + +test('a deleted step leaves its resources behind, and a deleted run does not', async (t) => { + if (needDb(t)) return + // TEAMS.md §2.5's correction, held as a test: `step_id` is SET NULL and it only + // works because the generated column reads `status` alone. If `step_id` were in + // that expression MariaDB would refuse the constraint outright (error 1901), and + // the migration would fail on a fresh install rather than here. + // + // The two directions are different on purpose. A record of what was changed in + // the WORLD must outlive the row that scheduled it — `engagement_sends`' + // argument — while a deleted RUN takes its ledger with it, because the ledger + // exists to answer questions about a run. + const a = await seedRun() + const stepId = await seedStep(a.runId) + const id = await insertResource(a.runId, { stepId, status: 'confirmed' }) + + await pool.query('DELETE FROM event_run_steps WHERE id = ?', [stepId]) + const orphaned = (await pool.query('SELECT step_id, status FROM event_run_resources WHERE id = ?', [id]))[0] + assert.equal(orphaned.step_id, null) + assert.equal(orphaned.status, 'confirmed') + + await pool.query('DELETE FROM event_runs WHERE id = ?', [a.runId]) + const gone = await pool.query('SELECT id FROM event_run_resources WHERE id = ?', [id]) + assert.equal(gone.length, 0) +}) + +test('the ledger model reserves, confirms, reverts and refuses for real', async (t) => { + if (needDb(t)) return + // Through the shipping module rather than a copy of its statements, like the + // budget tests above: `reserve` reads ER_DUP_ENTRY as a refusal and looks the + // holder up to name it, and both halves of that are the connector's behaviour + // rather than this file's. + const a = await seedRun() + const b = await seedRun() + + const first = await resourcesDb.reserve({ runId: a.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' }) + assert.equal(first.ok, true) + await resourcesDb.confirm(first.id) + + const second = await resourcesDb.reserve({ runId: b.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' }) + assert.equal(second.ok, false) + assert.equal(second.code, 'held') + assert.equal(Number(second.holder.run_id), Number(a.runId)) + assert.equal(second.holder.status, 'confirmed') + + // And once it is given back the second run gets it. + await resourcesDb.markReverted(first.id) + const third = await resourcesDb.reserve({ runId: b.runId, owner: 'demo', kind: 'override', ref: 'demo.rate' }) + assert.equal(third.ok, true) +}) + +test('failRevert increments and NEVER resets, and only a human clears it', async (t) => { + if (needDb(t)) return + // Engagement Phase 14's rule at the statement level: `revert_attempts = + // revert_attempts + 1` is written in one place, and the reset is a separate + // statement with an actor behind it. A `SET revert_attempts = ?` anywhere in + // the sweep would make the ceiling unreachable and the row cycle for ever. + const a = await seedRun() + const id = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' }) + .then((r) => r.id) + await resourcesDb.confirm(id) + + await resourcesDb.failRevert(id, 'the shard did not answer') + await resourcesDb.failRevert(id, 'still nothing') + let row = (await pool.query('SELECT * FROM event_run_resources WHERE id = ?', [id]))[0] + assert.equal(Number(row.revert_attempts), 2) + assert.equal(row.status, 'confirmed') + assert.equal(row.last_error, 'still nothing') + + assert.equal(await resourcesDb.resetAttempts(a.runId), 1) + row = (await pool.query('SELECT * FROM event_run_resources WHERE id = ?', [id]))[0] + assert.equal(Number(row.revert_attempts), 0) +}) + +test('claimRevert is a compare-and-set, and reverting is not re-claimable', async (t) => { + if (needDb(t)) return + // The cleanup leg and the manual cleanup route can both be working one run at + // once. `reverting` is deliberately not claimable — a row another pass is + // mid-revert on is left alone, exactly as a step with a live claim is. + const a = await seedRun() + const id = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' }) + .then((r) => r.id) + await resourcesDb.confirm(id) + + assert.equal(await resourcesDb.claimRevert(id), true) + assert.equal(await resourcesDb.claimRevert(id), false, 'a second pass must not take a row mid-revert') + + await resourcesDb.markReverted(id) + assert.equal(await resourcesDb.claimRevert(id), false, 'and a reverted row is finished') +}) + +test('the placeholder is resolvable exactly once, and only if it is a placeholder', async (t) => { + if (needDb(t)) return + // The `kind = '@step'` guard in the statement, not in JavaScript. It is what + // stops a bug elsewhere resolving a real resource — which would be core marking + // a live creature as given back without asking anyone. + const a = await seedRun() + const placeholder = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: resourcesDb.STEP_KIND, ref: 'k'.repeat(40) }) + .then((r) => r.id) + const real = await resourcesDb + .reserve({ runId: a.runId, owner: 'demo', kind: 'creature', ref: '0xA' }) + .then((r) => r.id) + + assert.equal(await resourcesDb.resolvePlaceholder(placeholder), true) + assert.equal(await resourcesDb.resolvePlaceholder(placeholder), false) + assert.equal(await resourcesDb.resolvePlaceholder(real), false) + const stillThere = (await pool.query('SELECT status FROM event_run_resources WHERE id = ?', [real]))[0] + assert.equal(stillThere.status, 'pending') +}) + +test('the unresolved reads see the five statuses that still want something', async (t) => { + if (needDb(t)) return + // `cleanup_status` is derived from this count, so what it includes IS the + // definition of "clean". A `drifted` row left out of it would let a run whose + // lease somebody else took call itself complete. + const a = await seedRun() + for (const status of ['pending', 'confirmed', 'reverting', 'reverted', 'orphaned', 'drifted']) { + await insertResource(a.runId, { ref: `ref-${status}`, status }) + } + assert.equal(await resourcesDb.unresolvedCount(a.runId), 5) + const rows = await resourcesDb.unresolvedForRun(a.runId) + assert.deepEqual( + rows.map((r) => r.status).sort(), + ['confirmed', 'drifted', 'orphaned', 'pending', 'reverting'], + ) + const counts = await resourcesDb.unresolvedCounts([a.runId]) + assert.equal(counts.get(a.runId) ?? counts.get(String(a.runId)), 5) +}) + +test('payload comes back as an object rather than as a string', async (t) => { + if (needDb(t)) return + // A lease's baseline lives in here and the cleanup sweep reads it out to pass + // to `restore`. The connector hands JSON back as text, so a missing hydration + // is a `restore(undefined)` — a lease put back to nothing, silently. + const a = await seedRun() + const id = await resourcesDb + .reserve({ + runId: a.runId, + owner: 'demo', + kind: 'override', + ref: 'demo.rate', + payload: { baseline: 1, applied: 3 }, + }) + .then((r) => r.id) + void id + const [row] = await resourcesDb.forRun(a.runId) + assert.deepEqual(row.payload, { baseline: 1, applied: 3 }) +}) + +test('the cleanup scan finds a run that owes something, whatever its status column says', async (t) => { + if (needDb(t)) return + await pool.query('ALTER TABLE event_runs ADD COLUMN IF NOT EXISTS cleanup_status ' + + "ENUM('not_required','pending','complete','incomplete') NOT NULL DEFAULT 'not_required'") + + // **`not_required` is in the scan because of a live-walk defect**, not for + // tidiness. A run whose only resource was a LEASE never went through the + // ledger's `markRunDirty` — `core.lease` reserves its own row — so its column + // stayed `not_required` and the lease was never given back at all. A terminal + // run with an unresolved row has work to do whatever any summary column says. + const lease = await seedRun({ status: 'completed' }) + await insertResource(lease.runId, { kind: 'override', ref: 'demo.rate', status: 'confirmed' }) + const marked = await seedRun({ status: 'cancelled' }) + await insertResource(marked.runId, { ref: '0xB', status: 'pending' }) + await pool.query("UPDATE event_runs SET cleanup_status = 'pending' WHERE id = ?", [marked.runId]) + + // Three that must NOT be selected, one per reason. + const running = await seedRun({ status: 'running' }) + await insertResource(running.runId, { ref: '0xC', status: 'confirmed' }) + const done = await seedRun({ status: 'completed' }) + await insertResource(done.runId, { ref: '0xD', status: 'reverted' }) + const givenUp = await seedRun({ status: 'completed' }) + await insertResource(givenUp.runId, { ref: '0xE', status: 'confirmed' }) + await pool.query("UPDATE event_runs SET cleanup_status = 'incomplete' WHERE id = ?", [givenUp.runId]) + + const found = (await resourcesDb.runsNeedingCleanup(10, 3)).map((r) => Number(r.id)) + assert.deepEqual(found.sort(), [Number(lease.runId), Number(marked.runId)].sort()) +}) + +test('the scan stops selecting a run whose attempts are spent', async (t) => { + if (needDb(t)) return + // Without the bound in the join, a run whose rows are all spent would be + // selected, worked over and found to have nothing to do on every tick for the + // rest of its life. + const a = await seedRun({ status: 'completed' }) + const id = await insertResource(a.runId, { status: 'confirmed' }) + await pool.query("UPDATE event_runs SET cleanup_status = 'pending' WHERE id = ?", [a.runId]) + assert.equal((await resourcesDb.runsNeedingCleanup(10, 3)).length, 1) + + await pool.query('UPDATE event_run_resources SET revert_attempts = 3 WHERE id = ?', [id]) + assert.equal((await resourcesDb.runsNeedingCleanup(10, 3)).length, 0) +}) + +test('setCleanupStatus is guarded, which is what stops a late row re-opening a swept run', async (t) => { + if (needDb(t)) return + const a = await seedRun({ status: 'completed' }) + assert.equal(await runsDb.setCleanupStatus(a.runId, 'complete'), true) + assert.equal(await runsDb.setCleanupStatus(a.runId, 'pending', ['not_required']), false) + assert.equal(await runsDb.setCleanupStatus(a.runId, 'pending'), true) +}) diff --git a/server/test/eventsAdmin.test.js b/server/test/eventsAdmin.test.js index ff3179d..0b77599 100644 --- a/server/test/eventsAdmin.test.js +++ b/server/test/eventsAdmin.test.js @@ -42,6 +42,13 @@ const logDb = require('../src/model/events/eventRunLog.db') // model needs a stub in every file that stubs that layer. const settingsDb = require('../src/model/events/eventActionSettings.db') const budgetDb = require('../src/model/events/eventRunBudget.db') +// Phase 8: the run console reads the resource ledger. The SAME rule, for the +// fourth time in this feature -- Phase 4's expansion leg, Phase 5's gate read, +// Phase 6's settings read and now this one. **Unstubbed it is not a wrong +// answer, it is a ten-second ECONNREFUSED against the dead port**, which is why +// one missing stub here cost the run detail test ten seconds and said nothing +// about the route it was testing. +const resourcesDb = require('../src/model/events/eventRunResources.db') const seriesDb = require('../src/model/events/eventSeries.db') const gatesDb = require('../src/model/events/eventPhaseGates.db') const activity = require('../src/model/activity/activity.model') @@ -63,6 +70,7 @@ for (const [name, mod] of [ ['gatesDb', gatesDb], ['settingsDb', settingsDb], ['budgetDb', budgetDb], + ['resourcesDb', resourcesDb], ['activity', activity], ]) { originals[name] = { mod, fns: { ...mod } } @@ -88,6 +96,7 @@ function installStubs() { gates: [], settings: new Map(), budget: new Map(), + resources: [], occurrences: new Set(), nextDefinition: 1, nextVersion: 1, @@ -341,6 +350,9 @@ function installStubs() { [...store.budget.values()] .filter((b) => Number(b.run_id) === Number(runId)) .sort((a, b) => a.dimension.localeCompare(b.dimension)) + + resourcesDb.forRun = async (runId) => + store.resources.filter((r) => Number(r.run_id) === Number(runId)) } // ── Fixtures ─────────────────────────────────────────────────────────────── @@ -415,23 +427,26 @@ test('the catalog serves the registry, callables stripped, with its vocabularies assert.equal(res.statusCode, 200) assert.deepEqual( res.body.actions.map((a) => a.id), - ['core.announce', 'core.wait', 'core.cue'], + ['core.announce', 'core.wait', 'core.cue', 'core.lease'], ) for (const action of res.body.actions) assert.equal(action.perform, undefined) assert.deepEqual(res.body.risks, ['notify', 'inspect', 'change', 'irreversible']) assert.deepEqual(res.body.onFailure, ['skip', 'pause', 'abort_run']) // The other three registrations of the module contract arrived in Phase 7, and // they are served BESIDE the actions because the editor needs all four to draw - // one step. Core declares no budgets and no leases of its own — its three - // actions cost nothing and hold nothing — so those are empty here, and that is - // the fact worth asserting: present and empty, not absent. + // one step. Core declares no budgets and no leases of its own — its four + // actions cost nothing, and `core.lease` BORROWS a lease rather than declaring + // one, which is the seam working the way round it is meant to: core owns the + // verb, a module owns the value. So both are empty here, and that is the fact + // worth asserting: present and empty, not absent. assert.deepEqual(res.body.budgets, []) assert.deepEqual(res.body.leases, []) - // One option source, and it is core's: `core.announce`'s leg param. It is here - // WITHOUT its resolver — the values are a request of their own. + // Two option sources, both core's: `core.announce`'s leg and `core.lease`'s + // lease. They are here WITHOUT their resolvers — the values are a request of + // their own. assert.deepEqual( res.body.optionSources.map((s) => s.id), - ['core.options.legs'], + ['core.options.legs', 'core.options.leases'], ) for (const s of res.body.optionSources) assert.equal(s.resolve, undefined) }) @@ -824,7 +839,12 @@ test('the board serves every registered action with its risk-class default, and assert.equal(res.statusCode, 200) const byId = Object.fromEntries(res.body.actions.map((a) => [a.id, a])) - assert.deepEqual(Object.keys(byId).sort(), ['core.announce', 'core.cue', 'core.wait']) + assert.deepEqual(Object.keys(byId).sort(), ['core.announce', 'core.cue', 'core.lease', 'core.wait']) + // And `core.lease` is the one core action the default-off rule bites: it is + // `change`, so a fresh deployment cannot borrow a value until an admin says so. + // §K's sentence, applied to core's own verb rather than only to a module's. + assert.equal(byId['core.lease'].enabled, false) + assert.equal(byId['core.lease'].changesWorld, true) // core.wait is `inspect`, and it arrives ENABLED. Read §K's sentence literally // and it would not, and every published event that waits would break on a fresh // deployment (org lead, 2026-09-03). @@ -843,6 +863,7 @@ test('the board never serves a callable', async () => { for (const a of res.body.actions) { assert.equal(a.perform, undefined) assert.equal(a.revert, undefined) + assert.equal(a.reconcile, undefined) assert.equal(a.cost, undefined) } }) diff --git a/server/test/eventsRoles.test.js b/server/test/eventsRoles.test.js index 9abd2dc..d0d64f5 100644 --- a/server/test/eventsRoles.test.js +++ b/server/test/eventsRoles.test.js @@ -162,6 +162,11 @@ const SURFACE = [ // Phase 6's switchboard — configuration that can break things. ['GET', '/events/actions', ['admin']], ['PUT', '/events/actions', ['admin']], + // Phase 8's cleanup, and it sits in the ADMIN column rather than with the live + // controls it is rendered beside. Re-running a teardown is not incident + // response — it asks core to write to the world again, which §K puts in the + // same row as the world-changing actions themselves. + ['POST', '/events/runs/1/cleanup', ['admin']], // Live control of a run in flight: admin and moderator, deliberately WIDER // than start. @@ -214,6 +219,19 @@ test('an editor may price an event but not publish or start it', async () => { assert.equal(await forbidden('POST', '/events/1/runs', 'editor'), true) }) +test('a moderator may stop a run but not re-run its cleanup', async () => { + // The same shape as start-and-stop above, one row further on, and held as its + // own claim for the same reason: the two controls sit next to each other on the + // run console and a later tidying pass that gave them one gate would have to + // delete an assertion that says why they do not share one. + // + // Cancelling is the 2am incident. Cleanup asks core to delete things in a live + // world, which is the narrower decision even though it is the tidier-sounding + // button. + assert.equal(await forbidden('POST', '/events/runs/1/cancel', 'moderator'), false) + assert.equal(await forbidden('POST', '/events/runs/1/cleanup', 'moderator'), true) +}) + test('the switchboard is admin only in both directions', async () => { // Reading which actions are enabled is as much `admin` as writing it: the board // is the deployment's posture, and §K puts it in the same row as the actions it