// ── event_run_resources — SQL only ───────────────────────────────────────── // // EVENTS.md §D and §L ("The ledger's two rules"), and Phase 8 of EVENTS_PLAN.md. // Everything one run created or leased, and what became of it. // // **Rule 1 lives in `reserve()`.** A resource is recorded BEFORE it is // confirmed, so the placeholder this writes is the row that exists while the // dispatch is in flight — and the row that SURVIVES when the acknowledgement is // lost. Recording on the answer instead would make every object whose ack went // missing invisible to cleanup for ever. // // **Rule 2 lives in the status column and in `failRevert()`.** A revert that // never succeeds leaves its row unreverted, with the error on it, and the run // completes with `cleanup_status = 'incomplete'` rather than being held open. // Loud and sticky. // // **The unique key is enforced by the database, not by a read.** `reserve()` // answers `{ ok: false, code: 'held' }` on a duplicate key rather than checking // first and then inserting — two runs entering the same tick would both pass the // check. It is the argument `event_run_budget.spend()` makes about the cap and // `runsDb.transition` makes about a status, in the third place it applies. const { query } = require('../../utils/db') const { parseJson } = require('./eventJson') // The one `kind` core owns. A module's kinds are opaque and stored verbatim; this // one is core's own, and `registries` refuses a module resource that claims it. const STEP_KIND = '@step' // The statuses that mean "core still believes this resource is this run's". They // are exactly the ones the `live_marker` generated column keeps non-NULL, so the // unique target key holds while a row is in one of them and releases when it // leaves. Duplicated here as a JavaScript list because the sweeps read by it too, // and a second copy that can drift is better than a query that cannot express it. const HELD = ['pending', 'confirmed', 'reverting'] // Every status that still wants a human or a retry: `HELD` plus the two that mean // "we let go, and not cleanly". This is what "unreverted" means everywhere in // this feature — the console's list, `cleanup_status`, and the manual retry. const UNRESOLVED = [...HELD, 'orphaned', 'drifted'] const COLUMNS = `id, run_id, step_id, owner_module, kind, ref, payload, lease_until, status, revert_attempts, last_error, member_key, created_at, updated_at` // `payload` is opaque to core and stored verbatim, but it comes back as a string // from the driver and every caller wants the object — the cleanup sweep reads a // lease's baseline out of it, and the console renders it. Hydrated here for the // same reason a step's params are: one place rather than at each read. const hydrate = (row) => row && { ...row, payload: parseJson(row.payload, null) } /** * Record a resource that does not exist yet. * * Answers `{ ok: true, id }`, or `{ ok: false, code: 'held', holder }` when the * target is already someone's — which is the lease conflict, surfaced as a * refusal rather than a failure because nothing is wrong with the system: another * run has the thing. * * **`ER_DUP_ENTRY` is the check.** The holder is looked up only to name it in the * refusal, and only after the insert has already lost the race. */ async function reserve({ runId, stepId = null, owner, kind, ref, payload = null, leaseUntil = null, memberKey = null }) { try { const result = await query( `INSERT INTO event_run_resources (run_id, step_id, owner_module, kind, ref, payload, lease_until, member_key, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')`, [runId, stepId, owner, kind, ref, payload === null ? null : JSON.stringify(payload), leaseUntil, memberKey], ) return { ok: true, id: Number(result.insertId) } } catch (err) { if (err && (err.code === 'ER_DUP_ENTRY' || err.errno === 1062)) { const [holder] = await query( `SELECT run_id, status FROM event_run_resources WHERE owner_module = ? AND kind = ? AND ref = ? AND status IN (?, ?, ?) LIMIT 1`, [owner, kind, ref, ...HELD], ) return { ok: false, code: 'held', holder: holder || null } } throw err } } /** * Promote a reserved row to `confirmed`, optionally attaching what the module * finally said about it. * * Guarded on `pending` so a late answer cannot un-revert a row cleanup has * already dealt with — the same reason every other write in this feature is a * compare-and-set rather than a read-then-write. */ async function confirm(id, { payload, leaseUntil, memberKey } = {}) { const sets = ["status = 'confirmed'"] const params = [] if (payload !== undefined) { sets.push('payload = ?') params.push(payload === null ? null : JSON.stringify(payload)) } if (leaseUntil !== undefined) { sets.push('lease_until = ?') params.push(leaseUntil) } if (memberKey !== undefined) { sets.push('member_key = ?') params.push(memberKey) } const result = await query( `UPDATE event_run_resources SET ${sets.join(', ')} WHERE id = ? AND status = 'pending'`, [...params, id], ) return (result.affectedRows || 0) > 0 } /** * Resolve a step placeholder once the module has named what it actually made. * * The placeholder's whole job is over at this point: the real rows exist, so the * `@step` row must stop being one of the things cleanup will try to revert. * `reverted` is the honest terminal state for it — there is nothing left to undo * that the rows it stood in for do not now cover — and it releases the * idempotency key for a later run, which matters because keys are per step and a * re-materialised step reuses its own. */ async function resolvePlaceholder(id) { const result = await query( `UPDATE event_run_resources SET status = 'reverted', last_error = NULL WHERE id = ? AND kind = ? AND status IN ('pending', 'confirmed')`, [id, STEP_KIND], ) return (result.affectedRows || 0) > 0 } /** * One row by its target, live or not — how a caller that lost the insert race * finds the row it meant to write. Newest first, so a target that has been held * and released several times answers with the current holder. */ async function findByTarget(owner, kind, ref) { const [row] = await query( `SELECT ${COLUMNS} FROM event_run_resources WHERE owner_module = ? AND kind = ? AND ref = ? ORDER BY id DESC LIMIT 1`, [owner, kind, ref], ) return hydrate(row) || null } /** One run's whole ledger, oldest first — the console's read. */ async function forRun(runId) { const rows = await query( `SELECT ${COLUMNS} FROM event_run_resources WHERE run_id = ? ORDER BY id`, [runId], ) return rows.map(hydrate) } /** The rows of one run that still want something: the cleanup sweep's input. */ async function unresolvedForRun(runId, { maxAttempts = null } = {}) { const params = [runId, ...UNRESOLVED] const attemptClause = maxAttempts === null ? '' : ' AND revert_attempts < ?' if (maxAttempts !== null) params.push(maxAttempts) const rows = await query( `SELECT ${COLUMNS} FROM event_run_resources WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)${attemptClause} ORDER BY id`, params, ) return rows.map(hydrate) } /** How many of one run's rows are still unresolved — what `cleanup_status` is derived from. */ async function unresolvedCount(runId) { const [row] = await query( `SELECT COUNT(*) AS n FROM event_run_resources WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`, [runId, ...UNRESOLVED], ) return Number(row?.n || 0) } /** Unresolved counts for several runs at once, keyed by run id — the run LIST's read. */ async function unresolvedCounts(runIds) { const ids = [...new Set(runIds || [])].filter(Boolean) if (!ids.length) return new Map() const rows = await query( `SELECT run_id, COUNT(*) AS n FROM event_run_resources WHERE run_id IN (${ids.map(() => '?').join(',')}) AND status IN (?, ?, ?, ?, ?) GROUP BY run_id`, [...ids, ...UNRESOLVED], ) return new Map(rows.map((r) => [r.run_id, Number(r.n)])) } /** * Claim one row for a revert: `pending | confirmed | orphaned | drifted → reverting`, * and `reverting` again once the claim on it has gone stale. * * The compare-and-set that keeps the cleanup leg and the manual cleanup route off * each other's rows. A row another pass is mid-revert on is left alone, exactly as * a step with a live claim is. * * **"Exactly as a step" has to include the expiry, and it did not until the Phase * 16 acceptance walk.** A step's claim carries `claim_expires_at`, so a step whose * process died is reclaimed once the lease lapses — that reclaim is the whole * reason §E's CAS survives §N4's single instance. A `reverting` row had no such * bound and nothing released it, so a process killed mid-teardown stranded the row * for good: the sweep skipped it every 15s forever, `cleanup_status` never left * `pending`, and `POST …/cleanup` — the recourse §I names — answered 200 and did * nothing, because it claims through this same function. Observed with a lease, * which then blocked the NEXT run of the same event from taking the value. * * The stale test is `updated_at`, not a new column: the row is stamped exactly * when it enters `reverting` and is not written again until the revert resolves, * so for a `reverting` row `updated_at` IS "when this claim was taken". The bound * is the run lease's, for the run lease's reason — it has to outlast a whole * tick's work on one run, and every revert in a sweep is bounded by its action's * own `budgetMs` long before this. * * `revert_attempts` is deliberately NOT incremented by reclaiming. A stale claim * is a process that died, not an attempt that failed, and counting it would burn * the retry budget on crashes — Engagement Phase 14's rule, one table over. * * **`updated_at` is re-stamped explicitly, and that is what keeps this a CAS.** * This connector sends `CLIENT_FOUND_ROWS`, so `affectedRows` counts rows MATCHED * rather than changed. For the four fresh statuses that is harmless — the winner * moves the row to `reverting` and the loser's `status IN (…)` no longer matches. * A stale `reverting` row has no such natural change: without re-stamping, the * row would still satisfy `status = 'reverting' AND updated_at < …` and a second * claimer would match it too. Writing the column is what makes the second one * miss. */ const REVERT_CLAIM_TTL_MS = Number(process.env.EVENT_REVERT_CLAIM_TTL_MS) || 15 * 60 * 1000 async function claimRevert(id) { const result = await query( `UPDATE event_run_resources SET status = 'reverting', updated_at = NOW() WHERE id = ? AND (status IN ('pending', 'confirmed', 'orphaned', 'drifted') OR (status = 'reverting' AND updated_at < (NOW() - INTERVAL ? MICROSECOND)))`, [id, REVERT_CLAIM_TTL_MS * 1000], ) return (result.affectedRows || 0) > 0 } /** The revert worked. `reverted` is terminal and releases the target. */ async function markReverted(id) { await query( `UPDATE event_run_resources SET status = 'reverted', last_error = NULL WHERE id = ?`, [id], ) } /** * The revert did not work, and the row goes back to being unresolved. * * `revert_attempts` is incremented here and NOWHERE else, and it is never reset by * a sweep — Engagement Phase 14's rule, whose defect was a reclaim that returned * every stale row to its start state and made the attempt ceiling unreachable, so * the row cycled for ever and was never eligible for any retention sweep. The one * thing that may reset it is a human pressing cleanup, which is the same licence * a human's step retry has. * * `restoreTo` is where the row lands: `drifted` when the module says somebody else * moved the value, `orphaned` when it says the thing is gone, and `confirmed` * otherwise — still ours, still out there, try again. */ async function failRevert(id, error, restoreTo = 'confirmed') { await query( `UPDATE event_run_resources SET status = ?, revert_attempts = revert_attempts + 1, last_error = ? WHERE id = ?`, [restoreTo, String(error || 'the revert did not answer').slice(0, 500), id], ) } /** * A human is trying again: clear the attempt counter on one run's unresolved rows. * * Only ever called from the cleanup route with an actor behind it. The automatic * leg must never do this (see `failRevert`). */ async function resetAttempts(runId) { const result = await query( `UPDATE event_run_resources SET revert_attempts = 0 WHERE run_id = ? AND status IN (?, ?, ?, ?, ?)`, [runId, ...UNRESOLVED], ) return result.affectedRows || 0 } /** Every live row one module owns, for the reconcile sweep. */ async function liveForModule(owner, { limit = 500 } = {}) { const rows = await query( `SELECT ${COLUMNS} FROM event_run_resources WHERE owner_module = ? AND status IN ('pending', 'confirmed') ORDER BY id LIMIT ?`, [owner, Number(limit)], ) return rows.map(hydrate) } /** Every module that currently owns a live row — who the reconcile sweep asks. */ async function modulesWithLiveRows() { const rows = await query( `SELECT DISTINCT owner_module FROM event_run_resources WHERE status IN ('pending', 'confirmed')`, ) return rows.map((r) => r.owner_module) } /** * The game no longer has it. Never reached by a revert — a revert that finds * nothing there is a SUCCESS (§L, and it is what a Rust wipe needs) — only by * reconcile, which is a different question: nobody asked for this to go. */ async function markOrphaned(id, detail = null) { await query( `UPDATE event_run_resources SET status = 'orphaned', last_error = ? WHERE id = ? AND status IN ('pending', 'confirmed', 'reverting')`, [detail === null ? null : String(detail).slice(0, 500), id], ) } /** * Terminal runs that still owe the world something — the cleanup leg's scan. * * **Both halves of the WHERE were live-walk findings, and they are opposite * mistakes.** * * `cleanup_status = 'pending'` alone missed a run whose only resource was a * LEASE: `core.lease` reserves its own row and never goes through the ledger's * `markRunDirty`, so the flag stayed `not_required` and the lease was never given * back at all. Hence `not_required` is in the list — a terminal run with an * unresolved row has something to do whatever any summary column says, and * treating that combination as work is the fail-safe direction. * * And the run status filter alone made `MAX_REVERT_ATTEMPTS` mean ONE attempt, * because the first failing sweep set `incomplete` and nothing looked at the run * again. That is fixed in `cleanupRun`, which now only writes `incomplete` once * there is nothing left it will try — so `incomplete` genuinely means "finished * with, and not finished", which is exactly what excludes both a run whose * retries are spent and a run an admin cancelled without cleanup. * * The attempt bound is in the join for a different reason: without it a run whose * rows are all spent would be selected, worked over and found to have nothing to * do on every tick for the rest of its life. */ async function runsNeedingCleanup(limit = 25, maxAttempts = 3) { return query( `SELECT DISTINCT r.id, r.status, r.cleanup_status, r.version_id, r.definition_id, r.scope FROM event_runs r JOIN event_run_resources res ON res.run_id = r.id WHERE r.status IN ('completed', 'cancelled', 'failed', 'missed') AND r.cleanup_status IN ('pending', 'not_required') AND res.status IN (?, ?, ?, ?, ?) AND res.revert_attempts < ? ORDER BY r.id LIMIT ?`, [...UNRESOLVED, Number(maxAttempts), Number(limit)], ) } module.exports = { STEP_KIND, HELD, UNRESOLVED, reserve, confirm, resolvePlaceholder, findByTarget, forRun, unresolvedForRun, unresolvedCount, unresolvedCounts, claimRevert, markReverted, failRevert, resetAttempts, liveForModule, modulesWithLiveRows, markOrphaned, runsNeedingCleanup, }