website/server/test/eventCleanup.test.js

// ── Giving back what a run took (EVENTS_PLAN.md Phase 8) ───────────────────
//
// The phase's shipped claim: **cleanup is generated from the ledger and runs on
// every terminal path.** An operator cannot be relied on to write the undo, and
// an aborted run never reaches the phase they wrote it in — so there is no
// cleanup phase in a spec, no `on_teardown` on an action, and one function that
// reads rows.
//
// The properties around it are §L's, and two of them are ones this codebase has
// already paid for once:
//
//   • **a revert that never succeeds stays visible rather than cycling** — rule 2,
//     and `MAX_REVERT_ATTEMPTS` is what stops the automatic retry. Only a human
//     clears the counter, which is Engagement Phase 14's rule stated a third time
//   • **reverting something that does not exist is a SUCCESS** — §L, and what a
//     Rust wipe needs
//   • **drift is not an error.** The module did exactly what it was asked and
//     found somebody else's value in place. A restore that wrote anyway would
//     silently revert an operator's manual fix
//   • **a resource the module no longer has becomes `orphaned`, never `reverted`** —
//     reverting it would be core recording that it put something back when what
//     happened is that the thing vanished
//   • **"I do not know" is never read as "it is gone".** Every unanswerable
//     reconcile leaves the ledger alone
//
// The sweep is driven against a stubbed db layer, exactly as `eventRunner.test.js`
// drives the runner: what a stub cannot prove is the SQL, and the unique key that
// makes two events unable to lease one target runs against a real MariaDB in
// `eventRunnerSql.test.js`.

process.env.DB_HOST = '127.0.0.1'
process.env.DB_PORT = '59999'

const { test, beforeEach, afterEach, after } = require('node:test')
const assert = require('node:assert/strict')

const registries = require('../src/modules/registries')
const cleanup = require('../src/events/cleanup')
const resourcesDb = require('../src/model/events/eventRunResources.db')
const runsDb = require('../src/model/events/eventRuns.db')
const stepsDb = require('../src/model/events/eventRunSteps.db')
const logDb = require('../src/model/events/eventRunLog.db')
const db = require('../src/utils/db')

after(() => db.close())

const UNRESOLVED = ['pending', 'confirmed', 'reverting', 'orphaned', 'drifted']

let store
const originals = {
  resourcesDb: { ...resourcesDb },
  runsDb: { ...runsDb },
  stepsDb: { ...stepsDb },
  logDb: { ...logDb },
}

beforeEach(() => {
  registries._reset()
  store = { rows: new Map(), steps: new Map(), log: [], next: 1, cleanupStatus: 'pending' }

  resourcesDb.unresolvedForRun = async (runId, { maxAttempts = null } = {}) =>
    [...store.rows.values()]
      .filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status))
      .filter((r) => maxAttempts === null || r.revert_attempts < maxAttempts)
      .map((r) => ({ ...r }))
  resourcesDb.unresolvedCount = async (runId) =>
    [...store.rows.values()].filter((r) => r.run_id === runId && UNRESOLVED.includes(r.status)).length
  resourcesDb.claimRevert = async (id) => {
    const r = store.rows.get(id)
    if (!r || !['pending', 'confirmed', 'orphaned', 'drifted'].includes(r.status)) return false
    r.status = 'reverting'
    return true
  }
  resourcesDb.markReverted = async (id) => {
    const r = store.rows.get(id)
    if (r) Object.assign(r, { status: 'reverted', last_error: null })
  }
  resourcesDb.failRevert = async (id, error, restoreTo = 'confirmed') => {
    const r = store.rows.get(id)
    if (r) Object.assign(r, { status: restoreTo, revert_attempts: r.revert_attempts + 1, last_error: String(error) })
  }
  resourcesDb.resetAttempts = async (runId) => {
    let n = 0
    for (const r of store.rows.values()) {
      if (r.run_id === runId && UNRESOLVED.includes(r.status)) {
        r.revert_attempts = 0
        n += 1
      }
    }
    return n
  }
  resourcesDb.markOrphaned = async (id, detail = null) => {
    const r = store.rows.get(id)
    if (r && ['pending', 'confirmed', 'reverting'].includes(r.status)) {
      Object.assign(r, { status: 'orphaned', last_error: detail })
    }
  }
  resourcesDb.liveForModule = async (owner) =>
    [...store.rows.values()].filter((r) => r.owner_module === owner && ['pending', 'confirmed'].includes(r.status)).map((r) => ({ ...r }))
  resourcesDb.modulesWithLiveRows = async () => [
    ...new Set([...store.rows.values()].filter((r) => ['pending', 'confirmed'].includes(r.status)).map((r) => r.owner_module)),
  ]
  resourcesDb.runsNeedingCleanup = async () => store.candidates || []
  resourcesDb.finishedRunsWithNothingLeft = async () => store.nothingLeft || []

  runsDb.setCleanupStatus = async (id, to, from = null) => {
    if (from && !from.includes(store.cleanupStatus)) return false
    store.cleanupStatus = to
    return true
  }
  stepsDb.getById = async (id) => store.steps.get(id) || null
  logDb.write = async (entry) => {
    store.log.push(entry)
  }
})

afterEach(() => {
  Object.assign(resourcesDb, originals.resourcesDb)
  Object.assign(runsDb, originals.runsDb)
  Object.assign(stepsDb, originals.stepsDb)
  Object.assign(logDb, originals.logDb)
  registries._reset()
})

const RUN = { id: 7, status: 'completed', cleanup_status: 'pending' }

function addStep(id, actionId, key = 'k'.repeat(40)) {
  store.steps.set(id, { id, run_id: RUN.id, action_id: actionId, idempotency_key: key, phase: 'p', seq: 0 })
}

function addResource(over = {}) {
  const id = store.next++
  const row = {
    id,
    run_id: RUN.id,
    step_id: 1,
    owner_module: 'demo',
    kind: 'creature',
    ref: `0x${id}`,
    payload: null,
    lease_until: null,
    status: 'confirmed',
    revert_attempts: 0,
    last_error: null,
    member_key: null,
    ...over,
  }
  store.rows.set(id, row)
  return row
}

function registerAction(over = {}) {
  const api = registries.stage('demo')
  api.registerEventActions([
    {
      id: 'demo.spawn',
      label: 'Spawn',
      risk: 'change',
      reversible: 'ledger',
      budgetMs: 500,
      perform: async () => ({ ok: true }),
      revert: async () => ({ ok: true }),
      ...over,
    },
  ])
  registries.apply(api.staged)
}

function registerLease(over = {}) {
  const api = registries.stage('demo')
  api.registerEventLeases([
    {
      id: 'demo.rate',
      label: 'Gather rate',
      type: 'float',
      min: 0.5,
      max: 5,
      maxDurationMs: 3_600_000,
      read: async () => ({ ok: true, value: 1 }),
      apply: async () => ({ ok: true }),
      restore: async () => ({ ok: true }),
      ...over,
    },
  ])
  registries.apply(api.staged)
}

const kinds = () => store.log.map((e) => e.kind)

// ── The shipped claim ──────────────────────────────────────────────────────

test('a run\'s ledger is given back in one call per step, and the run goes complete', async () => {
  // `revert` takes a LIST because that is what makes twelve creatures one round
  // trip rather than twelve. The grouping key is the STEP, because the resource
  // row records the module and the opaque names while the step records the verb.
  let seen = null
  registerAction({ revert: async (arg) => { seen = arg; return { ok: true } } })
  addStep(1, 'demo.spawn', 'key-one')
  addResource({ ref: '0xA' })
  addResource({ ref: '0xB' })

  const summary = await cleanup.cleanupRun(RUN)
  assert.deepEqual(summary, { attempted: 2, reverted: 2, drifted: 0, failed: 0, remaining: 0 })
  assert.equal(seen.runId, RUN.id)
  assert.equal(seen.idempotencyKey, 'key-one')
  assert.deepEqual(seen.resources.map((r) => r.ref), ['0xA', '0xB'])
  assert.equal(store.cleanupStatus, 'complete')
})

test('two steps are two calls, and one failing does not take the other down', async () => {
  const calls = []
  registerAction({
    revert: async ({ idempotencyKey, resources }) => {
      calls.push(idempotencyKey)
      return idempotencyKey === 'bad' ? { ok: false, error: 'the shard did not answer' } : { ok: true, resources }
    },
  })
  addStep(1, 'demo.spawn', 'good')
  addStep(2, 'demo.spawn', 'bad')
  addResource({ step_id: 1 })
  addResource({ step_id: 2 })

  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(calls.length, 2)
  assert.equal(summary.reverted, 1)
  assert.equal(summary.failed, 1)
  assert.equal(summary.remaining, 1)
  // **Still `pending`, because the failed row has retries left.** `incomplete`
  // means "finished with, and not finished" — it is what takes a run out of the
  // sweep's own scan, so writing it after the FIRST failure made
  // `MAX_REVERT_ATTEMPTS` quietly mean one attempt. Found by watching
  // `revert_attempts` sit at 1 through half a minute of live ticks.
  assert.equal(store.cleanupStatus, 'pending')
})

test('reverting something that does not exist is a success', async () => {
  // §L, and the Rust wipe: "gone, and that is fine". The module never has to
  // distinguish "I deleted it" from "it was not there" — which is also what makes
  // a placeholder for an object that may never have existed safe to write.
  registerAction({ revert: async () => ({ ok: true, detail: 'resource no longer exists' }) })
  addStep(1, 'demo.spawn')
  addResource()
  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.reverted, 1)
  assert.equal(store.cleanupStatus, 'complete')
})

test('a module may name the ones that did not come back', async () => {
  // Partial cleanup is the ordinary case — eleven of twelve creatures deleted —
  // and it is why the ledger is a row per object rather than a row per step.
  registerAction({ revert: async () => ({ ok: true, failed: ['0x2'] }) })
  addStep(1, 'demo.spawn')
  addResource({ ref: '0x1' })
  addResource({ ref: '0x2' })
  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.reverted, 1)
  assert.equal(summary.failed, 1)
  assert.match([...store.rows.values()].find((r) => r.ref === '0x2').last_error, /could not give "0x2" back/)
})

// ── Rule 2: loud and sticky ────────────────────────────────────────────────

test('a revert that never works stops retrying and stays visible', async () => {
  let calls = 0
  registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } })
  addStep(1, 'demo.spawn')
  addResource()

  for (let i = 0; i < 6; i++) await cleanup.cleanupRun(RUN)

  // Bounded at MAX_REVERT_ATTEMPTS, exactly like a step's attempts. A fourth ask
  // of a shard that has answered the same way three times is not new information,
  // and an unbounded counter is a row nothing can ever sweep.
  assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS)
  const row = [...store.rows.values()][0]
  assert.equal(row.revert_attempts, cleanup.MAX_REVERT_ATTEMPTS)
  assert.equal(row.status, 'confirmed')
  assert.equal(row.last_error, 'nope')
  // And ONLY now, with nothing left to try, does the run stop being the sweep's
  // business. §L: it does not stay `running` — an event whose world changes are
  // still up is a real state, and pretending the event is in progress hides it.
  assert.equal(store.cleanupStatus, 'incomplete')
})

test('the run goes back to pending each time it still has an attempt left', async () => {
  // The other half of the same rule, watched one pass at a time rather than at
  // the end. Each of the first two sweeps leaves the run in the scan; the third
  // takes it out. A test that only looked at the end state would pass against the
  // defect this replaced.
  registerAction({ revert: async () => ({ ok: false, error: 'nope' }) })
  addStep(1, 'demo.spawn')
  addResource()

  const seen = []
  for (let i = 0; i < 3; i++) {
    await cleanup.cleanupRun(RUN)
    seen.push(store.cleanupStatus)
  }
  assert.deepEqual(seen, ['pending', 'pending', 'incomplete'])
})

test('only a human clears the attempt counter', async () => {
  // Engagement Phase 14's defect, stated a third time: a SWEEP that returned every
  // stale row to its start state made the attempt ceiling unreachable, so the row
  // cycled for ever and was never eligible for any retention sweep. The automatic
  // leg must never do this; the cleanup route may, because a person asked.
  let calls = 0
  registerAction({ revert: async () => { calls += 1; return { ok: false, error: 'nope' } } })
  addStep(1, 'demo.spawn')
  addResource()

  for (let i = 0; i < 5; i++) await cleanup.cleanupRun(RUN)
  assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS)

  await cleanup.cleanupRun(RUN, { resetAttempts: true, actor: 9 })
  assert.equal(calls, cleanup.MAX_REVERT_ATTEMPTS + 1)
  assert.ok(store.log.some((e) => e.kind === 'cleanup.retry' && e.detail.by === 9))
})

test('a module that throws from revert is a transient failure, not a crashed sweep', async () => {
  registerAction({ revert: async () => { throw new Error('socket hung up') } })
  addStep(1, 'demo.spawn')
  addResource()
  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.failed, 1)
  assert.match([...store.rows.values()][0].last_error, /socket hung up/)
})

test('no shape a revert failure can take reads as success', () => {
  // `dispatch.classify`'s rule, applied to the other direction of the contract.
  // The expensive mistake here is the mirror of the one there: recording that a
  // world change was UNDONE when it was not.
  for (const raw of [null, undefined, 'ok', [], {}, { ok: 'yes' }, { ok: 1 }]) {
    assert.notEqual(cleanup.classifyRevert(raw, 'demo.spawn').outcome, 'done', JSON.stringify(raw))
  }
  assert.equal(cleanup.classifyRevert({ __timedOut: true, error: 'slow' }, 'x').outcome, 'retry')
  assert.equal(cleanup.classifyRevert({ ok: false, retry: false, error: 'never' }, 'x').outcome, 'terminal')
  assert.equal(cleanup.classifyRevert({ ok: true }, 'x').outcome, 'done')
})

test('an action whose module is gone leaves its rows unresolved with the reason', async () => {
  // Not a retry — nothing will change until an operator reinstalls it — and not
  // an orphan either, because core has no idea whether the thing is still there.
  addStep(1, 'demo.spawn')
  addResource()
  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.failed, 1)
  assert.match([...store.rows.values()][0].last_error, /no module registers "demo.spawn"/)
  // Retried like any other failure rather than given up on at once, and that is
  // the right uniformity here: "nothing registers this" stops being true the
  // moment an operator reinstalls the module, and three registry lookups cost
  // nothing. So it is `pending` until the attempts are spent.
  assert.equal(store.cleanupStatus, 'pending')
})

// ── Leases ─────────────────────────────────────────────────────────────────

test('a lease is restored through the LEASE registry, not through any action', async () => {
  // The split §F draws: core owns the duration and the conflict check, the module
  // owns reading and writing. It is why `core.lease` needs no `revert()` of its
  // own, and why an `override` row routes here rather than to its step's action.
  let seen = null
  registerLease({ restore: async (baseline, opts) => { seen = { baseline, opts }; return { ok: true } } })
  addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } })

  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.reverted, 1)
  assert.equal(seen.baseline, 1)
  // The drift check's input. `restore` MUST verify current === expected before
  // writing, and a lease whose restore wrote blindly would silently revert an
  // operator's manual fix.
  assert.equal(seen.opts.expected, 3)
})

test('drift is not an error: the world is left alone and the row says so', async () => {
  registerLease({ restore: async () => ({ ok: false, drifted: true, current: 4.5 }) })
  addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } })

  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.drifted, 1)
  assert.equal(summary.failed, 0)
  const row = [...store.rows.values()][0]
  assert.equal(row.status, 'drifted')
  assert.match(row.last_error, /now 4\.5 rather than what this run applied/)
  // Still surfaced. §L: "surfaced beside the unreverted ones" — the run does not
  // get to call itself clean because somebody else took the value. It is `pending`
  // rather than `incomplete` for one more reason worth keeping: drift is retried
  // like any other failure, because a GM who puts the value back between two ticks
  // should have the lease close cleanly.
  assert.equal(store.cleanupStatus, 'pending')
})

test('a lease whose module is uninstalled is unresolved, never assumed restored', async () => {
  addResource({ kind: 'override', ref: 'demo.rate', step_id: null, payload: { baseline: 1, applied: 3 } })
  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.failed, 1)
  assert.match([...store.rows.values()][0].last_error, /no module registers the lease "demo.rate"/)
})

test('a targeted lease is restored through the lease its ref NAMES, target in hand', async () => {
  // Phase 12b. A targeted row's ref is `<lease id>#<target>`, and both readers of
  // one — this sweep and `ledger.normalise` — looked their lease up by the whole
  // string. Left alone, every property lease would have come back "no module
  // registers the lease", which is the worst answer this table has: it leaves a
  // spawner turned up for good AND blames an uninstalled module for it.
  let seen = null
  registerLease({
    id: 'demo.spawner.count',
    type: 'int',
    min: 0,
    max: 50,
    target: { label: 'Which spawner' },
    restore: async (baseline, opts) => { seen = { baseline, opts }; return { ok: true } },
  })
  addResource({
    kind: 'override',
    ref: 'demo.spawner.count#003f11b8-9bfa',
    step_id: null,
    payload: { baseline: 3, applied: 30 },
  })

  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.reverted, 1)
  assert.equal(seen.baseline, 3)
  assert.equal(seen.opts.expected, 30)
  // The half that makes the restore land on the right object rather than on some
  // other run's spawner: the module is handed the target, not asked to parse the
  // ref core composed.
  assert.equal(seen.opts.target, '003f11b8-9bfa')
})

test('an unregistered TARGETED lease still names itself, ref and all', async () => {
  // The fail-closed direction is unchanged by the parser: a row whose module is
  // gone stays unresolved with the reason on it, and the reason quotes the ref
  // the operator will actually see in the console rather than the bare id.
  addResource({
    kind: 'override',
    ref: 'demo.spawner.count#003f11b8-9bfa',
    step_id: null,
    payload: { baseline: 3, applied: 30 },
  })
  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.failed, 1)
  assert.match(
    [...store.rows.values()][0].last_error,
    /no module registers the lease "demo\.spawner\.count#003f11b8-9bfa"/,
  )
})

// ── The sweep ──────────────────────────────────────────────────────────────

test('the sweep only touches TERMINAL runs', async () => {
  // A run still in flight has a ledger that is still growing, and reverting a
  // resource the next step is about to use would be core undoing an event while
  // it is happening.
  registerAction()
  addStep(1, 'demo.spawn')
  addResource()
  store.candidates = [{ id: RUN.id, status: 'running', cleanup_status: 'pending' }]
  assert.equal(await cleanup.sweep(), 0)
  assert.equal([...store.rows.values()][0].status, 'confirmed')

  store.candidates = [{ id: RUN.id, status: 'cancelled', cleanup_status: 'pending' }]
  assert.equal(await cleanup.sweep(), 1)
  assert.equal([...store.rows.values()][0].status, 'reverted')
})

// ── Reconcile ──────────────────────────────────────────────────────────────

test('a resource the module no longer has becomes orphaned, never reverted', async () => {
  // §L, and the distinction matters to the operator reading the console
  // afterwards: `reverted` says core put something back, `orphaned` says the
  // thing vanished while nobody was looking. Recording the second as the first
  // would be core claiming credit for a shard restart.
  registerAction({ reconcile: async () => ({ ok: true, inForce: ['0x1'] }) })
  addStep(1, 'demo.spawn')
  addResource({ ref: '0x1' })
  addResource({ ref: '0x2' })

  const summary = await cleanup.reconcileModule('demo')
  assert.deepEqual(summary, { asked: 2, inForce: 1, orphaned: 1, unanswered: 0 })
  assert.equal([...store.rows.values()].find((r) => r.ref === '0x2').status, 'orphaned')
  assert.equal([...store.rows.values()].find((r) => r.ref === '0x1').status, 'confirmed')
  assert.ok(store.log.some((e) => e.kind === 'resource.orphaned'))
})

test('"I do not know" is never read as "it is gone"', async () => {
  // Every unanswerable shape leaves the ledger exactly as it was. A reconcile
  // that read silence as absence would orphan a whole shard's worth of live
  // spawns the first time a sidecar was slow.
  for (const answer of [null, undefined, { ok: false }, { ok: true }, { ok: true, inForce: 'all' }, 'yes']) {
    store.rows.clear()
    store.log.length = 0
    registries._reset()
    registerAction({ reconcile: async () => answer })
    addStep(1, 'demo.spawn')
    addResource({ ref: '0x1' })
    const summary = await cleanup.reconcileModule('demo')
    assert.equal(summary.orphaned, 0, JSON.stringify(answer))
    assert.equal(summary.unanswered, 1, JSON.stringify(answer))
    assert.equal([...store.rows.values()][0].status, 'confirmed')
  }
})

test('a module with no reconcile is not broken; core keeps believing its ledger', async () => {
  // Optional where `revert` is required. A module that cannot answer leaves core
  // exactly where it was before this phase, which is a capability its deployment
  // does without rather than a boot it fails.
  registerAction()
  addStep(1, 'demo.spawn')
  addResource()
  const summary = await cleanup.reconcileModule('demo')
  assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 1 })
  assert.equal([...store.rows.values()][0].status, 'confirmed')
})

test('a reconcile that throws orphans nothing', async () => {
  registerAction({ reconcile: async () => { throw new Error('sidecar gone') } })
  addStep(1, 'demo.spawn')
  addResource()
  const summary = await cleanup.reconcileModule('demo')
  assert.equal(summary.unanswered, 1)
  assert.equal([...store.rows.values()][0].status, 'confirmed')
})

test('placeholders are not asked about, because there is nothing to ask yet', async () => {
  // A `@step` row names no object — it says "a dispatch was in flight and may
  // have made something". Asking a module whether it is in force is a question
  // with no answer, and reading a shrug as absence would resolve the one row whose
  // survival is the safety property.
  registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) })
  addStep(1, 'demo.spawn')
  addResource({ kind: resourcesDb.STEP_KIND, ref: 'a'.repeat(40), status: 'pending' })
  const summary = await cleanup.reconcileModule('demo')
  assert.deepEqual(summary, { asked: 0, inForce: 0, orphaned: 0, unanswered: 0 })
  assert.equal([...store.rows.values()][0].status, 'pending')
})

// ── Reconciling a LEASE (Phase 11b) ────────────────────────────────────────

/// A lease row's action is `core.lease`, so these need core's own registrations.
function registerCoreAnd(leaseOver = {}) {
  registries.registerCore()
  registerLease(leaseOver)
}

test('a lease row had no reconcile path at all until core.lease grew one', async () => {
  // The hole this phase closed, asserted from the outside. `reconcileModule`
  // resolves a resource to the action of the step that made it, and for a lease
  // that action is CORE's — a path no module can register anything on. So every
  // `override` row came back `unanswered` for the life of the run, and a lease the
  // shard had quietly dropped stayed in the ledger as live until teardown went
  // looking for a baseline nobody was holding.
  registerCoreAnd({ inForce: async () => ({ ok: true, held: false }) })
  addStep(1, 'core.lease')
  addResource({ kind: 'override', ref: 'demo.rate', payload: { baseline: 1, applied: 3 } })

  const summary = await cleanup.reconcileModule('demo')
  assert.deepEqual(summary, { asked: 1, inForce: 0, orphaned: 1, unanswered: 0 })
  assert.equal([...store.rows.values()][0].status, 'orphaned')
})

test('a lease the shard still has a record of stays put', async () => {
  registerCoreAnd({ inForce: async () => ({ ok: true, held: true }) })
  addStep(1, 'core.lease')
  addResource({ kind: 'override', ref: 'demo.rate', payload: { baseline: 1, applied: 3 } })

  const summary = await cleanup.reconcileModule('demo')
  assert.deepEqual(summary, { asked: 1, inForce: 1, orphaned: 0, unanswered: 0 })
  assert.equal([...store.rows.values()][0].status, 'confirmed')
})

test('a lease that cannot say is left alone, and DRIFT is not what this asks about', async () => {
  // Two properties in one walk. Every unanswerable shape leaves the row exactly as
  // it was, which is core's posture everywhere else — and a lease with no
  // `inForce()` at all is one of those shapes rather than a boot failure.
  //
  // The second is the reason `inForce` exists instead of a comparison against
  // `read()`: a value that differs from what the event applied is drift, and drift
  // is teardown's verdict to deliver through `restore` so the row lands `drifted`
  // with the current value beside it. A reconcile that inferred absence from a
  // changed value would orphan the row first and throw that away — telling the
  // operator the lease vanished rather than that somebody moved it.
  const shrugs = [
    undefined,                                   // no inForce() declared at all
    async () => null,
    async () => ({ ok: false, held: false }),    // could not ask; not an answer
    async () => ({ ok: true }),                  // answered without saying
    async () => { throw new Error('sidecar gone') },
    // The drift shape. `read()` would report 4.5 against an applied 3, and this
    // must NOT be read as "the lease is gone".
    async () => ({ ok: true, held: true, current: 4.5 }),
  ]

  for (const inForce of shrugs) {
    store.rows.clear()
    store.log.length = 0
    registries._reset()
    registerCoreAnd(inForce === undefined ? {} : { inForce })
    addStep(1, 'core.lease')
    addResource({ kind: 'override', ref: 'demo.rate', payload: { baseline: 1, applied: 3 } })

    const summary = await cleanup.reconcileModule('demo')
    assert.equal(summary.orphaned, 0, String(inForce))
    assert.equal([...store.rows.values()][0].status, 'confirmed', String(inForce))
  }
})

test('reconcileAll asks every module that owns a live row', async () => {
  registerAction({ reconcile: async () => ({ ok: true, inForce: [] }) })
  addStep(1, 'demo.spawn')
  addStep(2, 'other.thing')
  addResource({ owner_module: 'demo' })
  addResource({ owner_module: 'other', step_id: 2 })
  const out = await cleanup.reconcileAll()
  assert.deepEqual(Object.keys(out).sort(), ['demo', 'other'])
  assert.equal(out.demo.orphaned, 1)
  // The other module registers nothing, so its row is left alone rather than
  // orphaned by a module that is not there to be asked.
  assert.equal(out.other.unanswered, 1)
})

// ── Expiry (MODULE_API 1.11.0, Rust PLAN_FIXES D183) ─────────────────────────
//
// A game that ends something at its own deadline — a Rust zone erased when its
// time is up — says so through `ctx.events.expired`. The properties:
//
//   • **expired is terminal**: the sweep never tries to give it back, and it is
//     not one of the statuses that makes a run's cleanup unfinished
//   • **it is not orphaned**, and it wins over orphaned: a reconcile that ran
//     before the module heard the expiry filed the row as vanished
//   • **a module closes only its own rows**, and only the target it named
//   • **a revert already in flight is left to finish** — it finds nothing, which
//     §L calls a success, and settles as `reverted` by itself
//   • **a finished run whose last row this was is complete**, even one the sweep
//     had given up on as `incomplete`

function stubExpiry() {
  resourcesDb.expirableByTarget = async (owner, kind, ref) =>
    [...store.rows.values()]
      .filter((r) => r.owner_module === owner && r.kind === kind && r.ref === ref)
      .filter((r) => ['pending', 'confirmed', 'orphaned'].includes(r.status))
      .map((r) => ({ ...r }))
  resourcesDb.markExpired = async (id, detail = null) => {
    const r = store.rows.get(id)
    if (!r || !['pending', 'confirmed', 'orphaned'].includes(r.status)) return false
    Object.assign(r, { status: 'expired', last_error: detail })
    return true
  }
  store.run = { ...RUN }
  runsDb.getById = async (id) => (id === RUN.id ? store.run : null)
}

test('an expired resource is terminal: the sweep never tries to give it back', async () => {
  stubExpiry()
  let reverts = 0
  registerAction({ revert: async () => { reverts += 1; return { ok: true } } })
  addStep(1, 'demo.spawn')
  addResource({ kind: 'zone', ref: '3:rg-7-1' })

  assert.deepEqual(await cleanup.expireResource('demo', { kind: 'zone', ref: '3:rg-7-1' }), { expired: 1 })
  const row = [...store.rows.values()][0]
  assert.equal(row.status, 'expired')
  assert.ok(store.log.some((e) => e.kind === 'resource.expired' && e.detail.resource === 'zone:3:rg-7-1'))

  const summary = await cleanup.cleanupRun(RUN)
  assert.equal(summary.attempted, 0)
  assert.equal(reverts, 0)
  assert.equal(row.status, 'expired')
  assert.equal(store.cleanupStatus, 'complete')
})

test('an expiry wins over orphaned, and leaves a revert in flight alone', async () => {
  stubExpiry()
  addResource({ kind: 'zone', ref: 'a', status: 'orphaned' })
  addResource({ kind: 'zone', ref: 'b', status: 'reverting' })
  addResource({ kind: 'zone', ref: 'c', status: 'reverted' })

  for (const ref of ['a', 'b', 'c']) await cleanup.expireResource('demo', { kind: 'zone', ref })
  const status = (ref) => [...store.rows.values()].find((r) => r.ref === ref).status
  assert.equal(status('a'), 'expired')
  assert.equal(status('b'), 'reverting')
  assert.equal(status('c'), 'reverted')
})

test('a module closes only its own rows, and only the target it named', async () => {
  stubExpiry()
  addResource({ owner_module: 'demo', kind: 'zone', ref: 'x' })
  addResource({ owner_module: 'other', kind: 'zone', ref: 'x' })
  addResource({ owner_module: 'demo', kind: 'npc', ref: 'x' })

  assert.deepEqual(await cleanup.expireResource('demo', { kind: 'zone', ref: 'x' }), { expired: 1 })
  const rows = [...store.rows.values()]
  assert.deepEqual(rows.map((r) => r.status), ['expired', 'confirmed', 'confirmed'])
})

test('an expiry nobody ledgered, or a malformed one, is not an error', async () => {
  stubExpiry()
  addResource({ kind: 'zone', ref: 'x' })
  for (const bad of [undefined, {}, { kind: 'zone' }, { ref: 'x' }, { kind: 1, ref: 'x' }, { kind: 'zone', ref: 'y' },
    { kind: 'zone', ref: 'r'.repeat(191) }]) {
    assert.deepEqual(await cleanup.expireResource('demo', bad), { expired: 0 }, JSON.stringify(bad))
  }
  assert.equal([...store.rows.values()][0].status, 'confirmed')
  assert.equal(store.log.length, 0)
})

test('the last row of a finished run expiring completes an incomplete cleanup', async () => {
  stubExpiry()
  addResource({ kind: 'zone', ref: 'x' })
  store.cleanupStatus = 'incomplete'
  await cleanup.expireResource('demo', { kind: 'zone', ref: 'x' })
  assert.equal(store.cleanupStatus, 'complete')
})

test('a run still in flight keeps its cleanup status, and so does one with rows left', async () => {
  stubExpiry()
  addResource({ kind: 'zone', ref: 'x' })
  store.run = { ...RUN, status: 'running' }
  store.cleanupStatus = 'pending'
  await cleanup.expireResource('demo', { kind: 'zone', ref: 'x' })
  assert.equal(store.cleanupStatus, 'pending')

  store.rows.clear()
  addResource({ kind: 'zone', ref: 'y' })
  addResource({ kind: 'zone', ref: 'z' })
  store.run = { ...RUN }
  store.cleanupStatus = 'incomplete'
  await cleanup.expireResource('demo', { kind: 'zone', ref: 'y' })
  assert.equal(store.cleanupStatus, 'incomplete')
})

test('a run whose last resource expired while it ran is settled complete when it ends', async () => {
  // The step-2 walk's run 46: the zone expired while the run was still going, so
  // expireResource left its status to the terminal path — and the sweep's scan,
  // which joins on an unresolved row, would never have selected it. `pending`
  // for ever, on a ledger with nothing left in it.
  store.cleanupStatus = 'pending'
  store.candidates = []
  store.nothingLeft = [RUN.id]
  assert.equal(await cleanup.sweep(), 1)
  assert.equal(store.cleanupStatus, 'complete')

  // And never over an incomplete one — that is a human's to clear.
  store.cleanupStatus = 'incomplete'
  await cleanup.sweep()
  assert.equal(store.cleanupStatus, 'incomplete')
})