Files
Module-uo/server/test/shardIngest.eventReconcile.test.js
wtclaude 021f191f65
All checks were successful
PR Checks / client-build (pull_request) Successful in 20s
PR Checks / frozen-manifest (pull_request) Successful in 41s
PR Checks / server-tests (pull_request) Successful in 8m33s
fix(events): three defects the live rig found, two of them data loss
The whole-rig walk (ServUO + sidecar + website) against a real two-phase event.

- **A WS reconnect would have orphaned every live resource.** The backfill
  replays the last several `server.hello` frames in order — this rig saw three,
  each with a different `bootId` — so every replayed frame reads as a restart,
  and the intermediate ones compare a resource stamped with the CURRENT boot
  against a boot that ended hours ago. The row is then `orphaned`: a live crier
  line core will never take down again, lost to nothing worse than the website
  reconnecting. Gated on `!fromBackfill`, the rule the engagement fan-out and
  the SSE broadcast beside it already state. The website-was-down case is not
  missed — core asks every module at its own boot.
- **The shard explains its refusals and the run log dropped the explanation.**
  A 403 body reads `{"reason":"admin write plane disabled"}`; `legError` looks
  for `data.message`, finds nothing, and reports "sidecar responded 403". For a
  staff member clicking a button that is survivable. For an event that ran at
  four in the morning the run log is the only place anyone will learn why.
- **The "not retried" clause explained the wrong thing on a permanent status.**
  A 403 will not succeed on any attempt, so telling an operator it was not
  retried "because a repeat would announce twice" points them at a policy
  decision instead of at the switch they have to flip. The clause is now added
  only where a retry was genuinely given up, and 403/404 join the statuses the
  keyed verbs treat as terminal.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-09-04 07:36:03 -05:00

127 lines
5.9 KiB
JavaScript

// A shard restart makes the event resource ledger a claim about a world that no
// longer exists (EVENTS.md §F, EVENTS_PLAN.md Phases 8 and 9).
//
// Core cannot notice that on its own — it has no concept of the game being up —
// so the module says when, and `server.hello` carrying a *changed* `bootId` is
// the only signal that distinguishes a shard restart from a sidecar reconnect.
// Getting that wrong in either direction is a real failure: never asking leaves
// core believing a ledger of things that are gone, and asking on every reconnect
// makes core orphan rows that are perfectly alive.
const { test, beforeEach } = require('node:test')
const assert = require('node:assert/strict')
const shardIngest = require('../utils/shardIngest')
function makeDeps() {
const order = []
const noop = async () => {}
return {
order,
shardEvents: { append: noop },
shardState: { clearOnline: async () => { order.push('clearOnline') }, upsertOnline: noop, setOffline: noop },
shardLinks: {},
shardMarket: {},
uoLinkConfig: { recordStatus: async (row) => { order.push(`recordStatus:${row.bootId}`) } },
settings: { getInstanceName: async () => 'Rig' },
broadcast: () => {},
pushDispatch: () => {},
engagement: () => {},
eventsReconcile: () => { order.push('reconcile') },
log: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} },
}
}
const hello = (bootId) => ({ kind: 'server.hello', t: '2026-09-04T10:00:00Z', shard: 'Rig', bootId })
beforeEach(() => shardIngest.reset())
test('the first hello of a process is not a restart', async () => {
// The website has just come up and the shard has not moved. Everything in the
// ledger is still in force, and asking would be core spending a round trip per
// module to be told so.
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
assert.ok(!deps.order.includes('reconcile'))
})
test('a sidecar reconnect is not a restart either', async () => {
// `server.hello` is sent on EVERY reconnect, and the sidecar dropping its
// socket changes nothing in the game. Reconciling here would orphan every live
// row — the ledger would still be right and core would stop believing it.
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
await shardIngest.ingest(hello('boot-1'), deps)
assert.ok(!deps.order.includes('reconcile'))
})
test('a changed bootId asks every module to reconcile its ledger', async () => {
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
await shardIngest.ingest(hello('boot-2'), deps)
assert.equal(deps.order.filter((s) => s === 'reconcile').length, 1)
})
test('the reconcile happens AFTER the new bootId is recorded', async () => {
// The ordering is load-bearing rather than tidy. Every action decides what is
// still in force by comparing its stamp against the CURRENT boot id, which it
// reads back out of the row `recordStatus` writes. Asking first would compare
// every resource against the boot that has just ended — and every one of them
// would look live, which is the exact opposite of what a restart means.
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
await shardIngest.ingest(hello('boot-2'), deps)
const recordedAt = deps.order.lastIndexOf('recordStatus:boot-2')
const askedAt = deps.order.indexOf('reconcile')
assert.ok(recordedAt >= 0 && askedAt >= 0)
assert.ok(askedAt > recordedAt, 'reconcile must not run before the new boot id is stored')
})
test('a hello with no bootId at all changes nothing', async () => {
// An older plugin, or a frame that lost the field. Not knowing which boot this
// is cannot be allowed to read as "a new one".
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
await shardIngest.ingest({ kind: 'server.hello', t: '2026-09-04T10:00:00Z', shard: 'Rig' }, deps)
assert.ok(!deps.order.includes('reconcile'))
})
test('a backfill replay never reconciles, however many boots it walks through', async () => {
// **The defect the live rig found, and nothing else could.** A WS reconnect
// replays the last several `server.hello` frames in order — this rig saw three,
// each with a different `bootId` — so every replayed frame looks like a
// restart. Acting on the intermediate ones would compare a resource stamped
// with the CURRENT boot against a boot that ended hours ago and mark it
// `orphaned`: a live crier line core will never take down again, lost to
// nothing worse than the website reconnecting.
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
for (const boot of ['boot-2', 'boot-3', 'boot-4']) {
await shardIngest.ingest(hello(boot), { ...deps, fromBackfill: true })
}
assert.ok(!deps.order.includes('reconcile'))
// The replay still moves the tracked boot on, so the NEXT live hello is
// measured against where the replay left off rather than against boot-1.
assert.ok(deps.order.includes('recordStatus:boot-4'))
})
test('a live hello after a replay is still a restart', async () => {
// The gate is about the frame, not about the module going quiet: skipping the
// replay must not make the next genuine restart invisible.
const deps = makeDeps()
await shardIngest.ingest(hello('boot-1'), deps)
await shardIngest.ingest(hello('boot-2'), { ...deps, fromBackfill: true })
await shardIngest.ingest(hello('boot-3'), deps)
assert.equal(deps.order.filter((s) => s === 'reconcile').length, 1)
})
test('a reconcile that throws does not take the ingest down with it', async () => {
// Fire-and-forget by the contract, and the feed must survive one bad module:
// `ingest()` never throws, because a single event may not kill the socket.
const deps = makeDeps()
deps.eventsReconcile = () => { throw new Error('registry exploded') }
await shardIngest.ingest(hello('boot-1'), deps)
await assert.doesNotReject(() => shardIngest.ingest(hello('boot-2'), deps))
})