The whole-rig walk (ServUO + sidecar + website) against a real two-phase event.
- **A WS reconnect would have orphaned every live resource.** The backfill
replays the last several `server.hello` frames in order — this rig saw three,
each with a different `bootId` — so every replayed frame reads as a restart,
and the intermediate ones compare a resource stamped with the CURRENT boot
against a boot that ended hours ago. The row is then `orphaned`: a live crier
line core will never take down again, lost to nothing worse than the website
reconnecting. Gated on `!fromBackfill`, the rule the engagement fan-out and
the SSE broadcast beside it already state. The website-was-down case is not
missed — core asks every module at its own boot.
- **The shard explains its refusals and the run log dropped the explanation.**
A 403 body reads `{"reason":"admin write plane disabled"}`; `legError` looks
for `data.message`, finds nothing, and reports "sidecar responded 403". For a
staff member clicking a button that is survivable. For an event that ran at
four in the morning the run log is the only place anyone will learn why.
- **The "not retried" clause explained the wrong thing on a permanent status.**
A 403 will not succeed on any attempt, so telling an operator it was not
retried "because a repeat would announce twice" points them at a policy
decision instead of at the switch they have to flip. The clause is now added
only where a retry was genuinely given up, and 403/404 join the statuses the
keyed verbs treat as terminal.
Co-Authored-By: Claude <noreply@anthropic.com>
127 lines
5.9 KiB
JavaScript
127 lines
5.9 KiB
JavaScript
// A shard restart makes the event resource ledger a claim about a world that no
|
|
// longer exists (EVENTS.md §F, EVENTS_PLAN.md Phases 8 and 9).
|
|
//
|
|
// Core cannot notice that on its own — it has no concept of the game being up —
|
|
// so the module says when, and `server.hello` carrying a *changed* `bootId` is
|
|
// the only signal that distinguishes a shard restart from a sidecar reconnect.
|
|
// Getting that wrong in either direction is a real failure: never asking leaves
|
|
// core believing a ledger of things that are gone, and asking on every reconnect
|
|
// makes core orphan rows that are perfectly alive.
|
|
|
|
const { test, beforeEach } = require('node:test')
|
|
const assert = require('node:assert/strict')
|
|
|
|
const shardIngest = require('../utils/shardIngest')
|
|
|
|
function makeDeps() {
|
|
const order = []
|
|
const noop = async () => {}
|
|
return {
|
|
order,
|
|
shardEvents: { append: noop },
|
|
shardState: { clearOnline: async () => { order.push('clearOnline') }, upsertOnline: noop, setOffline: noop },
|
|
shardLinks: {},
|
|
shardMarket: {},
|
|
uoLinkConfig: { recordStatus: async (row) => { order.push(`recordStatus:${row.bootId}`) } },
|
|
settings: { getInstanceName: async () => 'Rig' },
|
|
broadcast: () => {},
|
|
pushDispatch: () => {},
|
|
engagement: () => {},
|
|
eventsReconcile: () => { order.push('reconcile') },
|
|
log: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} },
|
|
}
|
|
}
|
|
|
|
const hello = (bootId) => ({ kind: 'server.hello', t: '2026-09-04T10:00:00Z', shard: 'Rig', bootId })
|
|
|
|
beforeEach(() => shardIngest.reset())
|
|
|
|
test('the first hello of a process is not a restart', async () => {
|
|
// The website has just come up and the shard has not moved. Everything in the
|
|
// ledger is still in force, and asking would be core spending a round trip per
|
|
// module to be told so.
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
assert.ok(!deps.order.includes('reconcile'))
|
|
})
|
|
|
|
test('a sidecar reconnect is not a restart either', async () => {
|
|
// `server.hello` is sent on EVERY reconnect, and the sidecar dropping its
|
|
// socket changes nothing in the game. Reconciling here would orphan every live
|
|
// row — the ledger would still be right and core would stop believing it.
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
assert.ok(!deps.order.includes('reconcile'))
|
|
})
|
|
|
|
test('a changed bootId asks every module to reconcile its ledger', async () => {
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
await shardIngest.ingest(hello('boot-2'), deps)
|
|
assert.equal(deps.order.filter((s) => s === 'reconcile').length, 1)
|
|
})
|
|
|
|
test('the reconcile happens AFTER the new bootId is recorded', async () => {
|
|
// The ordering is load-bearing rather than tidy. Every action decides what is
|
|
// still in force by comparing its stamp against the CURRENT boot id, which it
|
|
// reads back out of the row `recordStatus` writes. Asking first would compare
|
|
// every resource against the boot that has just ended — and every one of them
|
|
// would look live, which is the exact opposite of what a restart means.
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
await shardIngest.ingest(hello('boot-2'), deps)
|
|
|
|
const recordedAt = deps.order.lastIndexOf('recordStatus:boot-2')
|
|
const askedAt = deps.order.indexOf('reconcile')
|
|
assert.ok(recordedAt >= 0 && askedAt >= 0)
|
|
assert.ok(askedAt > recordedAt, 'reconcile must not run before the new boot id is stored')
|
|
})
|
|
|
|
test('a hello with no bootId at all changes nothing', async () => {
|
|
// An older plugin, or a frame that lost the field. Not knowing which boot this
|
|
// is cannot be allowed to read as "a new one".
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
await shardIngest.ingest({ kind: 'server.hello', t: '2026-09-04T10:00:00Z', shard: 'Rig' }, deps)
|
|
assert.ok(!deps.order.includes('reconcile'))
|
|
})
|
|
|
|
test('a backfill replay never reconciles, however many boots it walks through', async () => {
|
|
// **The defect the live rig found, and nothing else could.** A WS reconnect
|
|
// replays the last several `server.hello` frames in order — this rig saw three,
|
|
// each with a different `bootId` — so every replayed frame looks like a
|
|
// restart. Acting on the intermediate ones would compare a resource stamped
|
|
// with the CURRENT boot against a boot that ended hours ago and mark it
|
|
// `orphaned`: a live crier line core will never take down again, lost to
|
|
// nothing worse than the website reconnecting.
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
for (const boot of ['boot-2', 'boot-3', 'boot-4']) {
|
|
await shardIngest.ingest(hello(boot), { ...deps, fromBackfill: true })
|
|
}
|
|
assert.ok(!deps.order.includes('reconcile'))
|
|
// The replay still moves the tracked boot on, so the NEXT live hello is
|
|
// measured against where the replay left off rather than against boot-1.
|
|
assert.ok(deps.order.includes('recordStatus:boot-4'))
|
|
})
|
|
|
|
test('a live hello after a replay is still a restart', async () => {
|
|
// The gate is about the frame, not about the module going quiet: skipping the
|
|
// replay must not make the next genuine restart invisible.
|
|
const deps = makeDeps()
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
await shardIngest.ingest(hello('boot-2'), { ...deps, fromBackfill: true })
|
|
await shardIngest.ingest(hello('boot-3'), deps)
|
|
assert.equal(deps.order.filter((s) => s === 'reconcile').length, 1)
|
|
})
|
|
|
|
test('a reconcile that throws does not take the ingest down with it', async () => {
|
|
// Fire-and-forget by the contract, and the feed must survive one bad module:
|
|
// `ingest()` never throws, because a single event may not kill the socket.
|
|
const deps = makeDeps()
|
|
deps.eventsReconcile = () => { throw new Error('registry exploded') }
|
|
await shardIngest.ingest(hello('boot-1'), deps)
|
|
await assert.doesNotReject(() => shardIngest.ingest(hello('boot-2'), deps))
|
|
})
|