feat(modules): boot/shutdown hook dispatch and the installed_modules reconcile

Phase 2, PR 5 of docs/website/MODULE_SYSTEM.md 2.7. api.onBoot/api.onShutdown
stop throwing, server.js gains one call on each side, and the 2.4 state machine
finally runs against real outcomes -- which is what makes 4.5's `disabled` 404
leg reachable for the first time.

Dispatch and reconcile live in src/modules/lifecycle.js rather than in the
loader, for the reason the schema replay does: routeManifest.js and swagger.js
both require app.js against a dead pool, so the loader may not reach the
database. The two halves meet at exactly one function, loader.setState(), so the
in-memory record the dispatch guard reads and the row the admin panel reads are
moved together and cannot disagree.

Four decisions, all recorded in MODULE_API.md 2.5 and 4.4:

- The loader classifies its failures by 4.3 step, so failure_stage says where a
  module broke instead of being a column nothing ever filled. The four steps
  readManifest covers in one pass label themselves; the rest are inferred from
  how far load() had got, and an unlabelled throw is recorded against the step
  that was running rather than guessed at.
- A row whose directory is gone is marked startup_failed rather than left
  claiming `enabled` -- the boot reset has just moved it there, and a row
  claiming to be enabled for a module that is not on the volume is the one state
  that is simply untrue. An uninstall leaves `disabled`, which the reset never
  touches, so this catches only a hand-deleted directory.
- Core's eight UO boot call sites stay in server.js until Phase 3. Unlike a
  registered announce leg, a boot call site already has somewhere to live, so
  moving it now would be extraction done early in a phase whose exit criterion
  is that nothing changes.
- onBoot gets no timeout. Shutdown races a SIGKILL and boot does not, and a slow
  onBoot delaying the listener is the contract's promise to a module that must
  warm up before it serves.

The operator's switch wins over everything: a disabled module is guarded, not
booted, and does not have its failure re-recorded, or an outcome would silently
switch it back on next boot. Every database write in the reconcile is
individually caught -- a row that will not update is worse reporting, never a
failed boot.

900 tests pass (17 new). routes.manifest.json is unchanged at 229 routes and the
OpenAPI spec regenerates byte-identical.

Co-Authored-By: Claude <noreply@anthropic.com>

server/test/moduleLoader.test.js

@@ -437,9 +437,10 @@ test('a registered module answers on its prefix; a failed one is simply absent',
 test('a module that fails AFTER mounting keeps its URLs and answers 503', async () => {
   // The right-hand column of §4.4, and the reason routes.manifest.json can be
   // generated off a dead database: the URL surface must not depend on whether a
-  // boot step succeeded on the generating machine. PR 3 (schema replay) and PR 5
-  // (onBoot) are the two things that will trip this in real life; here the state
-  // is moved by hand, because the loader is the thing under test.
+  // boot step succeeded on the generating machine. The schema replay and onBoot
+  // are the two things that trip this in real life (moduleSchema.test.js and
+  // moduleLifecycle.test.js cover both); here the state is moved by hand,
+  // because the loader is the thing under test.
   writeModule('later', oneRoute('/widgets'))
 
   const tiers = emptyTiers()
@@ -451,18 +452,23 @@ test('a module that fails AFTER mounting keeps its URLs and answers 503', async
 
     assert.equal(stateOf(loader, 'later').state, 'registered')
 
-    loader.setState('later', 'startup_failed', 'schema fragment blew up')
+    loader.setState('later', 'startup_failed', { stage: 'schema', reason: 'schema fragment blew up' })
     assert.equal((await fetch(`${app.url}/public/widgets`)).status, 503)
     assert.equal(stateOf(loader, 'later').reason, 'schema fragment blew up')
+    assert.equal(stateOf(loader, 'later').stage, 'schema')
 
-    // The 404 leg becomes reachable for real in PR 5, when the boot reconcile
-    // reads a `disabled` row out of installed_modules. A disabled module is
-    // mounted and guarded, never unmounted (§4.5) — same reason as the 503.
+    // The 404 leg is reached for real by the boot reconcile, when it finds a
+    // `disabled` row in installed_modules. A disabled module is mounted and
+    // guarded, never unmounted (§4.5) — same reason as the 503.
     loader.setState('later', 'disabled')
     assert.equal((await fetch(`${app.url}/public/widgets`)).status, 404)
 
     loader.setState('later', 'started')
     assert.equal((await fetch(`${app.url}/public/widgets`)).status, 200)
+    // Every non-failing move clears the failure, so a running module can never
+    // show the reason it failed two boots ago (§2.4).
+    assert.equal(stateOf(loader, 'later').reason, null)
+    assert.equal(stateOf(loader, 'later').stage, null)
   } finally {
     await app.close()
   }
@@ -506,19 +512,79 @@ test('ctx exposes exactly the documented surface, and is frozen', () => {
   assert.equal(probe.mutable, false, 'ctx members must be frozen')
 })
 
-test('the register calls PR 5 owns throw rather than silently accepting', () => {
-  // An accepting no-op would let a module believe it had registered a boot hook
-  // and fail silently at the far end.
-  for (const [call, pr] of [
-    ['onBoot', 5],
-    ['onShutdown', 5],
+// ── Lifecycle hooks ────────────────────────────────────────────────────────
+
+test('a lifecycle hook must be a function, and may be registered once', () => {
+  // Both are register-time failures, so they cost the module its mount entirely
+  // rather than surfacing at boot — the far end of a hook that was never really
+  // registered is a module that silently never warms up.
+  for (const [body, expected] of [
+    ['api.onBoot("later")', /onBoot: expected a function/],
+    ['api.onShutdown("later")', /onShutdown: expected a function/],
+    ['api.onBoot(() => {}); api.onBoot(() => {})', /onBoot\(\) called twice/],
   ]) {
     tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-modules-'))
-    writeModule('early', { server: `module.exports = (ctx, api) => api.${call}(() => {})` })
-    assert.match(
-      stateOf(freshLoader(tmpRoot), 'early').reason,
-      new RegExp(`${call}: not available until phase 2 PR ${pr}`),
-    )
+    writeModule('hooked', { server: `module.exports = (ctx, api) => { ${body} }` })
+    const state = stateOf(freshLoader(tmpRoot), 'hooked')
+    assert.match(state.reason, expected)
+    assert.equal(state.stage, 'register')
+  }
+})
+
+test('a module with no hooks is bootable, and offers nothing to shut down', () => {
+  writeModule('quiet', oneRoute('/widgets'))
+  const loader = freshLoader(tmpRoot)
+
+  // Listed with a null hook rather than filtered out: it still has to reach
+  // `started`, or the admin panel and the dispatch guard would disagree about
+  // whether it is serving.
+  assert.deepEqual(loader.bootable().map((b) => b.id), ['quiet'])
+  assert.equal(loader.bootable()[0].hook, null)
+  assert.deepEqual(loader.shutdownHooks(), [])
+})
+
+test('shutdown hooks come back in reverse order, and only for started modules', () => {
+  const hook = 'module.exports = (ctx, api) => api.onShutdown(async () => {})'
+  writeModule('aaa', { server: hook })
+  writeModule('bbb', { server: hook })
+  writeModule('ccc', { server: hook })
+  const loader = freshLoader(tmpRoot)
+
+  // Nothing has started yet, so there is nothing to tear down.
+  assert.deepEqual(loader.shutdownHooks(), [])
+
+  loader.setState('aaa', 'started')
+  loader.setState('bbb', 'startup_failed', { stage: 'boot', reason: 'never warmed up' })
+  loader.setState('ccc', 'started')
+
+  // Reverse registration order (§2.5), and `bbb` is absent: a module whose
+  // onBoot threw is mid-way through a warm-up it never finished, and handing it
+  // a half-built world to tear down is worse than not closing cleanly.
+  assert.deepEqual(loader.shutdownHooks().map((h) => h.id), ['ccc', 'aaa'])
+})
+
+// ── Failure stages ─────────────────────────────────────────────────────────
+
+test('a failure is recorded against the §4.3 step that produced it', () => {
+  // installed_modules.failure_stage exists so the admin panel can say WHERE a
+  // module broke. The four steps readManifest covers in one pass have to label
+  // themselves; the rest are inferred from how far load() had got.
+  const cases = [
+    ['a-manifest', { manifest: { nonsense: true } }, 'manifest'],
+    ['b-coreapi', { manifest: { coreApi: '^99.0.0' } }, 'core_api'],
+    ['c-mounts', { manifest: { mounts: { public: ['/bad prefix'] } } }, 'mounts'],
+    ['d-slots', { manifest: { extensions: ['no.such.slot'] } }, 'extensions'],
+    ['e-schema', { schema: 'DELETE FROM x;' }, 'schema'],
+    ['f-require', { server: 'throw new Error("boom")' }, 'require'],
+    ['g-register', { server: 'module.exports = (ctx, api) => { throw new Error("nope") }' }, 'register'],
+  ]
+  for (const [id, spec] of cases) writeModule(id, spec)
+  const loader = freshLoader(tmpRoot)
+
+  for (const [id, , stage] of cases) {
+    const state = stateOf(loader, id)
+    assert.equal(state.state, 'startup_failed', `${id} should have failed`)
+    assert.equal(state.stage, stage, `${id} should have failed at "${stage}"`)
   }
 })