feat(modules): boot/shutdown hook dispatch and the installed_modules reconcile

Phase 2, PR 5 of docs/website/MODULE_SYSTEM.md 2.7. api.onBoot/api.onShutdown
stop throwing, server.js gains one call on each side, and the 2.4 state machine
finally runs against real outcomes -- which is what makes 4.5's `disabled` 404
leg reachable for the first time.

Dispatch and reconcile live in src/modules/lifecycle.js rather than in the
loader, for the reason the schema replay does: routeManifest.js and swagger.js
both require app.js against a dead pool, so the loader may not reach the
database. The two halves meet at exactly one function, loader.setState(), so the
in-memory record the dispatch guard reads and the row the admin panel reads are
moved together and cannot disagree.

Four decisions, all recorded in MODULE_API.md 2.5 and 4.4:

- The loader classifies its failures by 4.3 step, so failure_stage says where a
  module broke instead of being a column nothing ever filled. The four steps
  readManifest covers in one pass label themselves; the rest are inferred from
  how far load() had got, and an unlabelled throw is recorded against the step
  that was running rather than guessed at.
- A row whose directory is gone is marked startup_failed rather than left
  claiming `enabled` -- the boot reset has just moved it there, and a row
  claiming to be enabled for a module that is not on the volume is the one state
  that is simply untrue. An uninstall leaves `disabled`, which the reset never
  touches, so this catches only a hand-deleted directory.
- Core's eight UO boot call sites stay in server.js until Phase 3. Unlike a
  registered announce leg, a boot call site already has somewhere to live, so
  moving it now would be extraction done early in a phase whose exit criterion
  is that nothing changes.
- onBoot gets no timeout. Shutdown races a SIGKILL and boot does not, and a slow
  onBoot delaying the listener is the contract's promise to a module that must
  warm up before it serves.

The operator's switch wins over everything: a disabled module is guarded, not
booted, and does not have its failure re-recorded, or an outcome would silently
switch it back on next boot. Every database write in the reconcile is
individually caught -- a row that will not update is worse reporting, never a
failed boot.

900 tests pass (17 new). routes.manifest.json is unchanged at 229 routes and the
OpenAPI spec regenerates byte-identical.

Co-Authored-By: Claude <noreply@anthropic.com>

server/test/moduleLifecycle.test.js

@@ -0,0 +1,376 @@
+// ── Boot and shutdown dispatch ─────────────────────────────────────────────
+//
+// Phase 2, PR 5. The contract is MODULE_API.md §2.5 (when the hooks run, in what
+// order, with what budget), §4.4 (a failure after mounting is a 503, not a
+// crash), §4.5 (a disabled module is guarded, never unmounted) and
+// MODULE_SYSTEM.md §2.4 (what a boot does to installed_modules).
+//
+// The property under test throughout, as in moduleLoader.test.js and
+// moduleSchema.test.js: **the failing module fails alone.** A hook that throws,
+// a hook that hangs, a row that will not write — none of them may cost the site
+// its boot or the next module its start.
+//
+// No database is involved: `boot()` takes the model as an injectable dependency
+// for exactly the reason `replayFragments` takes its query, and the fake below
+// records every call so the ORDER of the reconcile can be asserted — which is
+// the whole design, not an implementation detail.
+process.env.DB_HOST = '127.0.0.1'
+process.env.DB_PORT = '59999'
+
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+
+const { test, beforeEach, after } = require('node:test')
+const assert = require('node:assert/strict')
+
+const express = require('express')
+
+const db = require('../src/utils/db')
+const registries = require('../src/modules/registries')
+const lifecycle = require('../src/modules/lifecycle')
+const { startApp } = require('./_helper')
+
+after(() => db.close())
+
+let tmpRoot
+
+const emptyTiers = () => ({
+  public: express.Router(),
+  admin: express.Router(),
+  player: express.Router(),
+})
+
+function freshLoader(dir, tiers = emptyTiers()) {
+  process.env.MODULES_DIR = dir
+  registries._reset()
+  delete require.cache[require.resolve('../src/modules/loader')]
+  // eslint-disable-next-line global-require
+  const loader = require('../src/modules/loader')
+  loader.load(tiers)
+  return loader
+}
+
+/**
+ * A module whose hooks report themselves into a file.
+ *
+ * A file rather than a shared array because the module is `require`d from disk
+ * and cannot close over anything this file owns — the same trick the ctx probe
+ * in moduleLoader.test.js uses.
+ */
+function writeModule(id, { boot, shutdown, mounts, log: logFile } = {}) {
+  const dir = path.join(tmpRoot, id)
+  fs.mkdirSync(dir, { recursive: true })
+  fs.writeFileSync(path.join(dir, 'module.json'), JSON.stringify({
+    id,
+    name: `Module ${id}`,
+    version: '1.2.3',
+    coreApi: '^1.0.0',
+    server: 'index.js',
+    ...(mounts === undefined ? {} : { mounts }),
+  }))
+  const note = logFile
+    ? `const note = (what) => require('fs').appendFileSync(${JSON.stringify(logFile)}, what + '\\n')`
+    : 'const note = () => {}'
+  const register = mounts === undefined ? '' : `
+    const r = ctx.express.Router()
+    r.get('/', (req, res) => res.json({ ok: true }))
+    api.registerRoutes({ public: { '${(mounts.public || [])[0]}': r } })`
+  // `boot: ''` means "registers a hook that does nothing", which is a different
+  // module from one that registers no hook at all — hence the undefined check
+  // rather than a truthiness test.
+  fs.writeFileSync(path.join(dir, 'index.js'), `${note}
+    module.exports = (ctx, api) => {${register}
+      ${boot === undefined ? '' : `api.onBoot(async (c) => { note('boot:${id}' + (c && c.moduleId === '${id}' ? ':ctx' : ':NOCTX')); ${boot} })`}
+      ${shutdown === undefined ? '' : `api.onShutdown(async () => { note('shutdown:${id}'); ${shutdown} })`}
+    }`)
+  return dir
+}
+
+/** An in-memory stand-in for model/modules/modules.model.js. */
+function fakeModel(seed = []) {
+  const rows = new Map(seed.map((r) => [r.id, { failureStage: null, failureReason: null, ...r }]))
+  const calls = []
+  const model = {
+    rows,
+    calls,
+    async beginBoot() {
+      calls.push('beginBoot')
+      for (const row of rows.values()) {
+        if (row.state === 'disabled') continue
+        Object.assign(row, { state: 'enabled', failureStage: null, failureReason: null })
+      }
+    },
+    async recordInstalled({ id, name, version }) {
+      calls.push(`recordInstalled:${id}`)
+      const row = rows.get(id)
+      // Metadata is refreshed; state is deliberately left alone (§2.4).
+      if (row) Object.assign(row, { name, version })
+      else rows.set(id, { id, name, version, state: 'installed', failureStage: null, failureReason: null })
+    },
+    async list() {
+      calls.push('list')
+      return [...rows.values()]
+    },
+    async markStarted(id) {
+      calls.push(`markStarted:${id}`)
+      Object.assign(rows.get(id), { state: 'started', failureStage: null, failureReason: null })
+    },
+    async markStartupFailed(id, { stage, reason }) {
+      calls.push(`markStartupFailed:${id}`)
+      const row = rows.get(id)
+      // The model's own softening, reproduced because tests below depend on it:
+      // a disabled row is a no-op, or an outcome would overwrite the operator's
+      // decision and silently re-enable the module next boot.
+      if (!row || row.state === 'disabled') return
+      Object.assign(row, { state: 'startup_failed', failureStage: stage, failureReason: reason })
+    },
+  }
+  return model
+}
+
+const stateOf = (loader, id) => loader.list().find((m) => m.id === id)
+const noted = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n') : [])
+
+beforeEach(() => {
+  tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-lifecycle-'))
+})
+
+// ── The happy path ─────────────────────────────────────────────────────────
+
+test('every module is recorded, booted with its ctx and marked started, in scan order', async () => {
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('aaa', { boot: '', log: file })
+  writeModule('bbb', { boot: '', log: file })
+  const loader = freshLoader(tmpRoot)
+  const model = fakeModel()
+
+  await lifecycle.boot({ modules: loader, model })
+
+  // The reconcile order IS the design (§2.4): clear the last boot's outcomes
+  // first, so what is on display afterwards is what this boot did.
+  assert.equal(model.calls[0], 'beginBoot')
+  assert.deepEqual(model.calls.slice(1, 3), ['recordInstalled:aaa', 'recordInstalled:bbb'])
+  assert.deepEqual(noted(file), ['boot:aaa:ctx', 'boot:bbb:ctx'])
+  assert.equal(stateOf(loader, 'aaa').state, 'started')
+  assert.equal(model.rows.get('bbb').state, 'started')
+  // §2.4's metadata refresh: the row carries what the admin screen shows.
+  assert.equal(model.rows.get('aaa').name, 'Module aaa')
+  assert.equal(model.rows.get('aaa').version, '1.2.3')
+})
+
+test('a hand-placed directory gets a row with no provenance', async () => {
+  // §2.5 keeps a directory dropped on the volume by hand a supported install.
+  // Without a row it could never be disabled, and nothing could report it.
+  writeModule('byhand', { boot: '' })
+  const model = fakeModel()
+
+  await lifecycle.boot({ modules: freshLoader(tmpRoot), model })
+
+  const row = model.rows.get('byhand')
+  assert.equal(row.state, 'started')
+  assert.equal(row.source ?? null, null)
+  assert.equal(row.sha256 ?? null, null)
+})
+
+test('a module with no onBoot still reaches started', async () => {
+  writeModule('quiet', {})
+  const loader = freshLoader(tmpRoot)
+  const model = fakeModel()
+
+  await lifecycle.boot({ modules: loader, model })
+
+  // Nothing to warm up is not the same as never having started: the guard lets
+  // its routes through, so the row has to agree that it is serving.
+  assert.equal(stateOf(loader, 'quiet').state, 'started')
+  assert.equal(model.rows.get('quiet').state, 'started')
+})
+
+// ── Failure is a state ─────────────────────────────────────────────────────
+
+test('an onBoot that throws fails its own module and no one else', async () => {
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('aaa', { boot: '', log: file })
+  writeModule('bbb', { boot: 'throw new Error("cache warm-up failed")', log: file })
+  writeModule('ccc', { boot: '', log: file })
+  const loader = freshLoader(tmpRoot)
+  const model = fakeModel()
+
+  await lifecycle.boot({ modules: loader, model })
+
+  assert.equal(stateOf(loader, 'bbb').state, 'startup_failed')
+  assert.equal(stateOf(loader, 'bbb').stage, 'boot')
+  assert.match(stateOf(loader, 'bbb').reason, /cache warm-up failed/)
+  assert.equal(model.rows.get('bbb').failureStage, 'boot')
+  // The one that matters: the module AFTER the failure still booted.
+  assert.deepEqual(noted(file), ['boot:aaa:ctx', 'boot:bbb:ctx', 'boot:ccc:ctx'])
+  assert.equal(stateOf(loader, 'ccc').state, 'started')
+})
+
+test('a module whose onBoot failed keeps its URLs and answers 503', async () => {
+  // §4.4's right-hand column, reached by the real mechanism rather than a
+  // hand-moved state: routes.manifest.json must not depend on whether a boot
+  // hook happened to succeed on the machine that generated it.
+  writeModule('svc', { boot: 'throw new Error("no")', mounts: { public: ['/widgets'] } })
+  const tiers = emptyTiers()
+  const loader = freshLoader(tmpRoot, tiers)
+  const app = await startApp((a) => a.use('/public', tiers.public))
+
+  try {
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 200)
+    await lifecycle.boot({ modules: loader, model: fakeModel() })
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 503)
+  } finally {
+    await app.close()
+  }
+})
+
+test('a failure from load or schema replay is written down with its stage', async () => {
+  // Both happen before the database is reachable — load() at require time, the
+  // replay inside ensureSchema — so the boot reconcile is where they land.
+  writeModule('bad', {})
+  fs.writeFileSync(path.join(tmpRoot, 'bad', 'module.json'), JSON.stringify({
+    id: 'bad', name: 'bad', version: '1.0.0', coreApi: '^99.0.0',
+  }))
+  const loader = freshLoader(tmpRoot)
+  const model = fakeModel()
+
+  await lifecycle.boot({ modules: loader, model })
+
+  const row = model.rows.get('bad')
+  assert.equal(row.state, 'startup_failed')
+  assert.equal(row.failureStage, 'core_api')
+  assert.match(row.failureReason, /needs core API \^99\.0\.0/)
+})
+
+// ── The operator's switch ──────────────────────────────────────────────────
+
+test('a disabled row guards the module, skips its hook and is not overwritten', async () => {
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('off', { boot: '', shutdown: '', mounts: { public: ['/widgets'] }, log: file })
+  const tiers = emptyTiers()
+  const loader = freshLoader(tmpRoot, tiers)
+  const model = fakeModel([{ id: 'off', name: 'Module off', version: '1.2.3', state: 'disabled' }])
+  const app = await startApp((a) => a.use('/public', tiers.public))
+
+  try {
+    await lifecycle.boot({ modules: loader, model })
+
+    // §4.5's 404 leg, unreachable until this reconcile existed: mounted and
+    // guarded, never unmounted, so the URL surface stays a property of the
+    // volume rather than of a database row.
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 404)
+    assert.equal(stateOf(loader, 'off').state, 'disabled')
+    assert.deepEqual(noted(file), [], 'a disabled module must not be booted')
+    // Still disabled: an outcome must never overwrite a decision, or the next
+    // boot would silently switch it back on.
+    assert.equal(model.rows.get('off').state, 'disabled')
+    // And nothing to tear down, because it never started.
+    await lifecycle.shutdown({ modules: loader })
+    assert.deepEqual(noted(file), [])
+  } finally {
+    await app.close()
+  }
+})
+
+test('a row whose directory is gone is marked failed rather than left claiming enabled', async () => {
+  writeModule('here', { boot: '' })
+  const model = fakeModel([
+    { id: 'here', name: 'Module here', version: '1.2.3', state: 'started' },
+    { id: 'gone', name: 'Module gone', version: '0.9.0', state: 'started' },
+    // An uninstall leaves `disabled`, which beginBoot never touches — so this
+    // one is not an anomaly and must be left exactly as the operator left it.
+    { id: 'uninstalled', name: 'Module uninstalled', version: '0.1.0', state: 'disabled' },
+  ])
+
+  await lifecycle.boot({ modules: freshLoader(tmpRoot), model })
+
+  assert.equal(model.rows.get('here').state, 'started')
+  assert.equal(model.rows.get('gone').state, 'startup_failed')
+  assert.match(model.rows.get('gone').failureReason, /not present on the volume/)
+  assert.equal(model.rows.get('uninstalled').state, 'disabled')
+})
+
+// ── The site comes up regardless ───────────────────────────────────────────
+
+test('a database that will not take the bookkeeping still boots the modules', async () => {
+  // A row that will not update is bad — the admin panel shows the wrong thing —
+  // but it is strictly less bad than a site that will not start.
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('aaa', { boot: '', log: file })
+  const loader = freshLoader(tmpRoot)
+  const model = fakeModel()
+  for (const name of ['beginBoot', 'recordInstalled', 'list', 'markStarted']) {
+    model[name] = async () => { throw new Error('ER_LOCK_WAIT_TIMEOUT') }
+  }
+
+  await assert.doesNotReject(() => lifecycle.boot({ modules: loader, model }))
+
+  assert.deepEqual(noted(file), ['boot:aaa:ctx'])
+  assert.equal(stateOf(loader, 'aaa').state, 'started')
+})
+
+test('a process that never scanned writes nothing at all', async () => {
+  // `npm run seed` is exactly this: it calls ensureSchema() without ever
+  // requiring app.js. Reconciling against an empty scan would mark every
+  // installed module as missing from the volume.
+  process.env.MODULES_DIR = tmpRoot
+  delete require.cache[require.resolve('../src/modules/loader')]
+  // eslint-disable-next-line global-require
+  const unscanned = require('../src/modules/loader')
+  const model = fakeModel([{ id: 'real', name: 'Module real', version: '1.0.0', state: 'started' }])
+
+  await lifecycle.boot({ modules: unscanned, model })
+
+  assert.deepEqual(model.calls, [])
+  assert.equal(model.rows.get('real').state, 'started')
+})
+
+// ── Shutdown ───────────────────────────────────────────────────────────────
+
+test('shutdown runs started modules in reverse order and skips the rest', async () => {
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('aaa', { boot: '', shutdown: '', log: file })
+  writeModule('bbb', { boot: 'throw new Error("no")', shutdown: '', log: file })
+  writeModule('ccc', { boot: '', shutdown: '', log: file })
+  const loader = freshLoader(tmpRoot)
+
+  await lifecycle.boot({ modules: loader, model: fakeModel() })
+  fs.writeFileSync(file, '') // only the shutdown half is under test
+  await lifecycle.shutdown({ modules: loader })
+
+  // Reverse of boot order, and `bbb` absent: its onBoot threw, so it has a
+  // half-built world that its onShutdown was never written to tear down.
+  assert.deepEqual(noted(file), ['shutdown:ccc', 'shutdown:aaa'])
+})
+
+test('a hook that hangs costs its budget, not the shutdown', async () => {
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('aaa', { boot: '', shutdown: '', log: file })
+  writeModule('zzz', { boot: '', shutdown: 'await new Promise(() => {})', log: file })
+  const loader = freshLoader(tmpRoot)
+
+  await lifecycle.boot({ modules: loader, model: fakeModel() })
+  fs.writeFileSync(file, '')
+
+  const started = Date.now()
+  await lifecycle.shutdown({ modules: loader, budgetMs: 50 })
+
+  // zzz never returns; it is abandoned and aaa still gets its turn. The
+  // alternative is a host where `systemctl stop` hangs until SIGKILL.
+  assert.deepEqual(noted(file), ['shutdown:zzz', 'shutdown:aaa'])
+  assert.ok(Date.now() - started < 2000, 'shutdown must not wait on a hung hook')
+})
+
+test('a hook that throws does not stop the ones behind it', async () => {
+  const file = path.join(tmpRoot, 'log.txt')
+  writeModule('aaa', { boot: '', shutdown: '', log: file })
+  writeModule('zzz', { boot: '', shutdown: 'throw new Error("close failed")', log: file })
+  const loader = freshLoader(tmpRoot)
+
+  await lifecycle.boot({ modules: loader, model: fakeModel() })
+  fs.writeFileSync(file, '')
+
+  await assert.doesNotReject(() => lifecycle.shutdown({ modules: loader }))
+  assert.deepEqual(noted(file), ['shutdown:zzz', 'shutdown:aaa'])
+})