feat(modules): the filesystem module loader

Phase 2 PR 2 of docs/website/MODULE_SYSTEM.md 2.7. Adds
server/src/modules/{loader,semver,version}.js: the synchronous scan of
MODULES_DIR, manifest validation, prefix and table-name collision
rejection, per-module try/catch and the mount into the three tier
routers behind the MODULE_API.md 4.5 dispatch guard.

Two decisions the contract left open, both now written up there:

- The load trigger is one explicit modules.load(tierRouters) call in
  app.js, not a lazy scan (API 7.6). Accessors throw until it has run,
  because "no modules installed" is a real answer a caller must not be
  handed by accident.
- Whether core owns a prefix is asked of the live tier routers via
  express's own layer.match(), skipping root-mounted layers, rather than
  a hardcoded table -- the spike's was already stale when written
  (API 4.3).

Mounting is a second pass after every module is validated. Doing it
inside the scan loop makes the first module's layers indistinguishable
from core's, so the second module claiming a taken prefix is told it
collided with core and the module-versus-module check is unreachable.

registerExtension/NotificationStreams/AnnounceLeg and onBoot/onShutdown
throw "not available until phase 2 PR 4/5" rather than no-op; an
accepting stub would let a module believe it had registered something.
No schema replay, no boot dispatch, no installed_modules reconcile --
those are PRs 3 and 5, and until PR 5 a record's state is in memory only.

No module ships on the volume, so nothing an operator or client can see
changes: 842 tests pass, routes.manifest.json is unchanged at 229 routes
and swagger-output.json regenerates byte-identical.

Co-Authored-By: Claude <noreply@anthropic.com>

server/test/moduleLoader.test.js

@@ -0,0 +1,444 @@
+// ── The loader's failure guarantees ────────────────────────────────────────
+//
+// docs/website/MODULE_API.md §4.4 promises that a module which fails ANYWHERE in
+// its lifecycle fails alone: the site comes up, other modules are unaffected, and
+// the failure is recorded rather than thrown. That is the property most worth a
+// test, because the failure paths are the ones nobody exercises by hand — every
+// manual check runs the happy path.
+//
+// Each test builds a throwaway modules directory, points MODULES_DIR at it and
+// re-requires the loader with a clean cache, so the scan is genuinely redone.
+// MODULES_DIR is read into a const at require time (it has to be: the scan is
+// synchronous and happens during app.js's require), so busting the cache is the
+// only honest way to point the loader somewhere else.
+//
+// Point the pool at a closed port BEFORE requiring anything: buildCtx pulls in
+// the models, which build a mariadb pool at require time. No query is ever run.
+process.env.DB_HOST = '127.0.0.1'
+process.env.DB_PORT = '59999'
+
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+
+const { test, beforeEach, after } = require('node:test')
+const assert = require('node:assert/strict')
+
+const express = require('express')
+
+const db = require('../src/utils/db')
+const { startApp } = require('./_helper')
+
+after(() => db.close())
+
+let tmpRoot
+
+/** Three empty routers standing in for core's tiers — nothing owned, nothing gated. */
+const emptyTiers = () => ({
+  public: express.Router(),
+  admin: express.Router(),
+  player: express.Router(),
+})
+
+function freshLoader(dir, tiers = emptyTiers()) {
+  process.env.MODULES_DIR = dir
+  delete require.cache[require.resolve('../src/modules/loader')]
+  // eslint-disable-next-line global-require
+  const loader = require('../src/modules/loader')
+  loader.load(tiers)
+  return loader
+}
+
+function writeModule(id, { manifest = {}, server, schema } = {}) {
+  const dir = path.join(tmpRoot, id)
+  fs.mkdirSync(dir, { recursive: true })
+  const full = {
+    id,
+    name: id,
+    version: '1.0.0',
+    coreApi: '^1.0.0',
+    ...(server === undefined ? {} : { server: 'index.js' }),
+    ...(schema === undefined ? {} : { schema: 'schema.sql', purge: 'purge.sql' }),
+    ...manifest,
+  }
+  fs.writeFileSync(path.join(dir, 'module.json'), JSON.stringify(full))
+  if (server !== undefined) fs.writeFileSync(path.join(dir, 'index.js'), server)
+  if (schema !== undefined) {
+    fs.writeFileSync(path.join(dir, 'schema.sql'), schema)
+    fs.writeFileSync(path.join(dir, 'purge.sql'), '')
+  }
+  return dir
+}
+
+/** A module that registers one router answering 200 at its prefix root. */
+const oneRoute = (prefix, tier = 'public') => ({
+  manifest: { mounts: { [tier]: [prefix] } },
+  server: `module.exports = (ctx, api) => {
+    const r = ctx.express.Router()
+    r.get('/', (req, res) => res.json({ ok: true }))
+    api.registerRoutes({ ${tier}: { '${prefix}': r } })
+  }`,
+})
+
+const stateOf = (loader, id) => loader.list().find((m) => m.id === id)
+
+beforeEach(() => {
+  tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-modules-'))
+})
+
+// ── Discovery and the explicit trigger ─────────────────────────────────────
+
+test('a missing modules directory is the normal case, not an error', () => {
+  const loader = freshLoader(path.join(tmpRoot, 'does-not-exist'))
+  assert.deepEqual(loader.list(), [])
+})
+
+test('reading the module list before load() throws instead of answering []', () => {
+  // §7.6. The spike's scan was lazy and silent, so a caller that required the
+  // loader and read nothing got an empty list — indistinguishable from a core
+  // with no modules installed. It cost one confusing failure; it now costs an
+  // error naming the missing call.
+  process.env.MODULES_DIR = tmpRoot
+  delete require.cache[require.resolve('../src/modules/loader')]
+  // eslint-disable-next-line global-require
+  const loader = require('../src/modules/loader')
+  assert.throws(() => loader.list(), /modules\.list\(\) before modules\.load\(\)/)
+})
+
+test('load() refuses to run without all three tier routers', () => {
+  writeModule('aaa', { server: 'module.exports = () => {}' })
+  process.env.MODULES_DIR = tmpRoot
+  delete require.cache[require.resolve('../src/modules/loader')]
+  // eslint-disable-next-line global-require
+  const loader = require('../src/modules/loader')
+  // Not a module's failure — a wiring mistake in core, and the one thing in this
+  // file that is allowed to throw past the caller.
+  assert.throws(() => loader.load({ public: express.Router() }), /missing the "admin" tier router/)
+})
+
+test('load() is once-only, so a second call cannot double-mount', () => {
+  writeModule('aaa', oneRoute('/thing'))
+  const tiers = emptyTiers()
+  const loader = freshLoader(tmpRoot, tiers)
+  const before = tiers.public.stack.length
+
+  loader.load(tiers)
+  assert.equal(tiers.public.stack.length, before)
+  assert.equal(loader.list().length, 1)
+})
+
+test('modules load in alphabetical order, since nothing computes a precedence', () => {
+  for (const id of ['ccc', 'aaa', 'bbb']) writeModule(id, { server: 'module.exports = () => {}' })
+  const loader = freshLoader(tmpRoot)
+  assert.deepEqual(loader.list().map((m) => m.id), ['aaa', 'bbb', 'ccc'])
+})
+
+// ── The failing module fails alone ─────────────────────────────────────────
+
+test('a module whose entry point throws does not stop the others loading', () => {
+  writeModule('aaa', { server: 'module.exports = () => {}' })
+  writeModule('bbb', { server: 'throw new Error("boom")' })
+  writeModule('ccc', { server: 'module.exports = () => {}' })
+  const loader = freshLoader(tmpRoot)
+
+  assert.equal(stateOf(loader, 'aaa').state, 'registered')
+  assert.equal(stateOf(loader, 'ccc').state, 'registered')
+
+  const bad = stateOf(loader, 'bbb')
+  assert.equal(bad.state, 'startup_failed')
+  assert.match(bad.reason, /boom/)
+})
+
+test('an entry point that exports something other than a function is rejected', () => {
+  writeModule('notfn', { server: 'module.exports = { register: () => {} }' })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'notfn').reason, /does not export a function/)
+})
+
+test('a coreApi mismatch is refused before the module is required at all', () => {
+  // The entry point would throw if it ran; the version gate must run first.
+  writeModule('old', {
+    manifest: { coreApi: '^99.0.0' },
+    server: 'throw new Error("should never be required")',
+  })
+  const loader = freshLoader(tmpRoot)
+  const mod = stateOf(loader, 'old')
+  assert.equal(mod.state, 'startup_failed')
+  assert.match(mod.reason, /needs core API \^99\.0\.0/)
+})
+
+test('an unknown manifest key is rejected, not ignored', () => {
+  // A typo'd key must be loud: an operator who believes they configured
+  // something and silently did not is worse off than one who sees a failure.
+  writeModule('typo', { manifest: { mount: { public: ['/x'] } } })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'typo').reason, /unknown key "mount"/)
+})
+
+test('a module id that does not match its directory is rejected', () => {
+  writeModule('onedir', { manifest: { id: 'another' } })
+  const loader = freshLoader(tmpRoot)
+  // Recorded under the DIRECTORY name — the id it claimed is exactly what is
+  // not trusted here.
+  assert.match(stateOf(loader, 'onedir').reason, /does not match directory/)
+})
+
+test('an unknown extension slot is rejected; only core may declare a slot', () => {
+  writeModule('presumptuous', { manifest: { extensions: ['admin.users.detail', 'admin.invented'] } })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'presumptuous').reason, /unknown extension slot "admin\.invented"/)
+})
+
+// ── Prefix ownership ───────────────────────────────────────────────────────
+
+test('two modules cannot claim the same prefix; the first one wins', () => {
+  writeModule('aaa', oneRoute('/thing'))
+  writeModule('bbb', oneRoute('/thing'))
+  const loader = freshLoader(tmpRoot)
+
+  assert.equal(stateOf(loader, 'aaa').state, 'registered')
+  assert.match(stateOf(loader, 'bbb').reason, /already registered by module "aaa"/)
+})
+
+test('a module cannot take a prefix core owns, asked of the live tier routers', () => {
+  // Deliberately the REAL routers rather than a hardcoded prefix list. The spike
+  // hardcoded core's ~24 prefixes and they were already stale; asking express
+  // itself is what stops the check drifting the next time core adds a capability
+  // router.
+  /* eslint-disable global-require */
+  const real = {
+    public: require('../src/router/v1/public'),
+    admin: require('../src/router/v1/admin'),
+    player: require('../src/router/v1/player'),
+  }
+  /* eslint-enable global-require */
+
+  writeModule('greedy', { manifest: { mounts: { admin: ['/users'] } } })
+  writeModule('alsogreedy', { manifest: { mounts: { public: ['/wiki'] } } })
+  // Free in every tier core actually mounts, so it must be allowed through.
+  writeModule('polite', oneRoute('/widgets', 'player'))
+
+  const loader = freshLoader(tmpRoot, real)
+  assert.match(stateOf(loader, 'greedy').reason, /owned by core/)
+  assert.match(stateOf(loader, 'alsogreedy').reason, /owned by core/)
+  assert.equal(stateOf(loader, 'polite').state, 'registered')
+})
+
+test('a root-mounted core layer does not make every prefix look taken', () => {
+  // public/index.js ends with `use('/', siteRouter)` and admin with the dashboard
+  // router; both match every path. Counting them would report every prefix as
+  // owned and no module could ever mount.
+  const tiers = emptyTiers()
+  tiers.public.use('/posts', express.Router())
+  tiers.public.use('/', express.Router())
+
+  writeModule('fine', oneRoute('/widgets'))
+  writeModule('taken', oneRoute('/posts'))
+  const loader = freshLoader(tmpRoot, tiers)
+
+  assert.equal(stateOf(loader, 'fine').state, 'registered')
+  assert.match(stateOf(loader, 'taken').reason, /owned by core/)
+})
+
+test('a prefix with a slash or a parameter in it is rejected', () => {
+  // A prefix that could contain either would let a module reach outside the slot
+  // it was given.
+  writeModule('nested', { manifest: { mounts: { public: ['/a/b'] } } })
+  writeModule('parameterised', { manifest: { mounts: { public: ['/:id'] } } })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'nested').reason, /bad prefix "\/a\/b"/)
+  assert.match(stateOf(loader, 'parameterised').reason, /bad prefix "\/:id"/)
+})
+
+test('registering a prefix that was never declared is rejected', () => {
+  // module.json is what the admin panel, the collision check and the reviewer
+  // all read, so it has to be the truth rather than a hint.
+  writeModule('sneaky', {
+    manifest: { mounts: { public: ['/declared'] } },
+    server: `module.exports = (ctx, api) => api.registerRoutes({
+      public: { '/declared': ctx.express.Router(), '/undeclared': ctx.express.Router() },
+    })`,
+  })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'sneaky').reason, /registered public\/undeclared without declaring it/)
+})
+
+test('declaring a prefix and never registering it is rejected too', () => {
+  writeModule('forgetful', {
+    manifest: { mounts: { public: ['/a', '/b'] } },
+    server: "module.exports = (ctx, api) => api.registerRoutes({ public: { '/a': ctx.express.Router() } })",
+  })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'forgetful').reason, /declared public\/b but never registered it/)
+})
+
+test('registering the same thing twice is an error, not a silent overwrite', () => {
+  writeModule('twice', {
+    manifest: { mounts: { public: ['/x'] } },
+    server: `module.exports = (ctx, api) => {
+      api.registerRoutes({ public: { '/x': ctx.express.Router() } })
+      api.registerRoutes({ public: { '/x': ctx.express.Router() } })
+    }`,
+  })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'twice').reason, /registerRoutes\(\) called twice/)
+})
+
+// ── Schema fragment validation (the replay itself is PR 3) ─────────────────
+
+test('a schema fragment declaring a core table is rejected', () => {
+  writeModule('thief', { schema: 'CREATE TABLE IF NOT EXISTS users (id INT);' })
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'thief').reason, /declares core table "users"/)
+})
+
+test('a schema fragment table must carry the module id as a prefix', () => {
+  writeModule('mine', { schema: 'CREATE TABLE IF NOT EXISTS widgets (id INT);' })
+  assert.match(stateOf(freshLoader(tmpRoot), 'mine').reason, /not prefixed "mine_"/)
+
+  tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-modules-'))
+  writeModule('mine', { schema: 'CREATE TABLE IF NOT EXISTS mine_widgets (id INT);' })
+  assert.equal(stateOf(freshLoader(tmpRoot), 'mine').state, 'registered')
+})
+
+test('two modules cannot own the same table either', () => {
+  writeModule('aaa', { schema: 'CREATE TABLE IF NOT EXISTS aaa_shared (id INT);' })
+  writeModule('bbb', {
+    manifest: { id: 'bbb' },
+    schema: 'CREATE TABLE IF NOT EXISTS aaa_shared (id INT);',
+  })
+  const loader = freshLoader(tmpRoot)
+  assert.equal(stateOf(loader, 'aaa').state, 'registered')
+  assert.match(stateOf(loader, 'bbb').reason, /already owned by module "aaa"/)
+})
+
+test('declaring a schema without a purge is rejected', () => {
+  const dir = path.join(tmpRoot, 'noway')
+  fs.mkdirSync(dir, { recursive: true })
+  fs.writeFileSync(
+    path.join(dir, 'module.json'),
+    JSON.stringify({ id: 'noway', name: 'x', version: '1.0.0', coreApi: '^1.0.0', schema: 'schema.sql' }),
+  )
+  const loader = freshLoader(tmpRoot)
+  // A module that can create tables and cannot drop them leaves an operator with
+  // orphaned data and no supported way to remove it.
+  assert.match(stateOf(loader, 'noway').reason, /declares schema but no purge/)
+})
+
+test('a declared purge file that is not there is rejected', () => {
+  writeModule('gone', { schema: 'CREATE TABLE IF NOT EXISTS gone_x (id INT);' })
+  fs.unlinkSync(path.join(tmpRoot, 'gone', 'purge.sql'))
+  const loader = freshLoader(tmpRoot)
+  assert.match(stateOf(loader, 'gone').reason, /purge file "purge\.sql" is missing/)
+})
+
+// ── Mounting and the dispatch guard ────────────────────────────────────────
+
+test('a registered module answers on its prefix; a failed one is simply absent', async () => {
+  writeModule('good', oneRoute('/widgets'))
+  writeModule('bad', { ...oneRoute('/broken'), server: 'throw new Error("boom")' })
+
+  const tiers = emptyTiers()
+  freshLoader(tmpRoot, tiers)
+  const app = await startApp((a) => a.use('/public', tiers.public))
+
+  try {
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 200)
+    // Failed BEFORE mounting, so its routes are not absent-with-a-503 — they do
+    // not exist at all (§4.4, left-hand column).
+    assert.equal((await fetch(`${app.url}/public/broken`)).status, 404)
+  } finally {
+    await app.close()
+  }
+})
+
+test('a module that fails AFTER mounting keeps its URLs and answers 503', async () => {
+  // The right-hand column of §4.4, and the reason routes.manifest.json can be
+  // generated off a dead database: the URL surface must not depend on whether a
+  // boot step succeeded on the generating machine. PR 3 (schema replay) and PR 5
+  // (onBoot) are the two things that will trip this in real life; here the state
+  // is moved by hand, because the loader is the thing under test.
+  writeModule('later', oneRoute('/widgets'))
+
+  const tiers = emptyTiers()
+  const loader = freshLoader(tmpRoot, tiers)
+  const app = await startApp((a) => a.use('/public', tiers.public))
+
+  try {
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 200)
+
+    assert.equal(stateOf(loader, 'later').state, 'registered')
+
+    loader.setState('later', 'startup_failed', 'schema fragment blew up')
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 503)
+    assert.equal(stateOf(loader, 'later').reason, 'schema fragment blew up')
+
+    // The 404 leg becomes reachable for real in PR 5, when the boot reconcile
+    // reads a `disabled` row out of installed_modules. A disabled module is
+    // mounted and guarded, never unmounted (§4.5) — same reason as the 503.
+    loader.setState('later', 'disabled')
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 404)
+
+    loader.setState('later', 'started')
+    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 200)
+  } finally {
+    await app.close()
+  }
+})
+
+test('setState refuses a state that is not one, and shrugs at an unknown id', () => {
+  writeModule('here', oneRoute('/widgets'))
+  const loader = freshLoader(tmpRoot)
+
+  assert.throws(() => loader.setState('here', 'enabled'), /unknown module state "enabled"/)
+  // An id with no record is not the boot path's problem to escalate.
+  assert.doesNotThrow(() => loader.setState('never-installed', 'started'))
+})
+
+// ── ctx ────────────────────────────────────────────────────────────────────
+
+test('ctx exposes exactly the documented surface, and is frozen', () => {
+  const seen = path.join(tmpRoot, 'probe-out.json')
+  writeModule('probe', {
+    server: `const fs = require('fs')
+      module.exports = (ctx) => {
+        let mutable = true
+        try { ctx.db.query = null; mutable = ctx.db.query === null } catch { mutable = false }
+        fs.writeFileSync(${JSON.stringify(seen)}, JSON.stringify({
+          keys: Object.keys(ctx).sort(),
+          middleware: Object.keys(ctx.middleware).sort(),
+          moduleId: ctx.moduleId,
+          mutable,
+        }))
+      }`,
+  })
+  assert.equal(stateOf(freshLoader(tmpRoot), 'probe').state, 'registered')
+
+  const probe = JSON.parse(fs.readFileSync(seen, 'utf8'))
+  assert.deepEqual(probe.keys, [
+    'auth', 'db', 'express', 'log', 'middleware', 'moduleId', 'paths',
+    'posts', 'push', 'secretBox', 'settings', 'uploads', 'validator',
+  ])
+  assert.deepEqual(probe.middleware, ['noindex', 'requireAuth', 'requireRole', 'siteMode', 'validate'])
+  assert.equal(probe.moduleId, 'probe')
+  assert.equal(probe.mutable, false, 'ctx members must be frozen')
+})
+
+test('the register calls PR 4 and PR 5 own throw rather than silently accepting', () => {
+  // An accepting no-op would let a module believe it had registered a
+  // notification stream or a boot hook and fail silently at the far end.
+  for (const [call, pr] of [
+    ['registerExtension', 4],
+    ['registerNotificationStreams', 4],
+    ['registerAnnounceLeg', 4],
+    ['onBoot', 5],
+    ['onShutdown', 5],
+  ]) {
+    tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-modules-'))
+    writeModule('early', { server: `module.exports = (ctx, api) => api.${call}(() => {})` })
+    assert.match(
+      stateOf(freshLoader(tmpRoot), 'early').reason,
+      new RegExp(`${call}: not available until phase 2 PR ${pr}`),
+    )
+  }
+})