website/server/test/moduleLifecycle.test.js

// ── Boot and shutdown dispatch ─────────────────────────────────────────────
//
// Phase 2, PR 5. The contract is MODULE_API.md §2.5 (when the hooks run, in what
// order, with what budget), §4.4 (a failure after mounting is a 503, not a
// crash), §4.5 (a disabled module is guarded, never unmounted) and
// MODULE_SYSTEM.md §2.4 (what a boot does to installed_modules).
//
// The property under test throughout, as in moduleLoader.test.js and
// moduleSchema.test.js: **the failing module fails alone.** A hook that throws,
// a hook that hangs, a row that will not write — none of them may cost the site
// its boot or the next module its start.
//
// No database is involved: `boot()` takes the model as an injectable dependency
// for exactly the reason `replayFragments` takes its query, and the fake below
// records every call so the ORDER of the reconcile can be asserted — which is
// the whole design, not an implementation detail.
process.env.DB_HOST = '127.0.0.1'
process.env.DB_PORT = '59999'

const fs = require('fs')
const os = require('os')
const path = require('path')

const { test, beforeEach, after } = require('node:test')
const assert = require('node:assert/strict')

const express = require('express')

const db = require('../src/utils/db')
const registries = require('../src/modules/registries')
const lifecycle = require('../src/modules/lifecycle')
const { startApp } = require('./_helper')

after(() => db.close())

let tmpRoot

const emptyTiers = () => ({
  public: express.Router(),
  admin: express.Router(),
  player: express.Router(),
})

function freshLoader(dir, tiers = emptyTiers()) {
  process.env.MODULES_DIR = dir
  registries._reset()
  delete require.cache[require.resolve('../src/modules/loader')]
  // eslint-disable-next-line global-require
  const loader = require('../src/modules/loader')
  loader.load(tiers)
  return loader
}

/**
 * A module whose hooks report themselves into a file.
 *
 * A file rather than a shared array because the module is `require`d from disk
 * and cannot close over anything this file owns — the same trick the ctx probe
 * in moduleLoader.test.js uses.
 */
function writeModule(id, { boot, shutdown, mounts, log: logFile } = {}) {
  const dir = path.join(tmpRoot, id)
  fs.mkdirSync(dir, { recursive: true })
  fs.writeFileSync(path.join(dir, 'module.json'), JSON.stringify({
    id,
    name: `Module ${id}`,
    version: '1.2.3',
    coreApi: '^1.0.0',
    server: 'index.js',
    ...(mounts === undefined ? {} : { mounts }),
  }))
  const note = logFile
    ? `const note = (what) => require('fs').appendFileSync(${JSON.stringify(logFile)}, what + '\\n')`
    : 'const note = () => {}'
  const register = mounts === undefined ? '' : `
    const r = ctx.express.Router()
    r.get('/', (req, res) => res.json({ ok: true }))
    api.registerRoutes({ public: { '${(mounts.public || [])[0]}': r } })`
  // `boot: ''` means "registers a hook that does nothing", which is a different
  // module from one that registers no hook at all — hence the undefined check
  // rather than a truthiness test.
  fs.writeFileSync(path.join(dir, 'index.js'), `${note}
    module.exports = (ctx, api) => {${register}
      ${boot === undefined ? '' : `api.onBoot(async (c) => { note('boot:${id}' + (c && c.moduleId === '${id}' ? ':ctx' : ':NOCTX')); ${boot} })`}
      ${shutdown === undefined ? '' : `api.onShutdown(async () => { note('shutdown:${id}'); ${shutdown} })`}
    }`)
  return dir
}

/** An in-memory stand-in for model/modules/modules.model.js. */
function fakeModel(seed = []) {
  const rows = new Map(seed.map((r) => [r.id, { failureStage: null, failureReason: null, ...r }]))
  const calls = []
  const model = {
    rows,
    calls,
    async beginBoot() {
      calls.push('beginBoot')
      for (const row of rows.values()) {
        if (row.state === 'disabled') continue
        Object.assign(row, { state: 'enabled', failureStage: null, failureReason: null })
      }
    },
    async recordInstalled({ id, name, version }) {
      calls.push(`recordInstalled:${id}`)
      const row = rows.get(id)
      // Metadata is refreshed; state is deliberately left alone (§2.4).
      if (row) Object.assign(row, { name, version })
      else rows.set(id, { id, name, version, state: 'installed', failureStage: null, failureReason: null })
    },
    async list() {
      calls.push('list')
      return [...rows.values()]
    },
    async markStarted(id) {
      calls.push(`markStarted:${id}`)
      Object.assign(rows.get(id), { state: 'started', failureStage: null, failureReason: null })
    },
    async markStartupFailed(id, { stage, reason }) {
      calls.push(`markStartupFailed:${id}`)
      const row = rows.get(id)
      // The model's own softening, reproduced because tests below depend on it:
      // a disabled row is a no-op, or an outcome would overwrite the operator's
      // decision and silently re-enable the module next boot.
      if (!row || row.state === 'disabled') return
      Object.assign(row, { state: 'startup_failed', failureStage: stage, failureReason: reason })
    },
    async disable(id) {
      calls.push(`disable:${id}`)
      const row = rows.get(id)
      if (row) Object.assign(row, { state: 'disabled', failureStage: null, failureReason: null })
    },
  }
  return model
}

const stateOf = (loader, id) => loader.list().find((m) => m.id === id)
const noted = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n') : [])

beforeEach(() => {
  tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-lifecycle-'))
})

// ── The happy path ─────────────────────────────────────────────────────────

test('every module is recorded, booted with its ctx and marked started, in scan order', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', log: file })
  writeModule('bbb', { boot: '', log: file })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })

  // The reconcile order IS the design (§2.4): clear the last boot's outcomes
  // first, so what is on display afterwards is what this boot did.
  assert.equal(model.calls[0], 'beginBoot')
  assert.deepEqual(model.calls.slice(1, 3), ['recordInstalled:aaa', 'recordInstalled:bbb'])
  assert.deepEqual(noted(file), ['boot:aaa:ctx', 'boot:bbb:ctx'])
  assert.equal(stateOf(loader, 'aaa').state, 'started')
  assert.equal(model.rows.get('bbb').state, 'started')
  // §2.4's metadata refresh: the row carries what the admin screen shows.
  assert.equal(model.rows.get('aaa').name, 'Module aaa')
  assert.equal(model.rows.get('aaa').version, '1.2.3')
})

test('a hand-placed directory gets a row with no provenance', async () => {
  // §2.5 keeps a directory dropped on the volume by hand a supported install.
  // Without a row it could never be disabled, and nothing could report it.
  writeModule('byhand', { boot: '' })
  const model = fakeModel()

  await lifecycle.boot({ modules: freshLoader(tmpRoot), model })

  const row = model.rows.get('byhand')
  assert.equal(row.state, 'started')
  assert.equal(row.source ?? null, null)
  assert.equal(row.sha256 ?? null, null)
})

test('a module with no onBoot still reaches started', async () => {
  writeModule('quiet', {})
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })

  // Nothing to warm up is not the same as never having started: the guard lets
  // its routes through, so the row has to agree that it is serving.
  assert.equal(stateOf(loader, 'quiet').state, 'started')
  assert.equal(model.rows.get('quiet').state, 'started')
})

// ── Failure is a state ─────────────────────────────────────────────────────

test('an onBoot that throws fails its own module and no one else', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', log: file })
  writeModule('bbb', { boot: 'throw new Error("cache warm-up failed")', log: file })
  writeModule('ccc', { boot: '', log: file })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })

  assert.equal(stateOf(loader, 'bbb').state, 'startup_failed')
  assert.equal(stateOf(loader, 'bbb').stage, 'boot')
  assert.match(stateOf(loader, 'bbb').reason, /cache warm-up failed/)
  assert.equal(model.rows.get('bbb').failureStage, 'boot')
  // The one that matters: the module AFTER the failure still booted.
  assert.deepEqual(noted(file), ['boot:aaa:ctx', 'boot:bbb:ctx', 'boot:ccc:ctx'])
  assert.equal(stateOf(loader, 'ccc').state, 'started')
})

test('a module whose onBoot failed keeps its URLs and answers 503', async () => {
  // §4.4's right-hand column, reached by the real mechanism rather than a
  // hand-moved state: routes.manifest.json must not depend on whether a boot
  // hook happened to succeed on the machine that generated it.
  writeModule('svc', { boot: 'throw new Error("no")', mounts: { public: ['/widgets'] } })
  const tiers = emptyTiers()
  const loader = freshLoader(tmpRoot, tiers)
  const app = await startApp((a) => a.use('/public', tiers.public))

  try {
    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 200)
    await lifecycle.boot({ modules: loader, model: fakeModel() })
    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 503)
  } finally {
    await app.close()
  }
})

test('a failure from load or schema replay is written down with its stage', async () => {
  // Both happen before the database is reachable — load() at require time, the
  // replay inside ensureSchema — so the boot reconcile is where they land.
  writeModule('bad', {})
  fs.writeFileSync(path.join(tmpRoot, 'bad', 'module.json'), JSON.stringify({
    id: 'bad', name: 'bad', version: '1.0.0', coreApi: '^99.0.0',
  }))
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })

  const row = model.rows.get('bad')
  assert.equal(row.state, 'startup_failed')
  assert.equal(row.failureStage, 'core_api')
  assert.match(row.failureReason, /needs core API \^99\.0\.0/)
})

// ── The operator's switch ──────────────────────────────────────────────────

test('a disabled row guards the module, skips its hook and is not overwritten', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('off', { boot: '', shutdown: '', mounts: { public: ['/widgets'] }, log: file })
  const tiers = emptyTiers()
  const loader = freshLoader(tmpRoot, tiers)
  const model = fakeModel([{ id: 'off', name: 'Module off', version: '1.2.3', state: 'disabled' }])
  const app = await startApp((a) => a.use('/public', tiers.public))

  try {
    await lifecycle.boot({ modules: loader, model })

    // §4.5's 404 leg, unreachable until this reconcile existed: mounted and
    // guarded, never unmounted, so the URL surface stays a property of the
    // volume rather than of a database row.
    assert.equal((await fetch(`${app.url}/public/widgets`)).status, 404)
    assert.equal(stateOf(loader, 'off').state, 'disabled')
    assert.deepEqual(noted(file), [], 'a disabled module must not be booted')
    // Still disabled: an outcome must never overwrite a decision, or the next
    // boot would silently switch it back on.
    assert.equal(model.rows.get('off').state, 'disabled')
    // And nothing to tear down, because it never started.
    await lifecycle.shutdown({ modules: loader })
    assert.deepEqual(noted(file), [])
  } finally {
    await app.close()
  }
})

test('a row whose directory is gone is marked failed rather than left claiming enabled', async () => {
  writeModule('here', { boot: '' })
  const model = fakeModel([
    { id: 'here', name: 'Module here', version: '1.2.3', state: 'started' },
    { id: 'gone', name: 'Module gone', version: '0.9.0', state: 'started' },
    // An uninstall leaves `disabled`, which beginBoot never touches — so this
    // one is not an anomaly and must be left exactly as the operator left it.
    { id: 'uninstalled', name: 'Module uninstalled', version: '0.1.0', state: 'disabled' },
  ])

  await lifecycle.boot({ modules: freshLoader(tmpRoot), model })

  assert.equal(model.rows.get('here').state, 'started')
  assert.equal(model.rows.get('gone').state, 'startup_failed')
  assert.match(model.rows.get('gone').failureReason, /not present on the volume/)
  assert.equal(model.rows.get('uninstalled').state, 'disabled')
})

// ── The site comes up regardless ───────────────────────────────────────────

test('a database that will not take the bookkeeping still boots the modules', async () => {
  // A row that will not update is bad — the admin panel shows the wrong thing —
  // but it is strictly less bad than a site that will not start.
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', log: file })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()
  for (const name of ['beginBoot', 'recordInstalled', 'list', 'markStarted']) {
    model[name] = async () => { throw new Error('ER_LOCK_WAIT_TIMEOUT') }
  }

  await assert.doesNotReject(() => lifecycle.boot({ modules: loader, model }))

  assert.deepEqual(noted(file), ['boot:aaa:ctx'])
  assert.equal(stateOf(loader, 'aaa').state, 'started')
})

test('a process that never scanned writes nothing at all', async () => {
  // `npm run seed` is exactly this: it calls ensureSchema() without ever
  // requiring app.js. Reconciling against an empty scan would mark every
  // installed module as missing from the volume.
  process.env.MODULES_DIR = tmpRoot
  delete require.cache[require.resolve('../src/modules/loader')]
  // eslint-disable-next-line global-require
  const unscanned = require('../src/modules/loader')
  const model = fakeModel([{ id: 'real', name: 'Module real', version: '1.0.0', state: 'started' }])

  await lifecycle.boot({ modules: unscanned, model })

  assert.deepEqual(model.calls, [])
  assert.equal(model.rows.get('real').state, 'started')
})

// ── Shutdown ───────────────────────────────────────────────────────────────

test('shutdown runs started modules in reverse order and skips the rest', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', shutdown: '', log: file })
  writeModule('bbb', { boot: 'throw new Error("no")', shutdown: '', log: file })
  writeModule('ccc', { boot: '', shutdown: '', log: file })
  const loader = freshLoader(tmpRoot)

  await lifecycle.boot({ modules: loader, model: fakeModel() })
  fs.writeFileSync(file, '') // only the shutdown half is under test
  await lifecycle.shutdown({ modules: loader })

  // Reverse of boot order, and `bbb` absent: its onBoot threw, so it has a
  // half-built world that its onShutdown was never written to tear down.
  assert.deepEqual(noted(file), ['shutdown:ccc', 'shutdown:aaa'])
})

test('a hook that hangs costs its budget, not the shutdown', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', shutdown: '', log: file })
  writeModule('zzz', { boot: '', shutdown: 'await new Promise(() => {})', log: file })
  const loader = freshLoader(tmpRoot)

  await lifecycle.boot({ modules: loader, model: fakeModel() })
  fs.writeFileSync(file, '')

  const started = Date.now()
  await lifecycle.shutdown({ modules: loader, budgetMs: 50 })

  // zzz never returns; it is abandoned and aaa still gets its turn. The
  // alternative is a host where `systemctl stop` hangs until SIGKILL.
  assert.deepEqual(noted(file), ['shutdown:zzz', 'shutdown:aaa'])
  assert.ok(Date.now() - started < 2000, 'shutdown must not wait on a hung hook')
})

test('a hook that throws does not stop the ones behind it', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', shutdown: '', log: file })
  writeModule('zzz', { boot: '', shutdown: 'throw new Error("close failed")', log: file })
  const loader = freshLoader(tmpRoot)

  await lifecycle.boot({ modules: loader, model: fakeModel() })
  fs.writeFileSync(file, '')

  await assert.doesNotReject(() => lifecycle.shutdown({ modules: loader }))
  assert.deepEqual(noted(file), ['shutdown:zzz', 'shutdown:aaa'])
})

// ── stop(): the admin panel's Disable ──────────────────────────────────────
//
// Phase 4, §2.7.2 decision 3. Phase 2's disable moved a record and left the
// module running; the whole point of these tests is that it no longer does.

test('stop runs that one module\'s onShutdown and leaves the others alone', async () => {
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: '', shutdown: '', log: file })
  writeModule('bbb', { boot: '', shutdown: '', log: file })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })
  fs.writeFileSync(file, '')

  const result = await lifecycle.stop('aaa', { modules: loader, model })

  assert.deepEqual(result, { stopped: true, error: null })
  // Only aaa. This is the difference from shutdown(), which runs everything.
  assert.deepEqual(noted(file), ['shutdown:aaa'])
  assert.equal(stateOf(loader, 'aaa').state, 'disabled')
  assert.equal(stateOf(loader, 'bbb').state, 'started', 'the other module keeps running')
  assert.equal(model.rows.get('aaa').state, 'disabled')
})

test('stop moves the record only after the hook has run', async () => {
  // While onShutdown runs, the module is still `started` — the only state in
  // which its routes and the world it is tearing down agree with each other. The
  // module reports its own view of itself, so a record moved too early shows up
  // here as `disabled` instead of `started`.
  const file = path.join(tmpRoot, 'log.txt')
  const dir = path.join(tmpRoot, 'aaa')
  fs.mkdirSync(dir, { recursive: true })
  fs.writeFileSync(path.join(dir, 'module.json'), JSON.stringify({
    id: 'aaa', name: 'A', version: '1.0.0', coreApi: '^1.0.0', server: 'index.js',
  }))
  fs.writeFileSync(path.join(dir, 'index.js'), `
    module.exports = (ctx, api) => {
      api.onBoot(async () => {})
      api.onShutdown(async () => {
        const loader = require(${JSON.stringify(require.resolve('../src/modules/loader'))})
        const me = loader.list().find((m) => m.id === 'aaa')
        require('fs').appendFileSync(${JSON.stringify(file)}, 'state-during-hook:' + me.state + ${JSON.stringify('\n')})
      })
    }`)

  const loader = freshLoader(tmpRoot)
  await lifecycle.boot({ modules: loader, model: fakeModel() })
  await lifecycle.stop('aaa', { modules: loader, model: fakeModel([{ id: 'aaa', state: 'started' }]) })

  assert.deepEqual(noted(file), ['state-during-hook:started'])
  assert.equal(stateOf(loader, 'aaa').state, 'disabled')
})

test('a hook that throws does not prevent the disable', async () => {
  // The opposite of the boot path's rule, deliberately. There, a failure means
  // the module never became safe to use; here, the operator has asked for it to
  // stop answering and a module that could not close cleanly is a reason to log
  // loudly, not a reason to leave it serving.
  writeModule('aaa', { boot: '', shutdown: 'throw new Error("socket stuck")' })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })
  const result = await lifecycle.stop('aaa', { modules: loader, model })

  assert.equal(result.stopped, false)
  assert.match(result.error, /socket stuck/)
  assert.equal(stateOf(loader, 'aaa').state, 'disabled', 'disabled anyway')
  assert.equal(model.rows.get('aaa').state, 'disabled')
})

test('a hook that hangs costs its budget, and the module is still disabled', async () => {
  writeModule('aaa', { boot: '', shutdown: 'await new Promise(() => {})' })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })
  const started = Date.now()
  const result = await lifecycle.stop('aaa', { modules: loader, model, budgetMs: 50 })

  assert.equal(result.stopped, false)
  assert.match(result.error, /budget/)
  assert.equal(stateOf(loader, 'aaa').state, 'disabled')
  assert.ok(Date.now() - started < 2000)
})

test('stopping a module with no onShutdown still disables it', async () => {
  // A hookless module has nothing to run and must still stop answering, or the
  // guard and the row disagree about what is serving.
  writeModule('aaa', { boot: '' })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })
  const result = await lifecycle.stop('aaa', { modules: loader, model })

  assert.deepEqual(result, { stopped: false, error: null })
  assert.equal(stateOf(loader, 'aaa').state, 'disabled')
  assert.equal(model.rows.get('aaa').state, 'disabled')
})

test('stopping a module whose onBoot failed does not run its onShutdown', async () => {
  // Same rule shutdownHooks() applies: a module that never finished warming up
  // has a half-built world its onShutdown was not written for. It is still
  // disabled — it just is not asked to tear anything down.
  const file = path.join(tmpRoot, 'log.txt')
  writeModule('aaa', { boot: 'throw new Error("no")', shutdown: '', log: file })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()

  await lifecycle.boot({ modules: loader, model })
  const before = noted(file)
  const result = await lifecycle.stop('aaa', { modules: loader, model })

  assert.equal(result.stopped, false)
  // Compared against what was already there rather than against an empty file:
  // `noted` splits, so a blanked file reads as [''] and an emptiness assertion
  // would pass for the wrong reason.
  assert.deepEqual(noted(file), before, 'no new hook output')
  assert.ok(!noted(file).includes('shutdown:aaa'))
  assert.equal(stateOf(loader, 'aaa').state, 'disabled')
})

test('stopping an unknown id writes the row and does not throw', async () => {
  // A row can exist for a module that is not on the volume, and disabling it is
  // exactly what an operator would do about that.
  writeModule('aaa', { boot: '' })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel([{ id: 'ghost', state: 'startup_failed' }])

  await assert.doesNotReject(() => lifecycle.stop('ghost', { modules: loader, model }))
  assert.equal(model.rows.get('ghost').state, 'disabled')
})

test('a row that will not write does not stop the module from being disabled', async () => {
  // The same rule the boot path follows: bookkeeping failure is not the
  // operation failing. The guard is what stops traffic, and it has already moved.
  writeModule('aaa', { boot: '', shutdown: '' })
  const loader = freshLoader(tmpRoot)
  const model = fakeModel()
  await lifecycle.boot({ modules: loader, model })
  model.disable = async () => { throw new Error('database is gone') }

  await assert.doesNotReject(() => lifecycle.stop('aaa', { modules: loader, model }))
  assert.equal(stateOf(loader, 'aaa').state, 'disabled')
})