website/server/src/modules/loader.js

// ── The module loader ──────────────────────────────────────────────────────
//
// Phase 2, PR 2 of docs/website/MODULE_SYSTEM.md §2.7. The normative contract is
// docs/website/MODULE_API.md Part 4; where the two disagree, the contract wins.
//
// The one property this file exists to guarantee, and the reason it looks the
// way it does:
//
//   **The filesystem is the mounting source of truth, and mounting is
//   SYNCHRONOUS.** `scripts/routeManifest.js:38` and `swagger/swagger.js:29`
//   both require app.js with the pool pointed at a dead port. A loader that
//   awaited a database row before mounting would make every module route
//   invisible to the frozen-URL-surface test (§1.12). So: readdirSync at require
//   time, no database, no promises (§4.1).
//
// A module that fails ANYWHERE in this file fails alone. Nothing here may throw
// past its own try/catch — a bad module must cost the site its routes, never its
// boot (§4.4).
//
// What is deliberately NOT here yet, each landing with the PR that first calls
// it (§2.7): the three de-entanglement registries (PR 4), boot/shutdown hook
// dispatch and the `installed_modules` reconcile (PR 5), GET
// /api/v1/public/modules and the client chunk's static mount (PRs 6-7). Until
// PR 5 the state a module carries is in memory only.
//
// PR 3 added the fragment half of the schema story: this file VALIDATES a
// fragment (statement by statement, at load time, before anything is mounted)
// and publishes it through `fragments()`. Replaying it needs a database, so it
// belongs to modules/schema.js, which utils/db.js calls after core's schema.

const fs = require('fs')
const path = require('path')

const { MODULE_API_VERSION } = require('./version')
const semver = require('./semver')
const registries = require('./registries')
const { splitStatements } = require('../utils/sqlStatements')

const log = require('../utils/logger')('modules')

const REPO_ROOT = path.join(__dirname, '..', '..', '..')
const MODULES_DIR = process.env.MODULES_DIR || path.join(REPO_ROOT, 'modules')

// One segment, lowercase, no parameters. A module prefix that could contain a
// `/` or a `:` would let a module reach outside the slot it was given.
const ID = /^[a-z][a-z0-9-]{1,31}$/
const PREFIX = /^\/[a-z0-9][a-z0-9-]*$/
const TIERS = ['public', 'admin', 'player']

const MANIFEST_KEYS = new Set([
  'id', 'name', 'version', 'coreApi', 'server', 'client',
  'schema', 'purge', 'mounts', 'extensions', 'capabilities',
])

// Extension slots are declared by core, at require time, in the router that owns
// the resource (registries.declareSlot). The loader asks the registry which exist
// rather than keeping a list, for the same reason the prefix check probes the
// live tier routers: a second copy of the answer is a copy that drifts.

// id → record. Populated by load(), read by list().
const modules = new Map()
let loaded = false

// ── ctx ────────────────────────────────────────────────────────────────────

// Everything a module may reach in core, and nothing else (§2.3). Required
// lazily inside the factory rather than at file scope: this file is required by
// app.js, and hoisting these to the top would make the DB pool, the settings
// model and the upload directory startup-time dependencies of the loader itself.
function buildCtx(id, moduleRoot) {
  /* eslint-disable global-require */
  // The shared SERVER dependencies — the exact counterpart of window.__rg's
  // react/react-dom/react-router on the client, and load-bearing for the same
  // two reasons (§7.2).
  //
  //   1. A module lives at <repo>/modules/<id>/, OUTSIDE server/, so Node's
  //      resolver walks up from there and never sees server/node_modules. A
  //      module that required 'express' itself would fail to load — which is
  //      exactly how this was discovered.
  //   2. Even if it resolved, a second copy of express in the process is a
  //      second Router prototype and a second set of instanceof checks. One
  //      express, owned by core, is the same rule as one React.
  //
  // The consequence for a module author is the same on both sides: declare these
  // external, never bundle them, take them from what core hands you.
  const express = require('express')
  const validator = require('express-validator')
  const db = require('../utils/db')
  const settings = require('../model/settings/settings.model')
  const posts = require('../model/posts/posts.model')
  const auth = require('../utils/auth')
  const pushDispatch = require('../utils/pushDispatch')
  const secretBox = require('../utils/secretBox')
  const createLogger = require('../utils/logger')
  const { requireAuth, requireRole } = require('../auth/session.middleware')
  const siteMode = require('../middleware/siteMode')
  const validate = require('../middleware/validate')
  const noindex = require('../middleware/noindex')
  const uploads = require('../router/v1/admin/imageUpload')
  /* eslint-enable global-require */

  // Narrowed on purpose (§2.3): utils/auth also re-exports signToken,
  // setAuthCookie and the TOTP challenge primitives, and minting a session is
  // core's job. A module that needs an identity needs to READ one.
  const ctx = {
    moduleId: id,
    paths: { moduleRoot },
    express,
    validator,
    db: { query: db.query, pool: db.pool },
    log: (namespace) => createLogger(namespace ? `${id}:${namespace}` : id),
    settings: {
      get: settings.get,
      set: settings.set,
      getInstanceName: settings.getInstanceName,
    },
    auth: { getUserFromRequest: auth.getUserFromRequest },
    push: { publish: pushDispatch.publish },
    secretBox: { encrypt: secretBox.encrypt, decrypt: secretBox.decrypt },
    middleware: { requireAuth, requireRole, siteMode, validate, noindex },
    uploads,
    posts: {
      listAll: posts.listAll,
      getById: posts.getById,
      linkAnnounceJob: posts.linkAnnounceJob,
      markAnnounced: posts.markAnnounced,
    },
  }
  // A guard against accident, not against a hostile module — the boundary is
  // organisational, not a security boundary (MODULE_SYSTEM.md §2.2).
  for (const value of Object.values(ctx)) {
    if (value && typeof value === 'object') Object.freeze(value)
  }
  return Object.freeze(ctx)
}

// ── The registration api ───────────────────────────────────────────────────

// Collects what the module registers so validation can compare it against what
// module.json DECLARED. Declaration is the contract; a module that registers a
// prefix it did not declare is rejected, because module.json is what the admin
// panel, the collision check and the reviewer all read.
function buildApi(record) {
  const once = (name) => {
    if (record.called.has(name)) throw new Error(`${name}() called twice`)
    record.called.add(name)
  }
  // PR 5 brings the boot hooks. They throw rather than no-op: an accepting stub
  // would let a module believe it had registered something and fail silently at
  // the far end.
  const notYet = (name, pr) => () => {
    throw new Error(`${name}: not available until phase 2 PR ${pr}`)
  }
  return {
    registerRoutes(mounts) {
      once('registerRoutes')
      if (!mounts || typeof mounts !== 'object') throw new Error('registerRoutes: expected an object')
      for (const [tier, byPrefix] of Object.entries(mounts)) {
        if (!TIERS.includes(tier)) throw new Error(`registerRoutes: unknown tier "${tier}"`)
        for (const [prefix, router] of Object.entries(byPrefix)) {
          if (!PREFIX.test(prefix)) throw new Error(`registerRoutes: bad prefix "${prefix}"`)
          if (typeof router !== 'function') throw new Error(`registerRoutes: ${tier}${prefix} is not a router`)
          record.routes[tier].set(prefix, router)
        }
      }
    },
    // The three de-entanglement registries (§2.4). They live in registries.js
    // rather than here because core registers through the same staging area, and
    // core has no `api` object.
    //
    // These STAGE. Nothing a module registers is visible to core until the
    // second pass commits it, for the reason the second pass exists at all: a
    // module that throws halfway through register(), or fails checkDeclared
    // after it, must leave nothing behind. A half-registered stream catalog
    // would be worse than a missing one — it would be a subscribable stream
    // nothing will ever publish to.
    registerExtension: record.staged.registerExtension,
    registerNotificationStreams(streams) {
      once('registerNotificationStreams')
      record.staged.registerNotificationStreams(streams)
    },
    registerAnnounceLeg: record.staged.registerAnnounceLeg,
    onBoot: notYet('onBoot', 5),
    onShutdown: notYet('onShutdown', 5),
  }
}

// ── Validation ─────────────────────────────────────────────────────────────

// Table names a module may create despite not carrying its own id as a prefix.
//
// module-uo's twenty-seven tables predate the module system by two years, and
// renaming live tables is a data migration this workstream deliberately does not
// do (MODULE_SYSTEM.md §1.6). Grandfathering them by an explicit, per-module
// allowlist keeps the prefix rule real for every module written after this one —
// the alternative, dropping the rule, would leave the first name collision to be
// discovered by a module silently adopting someone else's table.
const LEGACY_TABLE_PREFIXES = { uo: ['shard_', 'uo_link_'] }

const CREATE_TABLE = /CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[`"]?(\w+)[`"]?/gi

/** Table names core's own schema.sql declares — a module may not touch these. */
let coreTables = null
function coreTableNames() {
  if (coreTables) return coreTables
  coreTables = new Set()
  try {
    const sql = fs.readFileSync(path.join(__dirname, '..', '..', 'db', 'schema.sql'), 'utf8')
    for (const m of sql.matchAll(CREATE_TABLE)) coreTables.add(m[1].toLowerCase())
  } catch (err) {
    log.warn('could not read core schema for the table-collision check', { message: err.message })
  }
  return coreTables
}

// The only leading verbs a fragment may use — an allowlist, not a DROP denylist.
//
// §2.6 bans `DROP`, but a denylist only ever bans what somebody thought of, and
// core's own schema.sql needs exactly four verbs: CREATE, ALTER, INSERT, UPDATE.
// Anything else in a file that is REPLAYED ON EVERY BOOT is a mistake worth
// failing on — TRUNCATE and DELETE would empty a table every restart, RENAME
// would break on the second one, and GRANT/SET/USE are core's business, not a
// module's. CREATE covers CREATE INDEX as well as CREATE TABLE.
//
// This is a leading-verb check and says so: `ALTER TABLE x DROP COLUMN y` passes
// it. Catching that needs a SQL parser, which is a large dependency to take on
// for a rule whose real job is stopping the obvious foot-gun early.
const ALLOWED_VERBS = new Set(['CREATE', 'ALTER', 'INSERT', 'UPDATE'])

const CREATE_TABLE_ANY = /^CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?/i
const CREATE_TABLE_GUARDED = /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+/i

/**
 * Read and validate a module's schema fragment; return every table it declares.
 *
 * Validation happens HERE, at load time, and not in modules/schema.js where the
 * fragment is replayed, because every rule §2.6 states is knowable by reading
 * the file — no database required. Failing at load means a module with a bad
 * fragment never mounts at all (§4.4's first column: routes and nav simply
 * absent), rather than mounting, 503ing, and leaving whatever its fragment did
 * manage to execute behind it.
 *
 * Throws if the file is unreadable or breaks a rule.
 */
function tablesOf(dir, manifest) {
  if (!manifest.schema) return new Set()
  const file = path.join(dir, manifest.schema)
  const sql = fs.readFileSync(file, 'utf8')

  for (const statement of splitStatements(sql)) {
    const verb = (statement.match(/^\w+/) || [''])[0].toUpperCase()
    if (!ALLOWED_VERBS.has(verb)) {
      throw new Error(`schema fragment statement starts with "${verb}" (allowed: ${[...ALLOWED_VERBS].join(', ')})`)
    }
    // A bare CREATE TABLE succeeds exactly once and fails every boot after it,
    // which presents as a module that worked until the first restart.
    if (CREATE_TABLE_ANY.test(statement) && !CREATE_TABLE_GUARDED.test(statement)) {
      throw new Error('schema fragment has a CREATE TABLE without IF NOT EXISTS')
    }
  }

  return new Set([...sql.matchAll(CREATE_TABLE)].map((m) => m[1].toLowerCase()))
}

function checkTableNames(id, tables) {
  const allowed = LEGACY_TABLE_PREFIXES[id] || []
  const core = coreTableNames()

  for (const table of tables) {
    if (core.has(table)) throw new Error(`schema fragment declares core table "${table}"`)
    for (const other of modules.values()) {
      if (other.tables.has(table)) {
        throw new Error(`schema fragment declares "${table}", already owned by module "${other.id}"`)
      }
    }
    const prefixed = table.startsWith(`${id}_`) || allowed.some((p) => table.startsWith(p))
    if (!prefixed) throw new Error(`schema fragment table "${table}" is not prefixed "${id}_"`)
  }
}

/**
 * Does core already own this prefix in this tier?
 *
 * Asked of the LIVE tier router rather than a hardcoded list, so the check
 * cannot drift the first time core adds a capability router — the spike's
 * hardcoded table was already one prefix stale when it was written. Modules are
 * loaded after every core mount, so the stack is complete by the time this runs,
 * and `layer.match` is express's own matcher rather than a second-guess at its
 * regexp grammar.
 *
 * Root-mounted layers are skipped: `use(noindex, requireAuth)` and the two
 * `use('/', singletonRouter)` mounts match every path, and counting them would
 * report every prefix as taken.
 */
function ownedByCore(tierRouter, prefix) {
  return (tierRouter.stack || []).some(
    (layer) => layer.regexp && !layer.regexp.fast_slash && layer.match(prefix),
  )
}

function readManifest(dir, id, tierRouters) {
  const file = path.join(dir, 'module.json')
  const manifest = JSON.parse(fs.readFileSync(file, 'utf8'))

  for (const key of Object.keys(manifest)) {
    // Rejected, not ignored: a typo'd key must be a loud failure rather than a
    // silently inert setting the operator believes they configured.
    if (!MANIFEST_KEYS.has(key)) throw new Error(`unknown key "${key}" in module.json`)
  }
  if (!ID.test(manifest.id || '')) throw new Error(`invalid id "${manifest.id}"`)
  if (manifest.id !== id) throw new Error(`id "${manifest.id}" does not match directory "${id}"`)
  if (!manifest.version) throw new Error('missing version')
  if (!manifest.coreApi) throw new Error('missing coreApi')
  if (!semver.satisfies(MODULE_API_VERSION, manifest.coreApi)) {
    throw new Error(`needs core API ${manifest.coreApi}, this core is ${MODULE_API_VERSION}`)
  }

  for (const [tier, prefixes] of Object.entries(manifest.mounts || {})) {
    if (!TIERS.includes(tier)) throw new Error(`unknown tier "${tier}" in mounts`)
    for (const prefix of prefixes) {
      if (!PREFIX.test(prefix)) throw new Error(`bad prefix "${prefix}" in mounts.${tier}`)
      if (ownedByCore(tierRouters[tier], prefix)) {
        throw new Error(`prefix ${tier}${prefix} is owned by core`)
      }
      for (const other of modules.values()) {
        if ((other.manifest.mounts?.[tier] || []).includes(prefix)) {
          throw new Error(`prefix ${tier}${prefix} already registered by module "${other.id}"`)
        }
      }
    }
  }

  for (const slot of manifest.extensions || []) {
    if (!registries.hasSlot(slot)) throw new Error(`unknown extension slot "${slot}"`)
  }

  if (manifest.schema && !manifest.purge) {
    // A module that can create tables and cannot drop them leaves an operator
    // with orphaned data and no supported way to remove it.
    throw new Error('declares schema but no purge')
  }
  if (manifest.purge && !fs.existsSync(path.join(dir, manifest.purge))) {
    throw new Error(`purge file "${manifest.purge}" is missing`)
  }
  return manifest
}

// What the module registered must equal what it declared — in both directions.
function checkDeclared(record) {
  const declared = record.manifest.mounts || {}
  for (const tier of TIERS) {
    const want = new Set(declared[tier] || [])
    const got = new Set(record.routes[tier].keys())
    for (const p of got) if (!want.has(p)) throw new Error(`registered ${tier}${p} without declaring it`)
    for (const p of want) if (!got.has(p)) throw new Error(`declared ${tier}${p} but never registered it`)
  }
}

// ── Load ───────────────────────────────────────────────────────────────────

/**
 * Discover, validate, register and mount every module under MODULES_DIR.
 *
 * **Called exactly once, explicitly, from app.js**, after the three tier routers
 * are required and before the app is exported. There is no lazy self-scan: the
 * spike's was lazy and silent, so requiring the loader and reading the module
 * list gave an empty array and no error (MODULE_API.md §7.6). Everything that
 * reads the module list now throws until this has run.
 *
 * The ordering is not incidental. Core's mounts must already be on the tier
 * routers, because that is what the prefix-collision check is asked about; and
 * modules mount after them, so first-match-wins means a module could not shadow
 * a core prefix even if the check were bypassed.
 *
 * Safe to call when the modules directory does not exist — that is the normal
 * case for a bare core, and it is the state this PR ships in.
 *
 * @param {{public: Router, admin: Router, player: Router}} tierRouters
 */
function load(tierRouters) {
  if (loaded) return
  for (const tier of TIERS) {
    if (!tierRouters || typeof tierRouters[tier] !== 'function') {
      throw new Error(`modules.load: missing the "${tier}" tier router`)
    }
  }
  loaded = true

  let entries = []
  try {
    entries = fs.readdirSync(MODULES_DIR, { withFileTypes: true })
      .filter((e) => e.isDirectory())
      .map((e) => e.name)
      .sort() // alphabetical: there is no dependency resolution, and any other
              // order would imply a precedence nothing computes (§4.2)
  } catch {
    return // no modules directory is the normal case for a bare core
  }

  for (const id of entries) {
    const dir = path.join(MODULES_DIR, id)
    if (!fs.existsSync(path.join(dir, 'module.json'))) continue

    const record = {
      id,
      dir,
      manifest: null,
      routes: { public: new Map(), admin: new Map(), player: new Map() },
      staged: registries.stage(id),
      tables: new Set(),
      called: new Set(),
      state: 'installed',
      reason: null,
    }

    try {
      record.manifest = readManifest(dir, id, tierRouters)
      record.tables = tablesOf(dir, record.manifest)
      checkTableNames(id, record.tables)
      if (record.manifest.server) {
        const entry = path.join(dir, record.manifest.server)
        // eslint-disable-next-line global-require, import/no-dynamic-require
        const register = require(entry)
        if (typeof register !== 'function') throw new Error(`${record.manifest.server} does not export a function`)
        register(buildCtx(id, dir), buildApi(record))
        checkDeclared(record)
      }
      record.state = 'registered'
      modules.set(id, record)
      log.info(`registered module "${id}" v${record.manifest.version}`, {
        mounts: record.manifest.mounts,
      })
    } catch (err) {
      // A failure here is BEFORE any route was mounted, so this module's routes
      // and nav are simply absent and the site comes up without it (§4.4).
      record.state = 'startup_failed'
      record.reason = err.message
      record.manifest = record.manifest || { id, version: 'unknown' }
      modules.set(id, record)
      log.error(`module "${id}" failed to load — continuing without it`, { reason: err.message })
    }
  }

  // Mounting is a SECOND pass, after every module has been validated, and not
  // because it reads better. `ownedByCore` asks the live tier router what is
  // already on it, so mounting inside the loop would make the first module's
  // layers indistinguishable from core's — the second module claiming a taken
  // prefix would be told it collided with core, naming the wrong culprit, and
  // the module-versus-module check below it could never be reached.
  for (const record of modules.values()) {
    if (record.state !== 'registered') continue
    try {
      // Commit what this module staged. Collisions with core or with an earlier
      // module surface here, in scan order, and cost only this module.
      registries.apply(record.staged.staged)
    } catch (err) {
      record.state = 'startup_failed'
      record.reason = err.message
      log.error(`module "${record.id}" failed to register — continuing without it`, {
        reason: err.message,
      })
      continue // unmounted, exactly like a validation failure in the first pass
    }
    mount(record, tierRouters)
  }
}

/**
 * Mount one module's routers onto the tier routers, behind the dispatch guard.
 *
 * The guard is the other half of §4.4. A module that fails BEFORE this point has
 * no routes at all; one that fails after — schema replay (PR 3), `onBoot`
 * (PR 5) — keeps its URLs and answers 503, so `routes.manifest.json` never
 * depends on whether a boot hook happened to succeed on the machine that
 * generated it. `disabled` is 404 and unreachable until PR 5 wires
 * `installed_modules` in; it is written here because the guard is the contract's
 * §4.5, not a later addition.
 */
function mount(record, tierRouters) {
  for (const tier of TIERS) {
    for (const [prefix, router] of record.routes[tier]) {
      tierRouters[tier].use(prefix, (req, res, next) => {
        if (record.state === 'startup_failed') {
          return res.status(503).json({ message: 'Module unavailable' })
        }
        if (record.state === 'disabled') return res.status(404).json({ message: 'Not found' })
        return next()
      }, router)
    }
  }
}

// ── State ──────────────────────────────────────────────────────────────────

// The states a loaded record may hold, deliberately a hardcoded subset rather
// than an import of model/modules/modules.model.js's STATES: that model reaches
// the database, and this file must stay require-able against a dead one.
// `installed` is not here because a record leaves load() resolved either way.
const RECORD_STATES = new Set(['registered', 'started', 'disabled', 'startup_failed'])

/**
 * Move a loaded module to a new state — the POST-mount transitions.
 *
 * Called by whoever ran the step that failed or the step that succeeded, because
 * only they can know: PR 3's `ensureSchema()` replays the fragments, PR 5's boot
 * dispatch runs `onBoot` and reconciles `installed_modules` (whose `disabled`
 * rows are what first make the guard's 404 leg reachable).
 *
 * Unknown ids are ignored rather than thrown on: a module can be absent from the
 * volume and still have a row, and a caller on the boot path must not turn that
 * into everyone's failure.
 */
function setState(id, state, reason = null) {
  if (!RECORD_STATES.has(state)) throw new Error(`unknown module state "${state}"`)
  const record = modules.get(id)
  if (!record) return
  record.state = state
  record.reason = reason
}

// ── Introspection ──────────────────────────────────────────────────────────

function assertLoaded(caller) {
  if (!loaded) throw new Error(`modules.${caller}() before modules.load()`)
}

/**
 * Has load() run in this process?
 *
 * The one legitimate reason to ask instead of just calling an accessor: a
 * process that never required app.js and so has no module list to be wrong
 * about. `npm run seed` (db/seed.js) is exactly that — it calls ensureSchema()
 * standalone, and the fragment replay has to be able to tell "this is the seed
 * script" from "the server booted and something is mis-ordered", which is the
 * distinction §7.6's throw exists to preserve everywhere else.
 */
const isLoaded = () => loaded

/**
 * Every module found on the volume, loaded or failed, in scan order.
 *
 * Throws rather than returning `[]` when load() has not run — the empty list is
 * a real answer for a core with no modules installed, and a caller cannot tell
 * the two apart (§7.6).
 */
function list() {
  assertLoaded('list')
  return [...modules.values()].map((r) => ({
    id: r.id,
    name: r.manifest.name || r.id,
    version: r.manifest.version,
    state: r.state,
    reason: r.reason,
    capabilities: r.manifest.capabilities || [],
  }))
}

/**
 * Every schema fragment waiting to be replayed, in scan order.
 *
 * `registered` only: a module that failed validation must not get its tables
 * created (it is not going to run), and one already `started` has had them. The
 * absolute path is resolved here rather than handed out as a manifest-relative
 * name, so the replay never has to know how a module directory is laid out.
 *
 * @returns {{id: string, file: string}[]}
 */
function fragments() {
  assertLoaded('fragments')
  return [...modules.values()]
    .filter((r) => r.state === 'registered' && r.manifest.schema)
    .map((r) => ({ id: r.id, file: path.join(r.dir, r.manifest.schema) }))
}

/** Absolute path of the modules directory. */
const dir = () => MODULES_DIR

module.exports = { load, list, setState, fragments, isLoaded, dir }