Module-Rust/server/scripts/checkBundle.js

#!/usr/bin/env node
// ── Does the release actually ship everything the module needs? ────────────
//
// `ci/bundle.json` says what a release copies. `server/index.js` says what the
// module requires. Nothing keeps two lists in agreement on its own, and the first
// module this project shipped proved it: `Module-uo` added `server/commands/` in a
// cutover, its include list did not learn about it, and v1.0.0 installed cleanly
// and then died on the operator's box with
//
//   module "uo" failed to load — {"stage":"register","reason":"Cannot find
//   module './commands/guild.command'"}
//
// Nothing caught it, because the PR checks install the module by copying the
// WHOLE repo into core — they only ever exercised a tree that had the file. **The
// subset only exists in the release**, and the release had no check that the
// subset was complete. This module has that check from its first release rather
// than after its first outage.
//
// It asks the question in the two places it can be asked:
//
//   --check (PR checks)      Every file reachable from the entry point by a
//                            relative require lives under something ci/bundle.json
//                            lists. Source-tree only, so it is fast and needs no
//                            assembled bundle — it fails on the PR that adds the
//                            directory, which is where the fix is cheapest.
//
//   --bundle <dir> (release) Every relative specifier inside an ASSEMBLED bundle
//                            resolves to a file that is in it. Asked of the
//                            artifact rather than of the source, so it also
//                            catches a copy that half-failed, a list that names a
//                            path that has moved, and anything else between the
//                            declaration and the tarball.
//
// The two are deliberately not the same question. The first is about the list
// being right; the second is about the tarball being right. A release runs both.
//
// ── The third question, which is this module's own ─────────────────────────
//
// `server/package.json` declares **no runtime dependencies**, and the release
// therefore runs no `npm ci` and packs no `node_modules`. That is a decision, not
// an accident (org lead, phase 2), and the whole value of it is that the day it
// stops being true is a loud day. So both modes also assert the declaration is
// still empty: add a `dependencies` entry without teaching release.yml to install
// and pack it, and the bundle ships an import of something that is not there —
// the missing-directory failure again, wearing a different hat.
//
// ── Why reachability, and not "require the entry point" ────────────────────
//
// The obvious check — require the bundle's entry and see if it throws — does not
// work here, and the reason is in index.js's own header: its requires are inside
// `register()` because require order is load-bearing (`core.init(ctx)` has to run
// before anything under `router/` is required). So requiring the entry evaluates
// exactly one line, `require('./core')`, and reports success on a bundle missing
// every router it has. Calling `register()` for real would need a fake `ctx`
// complete enough to satisfy the whole module — which is what `test/` is for, and
// `test/` does not ship. Walking the requires statically asks the same question
// without needing either.

const fs = require('fs')
const path = require('path')
const { stripCommentsAndTemplates } = require('./checkImports')

const MODULE_ROOT = path.resolve(__dirname, '..', '..')

// Only relative specifiers. A bare one is checkImports.js's question, not this
// one, and the two failures want different advice.
const RELATIVE = /(?:require\(|from\s+|import\()\s*['"](\.[^'"]+)['"]/g

/**
 * Resolve a relative specifier the way Node would, for the file cases that can
 * appear here: an exact path, `+.js`/`+.json`, or a directory's `index.js`.
 *
 * Returns null when nothing exists — which is the finding, not an error.
 */
function resolveFile(fromDir, specifier) {
  const base = path.resolve(fromDir, specifier)
  const candidates = [base, `${base}.js`, `${base}.json`, path.join(base, 'index.js')]
  for (const c of candidates) {
    if (fs.existsSync(c) && fs.statSync(c).isFile()) return c
  }
  return null
}

/**
 * Every file reachable from `entry` by following relative requires, plus every
 * specifier that resolved to nothing.
 *
 * Exported so the test can point it at fixtures — the same reason checkImports.js
 * exports `scan`. A check that has never been shown to fail is a check nobody
 * knows the state of, and this one is load-bearing for every release.
 */
function reachable(entry) {
  const seen = new Set()
  const missing = []
  const queue = [entry]

  while (queue.length) {
    const file = queue.shift()
    if (seen.has(file)) continue
    seen.add(file)

    // A .json dependency is a leaf: it is reached, it ships, and it has no
    // requires of its own to follow.
    if (file.endsWith('.json')) continue

    const source = stripCommentsAndTemplates(fs.readFileSync(file, 'utf8'))
    for (const [, specifier] of source.matchAll(RELATIVE)) {
      const target = resolveFile(path.dirname(file), specifier)
      if (target) queue.push(target)
      else missing.push({ file, specifier })
    }
  }

  return { files: [...seen], missing }
}

/**
 * Everything ci/bundle.json says ends up in the bundle, as absolute paths:
 * `server[]` relative to server/, `root[]` and `generated[]` relative to the
 * module root. All three are equally "in the tarball" as far as a require is
 * concerned — the only difference is how they get there.
 */
function declaredServerPaths(moduleRoot = MODULE_ROOT) {
  const manifest = JSON.parse(fs.readFileSync(path.join(moduleRoot, 'ci', 'bundle.json'), 'utf8'))
  return [
    ...manifest.server.map((p) => path.join(moduleRoot, 'server', p)),
    ...(manifest.root || []).map((p) => path.join(moduleRoot, p)),
    ...(manifest.generated || []).map((p) => path.join(moduleRoot, p)),
  ]
}

/**
 * The runtime dependencies the shipped half declares.
 *
 * Empty is the shape this module is built around, and the release packs no
 * `node_modules` because of it. Returned rather than asserted so both modes can
 * report it with their own advice.
 */
function runtimeDependencies(moduleRoot = MODULE_ROOT) {
  const pkgPath = path.join(moduleRoot, 'server', 'package.json')
  if (!fs.existsSync(pkgPath)) return []
  return Object.keys(JSON.parse(fs.readFileSync(pkgPath, 'utf8')).dependencies || {})
}

const DEPENDENCY_ADVICE =
  'The release packs no node_modules, because this module declared none. A dependency\n' +
  'listed here but not installed and copied by .gitea/workflows/release.yml ships as an\n' +
  'import of something that is not in the tarball — the module installs and then dies at\n' +
  'the register stage on the operator\'s box.\n\n' +
  'Either drop the dependency (everything the shipped half needs arrives on ctx — §2.3),\n' +
  'or add the `npm ci --omit=dev` + copy steps to release.yml and list "node_modules" in\n' +
  'ci/bundle.json\'s server[], then update this check.\n'

const covers = (declared, file) => declared.some((d) => file === d || file.startsWith(d + path.sep))

/**
 * --check: is ci/bundle.json's list sufficient for what the entry point reaches?
 *
 * Reports the top-level entry to ADD rather than the individual files, because
 * that is the edit: the list is stated in top-level paths, and a new directory
 * arrives with a dozen files in it.
 */
function checkDeclaration(moduleRoot = MODULE_ROOT) {
  const serverRoot = path.join(moduleRoot, 'server')
  const entry = path.join(serverRoot, 'index.js')
  const { files, missing } = reachable(entry)
  const declared = declaredServerPaths(moduleRoot)

  // Grouped by the entry that would have to be added, which is the top-level
  // path under server/ — or, for the rare reachable file outside it, the path
  // itself, since that one belongs in root[] instead.
  const uncovered = new Map()
  for (const file of files) {
    if (covers(declared, file)) continue
    const inServer = file.startsWith(serverRoot + path.sep)
    const key = inServer
      ? `server/${path.relative(serverRoot, file).split(path.sep)[0]}`
      : path.relative(moduleRoot, file).split(path.sep).join('/')
    if (!uncovered.has(key)) uncovered.set(key, [])
    uncovered.get(key).push(file)
  }

  return { uncovered, missing, reached: files.length, dependencies: runtimeDependencies(moduleRoot) }
}

/**
 * --bundle: does every relative specifier inside an assembled bundle resolve?
 *
 * Walks the bundle's own server tree rather than starting from the entry point,
 * so a file that ships but is broken is caught too.
 */
function checkBundle(bundleRoot) {
  const serverRoot = path.join(bundleRoot, 'server')
  const missing = []
  const files = []

  const walk = (dir) => {
    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
      const p = path.join(dir, e.name)
      if (e.isDirectory()) {
        // An installed dependency tree would be npm's business, not this
        // check's. This module ships none; the skip stays so that the day one
        // arrives, this is not also the thing that breaks.
        if (e.name !== 'node_modules') walk(p)
      } else if (/\.(js|mjs|cjs)$/.test(e.name)) {
        files.push(p)
      }
    }
  }
  walk(serverRoot)

  for (const file of files) {
    const source = stripCommentsAndTemplates(fs.readFileSync(file, 'utf8'))
    for (const [, specifier] of source.matchAll(RELATIVE)) {
      if (!resolveFile(path.dirname(file), specifier)) missing.push({ file, specifier })
    }
  }

  return { missing, scanned: files.length, dependencies: runtimeDependencies(bundleRoot) }
}

module.exports = {
  reachable,
  resolveFile,
  checkDeclaration,
  checkBundle,
  declaredServerPaths,
  runtimeDependencies,
}

// Required by a test, or run as the check? Only the second one exits.
if (require.main !== module) return

const bundleFlag = process.argv.indexOf('--bundle')

if (bundleFlag !== -1) {
  const root = process.argv[bundleFlag + 1]
  if (!root) {
    console.error('--bundle needs the path to an assembled bundle')
    process.exit(2)
  }
  const { missing, scanned, dependencies } = checkBundle(path.resolve(root))
  if (missing.length) {
    console.error(`\nThe assembled bundle is incomplete — ${missing.length} require(s) resolve to nothing:\n`)
    for (const m of missing) {
      console.error(`  ${path.relative(root, m.file)}\n    requires "${m.specifier}" — not in the bundle`)
    }
    console.error('\nAdd the missing path to ci/bundle.json.\n')
    process.exit(1)
  }
  if (dependencies.length) {
    console.error(`\nThe assembled bundle declares ${dependencies.length} runtime dependency(ies) it does not carry:\n`)
    for (const d of dependencies) console.error(`  ${d}`)
    console.error(`\n${DEPENDENCY_ADVICE}`)
    process.exit(1)
  }
  console.log(`OK — every relative require in the bundle resolves (${scanned} files scanned), and it needs no node_modules.`)
} else {
  const { uncovered, missing, reached, dependencies } = checkDeclaration()

  if (missing.length) {
    console.error(`\n${missing.length} require(s) resolve to nothing in the source tree:\n`)
    for (const m of missing) {
      console.error(`  ${path.relative(MODULE_ROOT, m.file)}\n    requires "${m.specifier}"`)
    }
    console.error('')
    process.exit(1)
  }

  if (uncovered.size) {
    console.error('\nci/bundle.json does not ship everything server/index.js reaches.\n')
    console.error('A release built from this list would install and then fail at the')
    console.error("register stage with \"Cannot find module\", on the operator's box.\n")
    for (const [key, files] of uncovered) {
      console.error(`  ${key}   (${files.length} file${files.length === 1 ? '' : 's'} reachable)`)
      for (const f of files.slice(0, 5)) console.error(`      ${path.relative(MODULE_ROOT, f)}`)
      if (files.length > 5) console.error(`      … and ${files.length - 5} more`)
    }
    // server[] is written relative to server/, so name the entry to add rather
    // than the path just displayed — they differ by exactly that prefix.
    const toServer = [...uncovered.keys()].filter((k) => k.startsWith('server/'))
    const toRoot = [...uncovered.keys()].filter((k) => !k.startsWith('server/'))
    if (toServer.length) {
      console.error(`\nAdd ${toServer.map((k) => `"${k.slice('server/'.length)}"`).join(', ')} to ci/bundle.json's server[].`)
    }
    if (toRoot.length) {
      console.error(`\nAdd ${toRoot.map((k) => `"${k}"`).join(', ')} to ci/bundle.json's root[].`)
    }
    console.error('')
    process.exit(1)
  }

  if (dependencies.length) {
    console.error(`\nserver/package.json declares ${dependencies.length} runtime dependency(ies):\n`)
    for (const d of dependencies) console.error(`  ${d}`)
    console.error(`\n${DEPENDENCY_ADVICE}`)
    process.exit(1)
  }

  console.log(`OK — ci/bundle.json ships every file server/index.js reaches (${reached} files), and no runtime dependency is declared.`)
}