Module-uo/server/utils/clilocSource.js

// Cliloc table — the filesystem layer.
//
// `clilocParse.js` holds the pure parsers; this module is the only thing that
// touches cliloc files on disk, and it is shared by both callers:
//
//   - the server, which refreshes the table on boot (`shardClilocs.model.js`)
//   - the admin panel, which can force a reimport without a restart
//
// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived
// from them is committed: the repo holds no string table, exactly as it holds no
// map snapshot and no artwork. That rule is why this module reads a configured
// path instead of a path inside the repo.
//
// ── Why this reads a SET of files, not one ────────────────────────────────
//
// Shards edit items and add new ones. Those carry cliloc ids that a stock client
// table does not have — and forcing a 5 MB client re-export every time an
// operator adds one item would be miserable enough that the table would simply
// go stale, which is the exact failure the spawn atlas was redesigned to avoid.
//
// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted
// client file) plus every operator-maintained OVERLAY beside it, all re-read on
// every boot and hash-gated as a SET. Adding, editing or removing any overlay
// counts as drift and re-imports. Later sources win, so an overlay both adds new
// ids and overrides stock ones.
//
// Measured on a real shard: the script tree references 16,434 cliloc ids and only
// 37 are absent from the stock client table. Tens of entries against a 67k base
// is what makes the overlay the right shape rather than a second full table.
//
// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so
// the boot path hashes first and only parses when something actually changed.

const crypto = require('crypto')
const fs = require('fs')
const path = require('path')

const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse')

/**
 * Filenames looked for as the BASE table when the configured path is a directory.
 *
 * Ordered by how specific they are: an explicitly converted file wins over
 * something that merely sits in a client folder, so an operator who dropped a
 * `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one
 * they made rather than the one that will be rejected.
 *
 * Matching is case-insensitive against the real directory listing, because the
 * client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where
 * a hardcoded lowercase open would simply miss.
 */
const CANDIDATE_NAMES = [
  'clilocs.tsv',
  'clilocs.csv',
  'clilocs.plain',
  'cliloc.plain',
  'cliloc.plain.enu',
  'cliloc.enu.plain',
  'clilocs.txt',
  'cliloc.enu',
]

/**
 * Where shard-specific additions and overrides live: a `custom/` directory
 * beside the base table.
 *
 * ServUO has **no server-side convention** for custom clilocs — they live in the
 * patched client file a shard distributes to its players, and nothing in the
 * tree declares them. There is therefore nothing to discover, and this is the
 * one place in the cliloc pipeline that is a convention we chose rather than one
 * the shard already has. It is a directory rather than a single file so an
 * operator can keep additions grouped however they like (per system, per patch)
 * without the site caring.
 */
const CUSTOM_DIR = 'custom'
const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain']

class ClilocSourceError extends Error {
  constructor(message, code) {
    super(message)
    this.name = 'ClilocSourceError'
    this.code = code
  }
}

function sha256(buffer) {
  return crypto.createHash('sha256').update(buffer).digest('hex')
}

/**
 * Resolve the configured path to `{ root, base }`.
 *
 * Accepts either a direct file path or a directory to search, because operators
 * reasonably supply both — "here is the file" and "here is the folder I put it
 * in" are equally natural answers to the admin panel's prompt. When it is a
 * file, `root` is the directory CONTAINING it, so overlays work either way: an
 * operator who pointed at a file should not have to re-point at its folder just
 * to add a `custom/` directory next to it.
 */
function resolveBase(configured) {
  if (!configured || String(configured).trim() === '') {
    throw new ClilocSourceError('No cliloc path configured', 'NO_PATH')
  }
  const target = String(configured).trim()

  let stat
  try {
    stat = fs.statSync(target)
  } catch {
    throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND')
  }

  if (stat.isFile()) return { root: path.dirname(target), base: target }

  if (!stat.isDirectory()) {
    throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND')
  }

  let listing
  try {
    listing = fs.readdirSync(target)
  } catch {
    throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND')
  }

  const byLower = new Map(listing.map((name) => [name.toLowerCase(), name]))
  for (const candidate of CANDIDATE_NAMES) {
    const actual = byLower.get(candidate)
    if (actual) return { root: target, base: path.join(target, actual) }
  }

  throw new ClilocSourceError(
    `No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`,
    'NO_FILE',
  )
}

/** Overlay files under `<root>/custom/`, sorted so precedence is deterministic. */
function listCustom(root) {
  const dir = path.join(root, CUSTOM_DIR)
  let listing
  try {
    listing = fs.readdirSync(dir, { withFileTypes: true })
  } catch (err) {
    // No overlay directory is the normal case, not an error.
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
    throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE')
  }
  return listing
    .filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase()))
    .map((e) => e.name)
    .sort()
    .map((name) => path.join(dir, name))
}

function readFileOrThrow(file) {
  try {
    return fs.readFileSync(file)
  } catch {
    throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE')
  }
}

/**
 * Read every cliloc source under the configured path.
 *
 * Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }`
 * with the base first and overlays after, in the order they must be merged.
 *
 * Labels are root-relative and forward-slashed so a hash map compares equal
 * across platforms — the same directory read on Windows and Linux must produce
 * the same fingerprint, or every boot would look like a change. (The same
 * reasoning, and the same bug, as `spawnAtlasSource.readSources`.)
 */
function readSources(configured) {
  const { root, base } = resolveBase(configured)

  const describe = (file, kind) => {
    const buffer = readFileOrThrow(file)
    return {
      label: path.relative(root, file).split(path.sep).join('/'),
      kind,
      file,
      buffer,
      sha256: sha256(buffer),
      bytes: buffer.length,
      compressed: isCompressedCliloc(buffer),
    }
  }

  const files = [describe(base, 'base')]
  for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom'))

  return { root, files }
}

/**
 * A fingerprint of every source: `{ "<label>": "<sha256>" }`, plus the base's
 * details for the admin panel.
 *
 * `compressed` is reported here rather than left to the parse because the admin
 * panel calls this and NOT `readCliloc` (parsing 5 MB on every status poll would
 * be wasteful). Without it, pointing the setting at an unconverted client
 * directory reports a perfectly readable file with pending drift — "ready to
 * import" — and the operator only learns otherwise when the import fails. The
 * check is four bytes of a buffer already in hand.
 */
function hashSources(configured) {
  const { root, files } = readSources(configured)
  const hashes = {}
  for (const file of files) hashes[file.label] = file.sha256
  const base = files[0]
  return {
    root,
    hashes,
    file: base.file,
    bytes: base.bytes,
    compressed: files.some((f) => f.compressed),
    customCount: files.length - 1,
  }
}

/** True when two source fingerprints describe the same set of files. */
function sameSources(a, b) {
  if (!a || !b) return false
  const aKeys = Object.keys(a).sort()
  const bKeys = Object.keys(b).sort()
  if (aKeys.length !== bKeys.length) return false
  return aKeys.every((key, i) => key === bKeys[i] && a[key] === b[key])
}

/**
 * Labels present in `loaded` that are absent from `current`.
 *
 * This is the multi-source hazard that a single file did not have. One corrupt
 * file fails the parse loudly, but a source that has simply VANISHED — an
 * unmounted volume, a half-copied deploy — parses perfectly and imports a table
 * quietly missing everything that file contributed. That is the same ambiguity
 * the spawn atlas escalates for a disappearing facet, so it is escalated here
 * too rather than applied.
 */
function missingSources(current, loaded) {
  if (!loaded) return []
  return Object.keys(loaded).filter((label) => !Object.hasOwn(current, label))
}

/**
 * Read and parse every source, merged into one entry list.
 *
 * Later sources win: the base client table first, then each overlay in sorted
 * order, so an overlay both ADDS ids the client never had and OVERRIDES stock
 * ones the shard has re-purposed.
 *
 * Returns `{ entries, source }`. Throws `ClilocSourceError` for anything about
 * the paths and `ClilocFormatError` for anything about the contents — different
 * problems for an operator (wrong place vs wrong file), and the admin panel says
 * which. A format error names the file it came from, because "which of my six
 * overlay files is malformed" is otherwise a guessing game.
 */
function readCliloc(configured) {
  const { root, files } = readSources(configured)

  const merged = new Map()
  const perSource = []

  for (const file of files) {
    let entries
    try {
      entries = parseCliloc(file.buffer)
    } catch (err) {
      if (err instanceof ClilocFormatError) {
        throw new ClilocFormatError(`${file.label}: ${err.message}`, err.code)
      }
      throw err
    }

    let added = 0
    let overrode = 0
    for (const entry of entries) {
      if (!Number.isInteger(entry.number)) continue
      if (merged.has(entry.number)) overrode++
      else added++
      merged.set(entry.number, entry)
    }
    perSource.push({ label: file.label, kind: file.kind, entries: entries.length, added, overrode })
  }

  return {
    entries: [...merged.values()],
    source: {
      root,
      file: files[0].file,
      sha256: files[0].sha256,
      bytes: files[0].bytes,
      hashes: Object.fromEntries(files.map((f) => [f.label, f.sha256])),
      parserVersion: PARSER_VERSION,
      sources: perSource,
    },
  }
}

module.exports = {
  ClilocFormatError,
  ClilocSourceError,
  PARSER_VERSION,
  CANDIDATE_NAMES,
  CUSTOM_DIR,
  CUSTOM_EXTENSIONS,
  resolveBase,
  listCustom,
  readSources,
  hashSources,
  sameSources,
  missingSources,
  readCliloc,
}