Module-uo/server/utils/clilocSource.js

// Cliloc table — the filesystem layer.
//
// `clilocParse.js` holds the pure parsers; this module is the only thing that
// touches cliloc files on disk, and it is shared by both callers:
//
//   - the server, which refreshes the table on boot (`shardClilocs.model.js`)
//   - the admin panel, which can force a reimport without a restart
//
// ── What protocol 8 took away, and what it left ───────────────────────────
//
// The BASE table no longer comes from here on a shard that has uo-link
// configured: `clilocBridge.js` asks the shard for it, because the shard has the
// operator's client files already and, since phase 2, the decompressor to read
// them (docs/link/v8.md §9). Nobody converts a file by hand any more.
//
// Two things keep this module alive rather than deleting it:
//
//   - **Overlays.** Shard-added items carry cliloc ids no client table has, and
//     ServUO has no server-side notion of a custom cliloc — there is nothing on
//     the shard to ask for. `custom/` is still a directory the site reads, and
//     `readOverlays` below is the entry point the bridge path uses.
//   - **Installs with no shard link**, and development. A site that has never
//     configured uo-link can still be pointed at a converted file; that path is
//     deprecated, not removed, and it stays the whole of this module's base-table
//     behaviour.
//
// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived
// from them is committed: the repo holds no string table, exactly as it holds no
// map snapshot and no artwork. That rule is why this module reads a configured
// path instead of a path inside the repo.
//
// ── Why this reads a SET of files, not one ────────────────────────────────
//
// Shards edit items and add new ones. Those carry cliloc ids that a stock client
// table does not have — and forcing a 5 MB client re-export every time an
// operator adds one item would be miserable enough that the table would simply
// go stale, which is the exact failure the spawn atlas was redesigned to avoid.
//
// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted
// client file) plus every operator-maintained OVERLAY beside it, all re-read on
// every boot and hash-gated as a SET. Adding, editing or removing any overlay
// counts as drift and re-imports. Later sources win, so an overlay both adds new
// ids and overrides stock ones.
//
// Measured on a real shard: the script tree references 16,434 cliloc ids and only
// 37 are absent from the stock client table. Tens of entries against a 67k base
// is what makes the overlay the right shape rather than a second full table.
//
// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so
// the boot path hashes first and only parses when something actually changed.

const crypto = require('crypto')
const fs = require('fs')
const path = require('path')

const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse')

/**
 * Filenames looked for as the BASE table when the configured path is a directory.
 *
 * Ordered by how specific they are: an explicitly converted file wins over
 * something that merely sits in a client folder, so an operator who dropped a
 * `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one
 * they made rather than the one that will be rejected.
 *
 * Matching is case-insensitive against the real directory listing, because the
 * client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where
 * a hardcoded lowercase open would simply miss.
 */
const CANDIDATE_NAMES = [
  'clilocs.tsv',
  'clilocs.csv',
  'clilocs.plain',
  'cliloc.plain',
  'cliloc.plain.enu',
  'cliloc.enu.plain',
  'clilocs.txt',
  'cliloc.enu',
]

/**
 * Where shard-specific additions and overrides live: a `custom/` directory
 * beside the base table.
 *
 * ServUO has **no server-side convention** for custom clilocs — they live in the
 * patched client file a shard distributes to its players, and nothing in the
 * tree declares them. There is therefore nothing to discover, and this is the
 * one place in the cliloc pipeline that is a convention we chose rather than one
 * the shard already has. It is a directory rather than a single file so an
 * operator can keep additions grouped however they like (per system, per patch)
 * without the site caring.
 */
const CUSTOM_DIR = 'custom'
const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain']

class ClilocSourceError extends Error {
  constructor(message, code) {
    super(message)
    this.name = 'ClilocSourceError'
    this.code = code
  }
}

function sha256(buffer) {
  return crypto.createHash('sha256').update(buffer).digest('hex')
}

/**
 * Resolve the configured path to `{ root, base }`.
 *
 * Accepts either a direct file path or a directory to search, because operators
 * reasonably supply both — "here is the file" and "here is the folder I put it
 * in" are equally natural answers to the admin panel's prompt. When it is a
 * file, `root` is the directory CONTAINING it, so overlays work either way: an
 * operator who pointed at a file should not have to re-point at its folder just
 * to add a `custom/` directory next to it.
 */
function resolveBase(configured) {
  if (!configured || String(configured).trim() === '') {
    throw new ClilocSourceError('No cliloc path configured', 'NO_PATH')
  }
  const target = String(configured).trim()

  let stat
  try {
    stat = fs.statSync(target)
  } catch {
    throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND')
  }

  if (stat.isFile()) return { root: path.dirname(target), base: target }

  if (!stat.isDirectory()) {
    throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND')
  }

  let listing
  try {
    listing = fs.readdirSync(target)
  } catch {
    throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND')
  }

  const byLower = new Map(listing.map((name) => [name.toLowerCase(), name]))
  for (const candidate of CANDIDATE_NAMES) {
    const actual = byLower.get(candidate)
    if (actual) return { root: target, base: path.join(target, actual) }
  }

  throw new ClilocSourceError(
    `No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`,
    'NO_FILE',
  )
}

/** Overlay files under `<root>/custom/`, sorted so precedence is deterministic. */
function listCustom(root) {
  const dir = path.join(root, CUSTOM_DIR)
  let listing
  try {
    listing = fs.readdirSync(dir, { withFileTypes: true })
  } catch (err) {
    // No overlay directory is the normal case, not an error.
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
    throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE')
  }
  return listing
    .filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase()))
    .map((e) => e.name)
    .sort()
    .map((name) => path.join(dir, name))
}

function readFileOrThrow(file) {
  try {
    return fs.readFileSync(file)
  } catch {
    throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE')
  }
}

/**
 * Read every cliloc source under the configured path.
 *
 * Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }`
 * with the base first and overlays after, in the order they must be merged.
 *
 * Labels are root-relative and forward-slashed so a hash map compares equal
 * across platforms — the same directory read on Windows and Linux must produce
 * the same fingerprint, or every boot would look like a change. (The same
 * reasoning, and the same bug, as `spawnAtlasSource.readSources`.)
 */
function readSources(configured) {
  const { root, base } = resolveBase(configured)

  const describe = (file, kind) => {
    const buffer = readFileOrThrow(file)
    return {
      label: path.relative(root, file).split(path.sep).join('/'),
      kind,
      file,
      buffer,
      sha256: sha256(buffer),
      bytes: buffer.length,
      compressed: isCompressedCliloc(buffer),
    }
  }

  const files = [describe(base, 'base')]
  for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom'))

  return { root, files }
}

/**
 * Read the OVERLAY files only, with no base table.
 *
 * The bridge path needs exactly this: the base arrives from the shard and the
 * `custom/` directory beside the configured path still has to be merged over it.
 * `readSources` cannot answer it, because resolving a base is the first thing it
 * does and there may not be one — an operator on the bridge is entitled to point
 * this setting at a directory that holds nothing but `custom/`.
 *
 * **Never throws.** A path that is blank, missing or unreadable is reported as a
 * `problem` string and an empty file list, because none of those may stop a base
 * table that arrived perfectly well from being imported. The model decides what
 * to do about it — and it has a real decision to make, since an overlay that was
 * loaded last time and is missing now is the vanished-source hazard, not a
 * config typo.
 */
function readOverlays(configured) {
  const target = String(configured ?? '').trim()
  if (target === '') return { root: null, files: [], problem: null }

  let root
  try {
    const stat = fs.statSync(target)
    root = stat.isFile() ? path.dirname(target) : target
  } catch {
    return { root: null, files: [], problem: `Cliloc path does not exist: ${target}` }
  }

  let overlays
  try {
    overlays = listCustom(root)
  } catch (err) {
    return { root, files: [], problem: err.message }
  }

  const files = []
  for (const file of overlays) {
    let buffer
    try {
      buffer = fs.readFileSync(file)
    } catch {
      return { root, files: [], problem: `Cliloc overlay is not readable: ${file}` }
    }
    files.push({
      label: path.relative(root, file).split(path.sep).join('/'),
      kind: 'custom',
      file,
      buffer,
      sha256: sha256(buffer),
      bytes: buffer.length,
      compressed: isCompressedCliloc(buffer),
    })
  }

  return { root, files, problem: null }
}

/**
 * A fingerprint of every source: `{ "<label>": "<sha256>" }`, plus the base's
 * details for the admin panel.
 *
 * `compressed` is reported here rather than left to the parse because the admin
 * panel calls this and NOT `readCliloc` (parsing 5 MB on every status poll would
 * be wasteful). Without it, pointing the setting at an unconverted client
 * directory reports a perfectly readable file with pending drift — "ready to
 * import" — and the operator only learns otherwise when the import fails. The
 * check is four bytes of a buffer already in hand.
 */
function hashSources(configured) {
  const { root, files } = readSources(configured)
  const hashes = {}
  for (const file of files) hashes[file.label] = file.sha256
  const base = files[0]
  return {
    root,
    hashes,
    file: base.file,
    bytes: base.bytes,
    compressed: files.some((f) => f.compressed),
    customCount: files.length - 1,
  }
}

/** True when two source fingerprints describe the same set of files. */
function sameSources(a, b) {
  if (!a || !b) return false
  const aKeys = Object.keys(a).sort()
  const bKeys = Object.keys(b).sort()
  if (aKeys.length !== bKeys.length) return false
  return aKeys.every((key, i) => key === bKeys[i] && a[key] === b[key])
}

/**
 * Labels present in `loaded` that are absent from `current`.
 *
 * This is the multi-source hazard that a single file did not have. One corrupt
 * file fails the parse loudly, but a source that has simply VANISHED — an
 * unmounted volume, a half-copied deploy — parses perfectly and imports a table
 * quietly missing everything that file contributed. That is the same ambiguity
 * the spawn atlas escalates for a disappearing facet, so it is escalated here
 * too rather than applied.
 */
function missingSources(current, loaded) {
  if (!loaded) return []
  return Object.keys(loaded).filter((label) => !Object.hasOwn(current, label))
}

/**
 * The same question asked of OVERLAYS only.
 *
 * Needed because the base table moved to the bridge. An install upgraded from the
 * file pipeline carries a base label (`clilocs.plain`, say) in its loaded
 * fingerprint, and that label is *supposed* to disappear when the base starts
 * arriving from the shard — reporting it as a vanished source would make every
 * first import after the upgrade demand an approval for a change the upgrade
 * itself made. Overlay labels are the ones whose absence is genuinely ambiguous,
 * and they are exactly the labels under `custom/`.
 */
function missingOverlays(current, loaded) {
  if (!loaded) return []
  const prefix = `${CUSTOM_DIR}/`
  return Object.keys(loaded).filter(
    (label) => label.startsWith(prefix) && !Object.hasOwn(current, label),
  )
}

/**
 * Read and parse every source, merged into one entry list.
 *
 * Later sources win: the base client table first, then each overlay in sorted
 * order, so an overlay both ADDS ids the client never had and OVERRIDES stock
 * ones the shard has re-purposed.
 *
 * Returns `{ entries, source }`. Throws `ClilocSourceError` for anything about
 * the paths and `ClilocFormatError` for anything about the contents — different
 * problems for an operator (wrong place vs wrong file), and the admin panel says
 * which. A format error names the file it came from, because "which of my six
 * overlay files is malformed" is otherwise a guessing game.
 */
function readCliloc(configured) {
  const { root, files } = readSources(configured)

  const merged = new Map()
  const perSource = []

  for (const file of files) {
    let entries
    try {
      entries = parseCliloc(file.buffer)
    } catch (err) {
      if (err instanceof ClilocFormatError) {
        throw new ClilocFormatError(`${file.label}: ${err.message}`, err.code)
      }
      throw err
    }

    let added = 0
    let overrode = 0
    for (const entry of entries) {
      if (!Number.isInteger(entry.number)) continue
      if (merged.has(entry.number)) overrode++
      else added++
      merged.set(entry.number, entry)
    }
    perSource.push({ label: file.label, kind: file.kind, entries: entries.length, added, overrode })
  }

  return {
    entries: [...merged.values()],
    source: {
      root,
      file: files[0].file,
      sha256: files[0].sha256,
      bytes: files[0].bytes,
      hashes: Object.fromEntries(files.map((f) => [f.label, f.sha256])),
      parserVersion: PARSER_VERSION,
      sources: perSource,
    },
  }
}

module.exports = {
  ClilocFormatError,
  ClilocSourceError,
  PARSER_VERSION,
  CANDIDATE_NAMES,
  CUSTOM_DIR,
  CUSTOM_EXTENSIONS,
  resolveBase,
  listCustom,
  readSources,
  readOverlays,
  hashSources,
  sameSources,
  missingSources,
  missingOverlays,
  readCliloc,
}