Module-uo/server/utils/spawnAtlasSource.js

// Spawn atlas — the filesystem layer over a ServUO tree.
//
// `spawnAtlasParse.js` holds the pure parsers; this module is the only thing
// that touches a ServUO tree on disk, and it is shared by both callers:
//
//   - the server, which refreshes the atlas on boot (`shardAtlas.model.js`)
//   - the CLI (`scripts/importSpawnAtlas.js`)
//
// **As of protocol 8 phase 7 it is no longer the only way in** (docs/link/v8.md
// §10). `treeBridge.js` reads the same five labelled groups off the SHARD, over
// the sidecar, and hands back files in exactly the shape `readSources` produces
// here — which is why `buildFromFiles` below is where the parse actually starts
// and both readers feed it. That closes the one place the platform's rule (only
// the sidecar bridges the shard) was broken, and broken by the component that
// faces the internet: this file's `SERVUO_PATH` required the WEBSITE to be able
// to read the shard's directories.
//
// The shard's own files are the single source of truth. Nothing is precomputed
// and committed, because a shard's maps change over its lifetime — facets get
// added, replaced or renamed — and a snapshot in the repo would silently go
// stale against the world players actually see.
//
// Reading and hashing the whole tree costs ~120 ms and a full parse ~400 ms, so
// the boot path hashes first and only parses when something actually changed.

const crypto = require('crypto')
const fs = require('fs')
const path = require('path')

const {
  parsePoints,
  parseRegions,
  parseLocations,
  parseChampions,
  parseDecoration,
  buildPlacementIndex,
  buildFacetIndex,
  resolveFacetName,
  resolveRegion,
  facetKey,
  slugify,
} = require('./spawnAtlasParse')

const REGIONS_FILE = path.join('Data', 'Regions.xml')
const LOCATIONS_DIR = path.join('Data', 'Locations')
const SPAWNS_DIR = 'Spawns'
const CHAMPIONS_FILE = path.join('Config', 'ChampionSpawns.xml')
const DECORATION_DIR = path.join('Data', 'Decoration')

class AtlasSourceError extends Error {
  constructor(message, code) {
    super(message)
    this.name = 'AtlasSourceError'
    this.code = code
  }
}

// ── Reading ────────────────────────────────────────────────────────────────

/**
 * The fingerprint of one source file, over its RAW BYTES.
 *
 * Bytes rather than the decoded string, so that this reader and the bridge
 * reader cannot disagree. The shard hashes what it sends; a hash taken here over
 * `text` would be a hash of a UTF-8 RE-ENCODING of what was read — identical for
 * every valid UTF-8 file, and different for one that is not, because Node's utf8
 * decode replaces each undecodable byte with U+FFFD and the re-encode never gets
 * them back. A spawn file with one Latin-1 character in a creature name would
 * then fingerprint differently depending on which end read it, and the drift gate
 * would report a change on every single import, forever, with the tree untouched.
 */
function sha256(bytes) {
  return crypto.createHash('sha256').update(bytes).digest('hex')
}

function listXml(dir) {
  try {
    return fs
      .readdirSync(dir)
      .filter((name) => name.toLowerCase().endsWith('.xml'))
      .sort()
  } catch (err) {
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
    throw err
  }
}

/**
 * Every `.cfg` under `dir`, recursively, tree-relative and forward-slashed.
 *
 * Recursive because `Data/Decoration` nests two deep in places
 * (`Magincia/Trammel`, `Stygian Abyss/Ter Mur`, `Old/Britannia`) and a flat read
 * would silently index a third of what the shard actually has — the failure
 * mode being a dropdown that is quietly missing whole expansions rather than an
 * error anyone would notice.
 */
function listCfgTree(dir, prefix = '') {
  let entries
  try {
    entries = fs.readdirSync(dir, { withFileTypes: true })
  } catch (err) {
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
    throw err
  }

  const out = []
  for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
    const rel = prefix ? `${prefix}/${entry.name}` : entry.name
    if (entry.isDirectory()) out.push(...listCfgTree(path.join(dir, entry.name), rel))
    else if (entry.name.toLowerCase().endsWith('.cfg')) out.push(rel)
  }
  return out
}

function readIfPresent(file) {
  try {
    return fs.readFileSync(file)
  } catch (err) {
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null
    throw err
  }
}

/**
 * Read every atlas source file under `root`.
 *
 * Returns `{ files: [{ label, text, sha256, bytes }] }`, labels being
 * tree-relative and forward-slashed so a hash map compares equal across
 * platforms — the same tree read on Windows and Linux must produce the same
 * fingerprint or every boot would look like a change.
 */
function readSources(root) {
  if (!root || String(root).trim() === '') {
    throw new AtlasSourceError('No ServUO path configured', 'NO_PATH')
  }
  if (!fs.existsSync(root)) {
    throw new AtlasSourceError(`ServUO path does not exist: ${root}`, 'NOT_FOUND')
  }

  const files = []
  const push = (label, file) => {
    const bytes = readIfPresent(file)
    if (bytes === null) return false
    files.push({
      label,
      text: bytes.toString('utf8'),
      sha256: sha256(bytes),
      bytes: bytes.length,
    })
    return true
  }

  if (!push('Data/Regions.xml', path.join(root, REGIONS_FILE))) {
    throw new AtlasSourceError(`Missing required file: ${REGIONS_FILE}`, 'NO_REGIONS')
  }

  for (const name of listXml(path.join(root, LOCATIONS_DIR))) {
    push(`Data/Locations/${name}`, path.join(root, LOCATIONS_DIR, name))
  }

  const spawnFiles = listXml(path.join(root, SPAWNS_DIR))
  if (spawnFiles.length === 0) {
    throw new AtlasSourceError(`No spawn files found in ${SPAWNS_DIR}`, 'NO_SPAWNS')
  }
  for (const name of spawnFiles) push(`Spawns/${name}`, path.join(root, SPAWNS_DIR, name))

  push('Config/ChampionSpawns.xml', path.join(root, CHAMPIONS_FILE))

  // Optional, like the champion file: a shard that has stripped its decoration still
  // has a usable atlas, it just cannot offer the decoration verb anything to place.
  for (const rel of listCfgTree(path.join(root, DECORATION_DIR))) {
    push(`Data/Decoration/${rel}`, path.join(root, DECORATION_DIR, rel))
  }

  return { files }
}

/**
 * A fingerprint of the tree: `{ "<label>": "<sha256>" }`.
 *
 * The boot path compares this against what was last imported and skips the
 * parse entirely when it matches, which is the normal case on every restart
 * that did not follow a map update.
 */
function hashSources(root) {
  const { files } = readSources(root)
  const hashes = {}
  for (const file of files) hashes[file.label] = file.sha256
  return hashes
}

/**
 * Bumped whenever the parser produces DIFFERENT data from IDENTICAL source
 * files — a fixed misreading, a new field, a changed unit.
 *
 * Without it the hash gate is a trap: an install whose tree has not changed
 * would keep serving what an older parser derived, indefinitely, because the
 * only thing the boot path compares is the tree. The version is stored beside
 * the source hashes and a mismatch counts as drift, so a deploy that corrects
 * the parse actually reaches the data.
 *
 *   2 — respawn delays normalised to seconds (they are per-record minutes OR
 *       seconds in the source, decided by `DelayInSec`).
 *   3 — the decoration index, from `Data/Decoration/**\/*.cfg`.
 *   4 — a spawn point keeps its `UniqueId`, which is what a property lease
 *       targets (Phase 12b). The bump is what re-reads a tree the boot path
 *       would otherwise skip on an unchanged hash — the source files have not
 *       changed, only what is kept from them.
 *   5 — and it did NOT keep it: version 4 bumped the parser and the aggregator
 *       below still discarded the field, so the intent above shipped as a
 *       comment. This bump is what makes an already-imported tree re-read now
 *       that the mapping keeps it; without it `sameSources` sees an unchanged
 *       tree and every existing install stays empty. Released in v1.2.2.
 *   6 — source files are parsed in one canonical label order (protocol 8 phase
 *       7). The decoration index keeps the first item id it sees for a type, so
 *       the read order decided a preview graphic; it now cannot differ between a
 *       tree read off a disk and the same tree read over the bridge. Identical
 *       sources, and for a handful of types a different answer, which is exactly
 *       what this number exists to make reach an install.
 *
 *       This was written as 5 on `edge` while 5 was being released from `main`
 *       meaning something else, so the cutover renumbered it: an install that
 *       imported under v1.2.2 already stores 5, and had the number not moved,
 *       `sameSources` would have called that tree current and this change would
 *       have reached nobody who was already running.
 */
const PARSER_VERSION = 6

/** True when two source fingerprints describe the same tree. */
function sameSources(a, b) {
  if (!a || !b) return false
  const aKeys = Object.keys(a).sort()
  const bKeys = Object.keys(b).sort()
  if (aKeys.length !== bKeys.length) return false
  return aKeys.every((key, i) => key === bKeys[i] && a[key] === b[key])
}

// ── Aggregation ────────────────────────────────────────────────────────────

/**
 * Choose one display spelling for a creature.
 *
 * Spawn files are not consistent about case — the same creature is `Lizardman`
 * in one file and `lizardman` in another. Slugging collapses them correctly, but
 * the display name would otherwise depend on file read order. Most frequent
 * spelling wins; ties break toward more capitals, then alphabetically.
 */
function displayName(spellings) {
  const capitals = (value) => (value.match(/[A-Z]/g) || []).length
  return [...spellings.entries()].sort((a, b) => {
    if (b[1] !== a[1]) return b[1] - a[1]
    const caps = capitals(b[0]) - capitals(a[0])
    if (caps !== 0) return caps
    return a[0].localeCompare(b[0])
  })[0][0]
}

/**
 * Roll spawn points up into per-type creature rows.
 *
 * `total` is the sum of each type's own max across every point that spawns it —
 * how many of this creature the world holds at once. `facets` is a per-facet
 * point count, so "where does this live" answers without touching the points.
 */
function aggregateCreatures(points) {
  const creatures = new Map()
  for (const point of points) {
    for (const entry of point.types) {
      const slug = slugify(entry.type)
      if (slug === '') continue
      let creature = creatures.get(slug)
      if (!creature) {
        creature = { slug, name: '', total: 0, points: 0, facets: {}, spellings: new Map() }
        creatures.set(slug, creature)
      }
      creature.total += entry.max
      creature.points += 1
      creature.facets[point.facet] = (creature.facets[point.facet] || 0) + 1
      creature.spellings.set(entry.type, (creature.spellings.get(entry.type) || 0) + 1)
    }
  }

  return [...creatures.values()]
    .map(({ spellings, ...creature }) => ({ ...creature, name: displayName(spellings) }))
    .sort((a, b) => a.slug.localeCompare(b.slug))
}

// ── Build ──────────────────────────────────────────────────────────────────

/**
 * Parse a ServUO tree into the full atlas.
 *
 * Pure with respect to the database — it reads files and returns data; nothing
 * here writes. `shardAtlas.model.js` decides what to do with the result.
 */
function buildAtlas(root, options = {}) {
  return buildFromFiles(readSources(root).files, options)
}

/**
 * The parse itself, over files that have already been read.
 *
 * Split out in phase 7 so that a tree which arrived over the sidecar and a tree
 * read off a local disk go through the SAME code from here on. The alternative —
 * a second build for the bridge — would have been a second place for the facet
 * reconciliation, the decoration case-folding and the disabled-spawner filter to
 * be subtly different, and the difference would only ever show up as one install
 * having a slightly wrong atlas.
 */
function buildFromFiles(files, options = {}) {
  // **Sorted here, once, whatever order the reader handed them over in.**
  //
  // Order is not cosmetic in this parse: the decoration index keeps the FIRST
  // item id it sees for a type and the first spelling of it, and `meta.source` is
  // written in iteration order. Both readers happen to agree on a stock tree, and
  // "happen to" is the problem — the filesystem reader walks each decoration
  // directory with `localeCompare` while the shard sorts whole relative paths,
  // and those two disagree the moment a directory mixes cases. A tree read over
  // the bridge would then produce a subtly different atlas from the same tree read
  // off a disk, in a way nothing reports and only a side-by-side diff would find.
  //
  // A plain ordinal comparison rather than `localeCompare`, because the answer
  // must not depend on the host's ICU data either.
  files = [...files].sort((a, b) => (a.label < b.label ? -1 : a.label > b.label ? 1 : 0))

  const byLabel = new Map(files.map((file) => [file.label, file]))

  if (!byLabel.has('Data/Regions.xml')) {
    throw new AtlasSourceError('Missing required file: Data/Regions.xml', 'NO_REGIONS')
  }

  if (!files.some((file) => file.label.startsWith('Spawns/'))) {
    throw new AtlasSourceError('No spawn files found in Spawns', 'NO_SPAWNS')
  }

  const source = {}
  for (const file of files) source[file.label] = { bytes: file.bytes, sha256: file.sha256 }

  const regions = parseRegions(byLabel.get('Data/Regions.xml').text)

  const rawLandmarks = []
  for (const file of files) {
    if (!file.label.startsWith('Data/Locations/')) continue
    const basename = path.basename(file.label, '.xml')
    rawLandmarks.push(...parseLocations(file.text, basename))
  }

  const rawPoints = []
  for (const file of files) {
    if (!file.label.startsWith('Spawns/')) continue
    rawPoints.push(...parsePoints(file.text))
  }

  // The facet set is whatever THIS tree declares — never a built-in list. A
  // shard may add facets, replace them outright, or rename them when its maps
  // are updated, and the atlas has to follow without a code change. Spawn
  // records and region definitions are the authority, because those are the
  // names everything else is keyed on.
  const facetIndex = buildFacetIndex([
    ...rawPoints.map((point) => point.facet),
    ...regions.map((region) => region.facet),
  ])

  // Landmark facets are then matched against that set, which is what absorbs the
  // `Ter Mur` / `Tokuno Islands` spelling drift between Locations and <Map>.
  const landmarks = rawLandmarks.map(({ facetLabel, ...landmark }) => {
    const fromFile = resolveFacetName(landmark.facet, facetIndex)
    const matchedFile = facetIndex.has(facetKey(fromFile))
    const resolved = matchedFile ? fromFile : resolveFacetName(facetLabel, facetIndex)
    return { ...landmark, facet: resolved || landmark.facet }
  })

  const placement = buildPlacementIndex(regions, landmarks)
  const resolveOpts = options.landmarkRadius ? { landmarkRadius: options.landmarkRadius } : {}

  const disabled = rawPoints.filter((point) => !point.running).length
  const points = rawPoints
    // A spawner switched off in-world produces nothing; advertising it would be
    // a straight lie to a player planning a hunt.
    .filter((point) => point.running)
    // A spawner with no types is a placeholder — nothing to show.
    .filter((point) => point.types.length > 0)
    .map((point) => {
      const place = resolveRegion(point.x, point.y, point.facet, placement, resolveOpts)
      return {
        name: point.name,
        // **The field this whole `PARSER_VERSION` note was about, and it was
        // dropped right here.** The parser has produced it since Phase 12b and
        // the column and the query have both been waiting for it, but this
        // mapping rebuilds each point from an explicit field list and `uniqueId`
        // was not on it — so every row landed with `unique_id` NULL, and
        // `listSpawners`, whose WHERE is `unique_id IS NOT NULL`, could only ever
        // answer empty. That made `uo.options.spawners` an empty dropdown and
        // every Phase 12b object-property lease unauthorable, with nothing on the
        // form to say why. Found by the Phase 16b walk against a released bundle.
        uniqueId: point.uniqueId,
        facet: point.facet,
        x: point.x,
        y: point.y,
        width: point.width,
        height: point.height,
        range: point.range,
        maxCount: point.maxCount,
        minDelay: point.minDelay,
        maxDelay: point.maxDelay,
        todStart: point.todStart,
        todEnd: point.todEnd,
        todMode: point.todMode,
        region: place.region,
        landmark: place.landmark,
        label: place.label,
        types: point.types,
      }
    })

  const championsFile = byLabel.get('Config/ChampionSpawns.xml')
  const champions = (championsFile ? parseChampions(championsFile.text) : []).map((champ) => {
    const facet = resolveFacetName(champ.facet, facetIndex) || champ.facet
    return {
      ...champ,
      facet,
      slug: slugify(`${facet}-${champ.name}`),
      label: resolveRegion(champ.x, champ.y, facet, placement, resolveOpts).label,
    }
  })

  // Decoration: what this shard already calls scenery, which is what makes the
  // authoring dropdown the operator's own vocabulary rather than our taste.
  //
  // **Keyed case-INSENSITIVELY, because the decoration files disagree with
  // themselves about casing.** Stock 57.4 names four types under two spellings
  // each — `CheckerBoard`/`Checkerboard`, `ChessBoard`/`Chessboard`,
  // `MetalChest`/`Metalchest`, `SpinningWheelEastAddon`/`SpinningwheelEastAddon`
  // — and in every pair exactly one is a real class, the other a mis-cased line
  // the shard's own loader resolves anyway. A case-sensitive Map keeps both, and
  // then `shard_decor_types.type` (a PRIMARY KEY under MariaDB's default
  // `..._ai_ci` collation, which folds case) rejects the second row and takes the
  // WHOLE import transaction down with it. That is not a decoration bug: with no
  // atlas, every option source answers empty and no world verb can be authored at
  // all. The shard end of this feature already knew — `BridgeWorld.cs` resolves a
  // decor type with `FindTypeByName(name, ignoreCase: true)` and says why — so
  // folding here is the two ends agreeing rather than a new rule.
  //
  // The first spelling seen wins, exactly as the first item id does. Either
  // spelling resolves on the shard, so which one survives is cosmetic.
  const decorUses = new Map()
  for (const file of files) {
    if (!file.label.startsWith('Data/Decoration/')) continue
    for (const entry of parseDecoration(file.text)) {
      const key = entry.type.toLowerCase()
      const seen = decorUses.get(key)
      if (seen) {
        seen.uses += 1
        continue
      }
      // The FIRST item id wins, and it is only a preview: a type appears under
      // as many ids as it has facings or variants, and picking one arbitrarily
      // is honest in a way that picking "the most used" would not be.
      decorUses.set(key, { type: entry.type, itemId: entry.itemId, uses: 1 })
    }
  }
  const decor = [...decorUses.values()].sort((a, b) => a.type.localeCompare(b.type))

  const creatures = aggregateCreatures(points)
  const facets = [...new Set(points.map((point) => point.facet))].sort()
  const unresolved = points.filter((point) => !point.region && !point.landmark).length

  return {
    meta: {
      generatedAt: new Date().toISOString(),
      parserVersion: PARSER_VERSION,
      landmarkRadius: options.landmarkRadius ?? undefined,
      counts: {
        facets: facets.length,
        points: points.length,
        pointsDisabled: disabled,
        creatures: creatures.length,
        regions: regions.length,
        landmarks: landmarks.length,
        champions: champions.length,
        decor: decor.length,
        unresolvedPoints: unresolved,
      },
      source,
    },
    facets,
    creatures,
    regions,
    landmarks,
    champions,
    points,
    decor,
  }
}

// ── Backends ───────────────────────────────────────────────────────────────
//
// One descriptor, two readers (§10, phase 7). A source is `{ kind: 'fs', root }`
// or `{ kind: 'bridge' }`, and everything above this line belongs to the first.
//
// `treeBridge` is required lazily and INSIDE the functions rather than at the top
// of the file, because this module is also loaded by `scripts/importSpawnAtlas.js`
// and by tests that have no sidecar, no core logger and no intention of touching
// either. A top-level require would drag the whole client stack into both.

/** `{ files }` from whichever end this source names. */
async function readFrom(source) {
  if (source?.kind === 'bridge') {
    const { files } = await require('./treeBridge').readSources()
    return { files }
  }

  return readSources(source?.root)
}

/**
 * The `{ label: sha256 }` fingerprint, from whichever end.
 *
 * The bridge answers this from the MANIFEST alone — no file bytes cross the wire
 * to answer "has anything changed", which is the whole reason the manifest is a
 * separate call. A stock tree is one page and about 32 KB.
 */
async function hashFrom(source) {
  if (source?.kind === 'bridge') {
    const treeBridge = require('./treeBridge')
    const listing = await treeBridge.manifest()
    return treeBridge.fingerprintOf(listing.files)
  }

  return hashSources(source?.root)
}

/** The full atlas, from whichever end. */
async function buildFrom(source, options = {}) {
  const { files } = await readFrom(source)
  return buildFromFiles(files, options)
}

module.exports = {
  AtlasSourceError,
  PARSER_VERSION,
  readSources,
  hashSources,
  sameSources,
  buildAtlas,
  buildFromFiles,
  readFrom,
  hashFrom,
  buildFrom,
  aggregateCreatures,
  displayName,
}