Module-uo/server/model/shardClilocs/shardClilocs.model.js

const db = require('./shardClilocs.db')
const { settings } = require('../../core')
const { displayText, parseCliloc } = require('../../utils/clilocParse')
const {
  ClilocFormatError,
  ClilocSourceError,
  PARSER_VERSION,
  hashSources,
  sameSources,
  missingSources,
  missingOverlays,
  readOverlays,
  readCliloc,
} = require('../../utils/clilocSource')
const bridge = require('../../utils/clilocBridge')
const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model')
const log = require('../../core').logger('shardClilocs')

// The cliloc table — UO's id → display-string map, refreshed from a file the
// operator converts once from their own client.
//
// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a
// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace
// listing sends one too. Without the table the UI can only print `id 1023721`
// where the game prints "quarter staff".
//
// Two rules govern the boot path, both inherited from the spawn atlas:
//
//   1. **It never blocks startup.** No configured path, an unreadable file, a
//      wrong-format file, a database error — all caught and logged. The site
//      comes up either way, serving whatever table it already had (or none, in
//      which case the UI falls back to item ids exactly as it did before).
//   2. **Nothing client-derived is committed.** The table is built from the
//      operator's own file at a configured path. The repo ships no strings.
//
// The table is built from a SET of sources — a base table plus every
// operator-maintained overlay beside it — because shards edit items and add new
// ones, and those carry cliloc ids no stock client table has. Later sources win,
// so an overlay both adds ids the client never had and overrides stock ones.
//
// ── Where the base comes from (protocol 8, docs/link/v8.md §9) ─────────
//
// **The shard**, on any install with uo-link configured. It has the operator's
// client files already — a ServUO server cannot boot without them — and since
// phase 2 it has the decompressor too, so `GET /cliloc` returns the table and
// nobody installs UOFiddler or copies a 5 MB file anywhere.
//
// **A file on disk** otherwise. That is the pipeline this replaces, kept for
// installs with no shard link and for development, and deprecated rather than
// removed: an operator who has one keeps working, and an operator who has a shard
// never builds one. Passing an explicit `path` to `refresh()` still selects it,
// which is the escape hatch for "import from this file, this once".
//
// **Overlays are always the filesystem's**, either way. There is nothing on the
// shard to ask for: ServUO has no server-side notion of a custom cliloc, so the
// `custom/` directory is the only place those ids exist.
//
// ── What that changed about WHEN this runs ───────────────────────
//
// Boot no longer imports on the shard path. The file path could hash 5 MB locally
// on every restart and skip; the shard path would mean a sidecar round trip in the
// boot sequence, for a table that changes when an operator patches their client —
// an event they know about and we do not. So on the bridge, importing is an admin
// action (Admin → Shard), and boot leaves whatever is loaded serving.
//
// That set is also why this has the atlas's escalation, in a lighter form. A
// single corrupt file fails the parse loudly, but a source that has simply
// VANISHED parses perfectly and imports a table quietly missing everything it
// contributed — the same ambiguity (real change vs half-copied mount) the atlas
// stages a facet removal for. So a disappearing source is refused and reported
// rather than applied.
//
// It is lighter than the atlas's because it needs to be: the atlas stores a
// pending decision in its own table and adds approve/reject endpoints, whereas
// here the decision is a single boolean an admin passes to the import they were
// already going to run. Re-parsing at approval time — the property that makes
// the atlas store only the decision — is automatic when there is nothing stored.

const SETTING_KEY = 'cliloc_client_path'

/**
 * Where the converted cliloc file lives.
 *
 * The admin setting wins over the environment so an operator can repoint it
 * without a redeploy, matching how the rest of the shard integration is
 * admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the
 * deploy-time default, since the path usually describes a mount the deployment
 * sets up.
 */
async function getClientPath() {
  try {
    const configured = await settings.get(SETTING_KEY)
    if (configured && String(configured).trim() !== '') return String(configured).trim()
  } catch {
    // Settings unavailable is not fatal — fall through to the env default.
  }
  const fromEnv = process.env.UO_CLIENT_PATH
  return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : ''
}

async function setClientPath(value, updatedBy = null) {
  const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy)
  invalidate()
  return result
}

// ── Refresh ────────────────────────────────────────────────────────────────

/** Was the loaded table built by THIS parser? */
const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION

/**
 * Refresh the cliloc table from the configured file.
 *
 * Returns a result describing what happened rather than throwing, so the caller
 * — including the boot path — can log it and move on:
 *
 *   `skipped`     no path configured
 *   `unavailable` path configured but missing / unreadable / not a cliloc file
 *   `unchanged`   source hashes match the loaded table; nothing parsed
 *   `imported`    parsed and applied
 *   `needsReview` a previously-present source has vanished; NOT applied
 *   `failed`      parsed or applied and something went wrong
 *
 * `force` skips the hash check (an admin asking for a reimport). `approve`
 * additionally accepts a vanished source.
 *
 * Which SOURCE it reads is decided here and nowhere else: the shard when uo-link
 * is configured and enabled, a file otherwise, and always a file when the caller
 * named one.
 */
async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
  const override = String(pathOverride ?? '').trim()

  if (override === '' && (await shardLinked())) {
    return refreshFromShard({ force, approve })
  }

  return refreshFromFile({ force, approve, path: override })
}

/**
 * Is there a shard to ask?
 *
 * Both halves matter. `baseUrl` alone is an install that has been configured and
 * then switched off, and calling it would spend a 12 s timeout to learn what the
 * row already says. Never throws: an unreadable config means "no shard", and the
 * file path is a working answer.
 */
async function shardLinked() {
  try {
    const config = await uoLinkConfig.getSafe()
    return Boolean(config?.enabled && config?.baseUrl)
  } catch {
    return false
  }
}

/**
 * Merge parsed sources in order, later winning.
 *
 * Shared by both paths, because the merge is the same question whichever end the
 * base arrived from: what did each source contribute, and what did it override.
 * The per-source breakdown is for the admin panel — an operator who adds an
 * overlay wants to see it took effect, and `overrode: 0` on a file meant to
 * re-label stock items says it did not.
 */
function mergeSources(groups) {
  const merged = new Map()
  const sources = []

  for (const group of groups) {
    let added = 0
    let overrode = 0

    for (const entry of group.entries) {
      if (!Number.isInteger(entry.number)) continue
      if (merged.has(entry.number)) overrode++
      else added++
      merged.set(entry.number, entry)
    }

    sources.push({
      label: group.label,
      kind: group.kind,
      entries: group.entries.length,
      added,
      overrode,
    })
  }

  return { entries: [...merged.values()], sources }
}

/** Overlay hashes as a `{ label: sha256 }` map, in merge order. */
function overlayHashes(files) {
  const hashes = {}
  for (const file of files) hashes[file.label] = file.sha256
  return hashes
}

/** The overlay half of a stored fingerprint — everything under `custom/`. */
function onlyOverlays(hashes) {
  if (!hashes) return null
  const out = {}
  for (const [label, sha] of Object.entries(hashes)) {
    if (label.startsWith('custom/')) out[label] = sha
  }
  return out
}

/**
 * Import with the shard as the base source.
 *
 * The gate is two-part, and neither part is something the shard can answer for
 * us: has the client file changed (size/mtime/sha256, plus the shard's own
 * `EXTRACTOR_VERSION`), and has any overlay beside the configured path changed.
 * Either is drift; neither is the normal case.
 */
async function refreshFromShard({ force = false, approve = false } = {}) {
  let fingerprint

  try {
    fingerprint = await bridge.fingerprint()
  } catch (err) {
    if (err instanceof bridge.ClilocBridgeError) {
      return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code }
    }
    return { status: 'failed', source: 'bridge', reason: err.message }
  }

  const configured = await getClientPath()
  const overlays = readOverlays(configured)
  const hashes = overlayHashes(overlays.files)

  const meta = await db.getMeta().catch(() => null)

  if (
    !force &&
    bridge.sameSource(fingerprint, meta?.base) &&
    sameSources(hashes, onlyOverlays(meta?.hashes)) &&
    currentParser(meta)
  ) {
    return {
      status: 'unchanged',
      source: 'bridge',
      file: fingerprint.file,
      count: meta.count ?? null,
      customCount: overlays.files.length,
      hashing: fingerprint.hashing,
    }
  }

  // An overlay that was loaded last time and is not there now is refused rather
  // than applied — an unmounted volume and a deliberate deletion look identical
  // from here, and the wrong guess silently drops every name that file gave.
  // The BASE is deliberately not part of this question: an install upgraded from
  // the file pipeline is *supposed* to stop having one.
  const gone = missingOverlays(hashes, meta?.hashes)

  if (gone.length > 0 && !approve) {
    return {
      status: 'needsReview',
      source: 'bridge',
      reason: `${gone.length} previously-loaded cliloc overlay(s) are missing; the existing table is unchanged`,
      missingSources: gone,
      file: fingerprint.file,
    }
  }

  let base

  try {
    base = await bridge.readCliloc({ lang: bridge.DEFAULT_LANGUAGE })
  } catch (err) {
    if (err instanceof bridge.ClilocBridgeError) {
      return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code }
    }
    return { status: 'failed', source: 'bridge', reason: err.message }
  }

  const groups = [{ label: base.source.file, kind: 'shard', entries: base.entries }]

  for (const file of overlays.files) {
    try {
      groups.push({ label: file.label, kind: 'custom', entries: parseCliloc(file.buffer) })
    } catch (err) {
      if (err instanceof ClilocFormatError) {
        // Named, because "which of my six overlay files is malformed" is
        // otherwise a guessing game.
        return {
          status: 'unavailable',
          source: 'bridge',
          reason: `${file.label}: ${err.message}`,
          code: err.code,
        }
      }
      return { status: 'failed', source: 'bridge', reason: err.message }
    }
  }

  const merged = mergeSources(groups)

  try {
    const applied = await db.replaceAll(merged.entries, {
      source: 'bridge',
      base: fingerprint,
      hashes,
      parserVersion: PARSER_VERSION,
      sources: merged.sources,
      file: base.source.file,
      bytes: fingerprint.size,
    })

    invalidate()

    return {
      status: 'imported',
      source: 'bridge',
      file: base.source.file,
      count: applied.count,
      parsed: merged.entries.length,
      blank: applied.blank,
      pages: base.source.pages,
      sources: merged.sources,
      // The shard says how many rows it holds; this is how many arrived. They
      // agree, or the walk is wrong in a way no count on its own would show.
      reported: base.source.reported,
      received: base.source.received,
      overlayProblem: overlays.problem ?? undefined,
      acceptedMissing: gone.length > 0 ? gone : undefined,
    }
  } catch (err) {
    return { status: 'failed', source: 'bridge', reason: err.message }
  }
}

/**
 * Import from a converted file on disk — the pre-protocol-8 pipeline, unchanged.
 *
 * Deprecated but supported: an install with no shard link has no other way to get
 * a table, and development without a running ServUO is the same case.
 */
async function refreshFromFile({ force = false, approve = false, path: pathOverride = '' } = {}) {
  // An explicit override wins outright — a one-off "use this file", which must
  // not be silently overruled by the configured path the way an env default is.
  const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
  if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' }

  let fingerprint
  try {
    fingerprint = hashSources(configured)
  } catch (err) {
    if (err instanceof ClilocSourceError) {
      return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
    }
    return { status: 'failed', reason: err.message, path: configured }
  }

  const meta = await db.getMeta().catch(() => null)

  // Two things make a loaded table stale: any source changed, or the PARSER did.
  // Only checking the sources would strand an install whose client never patches
  // on whatever an older build derived.
  if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) {
    return {
      status: 'unchanged',
      path: configured,
      file: fingerprint.file,
      count: meta.count ?? null,
      customCount: fingerprint.customCount,
    }
  }

  // A source that was there last import and is not there now is refused, not
  // applied — an unmounted volume and a deliberate deletion look identical from
  // here, and the wrong guess silently drops every name that file contributed.
  const gone = missingSources(fingerprint.hashes, meta?.hashes)
  if (gone.length > 0 && !approve) {
    return {
      status: 'needsReview',
      reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`,
      missingSources: gone,
      path: configured,
      file: fingerprint.file,
    }
  }

  let parsed
  try {
    parsed = readCliloc(configured)
  } catch (err) {
    if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) {
      return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
    }
    return { status: 'failed', reason: err.message, path: configured }
  }

  try {
    // `source: 'file'` is what lets the NEXT refresh — and `status()` — tell a
    // table built from a converted file from one built over the bridge. Without
    // it an install that gains a shard link looks like it already imported.
    const applied = await db.replaceAll(parsed.entries, { ...parsed.source, source: 'file' })
    invalidate()
    return {
      status: 'imported',
      path: configured,
      file: parsed.source.file,
      count: applied.count,
      parsed: parsed.entries.length,
      blank: applied.blank,
      // Per-source breakdown: how many entries each file contributed and how
      // many of them overrode something already merged. An operator who adds an
      // overlay wants to see it took effect, and "overrode: 0" on a file meant
      // to re-label stock items says it did not.
      sources: parsed.source.sources,
      acceptedMissing: gone.length > 0 ? gone : undefined,
    }
  } catch (err) {
    return { status: 'failed', reason: err.message, path: configured }
  }
}

/**
 * Boot hook. Best-effort by contract: it logs and returns, never throws, so a
 * missing or malformed cliloc file can never stop the site coming up.
 *
 * **On the bridge it imports nothing**, deliberately. The file path can hash a
 * local 5 MB file on every restart and skip in 14 ms; asking the shard would put
 * a sidecar round trip in the boot sequence to answer a question whose answer is
 * "no" every time except after a client patch — which is an operator action, and
 * therefore something an operator can press a button for. Whatever table is
 * loaded keeps serving, which is exactly what happens today when a restart finds
 * nothing changed.
 */
async function refreshOnBoot() {
  try {
    if (await shardLinked()) {
      log.info('cliloc table comes from the shard; import is admin-triggered (Admin → Shard)')
      return { status: 'skipped', source: 'bridge', reason: 'the shard is the cliloc source' }
    }

    const result = await refresh()
    switch (result.status) {
      case 'imported':
        log.info('cliloc table refreshed', {
          file: result.file,
          count: result.count,
          overlays: (result.sources || []).filter((s) => s.kind === 'custom').length,
        })
        break
      case 'needsReview':
        log.warn(
          'cliloc refresh staged for admin review — a previously-loaded source is missing; ' +
            'the existing table is unchanged',
          { missing: result.missingSources },
        )
        break
      case 'unavailable':
        // Deliberately a warning, not an error: an operator who has not supplied
        // a cliloc file is in a supported state (the UI shows item ids), and the
        // most common cause — pointing at the client's own compressed file —
        // needs the reason spelled out rather than a stack trace.
        log.warn('cliloc source unavailable (item names will show as ids)', {
          reason: result.reason,
          code: result.code,
          path: result.path,
        })
        break
      case 'failed':
        log.warn('cliloc refresh failed', { reason: result.reason })
        break
      default:
        break
    }
    return result
  } catch (err) {
    log.warn('cliloc refresh errored', { error: err.message })
    return { status: 'failed', reason: err.message }
  }
}

/**
 * Everything the admin panel needs to describe cliloc state.
 *
 * Two shapes, one per source, sharing every field a panel actually renders
 * (`count`, `drift`, `problem`, `sources`, `missingSources`, `importedAt`). What
 * differs is what `file` means and what a problem with it looks like: on the
 * bridge it is the shard's own client file and the problems are transport ones,
 * on disk it is a path an operator typed.
 */
async function status({ path: pathOverride = '' } = {}) {
  if (pathOverride.trim() === '' && (await shardLinked())) return shardStatus()

  const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
  const meta = await db.getMeta().catch(() => null)
  const loaded = await db.count().catch(() => 0)

  let fileReadable = false
  let file = null
  let drift = null
  let problem = null
  let code = null
  let sources = []
  let missing = []
  if (configured !== '') {
    try {
      const fingerprint = hashSources(configured)
      fileReadable = true
      file = fingerprint.file
      sources = Object.keys(fingerprint.hashes)
      missing = missingSources(fingerprint.hashes, meta?.hashes)
      // A compressed file is readable but not importable, and the panel has to
      // say so HERE — otherwise pointing at an unconverted client directory
      // reports a healthy file with pending drift ("ready to import") and the
      // operator only finds out when the import fails. `drift` stays null
      // because comparing hashes with an unusable file answers nothing.
      if (fingerprint.compressed) {
        problem =
          'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' +
          'Convert it to the plain format first — see docs/website/CLILOCS.md.'
        code = 'COMPRESSED'
      } else {
        drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta)
      }
    } catch (err) {
      fileReadable = false
      problem = err.message
      code = err.code ?? null
    }
  }

  return {
    configured: configured !== '',
    path: configured,
    file,
    fileReadable,
    problem,
    code,
    drift,
    count: loaded,
    // Every source found now (base first, then overlays), what each contributed
    // at the last import, and any that have since vanished — which is the state
    // an import will refuse without `approve`.
    sources,
    loadedSources: meta?.sources ?? null,
    missingSources: missing,
    importedAt: meta?.importedAt ?? null,
    sourceBytes: meta?.bytes ?? null,
  }
}

/**
 * Status when the shard is the source.
 *
 * The one thing worth knowing here that the file path has no equivalent of:
 * `hashing`. The shard reports a null `sha256` for a client file it has not
 * hashed yet — hashing the 343 MB of art and animation it also serves cannot fit
 * in a 10 s reply, so it happens on its own thread — and a null hash means "ask
 * again", never "changed". Drift falls back to (size, mtime) meanwhile, which is
 * the same gate the shard itself applies, so an operator is never blocked from
 * importing by a hash that has not landed.
 */
async function shardStatus() {
  const configured = await getClientPath()
  const meta = await db.getMeta().catch(() => null)
  const loaded = await db.count().catch(() => 0)

  const overlays = readOverlays(configured)
  const hashes = overlayHashes(overlays.files)

  let fingerprint = null
  let problem = overlays.problem ?? null
  let code = null

  try {
    fingerprint = await bridge.fingerprint()
  } catch (err) {
    problem = err.message
    code = err.code ?? null
  }

  const drift = fingerprint
    ? !bridge.sameSource(fingerprint, meta?.base) ||
      !sameSources(hashes, onlyOverlays(meta?.hashes)) ||
      !currentParser(meta)
    : null

  return {
    source: 'bridge',
    configured: true,
    // The overlay directory, which is all the path setting still selects on this
    // source. Reported so a panel can say where `custom/` is being read from.
    path: configured,
    file: fingerprint?.file ?? bridge.SOURCE_FILE,
    fileReadable: Boolean(fingerprint),
    problem,
    code,
    drift,
    count: loaded,
    shard: fingerprint
      ? {
          size: fingerprint.size,
          mtime: fingerprint.mtime,
          sha256: fingerprint.sha256,
          extractorVersion: fingerprint.extractorVersion,
          hashing: fingerprint.hashing,
          complete: fingerprint.complete,
        }
      : null,
    sources: Object.keys(hashes),
    loadedSources: meta?.sources ?? null,
    missingSources: missingOverlays(hashes, meta?.hashes),
    importedAt: meta?.importedAt ?? null,
    sourceBytes: meta?.base?.size ?? meta?.bytes ?? null,
  }
}

// ── Lookup ─────────────────────────────────────────────────────────────────
//
// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is
// ~123k rows and shipping it to a client would dwarf every page that uses it,
// and the Android app consumes the same JSON and would otherwise need its own
// copy. Callers get names, not ids-plus-a-table.

// A small write-through cache in front of the table. Item ids repeat heavily —
// one page of listings is mostly the same few hundred clilocs, and a character
// sheet re-resolves the same gear on every view — so this turns the steady state
// into zero queries. Capped so a pathological caller cannot grow it without
// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry,
// which is cheap and correct for a table that only changes on reimport.
const CACHE_MAX = 20000
let cache = new Map()

function invalidate() {
  cache = new Map()
}

/**
 * Resolve a batch of cliloc ids to display strings.
 *
 * Returns a `Map<number, string>` holding only the ids that resolved to
 * something displayable — an id with no row, or one whose text is nothing but
 * interpolated arguments we do not have, is simply absent. Callers fall back to
 * whatever they had (the item id), so "missing" and "unnamed" collapse into one
 * branch at the call site.
 *
 * Never throws: a cliloc lookup is decoration on someone's character sheet, and
 * a database blip must not fail the sheet.
 */
async function resolveMany(numbers) {
  const out = new Map()
  if (!Array.isArray(numbers)) return out

  const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))]
  if (wanted.length === 0) return out

  const missing = []
  for (const number of wanted) {
    if (cache.has(number)) {
      const hit = cache.get(number)
      if (hit !== '') out.set(number, hit)
    } else {
      missing.push(number)
    }
  }

  if (missing.length > 0) {
    try {
      const rows = await db.lookup(missing)
      const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)]))
      if (cache.size + missing.length > CACHE_MAX) invalidate()
      for (const number of missing) {
        // Cache the miss too ('' meaning "no usable name"), so an id absent from
        // the table does not re-query on every page view.
        const text = found.get(number) ?? ''
        cache.set(number, text)
        if (text !== '') out.set(number, text)
      }
    } catch (err) {
      log.warn('cliloc lookup failed', { message: err.message })
    }
  }

  return out
}

/** Single-id convenience. Returns `null` when there is no usable name. */
async function resolve(number) {
  const found = await resolveMany([number])
  return found.get(number) ?? null
}

module.exports = {
  SETTING_KEY,
  getClientPath,
  setClientPath,
  shardLinked,
  refresh,
  refreshOnBoot,
  status,
  resolveMany,
  resolve,
  invalidate,
}