Module-uo/server/utils/clilocBridge.js

// Cliloc table — the SHARD source (docs/link/v8.md §9, protocol 8 phase 2).
//
// `clilocSource.js` is the filesystem half of this story and predates it. This is
// the half that replaces the part of it nobody enjoyed: until protocol 8 the base
// table reached the site because an operator installed UOFiddler, built a
// converter against its `Ultima.dll`, ran it over their client's compressed
// `Cliloc.enu` and copied a five-megabyte file to the web host — every time they
// patched their client.
//
// The shard has always had those files (a ServUO server cannot boot without a UO
// client) and, as of phase 2, has the decompressor too. So the base table now
// arrives over the same request/reply path as every other shard read, and the
// operator installs nothing.
//
// **What is NOT here.** Overlays. Shard-added items carry cliloc ids no client
// table has, ServUO has no server-side notion of a custom cliloc, and there is
// therefore nothing on the shard to ask for. `custom/` stays a directory the site
// reads (`clilocSource.readOverlays`), and the model merges it OVER whatever
// arrives here. That division is the whole of CLILOCS.md §Shard-added items and
// it is unchanged by this file.
//
// ── Why this walks pages instead of asking for a table ────────────────────
//
// The sidecar's reply timeout is 10 s and its inbound line cap is 1 MiB, so a
// five-megabyte table cannot be one answer. The shard cuts pages at a 512 KiB
// byte budget and hands back a cursor; this walks them. A stock English table is
// about eleven pages.
//
// Three properties of that envelope are load-bearing and each has a check below:
//
//   - **Only `cut: 'end'` means finished.** A short page can equally mean the
//     budget was spent (`budget`) or the family stopped at its own limit
//     (`limit`). Treating a short page as the end would import a truncated table,
//     which is indistinguishable downstream from a complete one — some items
//     named, some not, exactly what "no table at all" looks like.
//   - **The cursor must advance.** A shard that answered the same cursor forever
//     would spin this loop until the request timeout with nothing to show.
//   - **The file must not change underneath the walk.** Every page echoes the
//     source's size and mtime; an operator patching their client mid-import would
//     otherwise produce one table stitched from two, with no error anywhere.

// Required as a namespace, not destructured: a test that stubs the sidecar
// replaces these on the module object, and a destructured copy taken at load
// time would keep calling the real one.
const uoLinkClient = require('./uoLinkClient')
const log = require('../core').logger('cliloc-bridge')

/** The client file the base table comes from, as `assets.sources` names it. */
const SOURCE_FILE = 'cliloc.enu'

const DEFAULT_LANGUAGE = 'enu'

// Bounds on the walk. Neither is expected to be reached — a stock table is ~11
// pages and ~67k rows — and both exist so that a shard answering nonsense costs a
// bounded amount of time rather than an unbounded amount of memory.
const MAX_PAGES = 200
const MAX_ROWS = 500000

// 425 is the ordinary answer during an import, not an error: the shard's asset
// plane serves one request at a time on purpose, because its outbound queue is
// bounded in lines rather than bytes. So a page that comes back busy is retried
// with a short backoff rather than failing the import.
const BUSY_RETRIES = 5
const BUSY_BACKOFF_MS = [200, 400, 800, 1600, 3200]

class ClilocBridgeError extends Error {
  constructor(message, code) {
    super(message)
    this.name = 'ClilocBridgeError'
    this.code = code
  }
}

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))

/**
 * Map a sidecar response onto one of this module's codes.
 *
 * The statuses are the ones `respond_assets` produces, and the distinction that
 * matters most to an operator is 403 vs 404: "you have not switched this on" and
 * "your client does not have that file" are different jobs, and both are things
 * they can fix.
 */
function describeFailure(res, what) {
  const reason = res?.data?.reason || res?.error || `sidecar responded ${res?.status}`

  switch (res?.status) {
    case 403:
      return new ClilocBridgeError(
        `The shard is refusing to serve client assets (Bridge.AssetsEnabled is off): ${reason}`,
        'DISABLED',
      )
    case 404:
      return new ClilocBridgeError(`The shard has no ${what}: ${reason}`, 'NO_SOURCE')
    case 409:
      return new ClilocBridgeError(
        `The sidecar refused the protocol version this build declares: ${reason}`,
        'PROTOCOL',
      )
    case 422:
      return new ClilocBridgeError(`The shard could not read its own ${what}: ${reason}`, 'UNREADABLE')
    case 425:
      return new ClilocBridgeError(
        'The shard is busy serving another asset request and stayed busy',
        'BUSY',
      )
    case 503:
    case 504:
      return new ClilocBridgeError(`The shard did not answer: ${reason}`, 'SHARD_DOWN')
    default:
      return new ClilocBridgeError(reason, 'UNAVAILABLE')
  }
}

/**
 * Stage 1: the fingerprint of the shard's own cliloc file.
 *
 * Returns `{ file, size, mtime, sha256, extractorVersion, hashing, complete }`.
 *
 * `sha256` may be **null** — the shard reports hashes only once it has computed
 * them off the request path, because hashing the client files it also serves
 * (343 MB of art and animation) cannot fit inside a 10 s reply. A null hash means
 * "not yet", never "changed", and `sameSource` below compares (size, mtime) in
 * that case, which is the same gate the shard itself uses.
 */
async function fingerprint() {
  const res = await uoLinkClient.getAssetSources()
  if (!res.ok) throw describeFailure(res, 'client file manifest')

  // Since protocol 8 phase 7 this call answers when EITHER plane is enabled, so
  // a 200 no longer means the client files are on offer. Without this check an
  // operator who switched client-file extraction off would read "your UO client
  // has no cliloc.enu" and go looking at their client install for a setting that
  // lives on their shard.
  if (res.data?.assetsEnabled === false) {
    throw new ClilocBridgeError(
      'The shard is refusing to serve client assets (Bridge.AssetsEnabled is off)',
      'DISABLED',
    )
  }

  const files = Array.isArray(res.data?.files) ? res.data.files : []
  const entry = files.find((f) => String(f?.name || '').toLowerCase() === SOURCE_FILE)

  if (!entry) {
    throw new ClilocBridgeError(
      `The shard's UO client has no ${SOURCE_FILE} (it reported ${files.length} client file(s))`,
      'NO_SOURCE',
    )
  }

  return {
    kind: 'bridge',
    file: entry.name,
    path: entry.path ?? null,
    size: Number(entry.size) || 0,
    mtime: Number(entry.mtime) || 0,
    sha256: entry.sha256 ?? null,
    extractorVersion: Number(res.data?.extractorVersion) || 0,
    hashing: Boolean(res.data?.hashing),
    complete: Boolean(res.data?.complete),
  }
}

/**
 * True when two fingerprints describe the same client file.
 *
 * Hash first when both sides have one, because a hash is the only thing that
 * catches a file rewritten with the same length and timestamp. Falls back to
 * (size, mtime) when either side's hash is missing, which is the case on the
 * first poll after a shard restart and the reason `hashing` exists at all.
 */
function sameSource(a, b) {
  if (!a || !b) return false
  if (a.extractorVersion !== b.extractorVersion) return false
  if (a.sha256 && b.sha256) return a.sha256 === b.sha256
  return a.size === b.size && a.mtime === b.mtime && a.size > 0
}

/** One page, with the 425 backoff. */
async function fetchPage({ lang, cursor }) {
  for (let attempt = 0; ; attempt++) {
    const res = await uoLinkClient.getClilocTable({ lang, cursor })
    if (res.ok) return res.data

    if (res.status === 425 && attempt < BUSY_RETRIES) {
      await sleep(BUSY_BACKOFF_MS[Math.min(attempt, BUSY_BACKOFF_MS.length - 1)])
      continue
    }

    throw describeFailure(res, `cliloc.${lang}`)
  }
}

/**
 * Walk the whole table.
 *
 * Returns `{ entries, source }` where `entries` is `[{ number, flag, text }]` in
 * the shape `clilocParse` produces, so the merge in `shardClilocs.model` does not
 * care which source an entry came from.
 *
 * Blanks are already gone: the shard drops the ~56,000 empty strings a stock
 * table carries before they reach the wire, since the site would drop them at
 * import anyway. Nothing downstream changes — `db.replaceAll` still filters, and
 * still would if a source ever sent one.
 */
async function readCliloc({ lang = DEFAULT_LANGUAGE } = {}) {
  const started = Date.now()
  const entries = []

  let cursor = null
  let pages = 0
  let first = null
  let finished = false
  let total = null

  while (pages < MAX_PAGES) {
    const page = await fetchPage({ lang, cursor })
    pages++

    if (!page || !Array.isArray(page.rows)) {
      throw new ClilocBridgeError('The shard sent a cliloc page with no rows array', 'MALFORMED')
    }

    if (first === null) {
      first = { size: Number(page.size) || 0, mtime: Number(page.mtime) || 0 }
      total = Number.isFinite(Number(page.total)) ? Number(page.total) : null
    } else if (Number(page.size) !== first.size || Number(page.mtime) !== first.mtime) {
      // The client was patched (or a different one mounted) between two pages.
      // Refusing is the only honest answer: half of what we hold is from a file
      // that no longer exists, and nothing later can tell which half.
      throw new ClilocBridgeError(
        'The shard\'s cliloc file changed while it was being read; nothing was imported',
        'SOURCE_CHANGED',
      )
    }

    for (const row of page.rows) {
      const number = Number(row?.n)
      if (!Number.isInteger(number)) continue
      entries.push({ number, flag: Number(row?.f) || 0, text: String(row?.t ?? '') })
    }

    if (entries.length > MAX_ROWS) {
      throw new ClilocBridgeError(
        `The shard sent more than ${MAX_ROWS} cliloc rows; refusing to keep reading`,
        'TOO_LARGE',
      )
    }

    if (!page.more) {
      // `cut` is the field that says WHY a page was the last one, and only one of
      // its values means the table ended. A shard that stopped for its own limit
      // has not finished, and importing what arrived would silently drop the tail.
      if (page.cut !== 'end') {
        throw new ClilocBridgeError(
          `The shard stopped sending cliloc rows after ${entries.length} (cut: ${page.cut || 'unknown'})`,
          'INCOMPLETE',
        )
      }
      finished = true
      break
    }

    if (!page.cursor || page.cursor === cursor) {
      // Either would loop forever: no cursor to advance with, or the same one
      // back again.
      throw new ClilocBridgeError(
        `The shard asked for another cliloc page without advancing its cursor (${page.cursor || 'none'})`,
        'STUCK',
      )
    }

    cursor = page.cursor
  }

  if (!finished) {
    throw new ClilocBridgeError(
      `The cliloc table did not end within ${MAX_PAGES} pages; nothing was imported`,
      'TOO_LARGE',
    )
  }

  log.info('cliloc table read from the shard', {
    lang,
    entries: entries.length,
    pages,
    ms: Date.now() - started,
  })

  return {
    entries,
    source: {
      kind: 'bridge',
      lang,
      file: `cliloc.${lang}`,
      size: first?.size ?? 0,
      mtime: first?.mtime ?? 0,
      pages,
      // What the shard said it holds, kept beside what actually arrived. They
      // agree or the walk is wrong, and an operator seeing them disagree in the
      // panel learns more than a single number would tell them.
      reported: total,
      received: entries.length,
    },
  }
}

module.exports = {
  ClilocBridgeError,
  SOURCE_FILE,
  DEFAULT_LANGUAGE,
  MAX_PAGES,
  MAX_ROWS,
  fingerprint,
  sameSource,
  readCliloc,
}