server/boot.js

@@ -80,11 +80,18 @@ async function onBoot() {
   // REMOVE a facet is staged for admin approval instead of being applied.
   await shardAtlas.refreshOnBoot()
 
-  // Refresh the cliloc table (UO's id → display-string map) from the file the
+  // Refresh the cliloc table (UO's id → display-string map).
-  // operator converted out of their own client. Same contract as the atlas:
+  //
-  // hash-gated so an unchanged file costs one read, and best-effort so a missing
+  // **On an install with uo-link configured this imports nothing** — protocol 8
-  // or wrong-format file never stops the site coming up — it just means item
+  // moved the base table to the shard, and asking for it would put a sidecar
-  // names render as ids, which is what they did before the table existed.
+  // round trip in the boot sequence to answer a question whose answer is "no"
+  // except after a client patch. That is an operator action, so importing is an
+  // operator action: Admin → Shard (docs/link/v8.md §9).
+  //
+  // Without a shard link it is the old file pipeline, unchanged: hash-gated so an
+  // unchanged file costs one read, and best-effort so a missing or wrong-format
+  // file never stops the site coming up — it just means item names render as ids,
+  // which is what they did before the table existed.
   const clilocResult = await shardClilocs.refreshOnBoot()
 
   // A cliloc import changes what item names RESOLVE to, and the marketplace

server/db/schema.sql

@@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS uo_link_config (
   base_url          VARCHAR(255) NULL,
   ws_url            VARCHAR(255) NULL,
   auth_token_enc    TEXT         NULL,
-  protocol          INT          NOT NULL DEFAULT 7,
+  protocol          INT          NOT NULL DEFAULT 8,
   enabled           TINYINT(1)   NOT NULL DEFAULT 0,
   status            VARCHAR(20)  NOT NULL DEFAULT 'disconnected',
   status_detail     VARCHAR(500) NULL,
@@ -837,3 +837,21 @@ UPDATE uo_link_config SET protocol = 7
   WHERE id = 1 AND protocol < 7
     AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_7_migrated');
 INSERT IGNORE INTO settings (`key`, value) VALUES ('uo_link_protocol_7_migrated', '1');
+
+-- The protocol pin at 8 -- the Asset Bridge (docs/link/v8.md), and the first bump this
+-- module takes IN the phase that consumes it rather than a phase or two later.
+--
+-- Phase 1 of that work moved `link`'s PROTOCOL_VERSION and the overlay's `overlay.toml`
+-- together, because the installer refuses to pair a sidecar and an overlay that disagree.
+-- Nothing enforces the third declaration -- this one -- and the block above is the record
+-- of what that costs: two phases of every REST call answered `409 protocol version
+-- mismatch`, invisible because both live walks had set the column by hand.
+--
+-- Phase 2 is where this module first calls a protocol-8 route (`GET /cliloc`), so it is
+-- where the pin moves. Same one-shot shape and the same `protocol < 8`, so an install
+-- that missed an earlier bump is carried the whole way rather than one step.
+ALTER TABLE uo_link_config MODIFY COLUMN protocol INT NOT NULL DEFAULT 8;
+UPDATE uo_link_config SET protocol = 8
+  WHERE id = 1 AND protocol < 8
+    AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_8_migrated');
+INSERT IGNORE INTO settings (`key`, value) VALUES ('uo_link_protocol_8_migrated', '1');

server/model/shardClilocs/shardClilocs.model.js

@@ -1,6 +1,6 @@
 const db = require('./shardClilocs.db')
 const { settings } = require('../../core')
-const { displayText } = require('../../utils/clilocParse')
+const { displayText, parseCliloc } = require('../../utils/clilocParse')
 const {
   ClilocFormatError,
   ClilocSourceError,
@@ -8,8 +8,12 @@ const {
   hashSources,
   sameSources,
   missingSources,
+  missingOverlays,
+  readOverlays,
   readCliloc,
 } = require('../../utils/clilocSource')
+const bridge = require('../../utils/clilocBridge')
+const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model')
 const log = require('../../core').logger('shardClilocs')
 
 // The cliloc table — UO's id → display-string map, refreshed from a file the
@@ -29,11 +33,35 @@ const log = require('../../core').logger('shardClilocs')
 //   2. **Nothing client-derived is committed.** The table is built from the
 //      operator's own file at a configured path. The repo ships no strings.
 //
-// The table is built from a SET of sources — the converted client table plus
+// The table is built from a SET of sources — a base table plus every
-// every operator-maintained overlay beside it — because shards edit items and
+// operator-maintained overlay beside it — because shards edit items and add new
-// add new ones, and those carry cliloc ids no stock client table has. All of
+// ones, and those carry cliloc ids no stock client table has. Later sources win,
-// them are re-read on every boot and hash-gated together, so adding one custom
+// so an overlay both adds ids the client never had and overrides stock ones.
-// item never means re-exporting a 5 MB client file. Later sources win.
+//
+// ── Where the base comes from (protocol 8, docs/link/v8.md §9) ─────────
+//
+// **The shard**, on any install with uo-link configured. It has the operator's
+// client files already — a ServUO server cannot boot without them — and since
+// phase 2 it has the decompressor too, so `GET /cliloc` returns the table and
+// nobody installs UOFiddler or copies a 5 MB file anywhere.
+//
+// **A file on disk** otherwise. That is the pipeline this replaces, kept for
+// installs with no shard link and for development, and deprecated rather than
+// removed: an operator who has one keeps working, and an operator who has a shard
+// never builds one. Passing an explicit `path` to `refresh()` still selects it,
+// which is the escape hatch for "import from this file, this once".
+//
+// **Overlays are always the filesystem's**, either way. There is nothing on the
+// shard to ask for: ServUO has no server-side notion of a custom cliloc, so the
+// `custom/` directory is the only place those ids exist.
+//
+// ── What that changed about WHEN this runs ───────────────────────
+//
+// Boot no longer imports on the shard path. The file path could hash 5 MB locally
+// on every restart and skip; the shard path would mean a sidecar round trip in the
+// boot sequence, for a table that changes when an operator patches their client —
+// an event they know about and we do not. So on the bridge, importing is an admin
+// action (Admin → Shard), and boot leaves whatever is loaded serving.
 //
 // That set is also why this has the atlas's escalation, in a lighter form. A
 // single corrupt file fails the parse loudly, but a source that has simply
@@ -96,8 +124,224 @@ const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION
  *
  * `force` skips the hash check (an admin asking for a reimport). `approve`
  * additionally accepts a vanished source.
+ *
+ * Which SOURCE it reads is decided here and nowhere else: the shard when uo-link
+ * is configured and enabled, a file otherwise, and always a file when the caller
+ * named one.
  */
 async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
+  const override = String(pathOverride ?? '').trim()
+
+  if (override === '' && (await shardLinked())) {
+    return refreshFromShard({ force, approve })
+  }
+
+  return refreshFromFile({ force, approve, path: override })
+}
+
+/**
+ * Is there a shard to ask?
+ *
+ * Both halves matter. `baseUrl` alone is an install that has been configured and
+ * then switched off, and calling it would spend a 12 s timeout to learn what the
+ * row already says. Never throws: an unreadable config means "no shard", and the
+ * file path is a working answer.
+ */
+async function shardLinked() {
+  try {
+    const config = await uoLinkConfig.getSafe()
+    return Boolean(config?.enabled && config?.baseUrl)
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Merge parsed sources in order, later winning.
+ *
+ * Shared by both paths, because the merge is the same question whichever end the
+ * base arrived from: what did each source contribute, and what did it override.
+ * The per-source breakdown is for the admin panel — an operator who adds an
+ * overlay wants to see it took effect, and `overrode: 0` on a file meant to
+ * re-label stock items says it did not.
+ */
+function mergeSources(groups) {
+  const merged = new Map()
+  const sources = []
+
+  for (const group of groups) {
+    let added = 0
+    let overrode = 0
+
+    for (const entry of group.entries) {
+      if (!Number.isInteger(entry.number)) continue
+      if (merged.has(entry.number)) overrode++
+      else added++
+      merged.set(entry.number, entry)
+    }
+
+    sources.push({
+      label: group.label,
+      kind: group.kind,
+      entries: group.entries.length,
+      added,
+      overrode,
+    })
+  }
+
+  return { entries: [...merged.values()], sources }
+}
+
+/** Overlay hashes as a `{ label: sha256 }` map, in merge order. */
+function overlayHashes(files) {
+  const hashes = {}
+  for (const file of files) hashes[file.label] = file.sha256
+  return hashes
+}
+
+/** The overlay half of a stored fingerprint — everything under `custom/`. */
+function onlyOverlays(hashes) {
+  if (!hashes) return null
+  const out = {}
+  for (const [label, sha] of Object.entries(hashes)) {
+    if (label.startsWith('custom/')) out[label] = sha
+  }
+  return out
+}
+
+/**
+ * Import with the shard as the base source.
+ *
+ * The gate is two-part, and neither part is something the shard can answer for
+ * us: has the client file changed (size/mtime/sha256, plus the shard's own
+ * `EXTRACTOR_VERSION`), and has any overlay beside the configured path changed.
+ * Either is drift; neither is the normal case.
+ */
+async function refreshFromShard({ force = false, approve = false } = {}) {
+  let fingerprint
+
+  try {
+    fingerprint = await bridge.fingerprint()
+  } catch (err) {
+    if (err instanceof bridge.ClilocBridgeError) {
+      return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code }
+    }
+    return { status: 'failed', source: 'bridge', reason: err.message }
+  }
+
+  const configured = await getClientPath()
+  const overlays = readOverlays(configured)
+  const hashes = overlayHashes(overlays.files)
+
+  const meta = await db.getMeta().catch(() => null)
+
+  if (
+    !force &&
+    bridge.sameSource(fingerprint, meta?.base) &&
+    sameSources(hashes, onlyOverlays(meta?.hashes)) &&
+    currentParser(meta)
+  ) {
+    return {
+      status: 'unchanged',
+      source: 'bridge',
+      file: fingerprint.file,
+      count: meta.count ?? null,
+      customCount: overlays.files.length,
+      hashing: fingerprint.hashing,
+    }
+  }
+
+  // An overlay that was loaded last time and is not there now is refused rather
+  // than applied — an unmounted volume and a deliberate deletion look identical
+  // from here, and the wrong guess silently drops every name that file gave.
+  // The BASE is deliberately not part of this question: an install upgraded from
+  // the file pipeline is *supposed* to stop having one.
+  const gone = missingOverlays(hashes, meta?.hashes)
+
+  if (gone.length > 0 && !approve) {
+    return {
+      status: 'needsReview',
+      source: 'bridge',
+      reason: `${gone.length} previously-loaded cliloc overlay(s) are missing; the existing table is unchanged`,
+      missingSources: gone,
+      file: fingerprint.file,
+    }
+  }
+
+  let base
+
+  try {
+    base = await bridge.readCliloc({ lang: bridge.DEFAULT_LANGUAGE })
+  } catch (err) {
+    if (err instanceof bridge.ClilocBridgeError) {
+      return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code }
+    }
+    return { status: 'failed', source: 'bridge', reason: err.message }
+  }
+
+  const groups = [{ label: base.source.file, kind: 'shard', entries: base.entries }]
+
+  for (const file of overlays.files) {
+    try {
+      groups.push({ label: file.label, kind: 'custom', entries: parseCliloc(file.buffer) })
+    } catch (err) {
+      if (err instanceof ClilocFormatError) {
+        // Named, because "which of my six overlay files is malformed" is
+        // otherwise a guessing game.
+        return {
+          status: 'unavailable',
+          source: 'bridge',
+          reason: `${file.label}: ${err.message}`,
+          code: err.code,
+        }
+      }
+      return { status: 'failed', source: 'bridge', reason: err.message }
+    }
+  }
+
+  const merged = mergeSources(groups)
+
+  try {
+    const applied = await db.replaceAll(merged.entries, {
+      source: 'bridge',
+      base: fingerprint,
+      hashes,
+      parserVersion: PARSER_VERSION,
+      sources: merged.sources,
+      file: base.source.file,
+      bytes: fingerprint.size,
+    })
+
+    invalidate()
+
+    return {
+      status: 'imported',
+      source: 'bridge',
+      file: base.source.file,
+      count: applied.count,
+      parsed: merged.entries.length,
+      blank: applied.blank,
+      pages: base.source.pages,
+      sources: merged.sources,
+      // The shard says how many rows it holds; this is how many arrived. They
+      // agree, or the walk is wrong in a way no count on its own would show.
+      reported: base.source.reported,
+      received: base.source.received,
+      overlayProblem: overlays.problem ?? undefined,
+      acceptedMissing: gone.length > 0 ? gone : undefined,
+    }
+  } catch (err) {
+    return { status: 'failed', source: 'bridge', reason: err.message }
+  }
+}
+
+/**
+ * Import from a converted file on disk — the pre-protocol-8 pipeline, unchanged.
+ *
+ * Deprecated but supported: an install with no shard link has no other way to get
+ * a table, and development without a running ServUO is the same case.
+ */
+async function refreshFromFile({ force = false, approve = false, path: pathOverride = '' } = {}) {
   // An explicit override wins outright — a one-off "use this file", which must
   // not be silently overruled by the configured path the way an env default is.
   const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
@@ -153,7 +397,10 @@ async function refresh({ force = false, approve = false, path: pathOverride = ''
   }
 
   try {
-    const applied = await db.replaceAll(parsed.entries, parsed.source)
+    // `source: 'file'` is what lets the NEXT refresh — and `status()` — tell a
+    // table built from a converted file from one built over the bridge. Without
+    // it an install that gains a shard link looks like it already imported.
+    const applied = await db.replaceAll(parsed.entries, { ...parsed.source, source: 'file' })
     invalidate()
     return {
       status: 'imported',
@@ -177,9 +424,22 @@ async function refresh({ force = false, approve = false, path: pathOverride = ''
 /**
  * Boot hook. Best-effort by contract: it logs and returns, never throws, so a
  * missing or malformed cliloc file can never stop the site coming up.
+ *
+ * **On the bridge it imports nothing**, deliberately. The file path can hash a
+ * local 5 MB file on every restart and skip in 14 ms; asking the shard would put
+ * a sidecar round trip in the boot sequence to answer a question whose answer is
+ * "no" every time except after a client patch — which is an operator action, and
+ * therefore something an operator can press a button for. Whatever table is
+ * loaded keeps serving, which is exactly what happens today when a restart finds
+ * nothing changed.
  */
 async function refreshOnBoot() {
   try {
+    if (await shardLinked()) {
+      log.info('cliloc table comes from the shard; import is admin-triggered (Admin → Shard)')
+      return { status: 'skipped', source: 'bridge', reason: 'the shard is the cliloc source' }
+    }
+
     const result = await refresh()
     switch (result.status) {
       case 'imported':
@@ -220,8 +480,18 @@ async function refreshOnBoot() {
   }
 }
 
-/** Everything the admin panel needs to describe cliloc state. */
+/**
+ * Everything the admin panel needs to describe cliloc state.
+ *
+ * Two shapes, one per source, sharing every field a panel actually renders
+ * (`count`, `drift`, `problem`, `sources`, `missingSources`, `importedAt`). What
+ * differs is what `file` means and what a problem with it looks like: on the
+ * bridge it is the shard's own client file and the problems are transport ones,
+ * on disk it is a path an operator typed.
+ */
 async function status({ path: pathOverride = '' } = {}) {
+  if (pathOverride.trim() === '' && (await shardLinked())) return shardStatus()
+
   const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
   const meta = await db.getMeta().catch(() => null)
   const loaded = await db.count().catch(() => 0)
@@ -280,6 +550,72 @@ async function status({ path: pathOverride = '' } = {}) {
   }
 }
 
+/**
+ * Status when the shard is the source.
+ *
+ * The one thing worth knowing here that the file path has no equivalent of:
+ * `hashing`. The shard reports a null `sha256` for a client file it has not
+ * hashed yet — hashing the 343 MB of art and animation it also serves cannot fit
+ * in a 10 s reply, so it happens on its own thread — and a null hash means "ask
+ * again", never "changed". Drift falls back to (size, mtime) meanwhile, which is
+ * the same gate the shard itself applies, so an operator is never blocked from
+ * importing by a hash that has not landed.
+ */
+async function shardStatus() {
+  const configured = await getClientPath()
+  const meta = await db.getMeta().catch(() => null)
+  const loaded = await db.count().catch(() => 0)
+
+  const overlays = readOverlays(configured)
+  const hashes = overlayHashes(overlays.files)
+
+  let fingerprint = null
+  let problem = overlays.problem ?? null
+  let code = null
+
+  try {
+    fingerprint = await bridge.fingerprint()
+  } catch (err) {
+    problem = err.message
+    code = err.code ?? null
+  }
+
+  const drift = fingerprint
+    ? !bridge.sameSource(fingerprint, meta?.base) ||
+      !sameSources(hashes, onlyOverlays(meta?.hashes)) ||
+      !currentParser(meta)
+    : null
+
+  return {
+    source: 'bridge',
+    configured: true,
+    // The overlay directory, which is all the path setting still selects on this
+    // source. Reported so a panel can say where `custom/` is being read from.
+    path: configured,
+    file: fingerprint?.file ?? bridge.SOURCE_FILE,
+    fileReadable: Boolean(fingerprint),
+    problem,
+    code,
+    drift,
+    count: loaded,
+    shard: fingerprint
+      ? {
+          size: fingerprint.size,
+          mtime: fingerprint.mtime,
+          sha256: fingerprint.sha256,
+          extractorVersion: fingerprint.extractorVersion,
+          hashing: fingerprint.hashing,
+          complete: fingerprint.complete,
+        }
+      : null,
+    sources: Object.keys(hashes),
+    loadedSources: meta?.sources ?? null,
+    missingSources: missingOverlays(hashes, meta?.hashes),
+    importedAt: meta?.importedAt ?? null,
+    sourceBytes: meta?.base?.size ?? meta?.bytes ?? null,
+  }
+}
+
 // ── Lookup ─────────────────────────────────────────────────────────────────
 //
 // Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is
@@ -359,6 +695,7 @@ module.exports = {
   SETTING_KEY,
   getClientPath,
   setClientPath,
+  shardLinked,
   refresh,
   refreshOnBoot,
   status,

server/model/uoLinkConfig/uoLinkConfig.model.js

@@ -11,8 +11,11 @@ const { secretBox } = require('../../core')
 // Only used before an admin has saved anything — the stored row wins once it exists,
 // and UOLINK_PROTOCOL still overrides for an operator running an older sidecar.
 //
-// This says 7 because this build speaks protocol 7: the idempotency key and the
+// This says 8 because this build speaks protocol 8: the idempotency key and the
-// participation ledger (6), and the world verbs plus the targeted lease planes (7).
+// participation ledger (6), the world verbs plus the targeted lease planes (7), and
+// the Asset Bridge (8) -- of which this module is the first consumer, importing the
+// cliloc table over `GET /cliloc` instead of reading a file an operator converted by
+// hand (docs/link/v8.md §9).
 //
 // It said 4 before 5, and 3 for a while after protocol 4 shipped — which is the bug this
 // constant was introduced to fix. A FRESH install pinned 3, the sidecar answered
@@ -32,7 +35,7 @@ const { secretBox } = require('../../core')
 // (`PROTOCOL_VERSION`) and the overlay's in `servuo-plugins/overlay.toml`; the thing that
 // actually pairs them is the installer's bundle check, at deploy time. So bumping this in
 // the same change as the emitters is still the discipline, and no test here replaces it.
-const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 7
+const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 8
 
 function toSafe(row) {
   if (!row) {

server/router/admin/shard.router.js

@@ -307,10 +307,17 @@ shardRouter.put(
 )
 
 // ── Cliloc table (admin only) ─────────────────────────────────────────────
-// UO's id → display-string map, converted once by the operator from their own
+// UO's id → display-string map, read from the shard's own UO client over the
-// client (docs/website/CLILOCS.md). Sits beside the atlas for the same reason:
+// bridge (docs/link/v8.md §9, docs/website/CLILOCS.md). Sits beside the atlas for
-// it is static content derived from operator-supplied files rather than anything
+// the same reason: it is static content derived from the operator's own files
-// the sidecar sends, and operating it is shard administration.
+// rather than anything the sidecar streams, and operating it is shard
+// administration.
+//
+// Protocol 8 changed where the base table comes from, not what these routes are:
+// the shard decompresses `Cliloc.enu` and serves it paged, so an operator no
+// longer converts anything by hand. Import stays an explicit admin action,
+// because the only thing that changes a client's table is an operator patching
+// their client.
 //
 // There is deliberately NO public counterpart. The table is never served as a
 // table — 123k rows would dwarf any page that used it, and the Android client
@@ -320,7 +327,7 @@ shardRouter.get(
   '/clilocs',
   // #swagger.tags = ['Admin · Shard']
   // #swagger.summary = 'Cliloc table status: sources, drift, entry count (admin only)'
-  // #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether the files on disk have drifted from them. The table is built from a SET of sources — the converted client table plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any source that was loaded before and is now gone; an import refuses that without `approve`. A shard with nothing configured is a supported state — item names simply render as ids.'
+  // #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether they have drifted from what is loaded. `source` says which pipeline is in use: `bridge` (the shard reads its own client — the normal case once uo-link is configured) or `file` (a converted file on disk, deprecated, kept for installs with no shard link). On the bridge, `shard` carries the client file’s size, mtime, hash and the shard’s extractor version, and `shard.hashing: true` means a null hash is “not computed yet”, not “changed”. The table is always a SET: the base plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any overlay that was loaded before and is now gone; an import refuses that without `approve`. A shard with no source at all is a supported state — item names simply render as ids.'
   // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
   /* #swagger.responses[200] = { description: 'Cliloc status', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocStatus" } } } } */
   /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
@@ -330,8 +337,8 @@ shardRouter.get(
 shardRouter.post(
   '/clilocs/import',
   // #swagger.tags = ['Admin · Shard']
-  // #swagger.summary = 'Re-import the cliloc table from its source files (admin only)'
+  // #swagger.summary = 'Re-import the cliloc table from its source (admin only)'
-  // #swagger.description = 'Applies a client patch, or a change to the shard’s own overlay files, without a restart. `force` reimports even when the source hashes match what is loaded. `approve` accepts a refresh in which a previously-loaded source has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. A missing path — or the common mistake of pointing at the client’s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.'
+  // #swagger.description = 'Applies a client patch, or a change to the shard’s own overlay files, without a restart. On the bridge this is the ONLY thing that imports — boot deliberately does not call the shard — so it is what an operator presses after patching their client. `force` reimports even when the sources are unchanged. `approve` accepts a refresh in which a previously-loaded overlay has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane the operator has switched off, a client with no cliloc file, or a malformed overlay all answer 200 with status "unavailable" and a reason naming what to fix.'
   // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
   /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the sources are unchanged." }, approve: { type: "boolean", description: "Accept a refresh in which a previously-loaded source has vanished." } } } } } } */
   /* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocRefreshResult" } } } } */
@@ -344,10 +351,10 @@ shardRouter.post(
 shardRouter.put(
   '/clilocs/path',
   // #swagger.tags = ['Admin · Shard']
-  // #swagger.summary = 'Set the cliloc source the site reads from (admin only)'
+  // #swagger.summary = 'Set the cliloc path the site reads overlays (and any file base) from (admin only)'
-  // #swagger.description = 'Accepts either the converted base file itself or a directory to search. Overlays are read from a `custom/` directory beside it either way — pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.'
+  // #swagger.description = 'On an install with uo-link configured this selects only where `custom/` overlays are read from — the base table comes from the shard. Without a shard link it is also where the converted base file is looked for, which is the deprecated pre-protocol-8 pipeline. Accepts either a file or a directory to search; overlays are read from a `custom/` directory beside it either way, so pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.'
   // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
-  /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." } } } } } } */
+  /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Directory holding the custom/ overlays (and, with no shard link, a converted base file). Blank clears it." } } } } } } */
   /* #swagger.responses[200] = { description: 'Cliloc status after the change', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocStatus" } } } } */
   adminOnly,
   body('path').isString().isLength({ max: 512 }),

server/router/admin/shardClilocs.controller.js

@@ -1,8 +1,8 @@
 // ── Admin · Cliloc table ───────────────────────────────────────────────────
 //
-// Operating the cliloc import: where the converted cliloc file is, whether it
+// Operating the cliloc import: which source the table comes from, whether it has
-// has drifted from what is loaded, and a forced reimport after a client patch
+// drifted from what is loaded, and a reimport after a client patch
-// (docs/website/CLILOCS.md).
+// (docs/link/v8.md §9, docs/website/CLILOCS.md).
 //
 // The policy lives in the model. This controller does three things and no more:
 // it validates input, it maps a refresh RESULT onto an HTTP status, and it
@@ -10,11 +10,16 @@
 //
 // **A refresh result is not an exception.** `shardClilocs.refresh()` reports
 // `unavailable` / `failed` rather than throwing, because the boot path must never
-// be stopped by a bad file. That contract is preserved here: a missing file, or
+// be stopped by a bad source. That contract is preserved here, and protocol 8
-// the single most likely operator mistake — pointing at the client's own
+// widened the set of things it covers: a shard that is down, an asset plane the
-// COMPRESSED `Cliloc.enu` — is a 200 carrying `status: 'unavailable'` and the
+// operator has switched off, a client with no cliloc file, a client patched
-// reason, not a 500. A 500 would say only "something broke"; the operator needs
+// halfway through the import — plus everything the file pipeline could already
-// to be told which file to convert.
+// report. Each is a 200 carrying `status: 'unavailable'` and a reason naming what
+// to fix, not a 500 that says only "something broke".
+//
+// **Import matters more than it used to.** On the bridge, boot deliberately does
+// not call the shard, so this endpoint is the only thing that refreshes the
+// table — the operator presses it after patching their client.
 
 const clilocs = require('../../model/shardClilocs/shardClilocs.model')
 const market = require('../../model/shardMarket/shardMarket.model')
@@ -69,6 +74,10 @@ async function importClilocs(req, res) {
         force,
         approve,
         status: result.status,
+        // Which pipeline actually ran. Worth having in the audit log for the
+        // same reason it is in the status: an operator debugging a stale table
+        // needs to know whether the site asked the shard or read a file.
+        source: result.source ?? null,
         count: result.count ?? null,
         missingSources: result.missingSources ?? result.acceptedMissing ?? null,
       },
@@ -80,12 +89,16 @@ async function importClilocs(req, res) {
   }
 }
 
-// PUT /admin/shard/clilocs/path — point the site at a different cliloc file.
+// PUT /admin/shard/clilocs/path — point the site at a different cliloc path.
+//
+// On an install with uo-link configured this selects where `custom/` OVERLAYS are
+// read from; the base table comes from the shard either way. Without a shard link
+// it is also where the converted base file is looked for.
 //
 // Persisted as a setting, which wins over the UO_CLIENT_PATH env default so an
 // operator can move the mount without a redeploy. Blank clears it, which turns
-// resolution off (boot skips, the loaded table keeps serving) — a legitimate
+// overlay resolution off (the loaded table keeps serving) — a legitimate thing to
-// thing to want, so it is allowed rather than validated away.
+// want, so it is allowed rather than validated away.
 //
 // Deliberately does NOT import as a side effect, for the same reason the atlas
 // path does not: changing where the table reads from and reloading it are

server/swagger/doc.js

@@ -490,21 +490,40 @@ module.exports = {
       UoClilocStatus: {
         type: 'object',
         description:
-          'Admin view of cliloc state: where the converted file is, whether it is readable, how many entries are loaded, and whether the file has drifted from them. `configured: false` is a supported state — item names then render as ids.',
+          'Admin view of cliloc state: which source the base table comes from, whether it can be read, how many entries are loaded, and whether anything has drifted from them. Nothing configured at all is a supported state — item names then render as ids.',
         properties: {
+          source: {
+            type: 'string',
+            enum: ['bridge', 'file'],
+            description: '`bridge`: the shard reads its own UO client (protocol 8, the normal case). `file`: a converted file on disk — the pre-protocol-8 pipeline, deprecated, kept for installs with no shard link.',
+            example: 'bridge',
+          },
           configured: { type: 'boolean', example: true },
-          path: { type: 'string', example: '/srv/uo-client' },
+          path: { type: 'string', description: 'On the bridge: where `custom/` overlays are read from. On a file source: the base path too.', example: '/srv/uo-client' },
-          file: { type: 'string', nullable: true, description: 'The file actually resolved, when the path is a directory.', example: '/srv/uo-client/clilocs.tsv' },
+          file: { type: 'string', nullable: true, description: 'The base file in use — the shard’s own `cliloc.enu` on the bridge, the resolved local file otherwise.', example: 'cliloc.enu' },
           fileReadable: { type: 'boolean', example: true },
-          problem: { type: 'string', nullable: true, description: 'Why the file cannot be used, when it cannot. Set (with code COMPRESSED) for a readable-but-unconverted client file.', example: null },
+          problem: { type: 'string', nullable: true, description: 'Why the base cannot be used, when it cannot: a shard that is down or has assets switched off, or (on a file source) a missing or still-compressed file.', example: null },
-          code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED'] },
+          code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'UNAVAILABLE'] },
+          shard: {
+            type: 'object',
+            nullable: true,
+            description: 'Present on the bridge: the shard’s own cliloc file as it is right now. `hashing: true` with a null `sha256` means the hash has not been computed yet — “ask again”, not “changed”.',
+            properties: {
+              size: { type: 'integer', example: 4989921 },
+              mtime: { type: 'integer', description: 'Unix milliseconds.', example: 1757462400000 },
+              sha256: { type: 'string', nullable: true },
+              extractorVersion: { type: 'integer', description: 'The version of the shard’s extraction code. A bump makes everything derived from it drift.', example: 1 },
+              hashing: { type: 'boolean', example: false },
+              complete: { type: 'boolean', description: 'Every client file has a hash.', example: true },
+            },
+          },
           drift: { type: 'boolean', nullable: true, description: 'True when any source hash differs from the loaded table. NULL when the sources could not be read or are not usable.', example: false },
           count: { type: 'integer', description: 'Entries currently loaded.', example: 67496 },
           sources: {
             type: 'array',
             items: { type: 'string' },
             description: 'Every source found now, root-relative, base first then overlays in merge order.',
-            example: ['clilocs.plain', 'custom/uomysticmoon.tsv'],
+            example: ['custom/uomysticmoon.tsv'],
           },
           loadedSources: {
             type: 'array',
@@ -514,7 +533,7 @@ module.exports = {
               type: 'object',
               properties: {
                 label: { type: 'string', example: 'custom/uomysticmoon.tsv' },
-                kind: { type: 'string', enum: ['base', 'custom'], example: 'custom' },
+                kind: { type: 'string', enum: ['shard', 'base', 'custom'], description: '`shard` is the table read over the bridge; `base` a converted file on disk.', example: 'custom' },
                 entries: { type: 'integer', example: 37 },
                 added: { type: 'integer', description: 'Ids this source introduced.', example: 25 },
                 overrode: { type: 'integer', description: 'Ids it replaced from an earlier source.', example: 12 },
@@ -543,17 +562,22 @@ module.exports = {
             example: 'imported',
           },
           reason: { type: 'string', nullable: true },
+          source: { type: 'string', nullable: true, enum: ['bridge', 'file'], description: 'Which source this refresh read.', example: 'bridge' },
           code: {
             type: 'string',
             nullable: true,
-            description: 'Machine-readable cause. `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.',
+            description: 'Machine-readable cause. Bridge codes describe the shard (`DISABLED`: the operator switched the asset plane off; `NO_SOURCE`: its client has no cliloc file; `SHARD_DOWN`; `SOURCE_CHANGED`: the client was patched mid-import, so nothing was applied). File codes describe the path — `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.',
-            enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER'],
+            enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER', 'DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'UNAVAILABLE', 'SOURCE_CHANGED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE'],
           },
           path: { type: 'string', nullable: true },
           file: { type: 'string', nullable: true },
           count: { type: 'integer', nullable: true, description: 'Entries stored (blank strings are dropped).', example: 67496 },
           parsed: { type: 'integer', nullable: true, description: 'Entries read across every source before blanks were dropped.', example: 123527 },
-          blank: { type: 'integer', nullable: true, example: 55994 },
+          blank: { type: 'integer', nullable: true, example: 0 },
+          pages: { type: 'integer', nullable: true, description: 'Bridge only: how many pages the table arrived in (a stock English table is about eleven).', example: 11 },
+          reported: { type: 'integer', nullable: true, description: 'Bridge only: how many rows the shard said it holds.', example: 67496 },
+          received: { type: 'integer', nullable: true, description: 'Bridge only: how many arrived. Disagreeing with `reported` means the walk is wrong.', example: 67496 },
+          overlayProblem: { type: 'string', nullable: true, description: 'The base imported, but the overlay directory could not be read. Reported rather than fatal.' },
           sources: {
             type: 'array',
             nullable: true,
@@ -562,7 +586,7 @@ module.exports = {
               type: 'object',
               properties: {
                 label: { type: 'string' },
-                kind: { type: 'string', enum: ['base', 'custom'] },
+                kind: { type: 'string', enum: ['shard', 'base', 'custom'] },
                 entries: { type: 'integer' },
                 added: { type: 'integer' },
                 overrode: { type: 'integer' },

server/test/clilocBridge.test.js

@@ -0,0 +1,303 @@
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+
+const uoLinkClient = require('../utils/uoLinkClient')
+const bridge = require('../utils/clilocBridge')
+
+// The walk over `GET /cliloc`, driven against a stubbed sidecar client.
+//
+// Everything asserted here is a way the shard can be wrong that leaves the
+// website holding a table it believes is complete. That is the failure worth
+// testing, because it is invisible downstream: a truncated cliloc table renders
+// some items with names and some with ids, which is exactly what NO table looks
+// like. None of these are hypothetical shapes — each corresponds to a field the
+// paging envelope carries specifically so this side can tell the difference
+// (docs/link/v8.md §3.4).
+
+const saved = {}
+
+function stub({ sources, pages }) {
+  saved.getAssetSources = uoLinkClient.getAssetSources
+  saved.getClilocTable = uoLinkClient.getClilocTable
+
+  const calls = []
+
+  uoLinkClient.getAssetSources = async () => sources
+  uoLinkClient.getClilocTable = async ({ lang, cursor } = {}) => {
+    calls.push({ lang, cursor: cursor ?? null })
+    const next = pages.shift()
+    if (!next) throw new Error('the walk asked for more pages than the test supplied')
+    return next
+  }
+
+  return calls
+}
+
+function restore() {
+  if (saved.getAssetSources) uoLinkClient.getAssetSources = saved.getAssetSources
+  if (saved.getClilocTable) uoLinkClient.getClilocTable = saved.getClilocTable
+}
+
+const ok = (data) => ({ ok: true, status: 200, data })
+
+/** One page of rows, with the source fingerprint every page echoes. */
+const page = (rows, extra = {}) =>
+  ok({
+    kind: 'cliloc.table.ok',
+    lang: 'enu',
+    file: 'cliloc.enu',
+    size: 4989921,
+    mtime: 1757462400000,
+    total: 3,
+    rows,
+    more: false,
+    cut: 'end',
+    ...extra,
+  })
+
+const sourcesReply = (file = {}) =>
+  ok({
+    kind: 'assets.sources.ok',
+    extractorVersion: 1,
+    imaging: { ok: true },
+    hashing: false,
+    complete: true,
+    files: [
+      { name: 'cliloc.enu', path: '/uo/cliloc.enu', size: 4989921, mtime: 1757462400000, sha256: 'abc', ...file },
+      { name: 'art.mul', path: '/uo/art.mul', size: 148000000, mtime: 1, sha256: null },
+    ],
+  })
+
+// ── Stage 1: the fingerprint ───────────────────────────────────────────────
+
+test('fingerprint picks the cliloc file out of the client manifest', async (t) => {
+  stub({ sources: sourcesReply(), pages: [] })
+  t.after(restore)
+
+  const fp = await bridge.fingerprint()
+
+  assert.equal(fp.file, 'cliloc.enu')
+  assert.equal(fp.size, 4989921)
+  assert.equal(fp.sha256, 'abc')
+  assert.equal(fp.extractorVersion, 1)
+})
+
+test('a client with no cliloc file is NO_SOURCE, not a crash', async (t) => {
+  stub({
+    sources: ok({ extractorVersion: 1, files: [{ name: 'art.mul', size: 1, mtime: 1 }] }),
+    pages: [],
+  })
+  t.after(restore)
+
+  await assert.rejects(bridge.fingerprint(), (err) => {
+    assert.equal(err.code, 'NO_SOURCE')
+    return true
+  })
+})
+
+test('the asset plane being switched off reads as a refusal, not a bug', async (t) => {
+  stub({
+    sources: { ok: false, status: 403, data: { reason: 'asset extraction is disabled on this shard' } },
+    pages: [],
+  })
+  t.after(restore)
+
+  await assert.rejects(bridge.fingerprint(), (err) => {
+    assert.equal(err.code, 'DISABLED')
+    return true
+  })
+})
+
+// A hash that has not been computed yet is the shard's ordinary first answer:
+// hashing the 343 MB of art and animation it also serves cannot fit in a 10 s
+// reply, so it happens off the request path. Treating a null hash as a CHANGE
+// would make the panel show drift forever on a shard nobody has imported from.
+test('a missing hash falls back to (size, mtime) rather than reading as drift', () => {
+  const before = { size: 10, mtime: 20, sha256: null, extractorVersion: 1 }
+  const after = { size: 10, mtime: 20, sha256: null, extractorVersion: 1 }
+
+  assert.equal(bridge.sameSource(before, after), true)
+  assert.equal(bridge.sameSource(before, { ...after, mtime: 21 }), false)
+})
+
+test('a hash on both sides beats size and mtime, which a patched-in-place file can preserve', () => {
+  const a = { size: 10, mtime: 20, sha256: 'aaa', extractorVersion: 1 }
+
+  assert.equal(bridge.sameSource(a, { ...a, sha256: 'bbb' }), false)
+  assert.equal(bridge.sameSource(a, { ...a, size: 11, mtime: 99 }), true)
+})
+
+test('the extractor version is part of the fingerprint, so a corrected reader drifts', () => {
+  const a = { size: 10, mtime: 20, sha256: 'aaa', extractorVersion: 1 }
+
+  assert.equal(bridge.sameSource(a, { ...a, extractorVersion: 2 }), false)
+})
+
+// ── Stage 2: the walk ──────────────────────────────────────────────────────
+
+test('a one-page table comes back whole', async (t) => {
+  const calls = stub({
+    sources: sourcesReply(),
+    pages: [page([{ n: 3, f: 0, t: 'c' }, { n: 1, f: 2, t: 'a' }])],
+  })
+  t.after(restore)
+
+  const { entries, source } = await bridge.readCliloc()
+
+  assert.deepEqual(entries, [
+    { number: 3, flag: 0, text: 'c' },
+    { number: 1, flag: 2, text: 'a' },
+  ])
+  assert.equal(source.pages, 1)
+  assert.equal(source.received, 2)
+  assert.equal(source.reported, 3)
+  assert.deepEqual(calls, [{ lang: 'enu', cursor: null }])
+})
+
+test('pages are walked by echoing the cursor back until more is false', async (t) => {
+  const calls = stub({
+    sources: sourcesReply(),
+    pages: [
+      page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }),
+      page([{ n: 2, f: 0, t: 'b' }], { more: true, cursor: 'n:2', cut: 'budget' }),
+      page([{ n: 3, f: 0, t: 'c' }]),
+    ],
+  })
+  t.after(restore)
+
+  const { entries, source } = await bridge.readCliloc()
+
+  assert.equal(entries.length, 3)
+  assert.equal(source.pages, 3)
+  assert.deepEqual(
+    calls.map((c) => c.cursor),
+    [null, 'n:1', 'n:2'],
+  )
+})
+
+// `cut` is the field that is easy to omit and expensive not to have. A short
+// page means the source ended, the byte budget was spent, or the family hit its
+// own limit — and only the first means finished.
+test('a last page that did not end the table is refused, not imported', async (t) => {
+  stub({
+    sources: sourcesReply(),
+    pages: [page([{ n: 1, f: 0, t: 'a' }], { more: false, cut: 'limit' })],
+  })
+  t.after(restore)
+
+  await assert.rejects(bridge.readCliloc(), (err) => {
+    assert.equal(err.code, 'INCOMPLETE')
+    return true
+  })
+})
+
+test('a shard that does not advance its cursor is stopped rather than spun on', async (t) => {
+  stub({
+    sources: sourcesReply(),
+    pages: [
+      page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }),
+      page([{ n: 2, f: 0, t: 'b' }], { more: true, cursor: 'n:1', cut: 'budget' }),
+    ],
+  })
+  t.after(restore)
+
+  await assert.rejects(bridge.readCliloc(), (err) => {
+    assert.equal(err.code, 'STUCK')
+    return true
+  })
+})
+
+test('more:true with no cursor at all is the same refusal', async (t) => {
+  stub({
+    sources: sourcesReply(),
+    pages: [page([{ n: 1, f: 0, t: 'a' }], { more: true, cut: 'budget' })],
+  })
+  t.after(restore)
+
+  await assert.rejects(bridge.readCliloc(), (err) => {
+    assert.equal(err.code, 'STUCK')
+    return true
+  })
+})
+
+// The one failure a count cannot catch: an operator patches their client while
+// the import is walking it. Half of what arrived is from a file that no longer
+// exists, and nothing later can tell which half.
+test('a client patched mid-walk aborts the whole import', async (t) => {
+  stub({
+    sources: sourcesReply(),
+    pages: [
+      page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }),
+      page([{ n: 2, f: 0, t: 'b' }], { size: 5000000, mtime: 1757470000000 }),
+    ],
+  })
+  t.after(restore)
+
+  await assert.rejects(bridge.readCliloc(), (err) => {
+    assert.equal(err.code, 'SOURCE_CHANGED')
+    return true
+  })
+})
+
+// 425 is flow control and the ORDINARY answer during an import — the shard's
+// asset plane serves one request at a time on purpose — so it is retried rather
+// than failed. (The backoff is real time, so this exercises one retry only.)
+test('a busy shard is retried, because the work is happening', async (t) => {
+  saved.getAssetSources = uoLinkClient.getAssetSources
+  saved.getClilocTable = uoLinkClient.getClilocTable
+  t.after(restore)
+
+  let attempts = 0
+  uoLinkClient.getAssetSources = async () => sourcesReply()
+  uoLinkClient.getClilocTable = async () => {
+    attempts++
+    if (attempts === 1) return { ok: false, status: 425, data: { kind: 'bridge.busy' } }
+    return page([{ n: 1, f: 0, t: 'a' }])
+  }
+
+  const { entries } = await bridge.readCliloc()
+
+  assert.equal(attempts, 2)
+  assert.equal(entries.length, 1)
+})
+
+test('a page with no rows array is malformed, not an empty table', async (t) => {
+  stub({ sources: sourcesReply(), pages: [ok({ kind: 'cliloc.table.ok', more: false, cut: 'end' })] })
+  t.after(restore)
+
+  await assert.rejects(bridge.readCliloc(), (err) => {
+    assert.equal(err.code, 'MALFORMED')
+    return true
+  })
+})
+
+test('a shard that never ends the table is bounded by the page cap', async (t) => {
+  saved.getAssetSources = uoLinkClient.getAssetSources
+  saved.getClilocTable = uoLinkClient.getClilocTable
+  t.after(restore)
+
+  let n = 0
+  uoLinkClient.getAssetSources = async () => sourcesReply()
+  uoLinkClient.getClilocTable = async () => {
+    n++
+    return page([{ n, f: 0, t: 'x' }], { more: true, cursor: `n:${n}`, cut: 'budget' })
+  }
+
+  await assert.rejects(bridge.readCliloc(), (err) => {
+    assert.equal(err.code, 'TOO_LARGE')
+    return true
+  })
+  assert.equal(n, bridge.MAX_PAGES)
+})
+
+test('rows with an unusable id are dropped rather than stored as NaN', async (t) => {
+  stub({
+    sources: sourcesReply(),
+    pages: [page([{ n: 'nonsense', f: 0, t: 'a' }, { n: 7, f: 0, t: 'b' }])],
+  })
+  t.after(restore)
+
+  const { entries } = await bridge.readCliloc()
+
+  assert.deepEqual(entries, [{ number: 7, flag: 0, text: 'b' }])
+})

server/test/clilocSourceSelection.test.js

@@ -0,0 +1,330 @@
+// Which cliloc source runs, and what the shard path does with the answer
+// (docs/link/v8.md §9, docs/website/CLILOCS.md).
+//
+// The model is the only place that decides between the two pipelines, so these
+// drive it with the shard, the database and the filesystem all stubbed. Nothing
+// here reaches the real sidecar or a real table.
+
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+
+const clilocs = require('../model/shardClilocs/shardClilocs.model')
+const db = require('../model/shardClilocs/shardClilocs.db')
+const bridge = require('../utils/clilocBridge')
+const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
+const { ctx } = require('./_setup')
+
+const saved = {
+  getMeta: db.getMeta,
+  replaceAll: db.replaceAll,
+  count: db.count,
+  fingerprint: bridge.fingerprint,
+  readCliloc: bridge.readCliloc,
+  getSafe: uoLinkConfig.getSafe,
+  settingsGet: ctx.settings.get,
+}
+
+function restore() {
+  db.getMeta = saved.getMeta
+  db.replaceAll = saved.replaceAll
+  db.count = saved.count
+  bridge.fingerprint = saved.fingerprint
+  bridge.readCliloc = saved.readCliloc
+  uoLinkConfig.getSafe = saved.getSafe
+  ctx.settings.get = saved.settingsGet
+}
+
+const FINGERPRINT = {
+  kind: 'bridge',
+  file: 'cliloc.enu',
+  size: 4989921,
+  mtime: 1757462400000,
+  sha256: 'abc',
+  extractorVersion: 1,
+  hashing: false,
+  complete: true,
+}
+
+/**
+ * A rig with the shard reachable (or not), the configured overlay path pointed
+ * at a temp directory, and every write captured rather than made.
+ */
+function rig({ linked = true, meta = null, clientPath = '', rows = [] } = {}) {
+  const applied = []
+
+  uoLinkConfig.getSafe = async () => ({ enabled: linked, baseUrl: linked ? 'http://127.0.0.1:8099' : null })
+  ctx.settings.get = async (key) => (key === clilocs.SETTING_KEY ? clientPath : null)
+
+  db.getMeta = async () => meta
+  db.count = async () => meta?.count ?? 0
+  db.replaceAll = async (entries, writtenMeta) => {
+    applied.push({ entries, meta: writtenMeta })
+    return { count: entries.length, blank: 0, duplicates: 0 }
+  }
+
+  bridge.fingerprint = async () => FINGERPRINT
+  bridge.readCliloc = async () => ({
+    entries: rows,
+    source: {
+      kind: 'bridge',
+      lang: 'enu',
+      file: 'cliloc.enu',
+      size: FINGERPRINT.size,
+      mtime: FINGERPRINT.mtime,
+      pages: 1,
+      reported: rows.length,
+      received: rows.length,
+    },
+  })
+
+  return applied
+}
+
+function tmpWithOverlay(contents) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cliloc-sel-'))
+  fs.mkdirSync(path.join(dir, 'custom'), { recursive: true })
+  if (contents !== undefined) fs.writeFileSync(path.join(dir, 'custom', 'shard.tsv'), contents)
+  return dir
+}
+
+// ── Which source runs ──────────────────────────────────────────────────────
+
+test('a configured shard is the base source, and the file path is not consulted', async (t) => {
+  const applied = rig({ rows: [{ number: 1, flag: 0, text: 'a' }] })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported')
+  assert.equal(result.source, 'bridge')
+  assert.equal(applied.length, 1)
+  assert.equal(applied[0].meta.source, 'bridge')
+})
+
+test('no shard link falls back to the file pipeline, unchanged', async (t) => {
+  rig({ linked: false })
+  t.after(restore)
+
+  // No path configured either, so the file path reports exactly what it always
+  // did — which is the assertion: the fallback is the OLD code, not a new one.
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'skipped')
+  assert.equal(result.reason, 'no cliloc path configured')
+})
+
+test('an explicit path is still an escape hatch, even with a shard linked', async (t) => {
+  let asked = false
+  rig({ linked: true })
+  bridge.fingerprint = async () => {
+    asked = true
+    return FINGERPRINT
+  }
+  t.after(restore)
+
+  const result = await clilocs.refresh({ path: path.join(os.tmpdir(), 'nope-does-not-exist') })
+
+  assert.equal(asked, false, 'the shard must not be asked when a file was named')
+  assert.equal(result.status, 'unavailable')
+})
+
+// Boot deliberately does not call the shard: it would put a sidecar round trip
+// in the startup sequence to answer a question whose answer is "no" except after
+// a client patch, which is an operator action.
+test('boot imports nothing over the bridge and leaves the loaded table serving', async (t) => {
+  let asked = false
+  rig({ linked: true })
+  bridge.fingerprint = async () => {
+    asked = true
+    return FINGERPRINT
+  }
+  t.after(restore)
+
+  const result = await clilocs.refreshOnBoot()
+
+  assert.equal(result.status, 'skipped')
+  assert.equal(result.source, 'bridge')
+  assert.equal(asked, false)
+})
+
+// ── The gate ───────────────────────────────────────────────────────────────
+
+test('an unchanged client file and no overlays is a no-op', async (t) => {
+  const applied = rig({
+    meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 },
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'unchanged')
+  assert.equal(result.count, 67496)
+  assert.equal(applied.length, 0)
+})
+
+test('a patched client re-imports', async (t) => {
+  const applied = rig({
+    meta: {
+      source: 'bridge',
+      base: { ...FINGERPRINT, sha256: 'older' },
+      hashes: {},
+      parserVersion: 1,
+      count: 10,
+    },
+    rows: [{ number: 1, flag: 0, text: 'a' }],
+  })
+  t.after(restore)
+
+  assert.equal((await clilocs.refresh()).status, 'imported')
+  assert.equal(applied.length, 1)
+})
+
+// The upgrade path. An install that used the converted-file pipeline carries its
+// base label in the stored fingerprint; on the bridge that label is SUPPOSED to
+// disappear. Counting it as a vanished source would make the first import after
+// the upgrade demand an approval for a change the upgrade itself made.
+test('the retired file base is not reported as a vanished source', async (t) => {
+  const applied = rig({
+    meta: {
+      source: 'file',
+      hashes: { 'clilocs.plain': 'aaa' },
+      parserVersion: 1,
+      count: 67496,
+    },
+    rows: [{ number: 1, flag: 0, text: 'a' }],
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported', result.reason)
+  assert.equal(applied.length, 1)
+})
+
+// An overlay is a different matter: it vanished, and an unmounted volume looks
+// exactly like a deliberate deletion from here.
+test('a vanished OVERLAY still stages for review', async (t) => {
+  const applied = rig({
+    meta: {
+      source: 'bridge',
+      base: FINGERPRINT,
+      hashes: { 'custom/shard.tsv': 'aaa' },
+      parserVersion: 1,
+      count: 5,
+    },
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'needsReview')
+  assert.deepEqual(result.missingSources, ['custom/shard.tsv'])
+  assert.equal(applied.length, 0)
+
+  const accepted = await clilocs.refresh({ approve: true })
+  assert.equal(accepted.status, 'imported')
+  assert.deepEqual(accepted.acceptedMissing, ['custom/shard.tsv'])
+})
+
+// ── The merge ──────────────────────────────────────────────────────────────
+
+test('an overlay overrides the shard table, and says so', async (t) => {
+  const dir = tmpWithOverlay('1023721\ta better staff\n900001\ta shard-only item\n')
+  const applied = rig({
+    clientPath: dir,
+    rows: [
+      { number: 1023721, flag: 0, text: 'quarter staff' },
+      { number: 3000001, flag: 0, text: 'Entering Britannia...' },
+    ],
+  })
+  t.after(() => {
+    restore()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported', result.reason)
+
+  const stored = new Map(applied[0].entries.map((e) => [e.number, e.text]))
+  assert.equal(stored.get(1023721), 'a better staff', 'the overlay must win')
+  assert.equal(stored.get(3000001), 'Entering Britannia...')
+  assert.equal(stored.get(900001), 'a shard-only item')
+
+  const overlay = result.sources.find((s) => s.kind === 'custom')
+  assert.equal(overlay.label, 'custom/shard.tsv')
+  assert.equal(overlay.added, 1)
+  assert.equal(overlay.overrode, 1)
+
+  // Only overlay hashes are stored now — the base is fingerprinted separately,
+  // and mixing them is what made the upgrade case above ambiguous.
+  assert.deepEqual(Object.keys(applied[0].meta.hashes), ['custom/shard.tsv'])
+  assert.equal(applied[0].meta.base.sha256, 'abc')
+})
+
+test('a malformed overlay names the file rather than failing the import namelessly', async (t) => {
+  const dir = tmpWithOverlay('not a cliloc file at all\n')
+  rig({ clientPath: dir, rows: [{ number: 1, flag: 0, text: 'a' }] })
+  t.after(() => {
+    restore()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'unavailable')
+  assert.match(result.reason, /custom\/shard\.tsv/)
+})
+
+// An overlay path an operator has mistyped must not stop a base table that
+// arrived perfectly well — but it must be visible, or the site silently serves a
+// table missing every shard-added name.
+test('an unreadable overlay path is reported beside a successful import', async (t) => {
+  const applied = rig({
+    clientPath: path.join(os.tmpdir(), 'cliloc-does-not-exist-at-all'),
+    rows: [{ number: 1, flag: 0, text: 'a' }],
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported')
+  assert.match(result.overlayProblem, /does not exist/)
+  assert.equal(applied.length, 1)
+})
+
+// ── Status ─────────────────────────────────────────────────────────────────
+
+test('status describes the shard source, hash state and drift', async (t) => {
+  rig({ meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 } })
+  t.after(restore)
+
+  const status = await clilocs.status()
+
+  assert.equal(status.source, 'bridge')
+  assert.equal(status.file, 'cliloc.enu')
+  assert.equal(status.fileReadable, true)
+  assert.equal(status.drift, false)
+  assert.equal(status.shard.extractorVersion, 1)
+  assert.equal(status.shard.hashing, false)
+})
+
+test('a shard that cannot be reached is a problem on the status, not a throw', async (t) => {
+  rig({})
+  bridge.fingerprint = async () => {
+    throw new bridge.ClilocBridgeError('The shard did not answer: timeout', 'SHARD_DOWN')
+  }
+  t.after(restore)
+
+  const status = await clilocs.status()
+
+  assert.equal(status.source, 'bridge')
+  assert.equal(status.fileReadable, false)
+  assert.equal(status.code, 'SHARD_DOWN')
+  // Null, not false: with no fingerprint there is nothing to compare, and
+  // reporting "no drift" would read as "up to date".
+  assert.equal(status.drift, null)
+})

server/utils/clilocBridge.js

@@ -0,0 +1,306 @@
+// Cliloc table — the SHARD source (docs/link/v8.md §9, protocol 8 phase 2).
+//
+// `clilocSource.js` is the filesystem half of this story and predates it. This is
+// the half that replaces the part of it nobody enjoyed: until protocol 8 the base
+// table reached the site because an operator installed UOFiddler, built a
+// converter against its `Ultima.dll`, ran it over their client's compressed
+// `Cliloc.enu` and copied a five-megabyte file to the web host — every time they
+// patched their client.
+//
+// The shard has always had those files (a ServUO server cannot boot without a UO
+// client) and, as of phase 2, has the decompressor too. So the base table now
+// arrives over the same request/reply path as every other shard read, and the
+// operator installs nothing.
+//
+// **What is NOT here.** Overlays. Shard-added items carry cliloc ids no client
+// table has, ServUO has no server-side notion of a custom cliloc, and there is
+// therefore nothing on the shard to ask for. `custom/` stays a directory the site
+// reads (`clilocSource.readOverlays`), and the model merges it OVER whatever
+// arrives here. That division is the whole of CLILOCS.md §Shard-added items and
+// it is unchanged by this file.
+//
+// ── Why this walks pages instead of asking for a table ────────────────────
+//
+// The sidecar's reply timeout is 10 s and its inbound line cap is 1 MiB, so a
+// five-megabyte table cannot be one answer. The shard cuts pages at a 512 KiB
+// byte budget and hands back a cursor; this walks them. A stock English table is
+// about eleven pages.
+//
+// Three properties of that envelope are load-bearing and each has a check below:
+//
+//   - **Only `cut: 'end'` means finished.** A short page can equally mean the
+//     budget was spent (`budget`) or the family stopped at its own limit
+//     (`limit`). Treating a short page as the end would import a truncated table,
+//     which is indistinguishable downstream from a complete one — some items
+//     named, some not, exactly what "no table at all" looks like.
+//   - **The cursor must advance.** A shard that answered the same cursor forever
+//     would spin this loop until the request timeout with nothing to show.
+//   - **The file must not change underneath the walk.** Every page echoes the
+//     source's size and mtime; an operator patching their client mid-import would
+//     otherwise produce one table stitched from two, with no error anywhere.
+
+// Required as a namespace, not destructured: a test that stubs the sidecar
+// replaces these on the module object, and a destructured copy taken at load
+// time would keep calling the real one.
+const uoLinkClient = require('./uoLinkClient')
+const log = require('../core').logger('cliloc-bridge')
+
+/** The client file the base table comes from, as `assets.sources` names it. */
+const SOURCE_FILE = 'cliloc.enu'
+
+const DEFAULT_LANGUAGE = 'enu'
+
+// Bounds on the walk. Neither is expected to be reached — a stock table is ~11
+// pages and ~67k rows — and both exist so that a shard answering nonsense costs a
+// bounded amount of time rather than an unbounded amount of memory.
+const MAX_PAGES = 200
+const MAX_ROWS = 500000
+
+// 425 is the ordinary answer during an import, not an error: the shard's asset
+// plane serves one request at a time on purpose, because its outbound queue is
+// bounded in lines rather than bytes. So a page that comes back busy is retried
+// with a short backoff rather than failing the import.
+const BUSY_RETRIES = 5
+const BUSY_BACKOFF_MS = [200, 400, 800, 1600, 3200]
+
+class ClilocBridgeError extends Error {
+  constructor(message, code) {
+    super(message)
+    this.name = 'ClilocBridgeError'
+    this.code = code
+  }
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Map a sidecar response onto one of this module's codes.
+ *
+ * The statuses are the ones `respond_assets` produces, and the distinction that
+ * matters most to an operator is 403 vs 404: "you have not switched this on" and
+ * "your client does not have that file" are different jobs, and both are things
+ * they can fix.
+ */
+function describeFailure(res, what) {
+  const reason = res?.data?.reason || res?.error || `sidecar responded ${res?.status}`
+
+  switch (res?.status) {
+    case 403:
+      return new ClilocBridgeError(
+        `The shard is refusing to serve client assets (Bridge.AssetsEnabled is off): ${reason}`,
+        'DISABLED',
+      )
+    case 404:
+      return new ClilocBridgeError(`The shard has no ${what}: ${reason}`, 'NO_SOURCE')
+    case 409:
+      return new ClilocBridgeError(
+        `The sidecar refused the protocol version this build declares: ${reason}`,
+        'PROTOCOL',
+      )
+    case 422:
+      return new ClilocBridgeError(`The shard could not read its own ${what}: ${reason}`, 'UNREADABLE')
+    case 425:
+      return new ClilocBridgeError(
+        'The shard is busy serving another asset request and stayed busy',
+        'BUSY',
+      )
+    case 503:
+    case 504:
+      return new ClilocBridgeError(`The shard did not answer: ${reason}`, 'SHARD_DOWN')
+    default:
+      return new ClilocBridgeError(reason, 'UNAVAILABLE')
+  }
+}
+
+/**
+ * Stage 1: the fingerprint of the shard's own cliloc file.
+ *
+ * Returns `{ file, size, mtime, sha256, extractorVersion, hashing, complete }`.
+ *
+ * `sha256` may be **null** — the shard reports hashes only once it has computed
+ * them off the request path, because hashing the client files it also serves
+ * (343 MB of art and animation) cannot fit inside a 10 s reply. A null hash means
+ * "not yet", never "changed", and `sameSource` below compares (size, mtime) in
+ * that case, which is the same gate the shard itself uses.
+ */
+async function fingerprint() {
+  const res = await uoLinkClient.getAssetSources()
+  if (!res.ok) throw describeFailure(res, 'client file manifest')
+
+  const files = Array.isArray(res.data?.files) ? res.data.files : []
+  const entry = files.find((f) => String(f?.name || '').toLowerCase() === SOURCE_FILE)
+
+  if (!entry) {
+    throw new ClilocBridgeError(
+      `The shard's UO client has no ${SOURCE_FILE} (it reported ${files.length} client file(s))`,
+      'NO_SOURCE',
+    )
+  }
+
+  return {
+    kind: 'bridge',
+    file: entry.name,
+    path: entry.path ?? null,
+    size: Number(entry.size) || 0,
+    mtime: Number(entry.mtime) || 0,
+    sha256: entry.sha256 ?? null,
+    extractorVersion: Number(res.data?.extractorVersion) || 0,
+    hashing: Boolean(res.data?.hashing),
+    complete: Boolean(res.data?.complete),
+  }
+}
+
+/**
+ * True when two fingerprints describe the same client file.
+ *
+ * Hash first when both sides have one, because a hash is the only thing that
+ * catches a file rewritten with the same length and timestamp. Falls back to
+ * (size, mtime) when either side's hash is missing, which is the case on the
+ * first poll after a shard restart and the reason `hashing` exists at all.
+ */
+function sameSource(a, b) {
+  if (!a || !b) return false
+  if (a.extractorVersion !== b.extractorVersion) return false
+  if (a.sha256 && b.sha256) return a.sha256 === b.sha256
+  return a.size === b.size && a.mtime === b.mtime && a.size > 0
+}
+
+/** One page, with the 425 backoff. */
+async function fetchPage({ lang, cursor }) {
+  for (let attempt = 0; ; attempt++) {
+    const res = await uoLinkClient.getClilocTable({ lang, cursor })
+    if (res.ok) return res.data
+
+    if (res.status === 425 && attempt < BUSY_RETRIES) {
+      await sleep(BUSY_BACKOFF_MS[Math.min(attempt, BUSY_BACKOFF_MS.length - 1)])
+      continue
+    }
+
+    throw describeFailure(res, `cliloc.${lang}`)
+  }
+}
+
+/**
+ * Walk the whole table.
+ *
+ * Returns `{ entries, source }` where `entries` is `[{ number, flag, text }]` in
+ * the shape `clilocParse` produces, so the merge in `shardClilocs.model` does not
+ * care which source an entry came from.
+ *
+ * Blanks are already gone: the shard drops the ~56,000 empty strings a stock
+ * table carries before they reach the wire, since the site would drop them at
+ * import anyway. Nothing downstream changes — `db.replaceAll` still filters, and
+ * still would if a source ever sent one.
+ */
+async function readCliloc({ lang = DEFAULT_LANGUAGE } = {}) {
+  const started = Date.now()
+  const entries = []
+
+  let cursor = null
+  let pages = 0
+  let first = null
+  let finished = false
+  let total = null
+
+  while (pages < MAX_PAGES) {
+    const page = await fetchPage({ lang, cursor })
+    pages++
+
+    if (!page || !Array.isArray(page.rows)) {
+      throw new ClilocBridgeError('The shard sent a cliloc page with no rows array', 'MALFORMED')
+    }
+
+    if (first === null) {
+      first = { size: Number(page.size) || 0, mtime: Number(page.mtime) || 0 }
+      total = Number.isFinite(Number(page.total)) ? Number(page.total) : null
+    } else if (Number(page.size) !== first.size || Number(page.mtime) !== first.mtime) {
+      // The client was patched (or a different one mounted) between two pages.
+      // Refusing is the only honest answer: half of what we hold is from a file
+      // that no longer exists, and nothing later can tell which half.
+      throw new ClilocBridgeError(
+        'The shard\'s cliloc file changed while it was being read; nothing was imported',
+        'SOURCE_CHANGED',
+      )
+    }
+
+    for (const row of page.rows) {
+      const number = Number(row?.n)
+      if (!Number.isInteger(number)) continue
+      entries.push({ number, flag: Number(row?.f) || 0, text: String(row?.t ?? '') })
+    }
+
+    if (entries.length > MAX_ROWS) {
+      throw new ClilocBridgeError(
+        `The shard sent more than ${MAX_ROWS} cliloc rows; refusing to keep reading`,
+        'TOO_LARGE',
+      )
+    }
+
+    if (!page.more) {
+      // `cut` is the field that says WHY a page was the last one, and only one of
+      // its values means the table ended. A shard that stopped for its own limit
+      // has not finished, and importing what arrived would silently drop the tail.
+      if (page.cut !== 'end') {
+        throw new ClilocBridgeError(
+          `The shard stopped sending cliloc rows after ${entries.length} (cut: ${page.cut || 'unknown'})`,
+          'INCOMPLETE',
+        )
+      }
+      finished = true
+      break
+    }
+
+    if (!page.cursor || page.cursor === cursor) {
+      // Either would loop forever: no cursor to advance with, or the same one
+      // back again.
+      throw new ClilocBridgeError(
+        `The shard asked for another cliloc page without advancing its cursor (${page.cursor || 'none'})`,
+        'STUCK',
+      )
+    }
+
+    cursor = page.cursor
+  }
+
+  if (!finished) {
+    throw new ClilocBridgeError(
+      `The cliloc table did not end within ${MAX_PAGES} pages; nothing was imported`,
+      'TOO_LARGE',
+    )
+  }
+
+  log.info('cliloc table read from the shard', {
+    lang,
+    entries: entries.length,
+    pages,
+    ms: Date.now() - started,
+  })
+
+  return {
+    entries,
+    source: {
+      kind: 'bridge',
+      lang,
+      file: `cliloc.${lang}`,
+      size: first?.size ?? 0,
+      mtime: first?.mtime ?? 0,
+      pages,
+      // What the shard said it holds, kept beside what actually arrived. They
+      // agree or the walk is wrong, and an operator seeing them disagree in the
+      // panel learns more than a single number would tell them.
+      reported: total,
+      received: entries.length,
+    },
+  }
+}
+
+module.exports = {
+  ClilocBridgeError,
+  SOURCE_FILE,
+  DEFAULT_LANGUAGE,
+  MAX_PAGES,
+  MAX_ROWS,
+  fingerprint,
+  sameSource,
+  readCliloc,
+}

server/utils/clilocSource.js

@@ -6,6 +6,24 @@
 //   - the server, which refreshes the table on boot (`shardClilocs.model.js`)
 //   - the admin panel, which can force a reimport without a restart
 //
+// ── What protocol 8 took away, and what it left ───────────────────────────
+//
+// The BASE table no longer comes from here on a shard that has uo-link
+// configured: `clilocBridge.js` asks the shard for it, because the shard has the
+// operator's client files already and, since phase 2, the decompressor to read
+// them (docs/link/v8.md §9). Nobody converts a file by hand any more.
+//
+// Two things keep this module alive rather than deleting it:
+//
+//   - **Overlays.** Shard-added items carry cliloc ids no client table has, and
+//     ServUO has no server-side notion of a custom cliloc — there is nothing on
+//     the shard to ask for. `custom/` is still a directory the site reads, and
+//     `readOverlays` below is the entry point the bridge path uses.
+//   - **Installs with no shard link**, and development. A site that has never
+//     configured uo-link can still be pointed at a converted file; that path is
+//     deprecated, not removed, and it stays the whole of this module's base-table
+//     behaviour.
+//
 // The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived
 // from them is committed: the repo holds no string table, exactly as it holds no
 // map snapshot and no artwork. That rule is why this module reads a configured
@@ -194,6 +212,63 @@ function readSources(configured) {
   return { root, files }
 }
 
+/**
+ * Read the OVERLAY files only, with no base table.
+ *
+ * The bridge path needs exactly this: the base arrives from the shard and the
+ * `custom/` directory beside the configured path still has to be merged over it.
+ * `readSources` cannot answer it, because resolving a base is the first thing it
+ * does and there may not be one — an operator on the bridge is entitled to point
+ * this setting at a directory that holds nothing but `custom/`.
+ *
+ * **Never throws.** A path that is blank, missing or unreadable is reported as a
+ * `problem` string and an empty file list, because none of those may stop a base
+ * table that arrived perfectly well from being imported. The model decides what
+ * to do about it — and it has a real decision to make, since an overlay that was
+ * loaded last time and is missing now is the vanished-source hazard, not a
+ * config typo.
+ */
+function readOverlays(configured) {
+  const target = String(configured ?? '').trim()
+  if (target === '') return { root: null, files: [], problem: null }
+
+  let root
+  try {
+    const stat = fs.statSync(target)
+    root = stat.isFile() ? path.dirname(target) : target
+  } catch {
+    return { root: null, files: [], problem: `Cliloc path does not exist: ${target}` }
+  }
+
+  let overlays
+  try {
+    overlays = listCustom(root)
+  } catch (err) {
+    return { root, files: [], problem: err.message }
+  }
+
+  const files = []
+  for (const file of overlays) {
+    let buffer
+    try {
+      buffer = fs.readFileSync(file)
+    } catch {
+      return { root, files: [], problem: `Cliloc overlay is not readable: ${file}` }
+    }
+    files.push({
+      label: path.relative(root, file).split(path.sep).join('/'),
+      kind: 'custom',
+      file,
+      buffer,
+      sha256: sha256(buffer),
+      bytes: buffer.length,
+      compressed: isCompressedCliloc(buffer),
+    })
+  }
+
+  return { root, files, problem: null }
+}
+
 /**
  * A fingerprint of every source: `{ "<label>": "<sha256>" }`, plus the base's
  * details for the admin panel.
@@ -244,6 +319,25 @@ function missingSources(current, loaded) {
   return Object.keys(loaded).filter((label) => !Object.hasOwn(current, label))
 }
 
+/**
+ * The same question asked of OVERLAYS only.
+ *
+ * Needed because the base table moved to the bridge. An install upgraded from the
+ * file pipeline carries a base label (`clilocs.plain`, say) in its loaded
+ * fingerprint, and that label is *supposed* to disappear when the base starts
+ * arriving from the shard — reporting it as a vanished source would make every
+ * first import after the upgrade demand an approval for a change the upgrade
+ * itself made. Overlay labels are the ones whose absence is genuinely ambiguous,
+ * and they are exactly the labels under `custom/`.
+ */
+function missingOverlays(current, loaded) {
+  if (!loaded) return []
+  const prefix = `${CUSTOM_DIR}/`
+  return Object.keys(loaded).filter(
+    (label) => label.startsWith(prefix) && !Object.hasOwn(current, label),
+  )
+}
+
 /**
  * Read and parse every source, merged into one entry list.
  *
@@ -309,8 +403,10 @@ module.exports = {
   resolveBase,
   listCustom,
   readSources,
+  readOverlays,
   hashSources,
   sameSources,
   missingSources,
+  missingOverlays,
   readCliloc,
 }

server/utils/uoLinkClient.js

@@ -178,6 +178,37 @@ const getPointsBoard = (system) => call(`/points/${encodeURIComponent(system)}`)
 const getMarket = ({ limit = 200, offset = 0 } = {}) =>
   call(`/market?limit=${encodeURIComponent(limit)}&offset=${encodeURIComponent(offset)}`)
 
+// ── Protocol 8: the Asset Bridge (docs/link/v8.md) ────────────────────────
+//
+// The shard reads the operator's own UO client files and hands the results over
+// this link, which is why nobody has to install UOFiddler any more.
+
+// Stage 1 of the import gate: what those client files currently ARE — size, mtime
+// and content hash of each, plus the version of the shard's extractor that would
+// read them. No pixels and no strings cross on this call; its whole job is to let
+// the site decide that nothing has changed and stop, which is the normal case on
+// every restart.
+//
+// `sha256` comes back NULL for a file the shard has not hashed yet (anim.mul is
+// 195 MB and hashing it cannot fit in a reply), with `hashing: true` alongside.
+// That is "ask again in a moment", not "the file changed".
+const getAssetSources = () => call('/assets/sources')
+
+// The cliloc table out of the shard's own client, PAGED: each reply carries `rows`
+// plus `more` / `cursor` / `cut`, and the caller echoes the cursor back until a
+// reply says `more: false`. Only `cut: 'end'` means the table is finished — a short
+// page can equally mean the byte budget was spent.
+//
+// `clilocBridge.js` is the thing that walks it; nothing else should call this
+// directly, because a half-walked table is worse than none.
+const getClilocTable = ({ lang, cursor } = {}) => {
+  const params = new URLSearchParams()
+  if (lang) params.set('lang', lang)
+  if (cursor) params.set('cursor', cursor)
+  const qs = params.toString()
+  return call(`/cliloc${qs ? `?${qs}` : ''}`)
+}
+
 // ── Commands ──────────────────────────────────────────────────────────────
 const confirmLink = (code, websiteUserId) =>
   call('/link/confirm', { method: 'POST', body: { code, websiteUserId: String(websiteUserId) } })
@@ -398,6 +429,8 @@ module.exports = {
   getPoints,
   getPointsBoard,
   getMarket,
+  getAssetSources,
+  getClilocTable,
   confirmLink,
   linkLookup,
   createAccount,

swagger-fragment.json

@@ -562,7 +562,7 @@
           "Admin · Shard"
         ],
         "summary": "Cliloc table status: sources, drift, entry count (admin only)",
-        "description": "Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether the files on disk have drifted from them. The table is built from a SET of sources — the converted client table plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any source that was loaded before and is now gone; an import refuses that without `approve`. A shard with nothing configured is a supported state — item names simply render as ids.",
+        "description": "Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether they have drifted from what is loaded. `source` says which pipeline is in use: `bridge` (the shard reads its own client — the normal case once uo-link is configured) or `file` (a converted file on disk, deprecated, kept for installs with no shard link). On the bridge, `shard` carries the client file’s size, mtime, hash and the shard’s extractor version, and `shard.hashing: true` means a null hash is “not computed yet”, not “changed”. The table is always a SET: the base plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any overlay that was loaded before and is now gone; an import refuses that without `approve`. A shard with no source at all is a supported state — item names simply render as ids.",
         "responses": {
           "200": {
             "description": "Cliloc status",
@@ -603,8 +603,8 @@
         "tags": [
           "Admin · Shard"
         ],
-        "summary": "Re-import the cliloc table from its source files (admin only)",
+        "summary": "Re-import the cliloc table from its source (admin only)",
-        "description": "Applies a client patch, or a change to the shard’s own overlay files, without a restart. `force` reimports even when the source hashes match what is loaded. `approve` accepts a refresh in which a previously-loaded source has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. A missing path — or the common mistake of pointing at the client’s own COMPRESSED Cliloc.enu — answers 200 with status \"unavailable\" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.",
+        "description": "Applies a client patch, or a change to the shard’s own overlay files, without a restart. On the bridge this is the ONLY thing that imports — boot deliberately does not call the shard — so it is what an operator presses after patching their client. `force` reimports even when the sources are unchanged. `approve` accepts a refresh in which a previously-loaded overlay has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane the operator has switched off, a client with no cliloc file, or a malformed overlay all answer 200 with status \"unavailable\" and a reason naming what to fix.",
         "responses": {
           "200": {
             "description": "What happened",
@@ -655,8 +655,8 @@
         "tags": [
           "Admin · Shard"
         ],
-        "summary": "Set the cliloc source the site reads from (admin only)",
+        "summary": "Set the cliloc path the site reads overlays (and any file base) from (admin only)",
-        "description": "Accepts either the converted base file itself or a directory to search. Overlays are read from a `custom/` directory beside it either way — pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.",
+        "description": "On an install with uo-link configured this selects only where `custom/` overlays are read from — the base table comes from the shard. Without a shard link it is also where the converted base file is looked for, which is the deprecated pre-protocol-8 pipeline. Accepts either a file or a directory to search; overlays are read from a `custom/` directory beside it either way, so pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.",
         "responses": {
           "200": {
             "description": "Cliloc status after the change",
@@ -692,7 +692,7 @@
                 "properties": {
                   "path": {
                     "type": "string",
-                    "description": "Path to the converted cliloc file, or a directory containing one. Blank disables resolution."
+                    "description": "Directory holding the custom/ overlays (and, with no shard link, a converted base file). Blank clears it."
                   }
                 }
               }
@@ -7273,11 +7273,38 @@
           },
           "description": {
             "type": "string",
-            "example": "Admin view of cliloc state: where the converted file is, whether it is readable, how many entries are loaded, and whether the file has drifted from them. `configured: false` is a supported state — item names then render as ids."
+            "example": "Admin view of cliloc state: which source the base table comes from, whether it can be read, how many entries are loaded, and whether anything has drifted from them. Nothing configured at all is a supported state — item names then render as ids."
           },
           "properties": {
             "type": "object",
             "properties": {
+              "source": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "string"
+                  },
+                  "enum": {
+                    "type": "array",
+                    "example": [
+                      "bridge",
+                      "file"
+                    ],
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "`bridge`: the shard reads its own UO client (protocol 8, the normal case). `file`: a converted file on disk — the pre-protocol-8 pipeline, deprecated, kept for installs with no shard link."
+                  },
+                  "example": {
+                    "type": "string",
+                    "example": "bridge"
+                  }
+                }
+              },
               "configured": {
                 "type": "object",
                 "properties": {
@@ -7298,6 +7325,10 @@
                     "type": "string",
                     "example": "string"
                   },
+                  "description": {
+                    "type": "string",
+                    "example": "On the bridge: where `custom/` overlays are read from. On a file source: the base path too."
+                  },
                   "example": {
                     "type": "string",
                     "example": "/srv/uo-client"
@@ -7317,11 +7348,11 @@
                   },
                   "description": {
                     "type": "string",
-                    "example": "The file actually resolved, when the path is a directory."
+                    "example": "The base file in use — the shard’s own `cliloc.enu` on the bridge, the resolved local file otherwise."
                   },
                   "example": {
                     "type": "string",
-                    "example": "/srv/uo-client/clilocs.tsv"
+                    "example": "cliloc.enu"
                   }
                 }
               },
@@ -7351,7 +7382,7 @@
                   },
                   "description": {
                     "type": "string",
-                    "example": "Why the file cannot be used, when it cannot. Set (with code COMPRESSED) for a readable-but-unconverted client file."
+                    "example": "Why the base cannot be used, when it cannot: a shard that is down or has assets switched off, or (on a file source) a missing or still-compressed file."
                   },
                   "example": {}
                 }
@@ -7378,7 +7409,13 @@
                       "NOT_FOUND",
                       "NO_FILE",
                       "UNREADABLE",
-                      "COMPRESSED"
+                      "COMPRESSED",
+                      "DISABLED",
+                      "NO_SOURCE",
+                      "SHARD_DOWN",
+                      "PROTOCOL",
+                      "BUSY",
+                      "UNAVAILABLE"
                     ],
                     "items": {
                       "type": "string"
@@ -7386,6 +7423,118 @@
                   }
                 }
               },
+              "shard": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "object"
+                  },
+                  "nullable": {
+                    "type": "boolean",
+                    "example": true
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "Present on the bridge: the shard’s own cliloc file as it is right now. `hashing: true` with a null `sha256` means the hash has not been computed yet — “ask again”, not “changed”."
+                  },
+                  "properties": {
+                    "type": "object",
+                    "properties": {
+                      "size": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "example": "integer"
+                          },
+                          "example": {
+                            "type": "number",
+                            "example": 4989921
+                          }
+                        }
+                      },
+                      "mtime": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "example": "integer"
+                          },
+                          "description": {
+                            "type": "string",
+                            "example": "Unix milliseconds."
+                          },
+                          "example": {
+                            "type": "number",
+                            "example": 1757462400000
+                          }
+                        }
+                      },
+                      "sha256": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "example": "string"
+                          },
+                          "nullable": {
+                            "type": "boolean",
+                            "example": true
+                          }
+                        }
+                      },
+                      "extractorVersion": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "example": "integer"
+                          },
+                          "description": {
+                            "type": "string",
+                            "example": "The version of the shard’s extraction code. A bump makes everything derived from it drift."
+                          },
+                          "example": {
+                            "type": "number",
+                            "example": 1
+                          }
+                        }
+                      },
+                      "hashing": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "example": "boolean"
+                          },
+                          "example": {
+                            "type": "boolean",
+                            "example": false
+                          }
+                        }
+                      },
+                      "complete": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "example": "boolean"
+                          },
+                          "description": {
+                            "type": "string",
+                            "example": "Every client file has a hash."
+                          },
+                          "example": {
+                            "type": "boolean",
+                            "example": true
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              },
               "drift": {
                 "type": "object",
                 "properties": {
@@ -7447,7 +7596,6 @@
                   "example": {
                     "type": "array",
                     "example": [
-                      "clilocs.plain",
                       "custom/uomysticmoon.tsv"
                     ],
                     "items": {
@@ -7504,6 +7652,7 @@
                               "enum": {
                                 "type": "array",
                                 "example": [
+                                  "shard",
                                   "base",
                                   "custom"
                                 ],
@@ -7511,6 +7660,10 @@
                                   "type": "string"
                                 }
                               },
+                              "description": {
+                                "type": "string",
+                                "example": "`shard` is the table read over the bridge; `base` a converted file on disk."
+                              },
                               "example": {
                                 "type": "string",
                                 "example": "custom"
@@ -7693,6 +7846,37 @@
                   }
                 }
               },
+              "source": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "string"
+                  },
+                  "nullable": {
+                    "type": "boolean",
+                    "example": true
+                  },
+                  "enum": {
+                    "type": "array",
+                    "example": [
+                      "bridge",
+                      "file"
+                    ],
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "Which source this refresh read."
+                  },
+                  "example": {
+                    "type": "string",
+                    "example": "bridge"
+                  }
+                }
+              },
               "code": {
                 "type": "object",
                 "properties": {
@@ -7706,7 +7890,7 @@
                   },
                   "description": {
                     "type": "string",
-                    "example": "Machine-readable cause. `COMPRESSED` means the client's own Cliloc.enu was supplied instead of a converted one."
+                    "example": "Machine-readable cause. Bridge codes describe the shard (`DISABLED`: the operator switched the asset plane off; `NO_SOURCE`: its client has no cliloc file; `SHARD_DOWN`; `SOURCE_CHANGED`: the client was patched mid-import, so nothing was applied). File codes describe the path — `COMPRESSED` means the client's own Cliloc.enu was supplied instead of a converted one."
                   },
                   "enum": {
                     "type": "array",
@@ -7718,7 +7902,18 @@
                       "COMPRESSED",
                       "TRUNCATED",
                       "EMPTY",
-                      "NOT_BUFFER"
+                      "NOT_BUFFER",
+                      "DISABLED",
+                      "NO_SOURCE",
+                      "SHARD_DOWN",
+                      "PROTOCOL",
+                      "BUSY",
+                      "UNAVAILABLE",
+                      "SOURCE_CHANGED",
+                      "INCOMPLETE",
+                      "STUCK",
+                      "MALFORMED",
+                      "TOO_LARGE"
                     ],
                     "items": {
                       "type": "string"
@@ -7807,7 +8002,87 @@
                   },
                   "example": {
                     "type": "number",
-                    "example": 55994
+                    "example": 0
+                  }
+                }
+              },
+              "pages": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "integer"
+                  },
+                  "nullable": {
+                    "type": "boolean",
+                    "example": true
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "Bridge only: how many pages the table arrived in (a stock English table is about eleven)."
+                  },
+                  "example": {
+                    "type": "number",
+                    "example": 11
+                  }
+                }
+              },
+              "reported": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "integer"
+                  },
+                  "nullable": {
+                    "type": "boolean",
+                    "example": true
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "Bridge only: how many rows the shard said it holds."
+                  },
+                  "example": {
+                    "type": "number",
+                    "example": 67496
+                  }
+                }
+              },
+              "received": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "integer"
+                  },
+                  "nullable": {
+                    "type": "boolean",
+                    "example": true
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "Bridge only: how many arrived. Disagreeing with `reported` means the walk is wrong."
+                  },
+                  "example": {
+                    "type": "number",
+                    "example": 67496
+                  }
+                }
+              },
+              "overlayProblem": {
+                "type": "object",
+                "properties": {
+                  "type": {
+                    "type": "string",
+                    "example": "string"
+                  },
+                  "nullable": {
+                    "type": "boolean",
+                    "example": true
+                  },
+                  "description": {
+                    "type": "string",
+                    "example": "The base imported, but the overlay directory could not be read. Reported rather than fatal."
                   }
                 }
               },
@@ -7855,6 +8130,7 @@
                               "enum": {
                                 "type": "array",
                                 "example": [
+                                  "shard",
                                   "base",
                                   "custom"
                                 ],