feat(cliloc): import the table from the shard, not from a file someone converted (Phase 2)

The base cliloc table now comes over the bridge. `clilocBridge.js` walks
`GET /cliloc` page by page and the model merges the `custom/` overlays over it —
overlays stay on disk because ServUO has no server-side notion of a custom
cliloc, so there is nothing on the shard to ask for.

**The shard wins whenever uo-link is configured and enabled**, with no mode
setting: there is no version of "which source?" an operator benefits from
answering. A file on disk remains the source only where there is no shard link,
plus a one-off explicit `path` — deprecated, not removed, and unchanged.

**Boot no longer imports on the bridge.** The file path could hash 5 MB locally
and skip in 14 ms; a shard round trip in the boot sequence would be spent
answering "no" on every restart but the one after a client patch — and patching a
client is an operator action, so importing became one. Admin → Shard → Import.
Whatever table is loaded keeps serving until then.

Three checks in the walk, each for a way a shard can hand back a table that looks
complete:

  * only `cut: 'end'` finishes it — a short page can equally be a spent budget,
    and a truncated table renders some items named and some not, which is exactly
    what NO table looks like;
  * the cursor must advance, or the walk stops rather than spinning;
  * every page echoes the source's size and mtime, so a client patched mid-import
    is refused outright rather than stitched from two files.

**The base is exempt from the vanished-source rule**, which is an upgrade detail
rather than a preference: an install that used the file pipeline carries its base
file's label in the stored fingerprint, and on the bridge that label is *supposed*
to disappear. Counting it as vanished would demand an approval for a change the
upgrade itself made. Overlays keep the rule in full.

**The protocol pin moves 7 → 8** — the third declaration site, and the one
nothing enforces. Phase 1 moved the sidecar and the overlay together because the
installer refuses a mismatched bundle; this one has to be moved by hand, in the
phase that first calls a protocol-8 route. The schema block above it is the
record of what forgetting costs: two phases of every REST call answered 409.

Verified against a live shard, sidecar and site: 12 pages, 67,496 rows imported
in 1.68 s, the operator's three-row overlay overriding stock strings on top of
it, and the next import correctly `unchanged`.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016wDDVXWMDz82WqE1i969r4

server/test/clilocSourceSelection.test.js

@@ -0,0 +1,330 @@
+// Which cliloc source runs, and what the shard path does with the answer
+// (docs/link/v8.md §9, docs/website/CLILOCS.md).
+//
+// The model is the only place that decides between the two pipelines, so these
+// drive it with the shard, the database and the filesystem all stubbed. Nothing
+// here reaches the real sidecar or a real table.
+
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+
+const clilocs = require('../model/shardClilocs/shardClilocs.model')
+const db = require('../model/shardClilocs/shardClilocs.db')
+const bridge = require('../utils/clilocBridge')
+const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
+const { ctx } = require('./_setup')
+
+const saved = {
+  getMeta: db.getMeta,
+  replaceAll: db.replaceAll,
+  count: db.count,
+  fingerprint: bridge.fingerprint,
+  readCliloc: bridge.readCliloc,
+  getSafe: uoLinkConfig.getSafe,
+  settingsGet: ctx.settings.get,
+}
+
+function restore() {
+  db.getMeta = saved.getMeta
+  db.replaceAll = saved.replaceAll
+  db.count = saved.count
+  bridge.fingerprint = saved.fingerprint
+  bridge.readCliloc = saved.readCliloc
+  uoLinkConfig.getSafe = saved.getSafe
+  ctx.settings.get = saved.settingsGet
+}
+
+const FINGERPRINT = {
+  kind: 'bridge',
+  file: 'cliloc.enu',
+  size: 4989921,
+  mtime: 1757462400000,
+  sha256: 'abc',
+  extractorVersion: 1,
+  hashing: false,
+  complete: true,
+}
+
+/**
+ * A rig with the shard reachable (or not), the configured overlay path pointed
+ * at a temp directory, and every write captured rather than made.
+ */
+function rig({ linked = true, meta = null, clientPath = '', rows = [] } = {}) {
+  const applied = []
+
+  uoLinkConfig.getSafe = async () => ({ enabled: linked, baseUrl: linked ? 'http://127.0.0.1:8099' : null })
+  ctx.settings.get = async (key) => (key === clilocs.SETTING_KEY ? clientPath : null)
+
+  db.getMeta = async () => meta
+  db.count = async () => meta?.count ?? 0
+  db.replaceAll = async (entries, writtenMeta) => {
+    applied.push({ entries, meta: writtenMeta })
+    return { count: entries.length, blank: 0, duplicates: 0 }
+  }
+
+  bridge.fingerprint = async () => FINGERPRINT
+  bridge.readCliloc = async () => ({
+    entries: rows,
+    source: {
+      kind: 'bridge',
+      lang: 'enu',
+      file: 'cliloc.enu',
+      size: FINGERPRINT.size,
+      mtime: FINGERPRINT.mtime,
+      pages: 1,
+      reported: rows.length,
+      received: rows.length,
+    },
+  })
+
+  return applied
+}
+
+function tmpWithOverlay(contents) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cliloc-sel-'))
+  fs.mkdirSync(path.join(dir, 'custom'), { recursive: true })
+  if (contents !== undefined) fs.writeFileSync(path.join(dir, 'custom', 'shard.tsv'), contents)
+  return dir
+}
+
+// ── Which source runs ──────────────────────────────────────────────────────
+
+test('a configured shard is the base source, and the file path is not consulted', async (t) => {
+  const applied = rig({ rows: [{ number: 1, flag: 0, text: 'a' }] })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported')
+  assert.equal(result.source, 'bridge')
+  assert.equal(applied.length, 1)
+  assert.equal(applied[0].meta.source, 'bridge')
+})
+
+test('no shard link falls back to the file pipeline, unchanged', async (t) => {
+  rig({ linked: false })
+  t.after(restore)
+
+  // No path configured either, so the file path reports exactly what it always
+  // did — which is the assertion: the fallback is the OLD code, not a new one.
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'skipped')
+  assert.equal(result.reason, 'no cliloc path configured')
+})
+
+test('an explicit path is still an escape hatch, even with a shard linked', async (t) => {
+  let asked = false
+  rig({ linked: true })
+  bridge.fingerprint = async () => {
+    asked = true
+    return FINGERPRINT
+  }
+  t.after(restore)
+
+  const result = await clilocs.refresh({ path: path.join(os.tmpdir(), 'nope-does-not-exist') })
+
+  assert.equal(asked, false, 'the shard must not be asked when a file was named')
+  assert.equal(result.status, 'unavailable')
+})
+
+// Boot deliberately does not call the shard: it would put a sidecar round trip
+// in the startup sequence to answer a question whose answer is "no" except after
+// a client patch, which is an operator action.
+test('boot imports nothing over the bridge and leaves the loaded table serving', async (t) => {
+  let asked = false
+  rig({ linked: true })
+  bridge.fingerprint = async () => {
+    asked = true
+    return FINGERPRINT
+  }
+  t.after(restore)
+
+  const result = await clilocs.refreshOnBoot()
+
+  assert.equal(result.status, 'skipped')
+  assert.equal(result.source, 'bridge')
+  assert.equal(asked, false)
+})
+
+// ── The gate ───────────────────────────────────────────────────────────────
+
+test('an unchanged client file and no overlays is a no-op', async (t) => {
+  const applied = rig({
+    meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 },
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'unchanged')
+  assert.equal(result.count, 67496)
+  assert.equal(applied.length, 0)
+})
+
+test('a patched client re-imports', async (t) => {
+  const applied = rig({
+    meta: {
+      source: 'bridge',
+      base: { ...FINGERPRINT, sha256: 'older' },
+      hashes: {},
+      parserVersion: 1,
+      count: 10,
+    },
+    rows: [{ number: 1, flag: 0, text: 'a' }],
+  })
+  t.after(restore)
+
+  assert.equal((await clilocs.refresh()).status, 'imported')
+  assert.equal(applied.length, 1)
+})
+
+// The upgrade path. An install that used the converted-file pipeline carries its
+// base label in the stored fingerprint; on the bridge that label is SUPPOSED to
+// disappear. Counting it as a vanished source would make the first import after
+// the upgrade demand an approval for a change the upgrade itself made.
+test('the retired file base is not reported as a vanished source', async (t) => {
+  const applied = rig({
+    meta: {
+      source: 'file',
+      hashes: { 'clilocs.plain': 'aaa' },
+      parserVersion: 1,
+      count: 67496,
+    },
+    rows: [{ number: 1, flag: 0, text: 'a' }],
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported', result.reason)
+  assert.equal(applied.length, 1)
+})
+
+// An overlay is a different matter: it vanished, and an unmounted volume looks
+// exactly like a deliberate deletion from here.
+test('a vanished OVERLAY still stages for review', async (t) => {
+  const applied = rig({
+    meta: {
+      source: 'bridge',
+      base: FINGERPRINT,
+      hashes: { 'custom/shard.tsv': 'aaa' },
+      parserVersion: 1,
+      count: 5,
+    },
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'needsReview')
+  assert.deepEqual(result.missingSources, ['custom/shard.tsv'])
+  assert.equal(applied.length, 0)
+
+  const accepted = await clilocs.refresh({ approve: true })
+  assert.equal(accepted.status, 'imported')
+  assert.deepEqual(accepted.acceptedMissing, ['custom/shard.tsv'])
+})
+
+// ── The merge ──────────────────────────────────────────────────────────────
+
+test('an overlay overrides the shard table, and says so', async (t) => {
+  const dir = tmpWithOverlay('1023721\ta better staff\n900001\ta shard-only item\n')
+  const applied = rig({
+    clientPath: dir,
+    rows: [
+      { number: 1023721, flag: 0, text: 'quarter staff' },
+      { number: 3000001, flag: 0, text: 'Entering Britannia...' },
+    ],
+  })
+  t.after(() => {
+    restore()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported', result.reason)
+
+  const stored = new Map(applied[0].entries.map((e) => [e.number, e.text]))
+  assert.equal(stored.get(1023721), 'a better staff', 'the overlay must win')
+  assert.equal(stored.get(3000001), 'Entering Britannia...')
+  assert.equal(stored.get(900001), 'a shard-only item')
+
+  const overlay = result.sources.find((s) => s.kind === 'custom')
+  assert.equal(overlay.label, 'custom/shard.tsv')
+  assert.equal(overlay.added, 1)
+  assert.equal(overlay.overrode, 1)
+
+  // Only overlay hashes are stored now — the base is fingerprinted separately,
+  // and mixing them is what made the upgrade case above ambiguous.
+  assert.deepEqual(Object.keys(applied[0].meta.hashes), ['custom/shard.tsv'])
+  assert.equal(applied[0].meta.base.sha256, 'abc')
+})
+
+test('a malformed overlay names the file rather than failing the import namelessly', async (t) => {
+  const dir = tmpWithOverlay('not a cliloc file at all\n')
+  rig({ clientPath: dir, rows: [{ number: 1, flag: 0, text: 'a' }] })
+  t.after(() => {
+    restore()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'unavailable')
+  assert.match(result.reason, /custom\/shard\.tsv/)
+})
+
+// An overlay path an operator has mistyped must not stop a base table that
+// arrived perfectly well — but it must be visible, or the site silently serves a
+// table missing every shard-added name.
+test('an unreadable overlay path is reported beside a successful import', async (t) => {
+  const applied = rig({
+    clientPath: path.join(os.tmpdir(), 'cliloc-does-not-exist-at-all'),
+    rows: [{ number: 1, flag: 0, text: 'a' }],
+  })
+  t.after(restore)
+
+  const result = await clilocs.refresh()
+
+  assert.equal(result.status, 'imported')
+  assert.match(result.overlayProblem, /does not exist/)
+  assert.equal(applied.length, 1)
+})
+
+// ── Status ─────────────────────────────────────────────────────────────────
+
+test('status describes the shard source, hash state and drift', async (t) => {
+  rig({ meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 } })
+  t.after(restore)
+
+  const status = await clilocs.status()
+
+  assert.equal(status.source, 'bridge')
+  assert.equal(status.file, 'cliloc.enu')
+  assert.equal(status.fileReadable, true)
+  assert.equal(status.drift, false)
+  assert.equal(status.shard.extractorVersion, 1)
+  assert.equal(status.shard.hashing, false)
+})
+
+test('a shard that cannot be reached is a problem on the status, not a throw', async (t) => {
+  rig({})
+  bridge.fingerprint = async () => {
+    throw new bridge.ClilocBridgeError('The shard did not answer: timeout', 'SHARD_DOWN')
+  }
+  t.after(restore)
+
+  const status = await clilocs.status()
+
+  assert.equal(status.source, 'bridge')
+  assert.equal(status.fileReadable, false)
+  assert.equal(status.code, 'SHARD_DOWN')
+  // Null, not false: with no fingerprint there is nothing to compare, and
+  // reporting "no drift" would read as "up to date".
+  assert.equal(status.drift, null)
+})