From 893a36618b31002d3e46b34e38f0d6aed0f5dd29 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Thu, 10 Sep 2026 11:13:24 -0500 Subject: [PATCH] feat(cliloc): import the table from the shard, not from a file someone converted (Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base cliloc table now comes over the bridge. `clilocBridge.js` walks `GET /cliloc` page by page and the model merges the `custom/` overlays over it — overlays stay on disk because ServUO has no server-side notion of a custom cliloc, so there is nothing on the shard to ask for. **The shard wins whenever uo-link is configured and enabled**, with no mode setting: there is no version of "which source?" an operator benefits from answering. A file on disk remains the source only where there is no shard link, plus a one-off explicit `path` — deprecated, not removed, and unchanged. **Boot no longer imports on the bridge.** The file path could hash 5 MB locally and skip in 14 ms; a shard round trip in the boot sequence would be spent answering "no" on every restart but the one after a client patch — and patching a client is an operator action, so importing became one. Admin → Shard → Import. Whatever table is loaded keeps serving until then. Three checks in the walk, each for a way a shard can hand back a table that looks complete: * only `cut: 'end'` finishes it — a short page can equally be a spent budget, and a truncated table renders some items named and some not, which is exactly what NO table looks like; * the cursor must advance, or the walk stops rather than spinning; * every page echoes the source's size and mtime, so a client patched mid-import is refused outright rather than stitched from two files. **The base is exempt from the vanished-source rule**, which is an upgrade detail rather than a preference: an install that used the file pipeline carries its base file's label in the stored fingerprint, and on the bridge that label is *supposed* to disappear. Counting it as vanished would demand an approval for a change the upgrade itself made. Overlays keep the rule in full. **The protocol pin moves 7 → 8** — the third declaration site, and the one nothing enforces. Phase 1 moved the sidecar and the overlay together because the installer refuses a mismatched bundle; this one has to be moved by hand, in the phase that first calls a protocol-8 route. The schema block above it is the record of what forgetting costs: two phases of every REST call answered 409. Verified against a live shard, sidecar and site: 12 pages, 67,496 rows imported in 1.68 s, the operator's three-row overlay overriding stock strings on top of it, and the next import correctly `unchanged`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016wDDVXWMDz82WqE1i969r4 --- server/boot.js | 17 +- server/db/schema.sql | 20 +- .../model/shardClilocs/shardClilocs.model.js | 353 +++++++++++++++++- .../model/uoLinkConfig/uoLinkConfig.model.js | 9 +- server/router/admin/shard.router.js | 27 +- .../router/admin/shardClilocs.controller.js | 35 +- server/swagger/doc.js | 46 ++- server/test/clilocBridge.test.js | 303 +++++++++++++++ server/test/clilocSourceSelection.test.js | 330 ++++++++++++++++ server/utils/clilocBridge.js | 306 +++++++++++++++ server/utils/clilocSource.js | 96 +++++ server/utils/uoLinkClient.js | 33 ++ swagger-fragment.json | 306 ++++++++++++++- 13 files changed, 1817 insertions(+), 64 deletions(-) create mode 100644 server/test/clilocBridge.test.js create mode 100644 server/test/clilocSourceSelection.test.js create mode 100644 server/utils/clilocBridge.js diff --git a/server/boot.js b/server/boot.js index 58bd97d..468a633 100644 --- a/server/boot.js +++ b/server/boot.js @@ -80,11 +80,18 @@ async function onBoot() { // REMOVE a facet is staged for admin approval instead of being applied. await shardAtlas.refreshOnBoot() - // Refresh the cliloc table (UO's id → display-string map) from the file the - // operator converted out of their own client. Same contract as the atlas: - // hash-gated so an unchanged file costs one read, and best-effort so a missing - // or wrong-format file never stops the site coming up — it just means item - // names render as ids, which is what they did before the table existed. + // Refresh the cliloc table (UO's id → display-string map). + // + // **On an install with uo-link configured this imports nothing** — protocol 8 + // moved the base table to the shard, and asking for it would put a sidecar + // round trip in the boot sequence to answer a question whose answer is "no" + // except after a client patch. That is an operator action, so importing is an + // operator action: Admin → Shard (docs/link/v8.md §9). + // + // Without a shard link it is the old file pipeline, unchanged: hash-gated so an + // unchanged file costs one read, and best-effort so a missing or wrong-format + // file never stops the site coming up — it just means item names render as ids, + // which is what they did before the table existed. const clilocResult = await shardClilocs.refreshOnBoot() // A cliloc import changes what item names RESOLVE to, and the marketplace diff --git a/server/db/schema.sql b/server/db/schema.sql index 862de6f..1563b40 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS uo_link_config ( base_url VARCHAR(255) NULL, ws_url VARCHAR(255) NULL, auth_token_enc TEXT NULL, - protocol INT NOT NULL DEFAULT 7, + protocol INT NOT NULL DEFAULT 8, enabled TINYINT(1) NOT NULL DEFAULT 0, status VARCHAR(20) NOT NULL DEFAULT 'disconnected', status_detail VARCHAR(500) NULL, @@ -837,3 +837,21 @@ UPDATE uo_link_config SET protocol = 7 WHERE id = 1 AND protocol < 7 AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_7_migrated'); INSERT IGNORE INTO settings (`key`, value) VALUES ('uo_link_protocol_7_migrated', '1'); + +-- The protocol pin at 8 -- the Asset Bridge (docs/link/v8.md), and the first bump this +-- module takes IN the phase that consumes it rather than a phase or two later. +-- +-- Phase 1 of that work moved `link`'s PROTOCOL_VERSION and the overlay's `overlay.toml` +-- together, because the installer refuses to pair a sidecar and an overlay that disagree. +-- Nothing enforces the third declaration -- this one -- and the block above is the record +-- of what that costs: two phases of every REST call answered `409 protocol version +-- mismatch`, invisible because both live walks had set the column by hand. +-- +-- Phase 2 is where this module first calls a protocol-8 route (`GET /cliloc`), so it is +-- where the pin moves. Same one-shot shape and the same `protocol < 8`, so an install +-- that missed an earlier bump is carried the whole way rather than one step. +ALTER TABLE uo_link_config MODIFY COLUMN protocol INT NOT NULL DEFAULT 8; +UPDATE uo_link_config SET protocol = 8 + WHERE id = 1 AND protocol < 8 + AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_8_migrated'); +INSERT IGNORE INTO settings (`key`, value) VALUES ('uo_link_protocol_8_migrated', '1'); diff --git a/server/model/shardClilocs/shardClilocs.model.js b/server/model/shardClilocs/shardClilocs.model.js index b4fd3ea..f3c8b53 100644 --- a/server/model/shardClilocs/shardClilocs.model.js +++ b/server/model/shardClilocs/shardClilocs.model.js @@ -1,6 +1,6 @@ const db = require('./shardClilocs.db') const { settings } = require('../../core') -const { displayText } = require('../../utils/clilocParse') +const { displayText, parseCliloc } = require('../../utils/clilocParse') const { ClilocFormatError, ClilocSourceError, @@ -8,8 +8,12 @@ const { hashSources, sameSources, missingSources, + missingOverlays, + readOverlays, readCliloc, } = require('../../utils/clilocSource') +const bridge = require('../../utils/clilocBridge') +const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model') const log = require('../../core').logger('shardClilocs') // The cliloc table — UO's id → display-string map, refreshed from a file the @@ -29,11 +33,35 @@ const log = require('../../core').logger('shardClilocs') // 2. **Nothing client-derived is committed.** The table is built from the // operator's own file at a configured path. The repo ships no strings. // -// The table is built from a SET of sources — the converted client table plus -// every operator-maintained overlay beside it — because shards edit items and -// add new ones, and those carry cliloc ids no stock client table has. All of -// them are re-read on every boot and hash-gated together, so adding one custom -// item never means re-exporting a 5 MB client file. Later sources win. +// The table is built from a SET of sources — a base table plus every +// operator-maintained overlay beside it — because shards edit items and add new +// ones, and those carry cliloc ids no stock client table has. Later sources win, +// so an overlay both adds ids the client never had and overrides stock ones. +// +// ── Where the base comes from (protocol 8, docs/link/v8.md §9) ───────── +// +// **The shard**, on any install with uo-link configured. It has the operator's +// client files already — a ServUO server cannot boot without them — and since +// phase 2 it has the decompressor too, so `GET /cliloc` returns the table and +// nobody installs UOFiddler or copies a 5 MB file anywhere. +// +// **A file on disk** otherwise. That is the pipeline this replaces, kept for +// installs with no shard link and for development, and deprecated rather than +// removed: an operator who has one keeps working, and an operator who has a shard +// never builds one. Passing an explicit `path` to `refresh()` still selects it, +// which is the escape hatch for "import from this file, this once". +// +// **Overlays are always the filesystem's**, either way. There is nothing on the +// shard to ask for: ServUO has no server-side notion of a custom cliloc, so the +// `custom/` directory is the only place those ids exist. +// +// ── What that changed about WHEN this runs ─────────────────────── +// +// Boot no longer imports on the shard path. The file path could hash 5 MB locally +// on every restart and skip; the shard path would mean a sidecar round trip in the +// boot sequence, for a table that changes when an operator patches their client — +// an event they know about and we do not. So on the bridge, importing is an admin +// action (Admin → Shard), and boot leaves whatever is loaded serving. // // That set is also why this has the atlas's escalation, in a lighter form. A // single corrupt file fails the parse loudly, but a source that has simply @@ -96,8 +124,224 @@ const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION * * `force` skips the hash check (an admin asking for a reimport). `approve` * additionally accepts a vanished source. + * + * Which SOURCE it reads is decided here and nowhere else: the shard when uo-link + * is configured and enabled, a file otherwise, and always a file when the caller + * named one. */ async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) { + const override = String(pathOverride ?? '').trim() + + if (override === '' && (await shardLinked())) { + return refreshFromShard({ force, approve }) + } + + return refreshFromFile({ force, approve, path: override }) +} + +/** + * Is there a shard to ask? + * + * Both halves matter. `baseUrl` alone is an install that has been configured and + * then switched off, and calling it would spend a 12 s timeout to learn what the + * row already says. Never throws: an unreadable config means "no shard", and the + * file path is a working answer. + */ +async function shardLinked() { + try { + const config = await uoLinkConfig.getSafe() + return Boolean(config?.enabled && config?.baseUrl) + } catch { + return false + } +} + +/** + * Merge parsed sources in order, later winning. + * + * Shared by both paths, because the merge is the same question whichever end the + * base arrived from: what did each source contribute, and what did it override. + * The per-source breakdown is for the admin panel — an operator who adds an + * overlay wants to see it took effect, and `overrode: 0` on a file meant to + * re-label stock items says it did not. + */ +function mergeSources(groups) { + const merged = new Map() + const sources = [] + + for (const group of groups) { + let added = 0 + let overrode = 0 + + for (const entry of group.entries) { + if (!Number.isInteger(entry.number)) continue + if (merged.has(entry.number)) overrode++ + else added++ + merged.set(entry.number, entry) + } + + sources.push({ + label: group.label, + kind: group.kind, + entries: group.entries.length, + added, + overrode, + }) + } + + return { entries: [...merged.values()], sources } +} + +/** Overlay hashes as a `{ label: sha256 }` map, in merge order. */ +function overlayHashes(files) { + const hashes = {} + for (const file of files) hashes[file.label] = file.sha256 + return hashes +} + +/** The overlay half of a stored fingerprint — everything under `custom/`. */ +function onlyOverlays(hashes) { + if (!hashes) return null + const out = {} + for (const [label, sha] of Object.entries(hashes)) { + if (label.startsWith('custom/')) out[label] = sha + } + return out +} + +/** + * Import with the shard as the base source. + * + * The gate is two-part, and neither part is something the shard can answer for + * us: has the client file changed (size/mtime/sha256, plus the shard's own + * `EXTRACTOR_VERSION`), and has any overlay beside the configured path changed. + * Either is drift; neither is the normal case. + */ +async function refreshFromShard({ force = false, approve = false } = {}) { + let fingerprint + + try { + fingerprint = await bridge.fingerprint() + } catch (err) { + if (err instanceof bridge.ClilocBridgeError) { + return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code } + } + return { status: 'failed', source: 'bridge', reason: err.message } + } + + const configured = await getClientPath() + const overlays = readOverlays(configured) + const hashes = overlayHashes(overlays.files) + + const meta = await db.getMeta().catch(() => null) + + if ( + !force && + bridge.sameSource(fingerprint, meta?.base) && + sameSources(hashes, onlyOverlays(meta?.hashes)) && + currentParser(meta) + ) { + return { + status: 'unchanged', + source: 'bridge', + file: fingerprint.file, + count: meta.count ?? null, + customCount: overlays.files.length, + hashing: fingerprint.hashing, + } + } + + // An overlay that was loaded last time and is not there now is refused rather + // than applied — an unmounted volume and a deliberate deletion look identical + // from here, and the wrong guess silently drops every name that file gave. + // The BASE is deliberately not part of this question: an install upgraded from + // the file pipeline is *supposed* to stop having one. + const gone = missingOverlays(hashes, meta?.hashes) + + if (gone.length > 0 && !approve) { + return { + status: 'needsReview', + source: 'bridge', + reason: `${gone.length} previously-loaded cliloc overlay(s) are missing; the existing table is unchanged`, + missingSources: gone, + file: fingerprint.file, + } + } + + let base + + try { + base = await bridge.readCliloc({ lang: bridge.DEFAULT_LANGUAGE }) + } catch (err) { + if (err instanceof bridge.ClilocBridgeError) { + return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code } + } + return { status: 'failed', source: 'bridge', reason: err.message } + } + + const groups = [{ label: base.source.file, kind: 'shard', entries: base.entries }] + + for (const file of overlays.files) { + try { + groups.push({ label: file.label, kind: 'custom', entries: parseCliloc(file.buffer) }) + } catch (err) { + if (err instanceof ClilocFormatError) { + // Named, because "which of my six overlay files is malformed" is + // otherwise a guessing game. + return { + status: 'unavailable', + source: 'bridge', + reason: `${file.label}: ${err.message}`, + code: err.code, + } + } + return { status: 'failed', source: 'bridge', reason: err.message } + } + } + + const merged = mergeSources(groups) + + try { + const applied = await db.replaceAll(merged.entries, { + source: 'bridge', + base: fingerprint, + hashes, + parserVersion: PARSER_VERSION, + sources: merged.sources, + file: base.source.file, + bytes: fingerprint.size, + }) + + invalidate() + + return { + status: 'imported', + source: 'bridge', + file: base.source.file, + count: applied.count, + parsed: merged.entries.length, + blank: applied.blank, + pages: base.source.pages, + sources: merged.sources, + // The shard says how many rows it holds; this is how many arrived. They + // agree, or the walk is wrong in a way no count on its own would show. + reported: base.source.reported, + received: base.source.received, + overlayProblem: overlays.problem ?? undefined, + acceptedMissing: gone.length > 0 ? gone : undefined, + } + } catch (err) { + return { status: 'failed', source: 'bridge', reason: err.message } + } +} + +/** + * Import from a converted file on disk — the pre-protocol-8 pipeline, unchanged. + * + * Deprecated but supported: an install with no shard link has no other way to get + * a table, and development without a running ServUO is the same case. + */ +async function refreshFromFile({ force = false, approve = false, path: pathOverride = '' } = {}) { // An explicit override wins outright — a one-off "use this file", which must // not be silently overruled by the configured path the way an env default is. const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() @@ -153,7 +397,10 @@ async function refresh({ force = false, approve = false, path: pathOverride = '' } try { - const applied = await db.replaceAll(parsed.entries, parsed.source) + // `source: 'file'` is what lets the NEXT refresh — and `status()` — tell a + // table built from a converted file from one built over the bridge. Without + // it an install that gains a shard link looks like it already imported. + const applied = await db.replaceAll(parsed.entries, { ...parsed.source, source: 'file' }) invalidate() return { status: 'imported', @@ -177,9 +424,22 @@ async function refresh({ force = false, approve = false, path: pathOverride = '' /** * Boot hook. Best-effort by contract: it logs and returns, never throws, so a * missing or malformed cliloc file can never stop the site coming up. + * + * **On the bridge it imports nothing**, deliberately. The file path can hash a + * local 5 MB file on every restart and skip in 14 ms; asking the shard would put + * a sidecar round trip in the boot sequence to answer a question whose answer is + * "no" every time except after a client patch — which is an operator action, and + * therefore something an operator can press a button for. Whatever table is + * loaded keeps serving, which is exactly what happens today when a restart finds + * nothing changed. */ async function refreshOnBoot() { try { + if (await shardLinked()) { + log.info('cliloc table comes from the shard; import is admin-triggered (Admin → Shard)') + return { status: 'skipped', source: 'bridge', reason: 'the shard is the cliloc source' } + } + const result = await refresh() switch (result.status) { case 'imported': @@ -220,8 +480,18 @@ async function refreshOnBoot() { } } -/** Everything the admin panel needs to describe cliloc state. */ +/** + * Everything the admin panel needs to describe cliloc state. + * + * Two shapes, one per source, sharing every field a panel actually renders + * (`count`, `drift`, `problem`, `sources`, `missingSources`, `importedAt`). What + * differs is what `file` means and what a problem with it looks like: on the + * bridge it is the shard's own client file and the problems are transport ones, + * on disk it is a path an operator typed. + */ async function status({ path: pathOverride = '' } = {}) { + if (pathOverride.trim() === '' && (await shardLinked())) return shardStatus() + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() const meta = await db.getMeta().catch(() => null) const loaded = await db.count().catch(() => 0) @@ -280,6 +550,72 @@ async function status({ path: pathOverride = '' } = {}) { } } +/** + * Status when the shard is the source. + * + * The one thing worth knowing here that the file path has no equivalent of: + * `hashing`. The shard reports a null `sha256` for a client file it has not + * hashed yet — hashing the 343 MB of art and animation it also serves cannot fit + * in a 10 s reply, so it happens on its own thread — and a null hash means "ask + * again", never "changed". Drift falls back to (size, mtime) meanwhile, which is + * the same gate the shard itself applies, so an operator is never blocked from + * importing by a hash that has not landed. + */ +async function shardStatus() { + const configured = await getClientPath() + const meta = await db.getMeta().catch(() => null) + const loaded = await db.count().catch(() => 0) + + const overlays = readOverlays(configured) + const hashes = overlayHashes(overlays.files) + + let fingerprint = null + let problem = overlays.problem ?? null + let code = null + + try { + fingerprint = await bridge.fingerprint() + } catch (err) { + problem = err.message + code = err.code ?? null + } + + const drift = fingerprint + ? !bridge.sameSource(fingerprint, meta?.base) || + !sameSources(hashes, onlyOverlays(meta?.hashes)) || + !currentParser(meta) + : null + + return { + source: 'bridge', + configured: true, + // The overlay directory, which is all the path setting still selects on this + // source. Reported so a panel can say where `custom/` is being read from. + path: configured, + file: fingerprint?.file ?? bridge.SOURCE_FILE, + fileReadable: Boolean(fingerprint), + problem, + code, + drift, + count: loaded, + shard: fingerprint + ? { + size: fingerprint.size, + mtime: fingerprint.mtime, + sha256: fingerprint.sha256, + extractorVersion: fingerprint.extractorVersion, + hashing: fingerprint.hashing, + complete: fingerprint.complete, + } + : null, + sources: Object.keys(hashes), + loadedSources: meta?.sources ?? null, + missingSources: missingOverlays(hashes, meta?.hashes), + importedAt: meta?.importedAt ?? null, + sourceBytes: meta?.base?.size ?? meta?.bytes ?? null, + } +} + // ── Lookup ───────────────────────────────────────────────────────────────── // // Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is @@ -359,6 +695,7 @@ module.exports = { SETTING_KEY, getClientPath, setClientPath, + shardLinked, refresh, refreshOnBoot, status, diff --git a/server/model/uoLinkConfig/uoLinkConfig.model.js b/server/model/uoLinkConfig/uoLinkConfig.model.js index 39d82a3..ab6c3c4 100644 --- a/server/model/uoLinkConfig/uoLinkConfig.model.js +++ b/server/model/uoLinkConfig/uoLinkConfig.model.js @@ -11,8 +11,11 @@ const { secretBox } = require('../../core') // Only used before an admin has saved anything — the stored row wins once it exists, // and UOLINK_PROTOCOL still overrides for an operator running an older sidecar. // -// This says 7 because this build speaks protocol 7: the idempotency key and the -// participation ledger (6), and the world verbs plus the targeted lease planes (7). +// This says 8 because this build speaks protocol 8: the idempotency key and the +// participation ledger (6), the world verbs plus the targeted lease planes (7), and +// the Asset Bridge (8) -- of which this module is the first consumer, importing the +// cliloc table over `GET /cliloc` instead of reading a file an operator converted by +// hand (docs/link/v8.md §9). // // It said 4 before 5, and 3 for a while after protocol 4 shipped — which is the bug this // constant was introduced to fix. A FRESH install pinned 3, the sidecar answered @@ -32,7 +35,7 @@ const { secretBox } = require('../../core') // (`PROTOCOL_VERSION`) and the overlay's in `servuo-plugins/overlay.toml`; the thing that // actually pairs them is the installer's bundle check, at deploy time. So bumping this in // the same change as the emitters is still the discipline, and no test here replaces it. -const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 7 +const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 8 function toSafe(row) { if (!row) { diff --git a/server/router/admin/shard.router.js b/server/router/admin/shard.router.js index c0a19af..04707f8 100644 --- a/server/router/admin/shard.router.js +++ b/server/router/admin/shard.router.js @@ -307,10 +307,17 @@ shardRouter.put( ) // ── Cliloc table (admin only) ───────────────────────────────────────────── -// UO's id → display-string map, converted once by the operator from their own -// client (docs/website/CLILOCS.md). Sits beside the atlas for the same reason: -// it is static content derived from operator-supplied files rather than anything -// the sidecar sends, and operating it is shard administration. +// UO's id → display-string map, read from the shard's own UO client over the +// bridge (docs/link/v8.md §9, docs/website/CLILOCS.md). Sits beside the atlas for +// the same reason: it is static content derived from the operator's own files +// rather than anything the sidecar streams, and operating it is shard +// administration. +// +// Protocol 8 changed where the base table comes from, not what these routes are: +// the shard decompresses `Cliloc.enu` and serves it paged, so an operator no +// longer converts anything by hand. Import stays an explicit admin action, +// because the only thing that changes a client's table is an operator patching +// their client. // // There is deliberately NO public counterpart. The table is never served as a // table — 123k rows would dwarf any page that used it, and the Android client @@ -320,7 +327,7 @@ shardRouter.get( '/clilocs', // #swagger.tags = ['Admin · Shard'] // #swagger.summary = 'Cliloc table status: sources, drift, entry count (admin only)' - // #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether the files on disk have drifted from them. The table is built from a SET of sources — the converted client table plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any source that was loaded before and is now gone; an import refuses that without `approve`. A shard with nothing configured is a supported state — item names simply render as ids.' + // #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether they have drifted from what is loaded. `source` says which pipeline is in use: `bridge` (the shard reads its own client — the normal case once uo-link is configured) or `file` (a converted file on disk, deprecated, kept for installs with no shard link). On the bridge, `shard` carries the client file’s size, mtime, hash and the shard’s extractor version, and `shard.hashing: true` means a null hash is “not computed yet”, not “changed”. The table is always a SET: the base plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any overlay that was loaded before and is now gone; an import refuses that without `approve`. A shard with no source at all is a supported state — item names simply render as ids.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] /* #swagger.responses[200] = { description: 'Cliloc status', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocStatus" } } } } */ /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ @@ -330,8 +337,8 @@ shardRouter.get( shardRouter.post( '/clilocs/import', // #swagger.tags = ['Admin · Shard'] - // #swagger.summary = 'Re-import the cliloc table from its source files (admin only)' - // #swagger.description = 'Applies a client patch, or a change to the shard’s own overlay files, without a restart. `force` reimports even when the source hashes match what is loaded. `approve` accepts a refresh in which a previously-loaded source has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. A missing path — or the common mistake of pointing at the client’s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.' + // #swagger.summary = 'Re-import the cliloc table from its source (admin only)' + // #swagger.description = 'Applies a client patch, or a change to the shard’s own overlay files, without a restart. On the bridge this is the ONLY thing that imports — boot deliberately does not call the shard — so it is what an operator presses after patching their client. `force` reimports even when the sources are unchanged. `approve` accepts a refresh in which a previously-loaded overlay has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane the operator has switched off, a client with no cliloc file, or a malformed overlay all answer 200 with status "unavailable" and a reason naming what to fix.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the sources are unchanged." }, approve: { type: "boolean", description: "Accept a refresh in which a previously-loaded source has vanished." } } } } } } */ /* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocRefreshResult" } } } } */ @@ -344,10 +351,10 @@ shardRouter.post( shardRouter.put( '/clilocs/path', // #swagger.tags = ['Admin · Shard'] - // #swagger.summary = 'Set the cliloc source the site reads from (admin only)' - // #swagger.description = 'Accepts either the converted base file itself or a directory to search. Overlays are read from a `custom/` directory beside it either way — pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.' + // #swagger.summary = 'Set the cliloc path the site reads overlays (and any file base) from (admin only)' + // #swagger.description = 'On an install with uo-link configured this selects only where `custom/` overlays are read from — the base table comes from the shard. Without a shard link it is also where the converted base file is looked for, which is the deprecated pre-protocol-8 pipeline. Accepts either a file or a directory to search; overlays are read from a `custom/` directory beside it either way, so pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] - /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." } } } } } } */ + /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Directory holding the custom/ overlays (and, with no shard link, a converted base file). Blank clears it." } } } } } } */ /* #swagger.responses[200] = { description: 'Cliloc status after the change', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocStatus" } } } } */ adminOnly, body('path').isString().isLength({ max: 512 }), diff --git a/server/router/admin/shardClilocs.controller.js b/server/router/admin/shardClilocs.controller.js index c35882a..8a6e750 100644 --- a/server/router/admin/shardClilocs.controller.js +++ b/server/router/admin/shardClilocs.controller.js @@ -1,8 +1,8 @@ // ── Admin · Cliloc table ─────────────────────────────────────────────────── // -// Operating the cliloc import: where the converted cliloc file is, whether it -// has drifted from what is loaded, and a forced reimport after a client patch -// (docs/website/CLILOCS.md). +// Operating the cliloc import: which source the table comes from, whether it has +// drifted from what is loaded, and a reimport after a client patch +// (docs/link/v8.md §9, docs/website/CLILOCS.md). // // The policy lives in the model. This controller does three things and no more: // it validates input, it maps a refresh RESULT onto an HTTP status, and it @@ -10,11 +10,16 @@ // // **A refresh result is not an exception.** `shardClilocs.refresh()` reports // `unavailable` / `failed` rather than throwing, because the boot path must never -// be stopped by a bad file. That contract is preserved here: a missing file, or -// the single most likely operator mistake — pointing at the client's own -// COMPRESSED `Cliloc.enu` — is a 200 carrying `status: 'unavailable'` and the -// reason, not a 500. A 500 would say only "something broke"; the operator needs -// to be told which file to convert. +// be stopped by a bad source. That contract is preserved here, and protocol 8 +// widened the set of things it covers: a shard that is down, an asset plane the +// operator has switched off, a client with no cliloc file, a client patched +// halfway through the import — plus everything the file pipeline could already +// report. Each is a 200 carrying `status: 'unavailable'` and a reason naming what +// to fix, not a 500 that says only "something broke". +// +// **Import matters more than it used to.** On the bridge, boot deliberately does +// not call the shard, so this endpoint is the only thing that refreshes the +// table — the operator presses it after patching their client. const clilocs = require('../../model/shardClilocs/shardClilocs.model') const market = require('../../model/shardMarket/shardMarket.model') @@ -69,6 +74,10 @@ async function importClilocs(req, res) { force, approve, status: result.status, + // Which pipeline actually ran. Worth having in the audit log for the + // same reason it is in the status: an operator debugging a stale table + // needs to know whether the site asked the shard or read a file. + source: result.source ?? null, count: result.count ?? null, missingSources: result.missingSources ?? result.acceptedMissing ?? null, }, @@ -80,12 +89,16 @@ async function importClilocs(req, res) { } } -// PUT /admin/shard/clilocs/path — point the site at a different cliloc file. +// PUT /admin/shard/clilocs/path — point the site at a different cliloc path. +// +// On an install with uo-link configured this selects where `custom/` OVERLAYS are +// read from; the base table comes from the shard either way. Without a shard link +// it is also where the converted base file is looked for. // // Persisted as a setting, which wins over the UO_CLIENT_PATH env default so an // operator can move the mount without a redeploy. Blank clears it, which turns -// resolution off (boot skips, the loaded table keeps serving) — a legitimate -// thing to want, so it is allowed rather than validated away. +// overlay resolution off (the loaded table keeps serving) — a legitimate thing to +// want, so it is allowed rather than validated away. // // Deliberately does NOT import as a side effect, for the same reason the atlas // path does not: changing where the table reads from and reloading it are diff --git a/server/swagger/doc.js b/server/swagger/doc.js index 1afa6cf..063f2ad 100644 --- a/server/swagger/doc.js +++ b/server/swagger/doc.js @@ -490,21 +490,40 @@ module.exports = { UoClilocStatus: { type: 'object', description: - 'Admin view of cliloc state: where the converted file is, whether it is readable, how many entries are loaded, and whether the file has drifted from them. `configured: false` is a supported state — item names then render as ids.', + 'Admin view of cliloc state: which source the base table comes from, whether it can be read, how many entries are loaded, and whether anything has drifted from them. Nothing configured at all is a supported state — item names then render as ids.', properties: { + source: { + type: 'string', + enum: ['bridge', 'file'], + description: '`bridge`: the shard reads its own UO client (protocol 8, the normal case). `file`: a converted file on disk — the pre-protocol-8 pipeline, deprecated, kept for installs with no shard link.', + example: 'bridge', + }, configured: { type: 'boolean', example: true }, - path: { type: 'string', example: '/srv/uo-client' }, - file: { type: 'string', nullable: true, description: 'The file actually resolved, when the path is a directory.', example: '/srv/uo-client/clilocs.tsv' }, + path: { type: 'string', description: 'On the bridge: where `custom/` overlays are read from. On a file source: the base path too.', example: '/srv/uo-client' }, + file: { type: 'string', nullable: true, description: 'The base file in use — the shard’s own `cliloc.enu` on the bridge, the resolved local file otherwise.', example: 'cliloc.enu' }, fileReadable: { type: 'boolean', example: true }, - problem: { type: 'string', nullable: true, description: 'Why the file cannot be used, when it cannot. Set (with code COMPRESSED) for a readable-but-unconverted client file.', example: null }, - code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED'] }, + problem: { type: 'string', nullable: true, description: 'Why the base cannot be used, when it cannot: a shard that is down or has assets switched off, or (on a file source) a missing or still-compressed file.', example: null }, + code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'UNAVAILABLE'] }, + shard: { + type: 'object', + nullable: true, + description: 'Present on the bridge: the shard’s own cliloc file as it is right now. `hashing: true` with a null `sha256` means the hash has not been computed yet — “ask again”, not “changed”.', + properties: { + size: { type: 'integer', example: 4989921 }, + mtime: { type: 'integer', description: 'Unix milliseconds.', example: 1757462400000 }, + sha256: { type: 'string', nullable: true }, + extractorVersion: { type: 'integer', description: 'The version of the shard’s extraction code. A bump makes everything derived from it drift.', example: 1 }, + hashing: { type: 'boolean', example: false }, + complete: { type: 'boolean', description: 'Every client file has a hash.', example: true }, + }, + }, drift: { type: 'boolean', nullable: true, description: 'True when any source hash differs from the loaded table. NULL when the sources could not be read or are not usable.', example: false }, count: { type: 'integer', description: 'Entries currently loaded.', example: 67496 }, sources: { type: 'array', items: { type: 'string' }, description: 'Every source found now, root-relative, base first then overlays in merge order.', - example: ['clilocs.plain', 'custom/uomysticmoon.tsv'], + example: ['custom/uomysticmoon.tsv'], }, loadedSources: { type: 'array', @@ -514,7 +533,7 @@ module.exports = { type: 'object', properties: { label: { type: 'string', example: 'custom/uomysticmoon.tsv' }, - kind: { type: 'string', enum: ['base', 'custom'], example: 'custom' }, + kind: { type: 'string', enum: ['shard', 'base', 'custom'], description: '`shard` is the table read over the bridge; `base` a converted file on disk.', example: 'custom' }, entries: { type: 'integer', example: 37 }, added: { type: 'integer', description: 'Ids this source introduced.', example: 25 }, overrode: { type: 'integer', description: 'Ids it replaced from an earlier source.', example: 12 }, @@ -543,17 +562,22 @@ module.exports = { example: 'imported', }, reason: { type: 'string', nullable: true }, + source: { type: 'string', nullable: true, enum: ['bridge', 'file'], description: 'Which source this refresh read.', example: 'bridge' }, code: { type: 'string', nullable: true, - description: 'Machine-readable cause. `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.', - enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER'], + description: 'Machine-readable cause. Bridge codes describe the shard (`DISABLED`: the operator switched the asset plane off; `NO_SOURCE`: its client has no cliloc file; `SHARD_DOWN`; `SOURCE_CHANGED`: the client was patched mid-import, so nothing was applied). File codes describe the path — `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.', + enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER', 'DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'UNAVAILABLE', 'SOURCE_CHANGED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE'], }, path: { type: 'string', nullable: true }, file: { type: 'string', nullable: true }, count: { type: 'integer', nullable: true, description: 'Entries stored (blank strings are dropped).', example: 67496 }, parsed: { type: 'integer', nullable: true, description: 'Entries read across every source before blanks were dropped.', example: 123527 }, - blank: { type: 'integer', nullable: true, example: 55994 }, + blank: { type: 'integer', nullable: true, example: 0 }, + pages: { type: 'integer', nullable: true, description: 'Bridge only: how many pages the table arrived in (a stock English table is about eleven).', example: 11 }, + reported: { type: 'integer', nullable: true, description: 'Bridge only: how many rows the shard said it holds.', example: 67496 }, + received: { type: 'integer', nullable: true, description: 'Bridge only: how many arrived. Disagreeing with `reported` means the walk is wrong.', example: 67496 }, + overlayProblem: { type: 'string', nullable: true, description: 'The base imported, but the overlay directory could not be read. Reported rather than fatal.' }, sources: { type: 'array', nullable: true, @@ -562,7 +586,7 @@ module.exports = { type: 'object', properties: { label: { type: 'string' }, - kind: { type: 'string', enum: ['base', 'custom'] }, + kind: { type: 'string', enum: ['shard', 'base', 'custom'] }, entries: { type: 'integer' }, added: { type: 'integer' }, overrode: { type: 'integer' }, diff --git a/server/test/clilocBridge.test.js b/server/test/clilocBridge.test.js new file mode 100644 index 0000000..4a30586 --- /dev/null +++ b/server/test/clilocBridge.test.js @@ -0,0 +1,303 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') + +const uoLinkClient = require('../utils/uoLinkClient') +const bridge = require('../utils/clilocBridge') + +// The walk over `GET /cliloc`, driven against a stubbed sidecar client. +// +// Everything asserted here is a way the shard can be wrong that leaves the +// website holding a table it believes is complete. That is the failure worth +// testing, because it is invisible downstream: a truncated cliloc table renders +// some items with names and some with ids, which is exactly what NO table looks +// like. None of these are hypothetical shapes — each corresponds to a field the +// paging envelope carries specifically so this side can tell the difference +// (docs/link/v8.md §3.4). + +const saved = {} + +function stub({ sources, pages }) { + saved.getAssetSources = uoLinkClient.getAssetSources + saved.getClilocTable = uoLinkClient.getClilocTable + + const calls = [] + + uoLinkClient.getAssetSources = async () => sources + uoLinkClient.getClilocTable = async ({ lang, cursor } = {}) => { + calls.push({ lang, cursor: cursor ?? null }) + const next = pages.shift() + if (!next) throw new Error('the walk asked for more pages than the test supplied') + return next + } + + return calls +} + +function restore() { + if (saved.getAssetSources) uoLinkClient.getAssetSources = saved.getAssetSources + if (saved.getClilocTable) uoLinkClient.getClilocTable = saved.getClilocTable +} + +const ok = (data) => ({ ok: true, status: 200, data }) + +/** One page of rows, with the source fingerprint every page echoes. */ +const page = (rows, extra = {}) => + ok({ + kind: 'cliloc.table.ok', + lang: 'enu', + file: 'cliloc.enu', + size: 4989921, + mtime: 1757462400000, + total: 3, + rows, + more: false, + cut: 'end', + ...extra, + }) + +const sourcesReply = (file = {}) => + ok({ + kind: 'assets.sources.ok', + extractorVersion: 1, + imaging: { ok: true }, + hashing: false, + complete: true, + files: [ + { name: 'cliloc.enu', path: '/uo/cliloc.enu', size: 4989921, mtime: 1757462400000, sha256: 'abc', ...file }, + { name: 'art.mul', path: '/uo/art.mul', size: 148000000, mtime: 1, sha256: null }, + ], + }) + +// ── Stage 1: the fingerprint ─────────────────────────────────────────────── + +test('fingerprint picks the cliloc file out of the client manifest', async (t) => { + stub({ sources: sourcesReply(), pages: [] }) + t.after(restore) + + const fp = await bridge.fingerprint() + + assert.equal(fp.file, 'cliloc.enu') + assert.equal(fp.size, 4989921) + assert.equal(fp.sha256, 'abc') + assert.equal(fp.extractorVersion, 1) +}) + +test('a client with no cliloc file is NO_SOURCE, not a crash', async (t) => { + stub({ + sources: ok({ extractorVersion: 1, files: [{ name: 'art.mul', size: 1, mtime: 1 }] }), + pages: [], + }) + t.after(restore) + + await assert.rejects(bridge.fingerprint(), (err) => { + assert.equal(err.code, 'NO_SOURCE') + return true + }) +}) + +test('the asset plane being switched off reads as a refusal, not a bug', async (t) => { + stub({ + sources: { ok: false, status: 403, data: { reason: 'asset extraction is disabled on this shard' } }, + pages: [], + }) + t.after(restore) + + await assert.rejects(bridge.fingerprint(), (err) => { + assert.equal(err.code, 'DISABLED') + return true + }) +}) + +// A hash that has not been computed yet is the shard's ordinary first answer: +// hashing the 343 MB of art and animation it also serves cannot fit in a 10 s +// reply, so it happens off the request path. Treating a null hash as a CHANGE +// would make the panel show drift forever on a shard nobody has imported from. +test('a missing hash falls back to (size, mtime) rather than reading as drift', () => { + const before = { size: 10, mtime: 20, sha256: null, extractorVersion: 1 } + const after = { size: 10, mtime: 20, sha256: null, extractorVersion: 1 } + + assert.equal(bridge.sameSource(before, after), true) + assert.equal(bridge.sameSource(before, { ...after, mtime: 21 }), false) +}) + +test('a hash on both sides beats size and mtime, which a patched-in-place file can preserve', () => { + const a = { size: 10, mtime: 20, sha256: 'aaa', extractorVersion: 1 } + + assert.equal(bridge.sameSource(a, { ...a, sha256: 'bbb' }), false) + assert.equal(bridge.sameSource(a, { ...a, size: 11, mtime: 99 }), true) +}) + +test('the extractor version is part of the fingerprint, so a corrected reader drifts', () => { + const a = { size: 10, mtime: 20, sha256: 'aaa', extractorVersion: 1 } + + assert.equal(bridge.sameSource(a, { ...a, extractorVersion: 2 }), false) +}) + +// ── Stage 2: the walk ────────────────────────────────────────────────────── + +test('a one-page table comes back whole', async (t) => { + const calls = stub({ + sources: sourcesReply(), + pages: [page([{ n: 3, f: 0, t: 'c' }, { n: 1, f: 2, t: 'a' }])], + }) + t.after(restore) + + const { entries, source } = await bridge.readCliloc() + + assert.deepEqual(entries, [ + { number: 3, flag: 0, text: 'c' }, + { number: 1, flag: 2, text: 'a' }, + ]) + assert.equal(source.pages, 1) + assert.equal(source.received, 2) + assert.equal(source.reported, 3) + assert.deepEqual(calls, [{ lang: 'enu', cursor: null }]) +}) + +test('pages are walked by echoing the cursor back until more is false', async (t) => { + const calls = stub({ + sources: sourcesReply(), + pages: [ + page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }), + page([{ n: 2, f: 0, t: 'b' }], { more: true, cursor: 'n:2', cut: 'budget' }), + page([{ n: 3, f: 0, t: 'c' }]), + ], + }) + t.after(restore) + + const { entries, source } = await bridge.readCliloc() + + assert.equal(entries.length, 3) + assert.equal(source.pages, 3) + assert.deepEqual( + calls.map((c) => c.cursor), + [null, 'n:1', 'n:2'], + ) +}) + +// `cut` is the field that is easy to omit and expensive not to have. A short +// page means the source ended, the byte budget was spent, or the family hit its +// own limit — and only the first means finished. +test('a last page that did not end the table is refused, not imported', async (t) => { + stub({ + sources: sourcesReply(), + pages: [page([{ n: 1, f: 0, t: 'a' }], { more: false, cut: 'limit' })], + }) + t.after(restore) + + await assert.rejects(bridge.readCliloc(), (err) => { + assert.equal(err.code, 'INCOMPLETE') + return true + }) +}) + +test('a shard that does not advance its cursor is stopped rather than spun on', async (t) => { + stub({ + sources: sourcesReply(), + pages: [ + page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }), + page([{ n: 2, f: 0, t: 'b' }], { more: true, cursor: 'n:1', cut: 'budget' }), + ], + }) + t.after(restore) + + await assert.rejects(bridge.readCliloc(), (err) => { + assert.equal(err.code, 'STUCK') + return true + }) +}) + +test('more:true with no cursor at all is the same refusal', async (t) => { + stub({ + sources: sourcesReply(), + pages: [page([{ n: 1, f: 0, t: 'a' }], { more: true, cut: 'budget' })], + }) + t.after(restore) + + await assert.rejects(bridge.readCliloc(), (err) => { + assert.equal(err.code, 'STUCK') + return true + }) +}) + +// The one failure a count cannot catch: an operator patches their client while +// the import is walking it. Half of what arrived is from a file that no longer +// exists, and nothing later can tell which half. +test('a client patched mid-walk aborts the whole import', async (t) => { + stub({ + sources: sourcesReply(), + pages: [ + page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }), + page([{ n: 2, f: 0, t: 'b' }], { size: 5000000, mtime: 1757470000000 }), + ], + }) + t.after(restore) + + await assert.rejects(bridge.readCliloc(), (err) => { + assert.equal(err.code, 'SOURCE_CHANGED') + return true + }) +}) + +// 425 is flow control and the ORDINARY answer during an import — the shard's +// asset plane serves one request at a time on purpose — so it is retried rather +// than failed. (The backoff is real time, so this exercises one retry only.) +test('a busy shard is retried, because the work is happening', async (t) => { + saved.getAssetSources = uoLinkClient.getAssetSources + saved.getClilocTable = uoLinkClient.getClilocTable + t.after(restore) + + let attempts = 0 + uoLinkClient.getAssetSources = async () => sourcesReply() + uoLinkClient.getClilocTable = async () => { + attempts++ + if (attempts === 1) return { ok: false, status: 425, data: { kind: 'bridge.busy' } } + return page([{ n: 1, f: 0, t: 'a' }]) + } + + const { entries } = await bridge.readCliloc() + + assert.equal(attempts, 2) + assert.equal(entries.length, 1) +}) + +test('a page with no rows array is malformed, not an empty table', async (t) => { + stub({ sources: sourcesReply(), pages: [ok({ kind: 'cliloc.table.ok', more: false, cut: 'end' })] }) + t.after(restore) + + await assert.rejects(bridge.readCliloc(), (err) => { + assert.equal(err.code, 'MALFORMED') + return true + }) +}) + +test('a shard that never ends the table is bounded by the page cap', async (t) => { + saved.getAssetSources = uoLinkClient.getAssetSources + saved.getClilocTable = uoLinkClient.getClilocTable + t.after(restore) + + let n = 0 + uoLinkClient.getAssetSources = async () => sourcesReply() + uoLinkClient.getClilocTable = async () => { + n++ + return page([{ n, f: 0, t: 'x' }], { more: true, cursor: `n:${n}`, cut: 'budget' }) + } + + await assert.rejects(bridge.readCliloc(), (err) => { + assert.equal(err.code, 'TOO_LARGE') + return true + }) + assert.equal(n, bridge.MAX_PAGES) +}) + +test('rows with an unusable id are dropped rather than stored as NaN', async (t) => { + stub({ + sources: sourcesReply(), + pages: [page([{ n: 'nonsense', f: 0, t: 'a' }, { n: 7, f: 0, t: 'b' }])], + }) + t.after(restore) + + const { entries } = await bridge.readCliloc() + + assert.deepEqual(entries, [{ number: 7, flag: 0, text: 'b' }]) +}) diff --git a/server/test/clilocSourceSelection.test.js b/server/test/clilocSourceSelection.test.js new file mode 100644 index 0000000..9b30f66 --- /dev/null +++ b/server/test/clilocSourceSelection.test.js @@ -0,0 +1,330 @@ +// Which cliloc source runs, and what the shard path does with the answer +// (docs/link/v8.md §9, docs/website/CLILOCS.md). +// +// The model is the only place that decides between the two pipelines, so these +// drive it with the shard, the database and the filesystem all stubbed. Nothing +// here reaches the real sidecar or a real table. + +const { test } = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const clilocs = require('../model/shardClilocs/shardClilocs.model') +const db = require('../model/shardClilocs/shardClilocs.db') +const bridge = require('../utils/clilocBridge') +const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model') +const { ctx } = require('./_setup') + +const saved = { + getMeta: db.getMeta, + replaceAll: db.replaceAll, + count: db.count, + fingerprint: bridge.fingerprint, + readCliloc: bridge.readCliloc, + getSafe: uoLinkConfig.getSafe, + settingsGet: ctx.settings.get, +} + +function restore() { + db.getMeta = saved.getMeta + db.replaceAll = saved.replaceAll + db.count = saved.count + bridge.fingerprint = saved.fingerprint + bridge.readCliloc = saved.readCliloc + uoLinkConfig.getSafe = saved.getSafe + ctx.settings.get = saved.settingsGet +} + +const FINGERPRINT = { + kind: 'bridge', + file: 'cliloc.enu', + size: 4989921, + mtime: 1757462400000, + sha256: 'abc', + extractorVersion: 1, + hashing: false, + complete: true, +} + +/** + * A rig with the shard reachable (or not), the configured overlay path pointed + * at a temp directory, and every write captured rather than made. + */ +function rig({ linked = true, meta = null, clientPath = '', rows = [] } = {}) { + const applied = [] + + uoLinkConfig.getSafe = async () => ({ enabled: linked, baseUrl: linked ? 'http://127.0.0.1:8099' : null }) + ctx.settings.get = async (key) => (key === clilocs.SETTING_KEY ? clientPath : null) + + db.getMeta = async () => meta + db.count = async () => meta?.count ?? 0 + db.replaceAll = async (entries, writtenMeta) => { + applied.push({ entries, meta: writtenMeta }) + return { count: entries.length, blank: 0, duplicates: 0 } + } + + bridge.fingerprint = async () => FINGERPRINT + bridge.readCliloc = async () => ({ + entries: rows, + source: { + kind: 'bridge', + lang: 'enu', + file: 'cliloc.enu', + size: FINGERPRINT.size, + mtime: FINGERPRINT.mtime, + pages: 1, + reported: rows.length, + received: rows.length, + }, + }) + + return applied +} + +function tmpWithOverlay(contents) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cliloc-sel-')) + fs.mkdirSync(path.join(dir, 'custom'), { recursive: true }) + if (contents !== undefined) fs.writeFileSync(path.join(dir, 'custom', 'shard.tsv'), contents) + return dir +} + +// ── Which source runs ────────────────────────────────────────────────────── + +test('a configured shard is the base source, and the file path is not consulted', async (t) => { + const applied = rig({ rows: [{ number: 1, flag: 0, text: 'a' }] }) + t.after(restore) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'imported') + assert.equal(result.source, 'bridge') + assert.equal(applied.length, 1) + assert.equal(applied[0].meta.source, 'bridge') +}) + +test('no shard link falls back to the file pipeline, unchanged', async (t) => { + rig({ linked: false }) + t.after(restore) + + // No path configured either, so the file path reports exactly what it always + // did — which is the assertion: the fallback is the OLD code, not a new one. + const result = await clilocs.refresh() + + assert.equal(result.status, 'skipped') + assert.equal(result.reason, 'no cliloc path configured') +}) + +test('an explicit path is still an escape hatch, even with a shard linked', async (t) => { + let asked = false + rig({ linked: true }) + bridge.fingerprint = async () => { + asked = true + return FINGERPRINT + } + t.after(restore) + + const result = await clilocs.refresh({ path: path.join(os.tmpdir(), 'nope-does-not-exist') }) + + assert.equal(asked, false, 'the shard must not be asked when a file was named') + assert.equal(result.status, 'unavailable') +}) + +// Boot deliberately does not call the shard: it would put a sidecar round trip +// in the startup sequence to answer a question whose answer is "no" except after +// a client patch, which is an operator action. +test('boot imports nothing over the bridge and leaves the loaded table serving', async (t) => { + let asked = false + rig({ linked: true }) + bridge.fingerprint = async () => { + asked = true + return FINGERPRINT + } + t.after(restore) + + const result = await clilocs.refreshOnBoot() + + assert.equal(result.status, 'skipped') + assert.equal(result.source, 'bridge') + assert.equal(asked, false) +}) + +// ── The gate ─────────────────────────────────────────────────────────────── + +test('an unchanged client file and no overlays is a no-op', async (t) => { + const applied = rig({ + meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 }, + }) + t.after(restore) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'unchanged') + assert.equal(result.count, 67496) + assert.equal(applied.length, 0) +}) + +test('a patched client re-imports', async (t) => { + const applied = rig({ + meta: { + source: 'bridge', + base: { ...FINGERPRINT, sha256: 'older' }, + hashes: {}, + parserVersion: 1, + count: 10, + }, + rows: [{ number: 1, flag: 0, text: 'a' }], + }) + t.after(restore) + + assert.equal((await clilocs.refresh()).status, 'imported') + assert.equal(applied.length, 1) +}) + +// The upgrade path. An install that used the converted-file pipeline carries its +// base label in the stored fingerprint; on the bridge that label is SUPPOSED to +// disappear. Counting it as a vanished source would make the first import after +// the upgrade demand an approval for a change the upgrade itself made. +test('the retired file base is not reported as a vanished source', async (t) => { + const applied = rig({ + meta: { + source: 'file', + hashes: { 'clilocs.plain': 'aaa' }, + parserVersion: 1, + count: 67496, + }, + rows: [{ number: 1, flag: 0, text: 'a' }], + }) + t.after(restore) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'imported', result.reason) + assert.equal(applied.length, 1) +}) + +// An overlay is a different matter: it vanished, and an unmounted volume looks +// exactly like a deliberate deletion from here. +test('a vanished OVERLAY still stages for review', async (t) => { + const applied = rig({ + meta: { + source: 'bridge', + base: FINGERPRINT, + hashes: { 'custom/shard.tsv': 'aaa' }, + parserVersion: 1, + count: 5, + }, + }) + t.after(restore) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'needsReview') + assert.deepEqual(result.missingSources, ['custom/shard.tsv']) + assert.equal(applied.length, 0) + + const accepted = await clilocs.refresh({ approve: true }) + assert.equal(accepted.status, 'imported') + assert.deepEqual(accepted.acceptedMissing, ['custom/shard.tsv']) +}) + +// ── The merge ────────────────────────────────────────────────────────────── + +test('an overlay overrides the shard table, and says so', async (t) => { + const dir = tmpWithOverlay('1023721\ta better staff\n900001\ta shard-only item\n') + const applied = rig({ + clientPath: dir, + rows: [ + { number: 1023721, flag: 0, text: 'quarter staff' }, + { number: 3000001, flag: 0, text: 'Entering Britannia...' }, + ], + }) + t.after(() => { + restore() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'imported', result.reason) + + const stored = new Map(applied[0].entries.map((e) => [e.number, e.text])) + assert.equal(stored.get(1023721), 'a better staff', 'the overlay must win') + assert.equal(stored.get(3000001), 'Entering Britannia...') + assert.equal(stored.get(900001), 'a shard-only item') + + const overlay = result.sources.find((s) => s.kind === 'custom') + assert.equal(overlay.label, 'custom/shard.tsv') + assert.equal(overlay.added, 1) + assert.equal(overlay.overrode, 1) + + // Only overlay hashes are stored now — the base is fingerprinted separately, + // and mixing them is what made the upgrade case above ambiguous. + assert.deepEqual(Object.keys(applied[0].meta.hashes), ['custom/shard.tsv']) + assert.equal(applied[0].meta.base.sha256, 'abc') +}) + +test('a malformed overlay names the file rather than failing the import namelessly', async (t) => { + const dir = tmpWithOverlay('not a cliloc file at all\n') + rig({ clientPath: dir, rows: [{ number: 1, flag: 0, text: 'a' }] }) + t.after(() => { + restore() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'unavailable') + assert.match(result.reason, /custom\/shard\.tsv/) +}) + +// An overlay path an operator has mistyped must not stop a base table that +// arrived perfectly well — but it must be visible, or the site silently serves a +// table missing every shard-added name. +test('an unreadable overlay path is reported beside a successful import', async (t) => { + const applied = rig({ + clientPath: path.join(os.tmpdir(), 'cliloc-does-not-exist-at-all'), + rows: [{ number: 1, flag: 0, text: 'a' }], + }) + t.after(restore) + + const result = await clilocs.refresh() + + assert.equal(result.status, 'imported') + assert.match(result.overlayProblem, /does not exist/) + assert.equal(applied.length, 1) +}) + +// ── Status ───────────────────────────────────────────────────────────────── + +test('status describes the shard source, hash state and drift', async (t) => { + rig({ meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 } }) + t.after(restore) + + const status = await clilocs.status() + + assert.equal(status.source, 'bridge') + assert.equal(status.file, 'cliloc.enu') + assert.equal(status.fileReadable, true) + assert.equal(status.drift, false) + assert.equal(status.shard.extractorVersion, 1) + assert.equal(status.shard.hashing, false) +}) + +test('a shard that cannot be reached is a problem on the status, not a throw', async (t) => { + rig({}) + bridge.fingerprint = async () => { + throw new bridge.ClilocBridgeError('The shard did not answer: timeout', 'SHARD_DOWN') + } + t.after(restore) + + const status = await clilocs.status() + + assert.equal(status.source, 'bridge') + assert.equal(status.fileReadable, false) + assert.equal(status.code, 'SHARD_DOWN') + // Null, not false: with no fingerprint there is nothing to compare, and + // reporting "no drift" would read as "up to date". + assert.equal(status.drift, null) +}) diff --git a/server/utils/clilocBridge.js b/server/utils/clilocBridge.js new file mode 100644 index 0000000..1dc8726 --- /dev/null +++ b/server/utils/clilocBridge.js @@ -0,0 +1,306 @@ +// Cliloc table — the SHARD source (docs/link/v8.md §9, protocol 8 phase 2). +// +// `clilocSource.js` is the filesystem half of this story and predates it. This is +// the half that replaces the part of it nobody enjoyed: until protocol 8 the base +// table reached the site because an operator installed UOFiddler, built a +// converter against its `Ultima.dll`, ran it over their client's compressed +// `Cliloc.enu` and copied a five-megabyte file to the web host — every time they +// patched their client. +// +// The shard has always had those files (a ServUO server cannot boot without a UO +// client) and, as of phase 2, has the decompressor too. So the base table now +// arrives over the same request/reply path as every other shard read, and the +// operator installs nothing. +// +// **What is NOT here.** Overlays. Shard-added items carry cliloc ids no client +// table has, ServUO has no server-side notion of a custom cliloc, and there is +// therefore nothing on the shard to ask for. `custom/` stays a directory the site +// reads (`clilocSource.readOverlays`), and the model merges it OVER whatever +// arrives here. That division is the whole of CLILOCS.md §Shard-added items and +// it is unchanged by this file. +// +// ── Why this walks pages instead of asking for a table ──────────────────── +// +// The sidecar's reply timeout is 10 s and its inbound line cap is 1 MiB, so a +// five-megabyte table cannot be one answer. The shard cuts pages at a 512 KiB +// byte budget and hands back a cursor; this walks them. A stock English table is +// about eleven pages. +// +// Three properties of that envelope are load-bearing and each has a check below: +// +// - **Only `cut: 'end'` means finished.** A short page can equally mean the +// budget was spent (`budget`) or the family stopped at its own limit +// (`limit`). Treating a short page as the end would import a truncated table, +// which is indistinguishable downstream from a complete one — some items +// named, some not, exactly what "no table at all" looks like. +// - **The cursor must advance.** A shard that answered the same cursor forever +// would spin this loop until the request timeout with nothing to show. +// - **The file must not change underneath the walk.** Every page echoes the +// source's size and mtime; an operator patching their client mid-import would +// otherwise produce one table stitched from two, with no error anywhere. + +// Required as a namespace, not destructured: a test that stubs the sidecar +// replaces these on the module object, and a destructured copy taken at load +// time would keep calling the real one. +const uoLinkClient = require('./uoLinkClient') +const log = require('../core').logger('cliloc-bridge') + +/** The client file the base table comes from, as `assets.sources` names it. */ +const SOURCE_FILE = 'cliloc.enu' + +const DEFAULT_LANGUAGE = 'enu' + +// Bounds on the walk. Neither is expected to be reached — a stock table is ~11 +// pages and ~67k rows — and both exist so that a shard answering nonsense costs a +// bounded amount of time rather than an unbounded amount of memory. +const MAX_PAGES = 200 +const MAX_ROWS = 500000 + +// 425 is the ordinary answer during an import, not an error: the shard's asset +// plane serves one request at a time on purpose, because its outbound queue is +// bounded in lines rather than bytes. So a page that comes back busy is retried +// with a short backoff rather than failing the import. +const BUSY_RETRIES = 5 +const BUSY_BACKOFF_MS = [200, 400, 800, 1600, 3200] + +class ClilocBridgeError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocBridgeError' + this.code = code + } +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Map a sidecar response onto one of this module's codes. + * + * The statuses are the ones `respond_assets` produces, and the distinction that + * matters most to an operator is 403 vs 404: "you have not switched this on" and + * "your client does not have that file" are different jobs, and both are things + * they can fix. + */ +function describeFailure(res, what) { + const reason = res?.data?.reason || res?.error || `sidecar responded ${res?.status}` + + switch (res?.status) { + case 403: + return new ClilocBridgeError( + `The shard is refusing to serve client assets (Bridge.AssetsEnabled is off): ${reason}`, + 'DISABLED', + ) + case 404: + return new ClilocBridgeError(`The shard has no ${what}: ${reason}`, 'NO_SOURCE') + case 409: + return new ClilocBridgeError( + `The sidecar refused the protocol version this build declares: ${reason}`, + 'PROTOCOL', + ) + case 422: + return new ClilocBridgeError(`The shard could not read its own ${what}: ${reason}`, 'UNREADABLE') + case 425: + return new ClilocBridgeError( + 'The shard is busy serving another asset request and stayed busy', + 'BUSY', + ) + case 503: + case 504: + return new ClilocBridgeError(`The shard did not answer: ${reason}`, 'SHARD_DOWN') + default: + return new ClilocBridgeError(reason, 'UNAVAILABLE') + } +} + +/** + * Stage 1: the fingerprint of the shard's own cliloc file. + * + * Returns `{ file, size, mtime, sha256, extractorVersion, hashing, complete }`. + * + * `sha256` may be **null** — the shard reports hashes only once it has computed + * them off the request path, because hashing the client files it also serves + * (343 MB of art and animation) cannot fit inside a 10 s reply. A null hash means + * "not yet", never "changed", and `sameSource` below compares (size, mtime) in + * that case, which is the same gate the shard itself uses. + */ +async function fingerprint() { + const res = await uoLinkClient.getAssetSources() + if (!res.ok) throw describeFailure(res, 'client file manifest') + + const files = Array.isArray(res.data?.files) ? res.data.files : [] + const entry = files.find((f) => String(f?.name || '').toLowerCase() === SOURCE_FILE) + + if (!entry) { + throw new ClilocBridgeError( + `The shard's UO client has no ${SOURCE_FILE} (it reported ${files.length} client file(s))`, + 'NO_SOURCE', + ) + } + + return { + kind: 'bridge', + file: entry.name, + path: entry.path ?? null, + size: Number(entry.size) || 0, + mtime: Number(entry.mtime) || 0, + sha256: entry.sha256 ?? null, + extractorVersion: Number(res.data?.extractorVersion) || 0, + hashing: Boolean(res.data?.hashing), + complete: Boolean(res.data?.complete), + } +} + +/** + * True when two fingerprints describe the same client file. + * + * Hash first when both sides have one, because a hash is the only thing that + * catches a file rewritten with the same length and timestamp. Falls back to + * (size, mtime) when either side's hash is missing, which is the case on the + * first poll after a shard restart and the reason `hashing` exists at all. + */ +function sameSource(a, b) { + if (!a || !b) return false + if (a.extractorVersion !== b.extractorVersion) return false + if (a.sha256 && b.sha256) return a.sha256 === b.sha256 + return a.size === b.size && a.mtime === b.mtime && a.size > 0 +} + +/** One page, with the 425 backoff. */ +async function fetchPage({ lang, cursor }) { + for (let attempt = 0; ; attempt++) { + const res = await uoLinkClient.getClilocTable({ lang, cursor }) + if (res.ok) return res.data + + if (res.status === 425 && attempt < BUSY_RETRIES) { + await sleep(BUSY_BACKOFF_MS[Math.min(attempt, BUSY_BACKOFF_MS.length - 1)]) + continue + } + + throw describeFailure(res, `cliloc.${lang}`) + } +} + +/** + * Walk the whole table. + * + * Returns `{ entries, source }` where `entries` is `[{ number, flag, text }]` in + * the shape `clilocParse` produces, so the merge in `shardClilocs.model` does not + * care which source an entry came from. + * + * Blanks are already gone: the shard drops the ~56,000 empty strings a stock + * table carries before they reach the wire, since the site would drop them at + * import anyway. Nothing downstream changes — `db.replaceAll` still filters, and + * still would if a source ever sent one. + */ +async function readCliloc({ lang = DEFAULT_LANGUAGE } = {}) { + const started = Date.now() + const entries = [] + + let cursor = null + let pages = 0 + let first = null + let finished = false + let total = null + + while (pages < MAX_PAGES) { + const page = await fetchPage({ lang, cursor }) + pages++ + + if (!page || !Array.isArray(page.rows)) { + throw new ClilocBridgeError('The shard sent a cliloc page with no rows array', 'MALFORMED') + } + + if (first === null) { + first = { size: Number(page.size) || 0, mtime: Number(page.mtime) || 0 } + total = Number.isFinite(Number(page.total)) ? Number(page.total) : null + } else if (Number(page.size) !== first.size || Number(page.mtime) !== first.mtime) { + // The client was patched (or a different one mounted) between two pages. + // Refusing is the only honest answer: half of what we hold is from a file + // that no longer exists, and nothing later can tell which half. + throw new ClilocBridgeError( + 'The shard\'s cliloc file changed while it was being read; nothing was imported', + 'SOURCE_CHANGED', + ) + } + + for (const row of page.rows) { + const number = Number(row?.n) + if (!Number.isInteger(number)) continue + entries.push({ number, flag: Number(row?.f) || 0, text: String(row?.t ?? '') }) + } + + if (entries.length > MAX_ROWS) { + throw new ClilocBridgeError( + `The shard sent more than ${MAX_ROWS} cliloc rows; refusing to keep reading`, + 'TOO_LARGE', + ) + } + + if (!page.more) { + // `cut` is the field that says WHY a page was the last one, and only one of + // its values means the table ended. A shard that stopped for its own limit + // has not finished, and importing what arrived would silently drop the tail. + if (page.cut !== 'end') { + throw new ClilocBridgeError( + `The shard stopped sending cliloc rows after ${entries.length} (cut: ${page.cut || 'unknown'})`, + 'INCOMPLETE', + ) + } + finished = true + break + } + + if (!page.cursor || page.cursor === cursor) { + // Either would loop forever: no cursor to advance with, or the same one + // back again. + throw new ClilocBridgeError( + `The shard asked for another cliloc page without advancing its cursor (${page.cursor || 'none'})`, + 'STUCK', + ) + } + + cursor = page.cursor + } + + if (!finished) { + throw new ClilocBridgeError( + `The cliloc table did not end within ${MAX_PAGES} pages; nothing was imported`, + 'TOO_LARGE', + ) + } + + log.info('cliloc table read from the shard', { + lang, + entries: entries.length, + pages, + ms: Date.now() - started, + }) + + return { + entries, + source: { + kind: 'bridge', + lang, + file: `cliloc.${lang}`, + size: first?.size ?? 0, + mtime: first?.mtime ?? 0, + pages, + // What the shard said it holds, kept beside what actually arrived. They + // agree or the walk is wrong, and an operator seeing them disagree in the + // panel learns more than a single number would tell them. + reported: total, + received: entries.length, + }, + } +} + +module.exports = { + ClilocBridgeError, + SOURCE_FILE, + DEFAULT_LANGUAGE, + MAX_PAGES, + MAX_ROWS, + fingerprint, + sameSource, + readCliloc, +} diff --git a/server/utils/clilocSource.js b/server/utils/clilocSource.js index 93ae061..9ed9d56 100644 --- a/server/utils/clilocSource.js +++ b/server/utils/clilocSource.js @@ -6,6 +6,24 @@ // - the server, which refreshes the table on boot (`shardClilocs.model.js`) // - the admin panel, which can force a reimport without a restart // +// ── What protocol 8 took away, and what it left ─────────────────────────── +// +// The BASE table no longer comes from here on a shard that has uo-link +// configured: `clilocBridge.js` asks the shard for it, because the shard has the +// operator's client files already and, since phase 2, the decompressor to read +// them (docs/link/v8.md §9). Nobody converts a file by hand any more. +// +// Two things keep this module alive rather than deleting it: +// +// - **Overlays.** Shard-added items carry cliloc ids no client table has, and +// ServUO has no server-side notion of a custom cliloc — there is nothing on +// the shard to ask for. `custom/` is still a directory the site reads, and +// `readOverlays` below is the entry point the bridge path uses. +// - **Installs with no shard link**, and development. A site that has never +// configured uo-link can still be pointed at a converted file; that path is +// deprecated, not removed, and it stays the whole of this module's base-table +// behaviour. +// // The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived // from them is committed: the repo holds no string table, exactly as it holds no // map snapshot and no artwork. That rule is why this module reads a configured @@ -194,6 +212,63 @@ function readSources(configured) { return { root, files } } +/** + * Read the OVERLAY files only, with no base table. + * + * The bridge path needs exactly this: the base arrives from the shard and the + * `custom/` directory beside the configured path still has to be merged over it. + * `readSources` cannot answer it, because resolving a base is the first thing it + * does and there may not be one — an operator on the bridge is entitled to point + * this setting at a directory that holds nothing but `custom/`. + * + * **Never throws.** A path that is blank, missing or unreadable is reported as a + * `problem` string and an empty file list, because none of those may stop a base + * table that arrived perfectly well from being imported. The model decides what + * to do about it — and it has a real decision to make, since an overlay that was + * loaded last time and is missing now is the vanished-source hazard, not a + * config typo. + */ +function readOverlays(configured) { + const target = String(configured ?? '').trim() + if (target === '') return { root: null, files: [], problem: null } + + let root + try { + const stat = fs.statSync(target) + root = stat.isFile() ? path.dirname(target) : target + } catch { + return { root: null, files: [], problem: `Cliloc path does not exist: ${target}` } + } + + let overlays + try { + overlays = listCustom(root) + } catch (err) { + return { root, files: [], problem: err.message } + } + + const files = [] + for (const file of overlays) { + let buffer + try { + buffer = fs.readFileSync(file) + } catch { + return { root, files: [], problem: `Cliloc overlay is not readable: ${file}` } + } + files.push({ + label: path.relative(root, file).split(path.sep).join('/'), + kind: 'custom', + file, + buffer, + sha256: sha256(buffer), + bytes: buffer.length, + compressed: isCompressedCliloc(buffer), + }) + } + + return { root, files, problem: null } +} + /** * A fingerprint of every source: `{ "