From bda031566a88bd7d1502f3853cd0897a1fbe5dcb Mon Sep 17 00:00:00 2001 From: wtclaude Date: Wed, 29 Jul 2026 06:46:12 -0500 Subject: [PATCH] feat(shard): read clilocs from a source SET so shard items get names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shards edit items and add new ones, and those carry cliloc ids no stock client table has. Reading exactly one converted file meant an operator had to re-export 5 MB every time they added one item — friction enough that the table would simply go stale, which is the failure the spawn atlas was redesigned to avoid in the first place. So this mirrors spawnAtlasSource.readSources(): a BASE (the converted client table) plus every operator-maintained overlay under `custom/`, all re-read on every boot and hash-gated as a SET. Later sources win, so an overlay both adds ids the client never had and overrides stock ones the shard re-purposed. Adding, editing or removing any overlay counts as drift. `custom/` is the one convention here that is ours rather than the shard's, and deliberately so: ServUO has no server-side notion of a custom cliloc — they live in the patched client a shard distributes, and nothing in the tree declares them. There is nothing to discover. (An operator who does patch their client cliloc needs no overlay: convert the patched file and the edits are in the base.) Scale, measured on the live shard: its script tree references 16,434 cliloc ids and only 37 are absent from stock — tens against a 67k base, which is why this is an overlay and not a second table. The set brings back a hazard a single file did not have, and it gets the atlas's answer. A corrupt source fails the parse loudly, but a source that has VANISHED parses perfectly and imports a table quietly missing everything it contributed — an unmounted volume is indistinguishable from a deliberate deletion. So it is staged, not applied (`needsReview`), reported by both the import and status(), and accepted with `{approve:true}`. That is a flag rather than the atlas's approve/reject pair because the atlas stores a pending decision SO THAT approving re-parses; here nothing is stored, so re-reading at approval time is automatic. Also reports a per-source breakdown (entries/added/overrode) on import and in status, which is how an operator confirms an overlay took effect — "overrode: 0" on a file meant to re-label stock items says it did not. Two bugs this surfaced, both found by running a shard-style overlay rather than by another stock-table fixture: - displayText tidied punctuation unconditionally, so a custom "Runic Gateway Sigil (v2)" rendered as "(v2". Stripping leftover brackets is right after a placeholder is removed and wrong otherwise — the same condition the `%` rule already had. - CANDIDATE_NAMES did not include `clilocs.plain`, which is the exact filename CLILOCS.md and the export tool's README tell operators to write. Pointing at the directory they were told to create failed with NO_FILE. Verified end to end against the live MariaDB and a real server boot: base-only import, overlay adding one id and overriding another (per-source breakdown correct), unchanged set as a no-op, an edited overlay re-importing and withdrawing its override, a vanished overlay refused with the table intact, status reporting missingSources, approve applying it, and a file-path configuration still finding overlays beside it. All three resolve correctly through the running server: shard-added, overridden and stock. 646 server tests pass (16 new in clilocSource.test.js, 3 new in clilocParse.test.js); swagger, routes.manifest.json and routes.guards.json regenerated. Co-Authored-By: Claude --- server/routes.guards.json | 2 +- .../src/model/shardClilocs/shardClilocs.db.js | 10 +- .../model/shardClilocs/shardClilocs.model.js | 105 +++-- server/src/router/v1/admin/shard.router.js | 15 +- .../v1/admin/shardClilocs.controller.js | 26 +- server/src/utils/clilocParse.js | 16 +- server/src/utils/clilocSource.js | 246 +++++++++--- server/swagger/swagger-output.json | 367 +++++++++++++++++- server/swagger/swagger.js | 66 +++- server/test/clilocParse.test.js | 12 + server/test/clilocSource.test.js | 194 +++++++++ 11 files changed, 939 insertions(+), 120 deletions(-) create mode 100644 server/test/clilocSource.test.js diff --git a/server/routes.guards.json b/server/routes.guards.json index ccd273e..c42ddc2 100644 --- a/server/routes.guards.json +++ b/server/routes.guards.json @@ -739,7 +739,7 @@ { "method": "POST", "path": "/api/v1/admin/shard/clilocs/import", - "handlers": 4, + "handlers": 5, "gates": [ "noindex", "requireAuth", diff --git a/server/src/model/shardClilocs/shardClilocs.db.js b/server/src/model/shardClilocs/shardClilocs.db.js index 530ec4b..d944524 100644 --- a/server/src/model/shardClilocs/shardClilocs.db.js +++ b/server/src/model/shardClilocs/shardClilocs.db.js @@ -34,10 +34,12 @@ async function replaceAll(entries, meta) { // identical content: the binary format carries the blanks explicitly and a // text export may or may not, depending on the tool. // - // Later duplicates win. The plain format permits a repeated id and the - // client's own loader resolves it the same way (its dictionary assignment - // overwrites), so collapsing here keeps the batch insert from failing on a - // primary-key collision for a file the game itself would load. + // Later duplicates win. Merging across sources already happened upstream in + // `readCliloc`, so in practice this collapses nothing — it is kept because + // the plain format permits a repeated id WITHIN one file and the client's + // own loader resolves it the same way (its dictionary assignment + // overwrites). Without it, a file the game itself would load happily would + // fail the batch insert on a primary-key collision. const byNumber = new Map() let blank = 0 for (const entry of entries) { diff --git a/server/src/model/shardClilocs/shardClilocs.model.js b/server/src/model/shardClilocs/shardClilocs.model.js index 8b72efa..6f193bd 100644 --- a/server/src/model/shardClilocs/shardClilocs.model.js +++ b/server/src/model/shardClilocs/shardClilocs.model.js @@ -5,7 +5,9 @@ const { ClilocFormatError, ClilocSourceError, PARSER_VERSION, - hashSource, + hashSources, + sameSources, + missingSources, readCliloc, } = require('../../utils/clilocSource') const log = require('../../utils/logger')('shardClilocs') @@ -27,13 +29,24 @@ const log = require('../../utils/logger')('shardClilocs') // 2. **Nothing client-derived is committed.** The table is built from the // operator's own file at a configured path. The repo ships no strings. // -// Unlike the atlas there is no staged-approval flow, and the difference is -// deliberate: the atlas stages a refresh that would REMOVE a facet because a -// half-copied tree and a real map change look identical from here. A cliloc file -// is a single file with a single hash, and the realistic corruption — a partial -// copy — makes the parser fail on a truncated record rather than yield a -// plausible-but-short table. The failure mode the atlas has to guess about is -// one this parser can simply detect. +// The table is built from a SET of sources — the converted client table plus +// every operator-maintained overlay beside it — because shards edit items and +// add new ones, and those carry cliloc ids no stock client table has. All of +// them are re-read on every boot and hash-gated together, so adding one custom +// item never means re-exporting a 5 MB client file. Later sources win. +// +// That set is also why this has the atlas's escalation, in a lighter form. A +// single corrupt file fails the parse loudly, but a source that has simply +// VANISHED parses perfectly and imports a table quietly missing everything it +// contributed — the same ambiguity (real change vs half-copied mount) the atlas +// stages a facet removal for. So a disappearing source is refused and reported +// rather than applied. +// +// It is lighter than the atlas's because it needs to be: the atlas stores a +// pending decision in its own table and adds approve/reject endpoints, whereas +// here the decision is a single boolean an admin passes to the import they were +// already going to run. Re-parsing at approval time — the property that makes +// the atlas store only the decision — is automatic when there is nothing stored. const SETTING_KEY = 'cliloc_client_path' @@ -76,13 +89,15 @@ const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION * * `skipped` no path configured * `unavailable` path configured but missing / unreadable / not a cliloc file - * `unchanged` source hash matches the loaded table; nothing parsed + * `unchanged` source hashes match the loaded table; nothing parsed * `imported` parsed and applied + * `needsReview` a previously-present source has vanished; NOT applied * `failed` parsed or applied and something went wrong * - * `force` skips the hash check (an admin asking for a reimport). + * `force` skips the hash check (an admin asking for a reimport). `approve` + * additionally accepts a vanished source. */ -async function refresh({ force = false, path: pathOverride = '' } = {}) { +async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) { // An explicit override wins outright — a one-off "use this file", which must // not be silently overruled by the configured path the way an env default is. const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() @@ -90,7 +105,7 @@ async function refresh({ force = false, path: pathOverride = '' } = {}) { let fingerprint try { - fingerprint = hashSource(configured) + fingerprint = hashSources(configured) } catch (err) { if (err instanceof ClilocSourceError) { return { status: 'unavailable', reason: err.message, code: err.code, path: configured } @@ -100,11 +115,31 @@ async function refresh({ force = false, path: pathOverride = '' } = {}) { const meta = await db.getMeta().catch(() => null) - // Two things make a loaded table stale: the file changed, or the PARSER did. - // Only checking the file would strand an install whose client never patches on - // whatever an older build derived. - if (!force && meta?.sha256 === fingerprint.sha256 && currentParser(meta)) { - return { status: 'unchanged', path: configured, file: fingerprint.file, count: meta.count ?? null } + // Two things make a loaded table stale: any source changed, or the PARSER did. + // Only checking the sources would strand an install whose client never patches + // on whatever an older build derived. + if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) { + return { + status: 'unchanged', + path: configured, + file: fingerprint.file, + count: meta.count ?? null, + customCount: fingerprint.customCount, + } + } + + // A source that was there last import and is not there now is refused, not + // applied — an unmounted volume and a deliberate deletion look identical from + // here, and the wrong guess silently drops every name that file contributed. + const gone = missingSources(fingerprint.hashes, meta?.hashes) + if (gone.length > 0 && !approve) { + return { + status: 'needsReview', + reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`, + missingSources: gone, + path: configured, + file: fingerprint.file, + } } let parsed @@ -118,7 +153,7 @@ async function refresh({ force = false, path: pathOverride = '' } = {}) { } try { - const applied = await db.replaceAll(parsed.entries, { ...parsed.source, mtime: fingerprint.mtime }) + const applied = await db.replaceAll(parsed.entries, parsed.source) invalidate() return { status: 'imported', @@ -127,7 +162,12 @@ async function refresh({ force = false, path: pathOverride = '' } = {}) { count: applied.count, parsed: parsed.entries.length, blank: applied.blank, - duplicates: applied.duplicates, + // Per-source breakdown: how many entries each file contributed and how + // many of them overrode something already merged. An operator who adds an + // overlay wants to see it took effect, and "overrode: 0" on a file meant + // to re-label stock items says it did not. + sources: parsed.source.sources, + acceptedMissing: gone.length > 0 ? gone : undefined, } } catch (err) { return { status: 'failed', reason: err.message, path: configured } @@ -143,7 +183,18 @@ async function refreshOnBoot() { const result = await refresh() switch (result.status) { case 'imported': - log.info('cliloc table refreshed', { file: result.file, count: result.count }) + log.info('cliloc table refreshed', { + file: result.file, + count: result.count, + overlays: (result.sources || []).filter((s) => s.kind === 'custom').length, + }) + break + case 'needsReview': + log.warn( + 'cliloc refresh staged for admin review — a previously-loaded source is missing; ' + + 'the existing table is unchanged', + { missing: result.missingSources }, + ) break case 'unavailable': // Deliberately a warning, not an error: an operator who has not supplied @@ -180,11 +231,15 @@ async function status({ path: pathOverride = '' } = {}) { let drift = null let problem = null let code = null + let sources = [] + let missing = [] if (configured !== '') { try { - const fingerprint = hashSource(configured) + const fingerprint = hashSources(configured) fileReadable = true file = fingerprint.file + sources = Object.keys(fingerprint.hashes) + missing = missingSources(fingerprint.hashes, meta?.hashes) // A compressed file is readable but not importable, and the panel has to // say so HERE — otherwise pointing at an unconverted client directory // reports a healthy file with pending drift ("ready to import") and the @@ -196,7 +251,7 @@ async function status({ path: pathOverride = '' } = {}) { 'Convert it to the plain format first — see docs/website/CLILOCS.md.' code = 'COMPRESSED' } else { - drift = meta?.sha256 !== fingerprint.sha256 || !currentParser(meta) + drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta) } } catch (err) { fileReadable = false @@ -214,6 +269,12 @@ async function status({ path: pathOverride = '' } = {}) { code, drift, count: loaded, + // Every source found now (base first, then overlays), what each contributed + // at the last import, and any that have since vanished — which is the state + // an import will refuse without `approve`. + sources, + loadedSources: meta?.sources ?? null, + missingSources: missing, importedAt: meta?.importedAt ?? null, sourceBytes: meta?.bytes ?? null, } diff --git a/server/src/router/v1/admin/shard.router.js b/server/src/router/v1/admin/shard.router.js index 4da5d7f..e886647 100644 --- a/server/src/router/v1/admin/shard.router.js +++ b/server/src/router/v1/admin/shard.router.js @@ -318,8 +318,8 @@ shardRouter.put( shardRouter.get( '/clilocs', // #swagger.tags = ['Admin · Shard'] - // #swagger.summary = 'Cliloc table status: path, drift, entry count (admin only)' - // #swagger.description = 'Where the converted cliloc file is, whether it can be read, how many entries are loaded, and whether the file on disk has drifted from them. A shard with no cliloc file configured is a supported state — item names simply render as ids.' + // #swagger.summary = 'Cliloc table status: sources, drift, entry count (admin only)' + // #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether the files on disk have drifted from them. The table is built from a SET of sources — the converted client table plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any source that was loaded before and is now gone; an import refuses that without `approve`. A shard with nothing configured is a supported state — item names simply render as ids.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] /* #swagger.responses[200] = { description: 'Cliloc status', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocStatus" } } } } */ /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ @@ -329,21 +329,22 @@ shardRouter.get( shardRouter.post( '/clilocs/import', // #swagger.tags = ['Admin · Shard'] - // #swagger.summary = 'Re-import the cliloc table from the converted file (admin only)' - // #swagger.description = 'Applies a client patch without a restart. `force` reimports even when the source hash matches what is loaded. A missing file — or the common mistake of pointing at the client\'s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.' + // #swagger.summary = 'Re-import the cliloc table from its source files (admin only)' + // #swagger.description = 'Applies a client patch, or a change to the shard\'s own overlay files, without a restart. `force` reimports even when the source hashes match what is loaded. `approve` accepts a refresh in which a previously-loaded source has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. A missing path — or the common mistake of pointing at the client\'s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] - /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the file is unchanged." } } } } } } */ + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the sources are unchanged." }, approve: { type: "boolean", description: "Accept a refresh in which a previously-loaded source has vanished." } } } } } } */ /* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocRefreshResult" } } } } */ adminOnly, body('force').optional().isBoolean(), + body('approve').optional().isBoolean(), validate, shardClilocs.importClilocs, ) shardRouter.put( '/clilocs/path', // #swagger.tags = ['Admin · Shard'] - // #swagger.summary = 'Set the cliloc file the site reads from (admin only)' - // #swagger.description = 'Accepts either the converted file itself or a directory to search. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.' + // #swagger.summary = 'Set the cliloc source the site reads from (admin only)' + // #swagger.description = 'Accepts either the converted base file itself or a directory to search. Overlays are read from a `custom/` directory beside it either way — pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.' // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." } } } } } } */ /* #swagger.responses[200] = { description: 'Cliloc status after the change', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocStatus" } } } } */ diff --git a/server/src/router/v1/admin/shardClilocs.controller.js b/server/src/router/v1/admin/shardClilocs.controller.js index 6004eae..dc3eb55 100644 --- a/server/src/router/v1/admin/shardClilocs.controller.js +++ b/server/src/router/v1/admin/shardClilocs.controller.js @@ -33,17 +33,33 @@ async function getStatus(req, res) { } } -// POST /admin/shard/clilocs/import — reload after a client patch without a -// restart. `force` reimports even when the source hash matches what is loaded -// (the escape hatch for "the database is wrong but the file is not"). +// POST /admin/shard/clilocs/import — reload after a client patch or a change to +// the shard's own overlay files, without a restart. +// +// `force` reimports even when the source hashes match what is loaded (the escape +// hatch for "the database is wrong but the files are not"). +// +// `approve` accepts a refresh in which a previously-loaded source has VANISHED. +// That is refused by default because an unmounted volume and a deliberate +// deletion look identical from the server — the lighter cousin of the atlas's +// approve/reject flow, and the reason it can be a flag here rather than a +// pending table is that nothing is stored to approve: the import re-reads the +// files at approval time by construction. async function importClilocs(req, res) { try { const force = !!req.body?.force - const result = await clilocs.refresh({ force }) + const approve = !!req.body?.approve + const result = await clilocs.refresh({ force, approve }) await activity.log({ req, action: 'shard.clilocs.import', - detail: { force, status: result.status, count: result.count ?? null }, + detail: { + force, + approve, + status: result.status, + count: result.count ?? null, + missingSources: result.missingSources ?? result.acceptedMissing ?? null, + }, }) return res.json(result) } catch (err) { diff --git a/server/src/utils/clilocParse.js b/server/src/utils/clilocParse.js index 37a1189..cf1a113 100644 --- a/server/src/utils/clilocParse.js +++ b/server/src/utils/clilocParse.js @@ -243,27 +243,31 @@ const PLACEHOLDER_RE = /~\d+_[^~]*~/g * `"[~1_stuff~]"` becomes `""` (correctly nothing — the whole string was the * argument) and `"cold damage ~1_val~%"` becomes `"cold damage"`. * - * The trailing `%` in that second example is only stripped BECAUSE a placeholder - * was removed — it is the unit belonging to the number we never had. Stripping - * `%` unconditionally would corrupt a string that legitimately ends in one. + * **Punctuation is only tidied when a placeholder was actually removed.** The + * trailing `%` above is the unit belonging to the number we never had, and the + * brackets in `[~1_stuff~]` only ever wrapped the argument — but a string with + * no placeholder has no such debris, and trimming it anyway corrupts real names. + * A shard's `"Runic Gateway Sigil (v2)"` came back as `"(v2"` while this was + * unconditional. * * Returns `''` when nothing survives, which callers treat as "no name" and fall * back to the item id — better than showing a bracket. */ +const DEBRIS = /^[\s\-–—,.;:%[\]()]+|[\s\-–—,.;:%[\]()]+$/g + function displayText(raw) { if (raw == null) return '' const source = String(raw) const hadPlaceholder = PLACEHOLDER_RE.test(source) PLACEHOLDER_RE.lastIndex = 0 // the regex is global; `test` advances it - const trailing = hadPlaceholder ? /[\s\-–—,.;:%[\]()]+$/ : /[\s\-–—,.;:[\]()]+$/ + if (!hadPlaceholder) return source.replace(/\s+/g, ' ').trim() return source .replace(PLACEHOLDER_RE, ' ') .replace(/\s+/g, ' ') .replace(/\s+([,.;:!?])/g, '$1') - .replace(/^[\s\-–—,.;:[\]()]+/, '') - .replace(trailing, '') + .replace(DEBRIS, '') .trim() } diff --git a/server/src/utils/clilocSource.js b/server/src/utils/clilocSource.js index d26fda9..93ae061 100644 --- a/server/src/utils/clilocSource.js +++ b/server/src/utils/clilocSource.js @@ -1,15 +1,32 @@ // Cliloc table — the filesystem layer. // // `clilocParse.js` holds the pure parsers; this module is the only thing that -// touches the converted cliloc file on disk, and it is shared by both callers: +// touches cliloc files on disk, and it is shared by both callers: // // - the server, which refreshes the table on boot (`shardClilocs.model.js`) // - the admin panel, which can force a reimport without a restart // -// The file is the OPERATOR'S, produced once from their own UO client (see -// docs/website/CLILOCS.md). Nothing derived from it is committed: the repo holds -// no string table, exactly as it holds no map snapshot and no artwork. That rule -// is why this module reads a configured path instead of a path inside the repo. +// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived +// from them is committed: the repo holds no string table, exactly as it holds no +// map snapshot and no artwork. That rule is why this module reads a configured +// path instead of a path inside the repo. +// +// ── Why this reads a SET of files, not one ──────────────────────────────── +// +// Shards edit items and add new ones. Those carry cliloc ids that a stock client +// table does not have — and forcing a 5 MB client re-export every time an +// operator adds one item would be miserable enough that the table would simply +// go stale, which is the exact failure the spawn atlas was redesigned to avoid. +// +// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted +// client file) plus every operator-maintained OVERLAY beside it, all re-read on +// every boot and hash-gated as a SET. Adding, editing or removing any overlay +// counts as drift and re-imports. Later sources win, so an overlay both adds new +// ids and overrides stock ones. +// +// Measured on a real shard: the script tree references 16,434 cliloc ids and only +// 37 are absent from the stock client table. Tens of entries against a 67k base +// is what makes the overlay the right shape rather than a second full table. // // Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so // the boot path hashes first and only parses when something actually changed. @@ -21,7 +38,7 @@ const path = require('path') const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse') /** - * Filenames looked for when the configured path is a DIRECTORY. + * Filenames looked for as the BASE table when the configured path is a directory. * * Ordered by how specific they are: an explicitly converted file wins over * something that merely sits in a client folder, so an operator who dropped a @@ -35,12 +52,29 @@ const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = r const CANDIDATE_NAMES = [ 'clilocs.tsv', 'clilocs.csv', + 'clilocs.plain', + 'cliloc.plain', 'cliloc.plain.enu', 'cliloc.enu.plain', 'clilocs.txt', 'cliloc.enu', ] +/** + * Where shard-specific additions and overrides live: a `custom/` directory + * beside the base table. + * + * ServUO has **no server-side convention** for custom clilocs — they live in the + * patched client file a shard distributes to its players, and nothing in the + * tree declares them. There is therefore nothing to discover, and this is the + * one place in the cliloc pipeline that is a convention we chose rather than one + * the shard already has. It is a directory rather than a single file so an + * operator can keep additions grouped however they like (per system, per patch) + * without the site caring. + */ +const CUSTOM_DIR = 'custom' +const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain'] + class ClilocSourceError extends Error { constructor(message, code) { super(message) @@ -54,13 +88,16 @@ function sha256(buffer) { } /** - * Resolve the configured path to an actual file. + * Resolve the configured path to `{ root, base }`. * * Accepts either a direct file path or a directory to search, because operators * reasonably supply both — "here is the file" and "here is the folder I put it - * in" are equally natural answers to the admin panel's prompt. + * in" are equally natural answers to the admin panel's prompt. When it is a + * file, `root` is the directory CONTAINING it, so overlays work either way: an + * operator who pointed at a file should not have to re-point at its folder just + * to add a `custom/` directory next to it. */ -function resolveFile(configured) { +function resolveBase(configured) { if (!configured || String(configured).trim() === '') { throw new ClilocSourceError('No cliloc path configured', 'NO_PATH') } @@ -73,7 +110,7 @@ function resolveFile(configured) { throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND') } - if (stat.isFile()) return target + if (stat.isFile()) return { root: path.dirname(target), base: target } if (!stat.isDirectory()) { throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND') @@ -89,7 +126,7 @@ function resolveFile(configured) { const byLower = new Map(listing.map((name) => [name.toLowerCase(), name])) for (const candidate of CANDIDATE_NAMES) { const actual = byLower.get(candidate) - if (actual) return path.join(target, actual) + if (actual) return { root: target, base: path.join(target, actual) } } throw new ClilocSourceError( @@ -98,12 +135,68 @@ function resolveFile(configured) { ) } +/** Overlay files under `/custom/`, sorted so precedence is deterministic. */ +function listCustom(root) { + const dir = path.join(root, CUSTOM_DIR) + let listing + try { + listing = fs.readdirSync(dir, { withFileTypes: true }) + } catch (err) { + // No overlay directory is the normal case, not an error. + if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return [] + throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE') + } + return listing + .filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase())) + .map((e) => e.name) + .sort() + .map((name) => path.join(dir, name)) +} + +function readFileOrThrow(file) { + try { + return fs.readFileSync(file) + } catch { + throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE') + } +} + /** - * A fingerprint of the source file: `{ file, sha256, bytes, mtime, compressed }`. + * Read every cliloc source under the configured path. * - * The boot path compares the hash against what was last imported and skips the - * parse entirely when it matches — the normal case on every restart that did not - * follow a client patch. + * Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }` + * with the base first and overlays after, in the order they must be merged. + * + * Labels are root-relative and forward-slashed so a hash map compares equal + * across platforms — the same directory read on Windows and Linux must produce + * the same fingerprint, or every boot would look like a change. (The same + * reasoning, and the same bug, as `spawnAtlasSource.readSources`.) + */ +function readSources(configured) { + const { root, base } = resolveBase(configured) + + const describe = (file, kind) => { + const buffer = readFileOrThrow(file) + return { + label: path.relative(root, file).split(path.sep).join('/'), + kind, + file, + buffer, + sha256: sha256(buffer), + bytes: buffer.length, + compressed: isCompressedCliloc(buffer), + } + } + + const files = [describe(base, 'base')] + for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom')) + + return { root, files } +} + +/** + * A fingerprint of every source: `{ "