diff --git a/.gitignore b/.gitignore index 622c8c6..9d33164 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,17 @@ logs/ # See docs/website/SPAWN_ATLAS.md and db/data/spawnAtlas.art.example.json. server/db/data/spawnAtlas.art.json +# Operator-supplied cliloc table. UO's localization strings are EA's, extracted +# from the operator's own client and converted once (docs/website/CLILOCS.md); +# the repo ships no string table, for the same reason it ships no artwork and no +# map snapshot. This covers the conventional in-repo location — the supported +# arrangement is a path OUTSIDE the repo, set from Admin → Shard. +server/db/data/cliloc* +server/db/data/clilocs.* +# The build output of tools/cliloc-export (a throwaway helper, not a package). +server/tools/cliloc-export/bin/ +server/tools/cliloc-export/obj/ + # reference material (extracted from the provided archives) _reference/ diff --git a/client/src/components/CharacterSheet.jsx b/client/src/components/CharacterSheet.jsx index 1a27376..58c8fb4 100644 --- a/client/src/components/CharacterSheet.jsx +++ b/client/src/components/CharacterSheet.jsx @@ -10,21 +10,42 @@ import ShardAccountActions from './ShardAccountActions.jsx' const RESIST_LABELS = { phys: 'Physical', fire: 'Fire', cold: 'Cold', pois: 'Poison', energy: 'Energy' } +// What to call an equipped item. +// +// Items on the wire carry a `LabelNumber`, not a name, so this used to be able +// to show nothing but the layer and `id 12345`. The server now resolves the +// cliloc against its own table and attaches `clilocName` (see +// docs/website/CLILOCS.md); a shard with no cliloc file configured sends none, +// and the layer fallback below is exactly what the sheet did before. +// +// A player-given `name` outranks the resolved type name — "Bob's lucky axe" +// should not be relabelled "hatchet" — and the server applies the same +// precedence, so this only re-states it for a profile that arrived with both. +const itemName = (it) => it.name || it.clilocName || it.layer || 'Item' + // The char.profile `titles` block (Protocol 2.0). fameKarma/skill are already // computed display strings; reward entries may be a cliloc NUMBER-as-string or a -// literal string. Without a cliloc table on the site we can only show literals, so -// numeric reward entries are skipped rather than shown as a raw number. Returns a -// de-duped list of human-readable title chips. +// literal string. +// +// `rewardResolved` is the server's parallel array with the numeric entries turned +// into words (null where the cliloc table had nothing, or is not configured at +// all). Prefer it, and keep the literal-only path as the fallback for a profile +// served before the cliloc table existed — a numeric entry with no resolution is +// still skipped rather than shown as a raw number. function displayTitles(titles) { if (!titles) return [] const out = [] if (titles.fameKarma) out.push(titles.fameKarma) if (titles.skill) out.push(titles.skill) - const reward = Array.isArray(titles.reward) ? titles.reward : [] + const raw = Array.isArray(titles.reward) ? titles.reward : [] + const resolved = Array.isArray(titles.rewardResolved) ? titles.rewardResolved : null + const reward = raw.map((r, i) => resolved?.[i] ?? (/^\d+$/.test(String(r)) ? null : String(r))) const sel = typeof titles.selected === 'number' ? titles.selected : -1 - // Prefer the selected reward title; fall back to the first literal one. - const candidate = sel >= 0 && sel < reward.length ? reward[sel] : reward.find((r) => r && !/^\d+$/.test(String(r))) - if (candidate && !/^\d+$/.test(String(candidate))) out.push(String(candidate)) + // Prefer the selected reward title; fall back to the first one that resolved. + // The `??` matters: a selected title whose cliloc did not resolve must fall + // through to the fallback rather than suppress the chip entirely. + const candidate = (sel >= 0 && sel < reward.length ? reward[sel] : null) ?? reward.find(Boolean) + if (candidate) out.push(String(candidate)) return [...new Set(out.filter(Boolean))] } @@ -241,12 +262,18 @@ export default function CharacterSheet({ char, moderation = false }) {
Equipment
- {equipment.map((it) => ( + {equipment.map((it) => { + const label = itemName(it) + const layer = it.layer || 'Item' + // The layer only earns its own line once the headline is a real + // name; when it IS the headline, repeating it is just noise. + const detail = [label === layer ? null : layer, `id ${it.itemId}`, it.hue ? `hue ${it.hue}` : null] + return (
-
{it.layer || 'Item'}
-
id {it.itemId}{it.hue ? ` · hue ${it.hue}` : ''}
+
{label}
+
{detail.filter(Boolean).join(' · ')}
{it.mods && Object.keys(it.mods).length > 0 && (
@@ -256,7 +283,8 @@ export default function CharacterSheet({ char, moderation = false }) {
)}
- ))} + ) + })}
)} diff --git a/server/db/schema.sql b/server/db/schema.sql index 8241434..1c5df9f 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -1180,6 +1180,39 @@ CREATE TABLE IF NOT EXISTS shard_champion_spawns ( INDEX idx_shard_champion_spawns_facet (facet) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- UO's localization table: cliloc id -> display string. Items carry a +-- `LabelNumber` rather than a name, so without this the site can only render +-- `id 1023721` where the game shows "quarter staff". The shard has always sent +-- the id (char.profile's `cliloc`, and one per marketplace listing) — the number +-- was never the missing piece, the table was. +-- +-- Sourced from a file the OPERATOR converts once from their own UO client and +-- points the site at (docs/website/CLILOCS.md); nothing derived from the client +-- is committed, the same rule the spawn atlas and the creature art map follow. +-- A shard with no cliloc file configured simply renders item ids, which is what +-- it did before this table existed. +-- +-- `text` is TEXT, not VARCHAR: real tables top out around 12 KB for the long +-- property descriptions, and truncating them silently would be worse than +-- storing them. Item NAMES are all short — the index that matters for search is +-- on the denormalized `shard_vendor_items.display_name`, not here. +CREATE TABLE IF NOT EXISTS shard_clilocs ( + number INT NOT NULL PRIMARY KEY, + flag SMALLINT NOT NULL DEFAULT 0, + text TEXT NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Singleton (id = 1) describing the cliloc table currently loaded: the source +-- file, its sha256, the entry count and the parser version. The boot path +-- compares the stored hash against the file on disk and skips the parse when +-- they match, which is every restart that did not follow a client patch. +CREATE TABLE IF NOT EXISTS shard_cliloc_meta ( + id TINYINT NOT NULL PRIMARY KEY DEFAULT 1, + payload JSON NOT NULL, + imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_cliloc_meta_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- Singleton (id = 1) describing the artifact currently loaded: when it was -- built, its counts, and a sha256 per ServUO source file. The admin drift check -- compares this against db/data/spawnAtlas.meta.json to report when the database diff --git a/server/routes.guards.json b/server/routes.guards.json index 23fbad8..c42ddc2 100644 --- a/server/routes.guards.json +++ b/server/routes.guards.json @@ -727,6 +727,37 @@ "validate" ] }, + { + "method": "GET", + "path": "/api/v1/admin/shard/clilocs", + "handlers": 2, + "gates": [ + "noindex", + "requireAuth" + ] + }, + { + "method": "POST", + "path": "/api/v1/admin/shard/clilocs/import", + "handlers": 5, + "gates": [ + "noindex", + "requireAuth", + "middleware", + "validate" + ] + }, + { + "method": "PUT", + "path": "/api/v1/admin/shard/clilocs/path", + "handlers": 4, + "gates": [ + "noindex", + "requireAuth", + "middleware", + "validate" + ] + }, { "method": "GET", "path": "/api/v1/admin/shard/houses", diff --git a/server/routes.manifest.json b/server/routes.manifest.json index f1e55e7..6c6544b 100644 --- a/server/routes.manifest.json +++ b/server/routes.manifest.json @@ -293,6 +293,18 @@ "method": "GET", "path": "/api/v1/admin/shard/char/:serial" }, + { + "method": "GET", + "path": "/api/v1/admin/shard/clilocs" + }, + { + "method": "POST", + "path": "/api/v1/admin/shard/clilocs/import" + }, + { + "method": "PUT", + "path": "/api/v1/admin/shard/clilocs/path" + }, { "method": "GET", "path": "/api/v1/admin/shard/houses" diff --git a/server/src/model/shardClilocs/shardClilocs.db.js b/server/src/model/shardClilocs/shardClilocs.db.js new file mode 100644 index 0000000..d944524 --- /dev/null +++ b/server/src/model/shardClilocs/shardClilocs.db.js @@ -0,0 +1,108 @@ +const { pool, query } = require('../../utils/db') + +// Raw SQL for the cliloc table. `shard_clilocs` is IMPORT-OWNED: `replaceAll` +// empties and refills it inside one transaction, and nothing else in the +// codebase writes to it. No foreign keys, consistent with every other shard_* +// table. + +const BATCH = 1000 + +/** + * Replace the entire cliloc table in one transaction. + * + * All-or-nothing on purpose: a failed reload must leave the previous table + * intact rather than a half-loaded one, because a partially-imported cliloc + * table is indistinguishable from a complete one to anyone reading it — you + * would just see some items named and some not, which is also what "no table at + * all" looks like. + * + * `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly + * commits, which would defeat exactly that guarantee. (The same trap the spawn + * atlas import documents; at ~123k rows `DELETE` is still well under a second.) + */ +async function replaceAll(entries, meta) { + const conn = await pool.getConnection() + try { + await conn.beginTransaction() + await conn.query('DELETE FROM shard_clilocs') + + // Blank entries are dropped rather than stored. Roughly HALF of a real + // cliloc table is empty strings — ids the client reserves and never uses — + // and a row that resolves to no name is indistinguishable from no row at + // all to every caller. Dropping them halves the table (123,490 → ~67,500) + // and, more importantly, makes the binary and text imports converge on + // identical content: the binary format carries the blanks explicitly and a + // text export may or may not, depending on the tool. + // + // Later duplicates win. Merging across sources already happened upstream in + // `readCliloc`, so in practice this collapses nothing — it is kept because + // the plain format permits a repeated id WITHIN one file and the client's + // own loader resolves it the same way (its dictionary assignment + // overwrites). Without it, a file the game itself would load happily would + // fail the batch insert on a primary-key collision. + const byNumber = new Map() + let blank = 0 + for (const entry of entries) { + if (!Number.isInteger(entry.number)) continue + if (String(entry.text ?? '').trim() === '') { + blank++ + continue + } + byNumber.set(entry.number, entry) + } + + const rows = [...byNumber.values()].map((e) => [e.number, e.flag ?? 0, e.text]) + for (let i = 0; i < rows.length; i += BATCH) { + await conn.batch('INSERT INTO shard_clilocs (number, flag, text) VALUES (?,?,?)', rows.slice(i, i + BATCH)) + } + + await conn.query( + 'INSERT INTO shard_cliloc_meta (id, payload) VALUES (1, ?) ' + + 'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP', + [JSON.stringify({ ...meta, count: rows.length })], + ) + + await conn.commit() + return { count: rows.length, blank, duplicates: entries.length - blank - rows.length } + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +async function getMeta() { + const rows = await query('SELECT payload, imported_at FROM shard_cliloc_meta WHERE id = 1') + if (rows.length === 0) return null + const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload + return { ...payload, importedAt: rows[0].imported_at } +} + +/** + * Look up a batch of ids. + * + * Batched rather than one-at-a-time because every caller has a LIST: a character + * sheet resolves a dozen equipment ids at once, and a page of marketplace + * listings resolves fifty. `IN (...)` with generated placeholders keeps it one + * round trip and one parameterized statement. + */ +async function lookup(numbers) { + if (!Array.isArray(numbers) || numbers.length === 0) return [] + const ids = [...new Set(numbers.filter((n) => Number.isInteger(n)))] + if (ids.length === 0) return [] + const placeholders = ids.map(() => '?').join(',') + return query(`SELECT number, text FROM shard_clilocs WHERE number IN (${placeholders})`, ids) +} + +async function count() { + const rows = await query('SELECT COUNT(*) AS n FROM shard_clilocs') + return Number(rows[0]?.n) || 0 +} + +module.exports = { + replaceAll, + getMeta, + lookup, + count, +} diff --git a/server/src/model/shardClilocs/shardClilocs.model.js b/server/src/model/shardClilocs/shardClilocs.model.js new file mode 100644 index 0000000..6f193bd --- /dev/null +++ b/server/src/model/shardClilocs/shardClilocs.model.js @@ -0,0 +1,368 @@ +const db = require('./shardClilocs.db') +const settings = require('../settings/settings.model') +const { displayText } = require('../../utils/clilocParse') +const { + ClilocFormatError, + ClilocSourceError, + PARSER_VERSION, + hashSources, + sameSources, + missingSources, + readCliloc, +} = require('../../utils/clilocSource') +const log = require('../../utils/logger')('shardClilocs') + +// The cliloc table — UO's id → display-string map, refreshed from a file the +// operator converts once from their own client. +// +// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a +// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace +// listing sends one too. Without the table the UI can only print `id 1023721` +// where the game prints "quarter staff". +// +// Two rules govern the boot path, both inherited from the spawn atlas: +// +// 1. **It never blocks startup.** No configured path, an unreadable file, a +// wrong-format file, a database error — all caught and logged. The site +// comes up either way, serving whatever table it already had (or none, in +// which case the UI falls back to item ids exactly as it did before). +// 2. **Nothing client-derived is committed.** The table is built from the +// operator's own file at a configured path. The repo ships no strings. +// +// The table is built from a SET of sources — the converted client table plus +// every operator-maintained overlay beside it — because shards edit items and +// add new ones, and those carry cliloc ids no stock client table has. All of +// them are re-read on every boot and hash-gated together, so adding one custom +// item never means re-exporting a 5 MB client file. Later sources win. +// +// That set is also why this has the atlas's escalation, in a lighter form. A +// single corrupt file fails the parse loudly, but a source that has simply +// VANISHED parses perfectly and imports a table quietly missing everything it +// contributed — the same ambiguity (real change vs half-copied mount) the atlas +// stages a facet removal for. So a disappearing source is refused and reported +// rather than applied. +// +// It is lighter than the atlas's because it needs to be: the atlas stores a +// pending decision in its own table and adds approve/reject endpoints, whereas +// here the decision is a single boolean an admin passes to the import they were +// already going to run. Re-parsing at approval time — the property that makes +// the atlas store only the decision — is automatic when there is nothing stored. + +const SETTING_KEY = 'cliloc_client_path' + +/** + * Where the converted cliloc file lives. + * + * The admin setting wins over the environment so an operator can repoint it + * without a redeploy, matching how the rest of the shard integration is + * admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the + * deploy-time default, since the path usually describes a mount the deployment + * sets up. + */ +async function getClientPath() { + try { + const configured = await settings.get(SETTING_KEY) + if (configured && String(configured).trim() !== '') return String(configured).trim() + } catch { + // Settings unavailable is not fatal — fall through to the env default. + } + const fromEnv = process.env.UO_CLIENT_PATH + return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : '' +} + +async function setClientPath(value, updatedBy = null) { + const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy) + invalidate() + return result +} + +// ── Refresh ──────────────────────────────────────────────────────────────── + +/** Was the loaded table built by THIS parser? */ +const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION + +/** + * Refresh the cliloc table from the configured file. + * + * Returns a result describing what happened rather than throwing, so the caller + * — including the boot path — can log it and move on: + * + * `skipped` no path configured + * `unavailable` path configured but missing / unreadable / not a cliloc file + * `unchanged` source hashes match the loaded table; nothing parsed + * `imported` parsed and applied + * `needsReview` a previously-present source has vanished; NOT applied + * `failed` parsed or applied and something went wrong + * + * `force` skips the hash check (an admin asking for a reimport). `approve` + * additionally accepts a vanished source. + */ +async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) { + // An explicit override wins outright — a one-off "use this file", which must + // not be silently overruled by the configured path the way an env default is. + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() + if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' } + + let fingerprint + try { + fingerprint = hashSources(configured) + } catch (err) { + if (err instanceof ClilocSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: configured } + } + return { status: 'failed', reason: err.message, path: configured } + } + + const meta = await db.getMeta().catch(() => null) + + // Two things make a loaded table stale: any source changed, or the PARSER did. + // Only checking the sources would strand an install whose client never patches + // on whatever an older build derived. + if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) { + return { + status: 'unchanged', + path: configured, + file: fingerprint.file, + count: meta.count ?? null, + customCount: fingerprint.customCount, + } + } + + // A source that was there last import and is not there now is refused, not + // applied — an unmounted volume and a deliberate deletion look identical from + // here, and the wrong guess silently drops every name that file contributed. + const gone = missingSources(fingerprint.hashes, meta?.hashes) + if (gone.length > 0 && !approve) { + return { + status: 'needsReview', + reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`, + missingSources: gone, + path: configured, + file: fingerprint.file, + } + } + + let parsed + try { + parsed = readCliloc(configured) + } catch (err) { + if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: configured } + } + return { status: 'failed', reason: err.message, path: configured } + } + + try { + const applied = await db.replaceAll(parsed.entries, parsed.source) + invalidate() + return { + status: 'imported', + path: configured, + file: parsed.source.file, + count: applied.count, + parsed: parsed.entries.length, + blank: applied.blank, + // Per-source breakdown: how many entries each file contributed and how + // many of them overrode something already merged. An operator who adds an + // overlay wants to see it took effect, and "overrode: 0" on a file meant + // to re-label stock items says it did not. + sources: parsed.source.sources, + acceptedMissing: gone.length > 0 ? gone : undefined, + } + } catch (err) { + return { status: 'failed', reason: err.message, path: configured } + } +} + +/** + * Boot hook. Best-effort by contract: it logs and returns, never throws, so a + * missing or malformed cliloc file can never stop the site coming up. + */ +async function refreshOnBoot() { + try { + const result = await refresh() + switch (result.status) { + case 'imported': + log.info('cliloc table refreshed', { + file: result.file, + count: result.count, + overlays: (result.sources || []).filter((s) => s.kind === 'custom').length, + }) + break + case 'needsReview': + log.warn( + 'cliloc refresh staged for admin review — a previously-loaded source is missing; ' + + 'the existing table is unchanged', + { missing: result.missingSources }, + ) + break + case 'unavailable': + // Deliberately a warning, not an error: an operator who has not supplied + // a cliloc file is in a supported state (the UI shows item ids), and the + // most common cause — pointing at the client's own compressed file — + // needs the reason spelled out rather than a stack trace. + log.warn('cliloc source unavailable (item names will show as ids)', { + reason: result.reason, + code: result.code, + path: result.path, + }) + break + case 'failed': + log.warn('cliloc refresh failed', { reason: result.reason }) + break + default: + break + } + return result + } catch (err) { + log.warn('cliloc refresh errored', { error: err.message }) + return { status: 'failed', reason: err.message } + } +} + +/** Everything the admin panel needs to describe cliloc state. */ +async function status({ path: pathOverride = '' } = {}) { + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() + const meta = await db.getMeta().catch(() => null) + const loaded = await db.count().catch(() => 0) + + let fileReadable = false + let file = null + let drift = null + let problem = null + let code = null + let sources = [] + let missing = [] + if (configured !== '') { + try { + const fingerprint = hashSources(configured) + fileReadable = true + file = fingerprint.file + sources = Object.keys(fingerprint.hashes) + missing = missingSources(fingerprint.hashes, meta?.hashes) + // A compressed file is readable but not importable, and the panel has to + // say so HERE — otherwise pointing at an unconverted client directory + // reports a healthy file with pending drift ("ready to import") and the + // operator only finds out when the import fails. `drift` stays null + // because comparing hashes with an unusable file answers nothing. + if (fingerprint.compressed) { + problem = + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.' + code = 'COMPRESSED' + } else { + drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta) + } + } catch (err) { + fileReadable = false + problem = err.message + code = err.code ?? null + } + } + + return { + configured: configured !== '', + path: configured, + file, + fileReadable, + problem, + code, + drift, + count: loaded, + // Every source found now (base first, then overlays), what each contributed + // at the last import, and any that have since vanished — which is the state + // an import will refuse without `approve`. + sources, + loadedSources: meta?.sources ?? null, + missingSources: missing, + importedAt: meta?.importedAt ?? null, + sourceBytes: meta?.bytes ?? null, + } +} + +// ── Lookup ───────────────────────────────────────────────────────────────── +// +// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is +// ~123k rows and shipping it to a client would dwarf every page that uses it, +// and the Android app consumes the same JSON and would otherwise need its own +// copy. Callers get names, not ids-plus-a-table. + +// A small write-through cache in front of the table. Item ids repeat heavily — +// one page of listings is mostly the same few hundred clilocs, and a character +// sheet re-resolves the same gear on every view — so this turns the steady state +// into zero queries. Capped so a pathological caller cannot grow it without +// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry, +// which is cheap and correct for a table that only changes on reimport. +const CACHE_MAX = 20000 +let cache = new Map() + +function invalidate() { + cache = new Map() +} + +/** + * Resolve a batch of cliloc ids to display strings. + * + * Returns a `Map` holding only the ids that resolved to + * something displayable — an id with no row, or one whose text is nothing but + * interpolated arguments we do not have, is simply absent. Callers fall back to + * whatever they had (the item id), so "missing" and "unnamed" collapse into one + * branch at the call site. + * + * Never throws: a cliloc lookup is decoration on someone's character sheet, and + * a database blip must not fail the sheet. + */ +async function resolveMany(numbers) { + const out = new Map() + if (!Array.isArray(numbers)) return out + + const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))] + if (wanted.length === 0) return out + + const missing = [] + for (const number of wanted) { + if (cache.has(number)) { + const hit = cache.get(number) + if (hit !== '') out.set(number, hit) + } else { + missing.push(number) + } + } + + if (missing.length > 0) { + try { + const rows = await db.lookup(missing) + const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)])) + if (cache.size + missing.length > CACHE_MAX) invalidate() + for (const number of missing) { + // Cache the miss too ('' meaning "no usable name"), so an id absent from + // the table does not re-query on every page view. + const text = found.get(number) ?? '' + cache.set(number, text) + if (text !== '') out.set(number, text) + } + } catch (err) { + log.warn('cliloc lookup failed', { message: err.message }) + } + } + + return out +} + +/** Single-id convenience. Returns `null` when there is no usable name. */ +async function resolve(number) { + const found = await resolveMany([number]) + return found.get(number) ?? null +} + +module.exports = { + SETTING_KEY, + getClientPath, + setClientPath, + refresh, + refreshOnBoot, + status, + resolveMany, + resolve, + invalidate, +} diff --git a/server/src/router/v1/admin/shard.router.js b/server/src/router/v1/admin/shard.router.js index ed03ff7..e886647 100644 --- a/server/src/router/v1/admin/shard.router.js +++ b/server/src/router/v1/admin/shard.router.js @@ -25,6 +25,7 @@ const { body, param } = require('express-validator') const shardOps = require('./shardOps.controller') const shardVisibility = require('./shardVisibility.controller') const shardAtlas = require('./shardAtlas.controller') +const shardClilocs = require('./shardClilocs.controller') const selfShard = require('../player/shard.controller') const { requireRole } = require('../../../utils/auth') const validate = require('../../../middleware/validate') @@ -304,6 +305,55 @@ shardRouter.put( shardAtlas.setPath, ) +// ── Cliloc table (admin only) ───────────────────────────────────────────── +// UO's id → display-string map, converted once by the operator from their own +// client (docs/website/CLILOCS.md). Sits beside the atlas for the same reason: +// it is static content derived from operator-supplied files rather than anything +// the sidecar sends, and operating it is shard administration. +// +// There is deliberately NO public counterpart. The table is never served as a +// table — 123k rows would dwarf any page that used it, and the Android client +// consumes the same already-resolved JSON. Names are applied server-side to the +// responses that need them. +shardRouter.get( + '/clilocs', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Cliloc table status: sources, drift, entry count (admin only)' + // #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether the files on disk have drifted from them. The table is built from a SET of sources — the converted client table plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any source that was loaded before and is now gone; an import refuses that without `approve`. A shard with nothing configured is a supported state — item names simply render as ids.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.responses[200] = { description: 'Cliloc status', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocStatus" } } } } */ + /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + adminOnly, + shardClilocs.getStatus, +) +shardRouter.post( + '/clilocs/import', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Re-import the cliloc table from its source files (admin only)' + // #swagger.description = 'Applies a client patch, or a change to the shard\'s own overlay files, without a restart. `force` reimports even when the source hashes match what is loaded. `approve` accepts a refresh in which a previously-loaded source has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. A missing path — or the common mistake of pointing at the client\'s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the sources are unchanged." }, approve: { type: "boolean", description: "Accept a refresh in which a previously-loaded source has vanished." } } } } } } */ + /* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocRefreshResult" } } } } */ + adminOnly, + body('force').optional().isBoolean(), + body('approve').optional().isBoolean(), + validate, + shardClilocs.importClilocs, +) +shardRouter.put( + '/clilocs/path', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Set the cliloc source the site reads from (admin only)' + // #swagger.description = 'Accepts either the converted base file itself or a directory to search. Overlays are read from a `custom/` directory beside it either way — pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." } } } } } } */ + /* #swagger.responses[200] = { description: 'Cliloc status after the change', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocStatus" } } } } */ + adminOnly, + body('path').isString().isLength({ max: 512 }), + validate, + shardClilocs.setPath, +) + // ── Feature visibility (admin only) ─────────────────────────────────── // Who can see which shard surface, and which sensitive fields within it. This // decides what ANONYMOUS visitors get, so it sits above the moderator tier. diff --git a/server/src/router/v1/admin/shardClilocs.controller.js b/server/src/router/v1/admin/shardClilocs.controller.js new file mode 100644 index 0000000..dc3eb55 --- /dev/null +++ b/server/src/router/v1/admin/shardClilocs.controller.js @@ -0,0 +1,94 @@ +// ── Admin · Cliloc table ─────────────────────────────────────────────────── +// +// Operating the cliloc import: where the converted cliloc file is, whether it +// has drifted from what is loaded, and a forced reimport after a client patch +// (docs/website/CLILOCS.md). +// +// The policy lives in the model. This controller does three things and no more: +// it validates input, it maps a refresh RESULT onto an HTTP status, and it +// records the action in the admin activity log. +// +// **A refresh result is not an exception.** `shardClilocs.refresh()` reports +// `unavailable` / `failed` rather than throwing, because the boot path must never +// be stopped by a bad file. That contract is preserved here: a missing file, or +// the single most likely operator mistake — pointing at the client's own +// COMPRESSED `Cliloc.enu` — is a 200 carrying `status: 'unavailable'` and the +// reason, not a 500. A 500 would say only "something broke"; the operator needs +// to be told which file to convert. + +const clilocs = require('../../../model/shardClilocs/shardClilocs.model') +const activity = require('../../../model/activity/activity.model') + +const log = require('../../../utils/logger')('admin-shard-clilocs') + +// GET /admin/shard/clilocs — what is loaded, what the file looks like, whether +// they disagree. There is no public counterpart: the cliloc table is never +// served as a table, only applied to names the site already returns. +async function getStatus(req, res) { + try { + return res.json(await clilocs.status()) + } catch (err) { + log.error('getStatus', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +// POST /admin/shard/clilocs/import — reload after a client patch or a change to +// the shard's own overlay files, without a restart. +// +// `force` reimports even when the source hashes match what is loaded (the escape +// hatch for "the database is wrong but the files are not"). +// +// `approve` accepts a refresh in which a previously-loaded source has VANISHED. +// That is refused by default because an unmounted volume and a deliberate +// deletion look identical from the server — the lighter cousin of the atlas's +// approve/reject flow, and the reason it can be a flag here rather than a +// pending table is that nothing is stored to approve: the import re-reads the +// files at approval time by construction. +async function importClilocs(req, res) { + try { + const force = !!req.body?.force + const approve = !!req.body?.approve + const result = await clilocs.refresh({ force, approve }) + await activity.log({ + req, + action: 'shard.clilocs.import', + detail: { + force, + approve, + status: result.status, + count: result.count ?? null, + missingSources: result.missingSources ?? result.acceptedMissing ?? null, + }, + }) + return res.json(result) + } catch (err) { + log.error('importClilocs', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +// PUT /admin/shard/clilocs/path — point the site at a different cliloc file. +// +// Persisted as a setting, which wins over the UO_CLIENT_PATH env default so an +// operator can move the mount without a redeploy. Blank clears it, which turns +// resolution off (boot skips, the loaded table keeps serving) — a legitimate +// thing to want, so it is allowed rather than validated away. +// +// Deliberately does NOT import as a side effect, for the same reason the atlas +// path does not: changing where the table reads from and reloading it are +// separate decisions. The response carries the refreshed status so the panel can +// offer the import immediately. +async function setPath(req, res) { + try { + const value = String(req.body?.path ?? '').trim() + await clilocs.setClientPath(value, req.user?.id ?? null) + await activity.log({ req, action: 'shard.clilocs.path', detail: { path: value } }) + return res.json(await clilocs.status()) + } catch (err) { + log.error('setClilocPath', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +module.exports = { getStatus, importClilocs, setPath } diff --git a/server/src/router/v1/player/shard.controller.js b/server/src/router/v1/player/shard.controller.js index 1603000..9f7c5d3 100644 --- a/server/src/router/v1/player/shard.controller.js +++ b/server/src/router/v1/player/shard.controller.js @@ -10,6 +10,7 @@ const uoLinkClient = require('../../../utils/uoLinkClient') const shardLinks = require('../../../model/shardLinks/shardLinks.model') const shardState = require('../../../model/shardState/shardState.model') +const shardClilocs = require('../../../model/shardClilocs/shardClilocs.model') const settings = require('../../../model/settings/settings.model') const { salesForAccounts } = require('../../../utils/shardSales') const activity = require('../../../model/activity/activity.model') @@ -18,9 +19,62 @@ const log = require('../../../utils/logger')('player-shard') const SERIAL_RE = /^0x[0-9a-fA-F]+$/ +/** + * Resolve the cliloc ids on a profile into display names. + * + * Items on the wire carry a `LabelNumber`, not a name — `BridgeProfile.WriteItem` + * sends `cliloc` on every equipment entry and `name` only for the minority of + * items a player has renamed. Reward titles are the same shape: the shard sends + * a cliloc number as a string, which the sheet previously had to SKIP because it + * had no way to turn it into words. + * + * Resolution happens here rather than in the browser because the table is ~123k + * rows: shipping it to render a dozen names would dwarf the page, and the + * Android client consumes this same JSON and would otherwise need its own copy. + * + * A shard with no cliloc table configured resolves nothing and the sheet renders + * ids exactly as it did before — this is decoration, and it is applied in the + * same best-effort block as the guild/governor cross-links. + */ +async function resolveProfileClilocs(profile) { + const wanted = [] + + const equipment = Array.isArray(profile.equipment) ? profile.equipment : [] + for (const item of equipment) { + if (Number.isInteger(item?.cliloc)) wanted.push(item.cliloc) + } + + // Reward titles arrive as strings that may be either a literal ("Knight of + // Trinsic") or a cliloc number in string form. Only the numeric ones need us. + const reward = Array.isArray(profile.titles?.reward) ? profile.titles.reward : [] + const rewardNumbers = reward.map((r) => (/^\d+$/.test(String(r)) ? Number(r) : null)) + for (const n of rewardNumbers) if (n !== null) wanted.push(n) + + if (wanted.length === 0) return + + const names = await shardClilocs.resolveMany(wanted) + if (names.size === 0) return + + for (const item of equipment) { + // A player-given name always wins over the type name: an item called "Bob's + // lucky axe" should not be relabelled "hatchet". + if (item?.name) continue + const resolved = names.get(item?.cliloc) + if (resolved) item.clilocName = resolved + } + + if (rewardNumbers.some((n) => n !== null)) { + profile.titles.rewardResolved = reward.map((raw, i) => { + const n = rewardNumbers[i] + return n === null ? String(raw) : names.get(n) ?? null + }) + } +} + // Decorate a char.profile with cross-links from our own board data: the guild the -// character leads and any city governorship on its account. Best-effort — a -// failure here never fails the profile (it's a nicety, not the sheet). +// character leads and any city governorship on its account, plus resolved cliloc +// names. Best-effort — a failure here never fails the profile (it's a nicety, +// not the sheet). async function enrichCharProfile(profile) { if (!profile) return profile try { @@ -30,6 +84,7 @@ async function enrichCharProfile(profile) { const govs = await shardState.listGovernorshipsForAccounts([profile.acct]) if (govs.length) profile.governorOf = govs.map((g) => g.city) } + await resolveProfileClilocs(profile) } catch (err) { log.warn('enrichCharProfile failed', { serial: profile.serial, message: err.message }) } diff --git a/server/src/server.js b/server/src/server.js index ae06133..e2ecadd 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -15,6 +15,7 @@ const settings = require('./model/settings/settings.model') const revokedSessions = require('./model/revokedSessions/revokedSessions.model') const mobileAuthBridge = require('./model/mobileAuthBridge/mobileAuthBridge.model') const shardAtlas = require('./model/shardAtlas/shardAtlas.model') +const shardClilocs = require('./model/shardClilocs/shardClilocs.model') const createLogger = require('./utils/logger') const { evaluateBotInternalKey } = require('./utils/botInternalKey') const brand = require('./config/brand') @@ -89,6 +90,13 @@ async function start() { // REMOVE a facet is staged for admin approval instead of being applied. await shardAtlas.refreshOnBoot() + // Refresh the cliloc table (UO's id → display-string map) from the file the + // operator converted out of their own client. Same contract as the atlas: + // hash-gated so an unchanged file costs one read, and best-effort so a missing + // or wrong-format file never stops the site coming up — it just means item + // names render as ids, which is what they did before the table existed. + await shardClilocs.refreshOnBoot() + const mode = await settings.get('site_mode') log.info(`site mode: ${String(mode || 'live').toUpperCase()}`) diff --git a/server/src/utils/clilocParse.js b/server/src/utils/clilocParse.js new file mode 100644 index 0000000..cf1a113 --- /dev/null +++ b/server/src/utils/clilocParse.js @@ -0,0 +1,287 @@ +// Cliloc parsing — the pure half. +// +// A "cliloc" is UO's localization table: an integer id mapped to a display +// string. Items carry a `LabelNumber` rather than a name, so without this table +// the site can only render `id 1023721` where the game shows "quarter staff". +// The shard already sends the id on every equipment entry (`char.profile`'s +// `cliloc` field) and will send one per marketplace listing — the *number* was +// never the missing piece, the *table* was. +// +// This module is fs-free on purpose, exactly like `spawnAtlasParse.js`: the +// suite runs in CI where there is no UO client, so every parser here is driven +// from inline fixtures. `clilocSource.js` is the only thing that touches disk. +// +// ── Two input formats, and why ───────────────────────────────────────────── +// +// The client's own `Cliloc.enu` is COMPRESSED (Mythic format) on any modern +// client, and decompressing it is a bit-level port of an inverse-BWT coder that +// nothing in this stack needs at runtime. ServUO's own bundled `Ultima.StringList` +// cannot read it either — which is why `VendorSearch.GetItemName` is already inert +// on such a shard and the plugin could not supply names even if we asked it to. +// +// So the operator converts once, from their own client, and points the site at +// the result (see docs/website/CLILOCS.md). Two shapes are accepted because +// different tools produce different things: +// +// • PLAIN BINARY — the pre-compression cliloc layout: a 6-byte header, then +// records of {int32 number, byte flag, uint16 length, UTF-8 bytes}. +// • DELIMITED TEXT — `numbertext` per line, which is what the common +// GUI exports emit. Quoted CSV fields and a header row are tolerated. +// +// Nothing derived from the client is ever committed: the converted file lives at +// an operator-supplied path and is gitignored, the same rule the spawn atlas art +// map already follows. + +/** Raised for a file we can identify but deliberately refuse to guess at. */ +class ClilocFormatError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocFormatError' + this.code = code + } +} + +/** + * Bumped when this parser produces DIFFERENT data from an IDENTICAL source file. + * + * Stored beside the source hash so the boot path can tell "same file, but the + * parser moved on" from "same file, nothing to do". Without it a corrected parse + * would ship and never reach an install whose cliloc file never changes — the + * trap `spawnAtlasSource.PARSER_VERSION` documents. + */ +const PARSER_VERSION = 1 + +// The plain layout's header is `02 00 00 00 01 00` — a 4-byte version and a +// 2-byte language marker. Only the size matters for parsing; the values are +// checked to sniff the format, not to validate it. +const HEADER_BYTES = 6 +const RECORD_HEADER_BYTES = 7 // int32 number + byte flag + uint16 length + +// Every compressed cliloc file the client ships begins with a DWORD whose high +// byte is 0x8E (the XOR key UOFiddler calls `HeaderXorKey`, 0x8E2C9A3D). That is +// the single cheapest way to tell an operator they exported the wrong file — +// without it, the plain parser happily reads compressed bytes as ~19k records of +// negative ids and 60 KB "strings" before dying somewhere in the middle, and the +// resulting error names the wrong problem. +const MYTHIC_HIGH_BYTE = 0x8e + +/** True when `buffer` is a Mythic-compressed cliloc rather than the plain layout. */ +function isCompressedCliloc(buffer) { + return buffer.length >= 4 && buffer[3] === MYTHIC_HIGH_BYTE +} + +/** + * Parse the plain binary cliloc layout. + * + * Strict about truncation, and that strictness is load-bearing: a half-copied or + * partly-written file is the realistic failure here, and it must fail loudly + * rather than import a silently short table that then renders half the world as + * `id 1023721`. A record that runs past the end of the buffer throws. + */ +function parseClilocBinary(buffer) { + if (!Buffer.isBuffer(buffer)) throw new ClilocFormatError('Not a buffer', 'NOT_BUFFER') + if (isCompressedCliloc(buffer)) { + throw new ClilocFormatError( + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.', + 'COMPRESSED', + ) + } + if (buffer.length < HEADER_BYTES) { + throw new ClilocFormatError('File is shorter than a cliloc header', 'TRUNCATED') + } + + const entries = [] + let offset = HEADER_BYTES + + while (offset < buffer.length) { + if (offset + RECORD_HEADER_BYTES > buffer.length) { + throw new ClilocFormatError( + `Truncated record header at byte ${offset} (${entries.length} entries read)`, + 'TRUNCATED', + ) + } + const number = buffer.readInt32LE(offset) + const flag = buffer.readUInt8(offset + 4) + // The length is written by the client as an unsigned 16-bit value. Reading it + // signed (as ServUO's own SDK does) turns any string over 32 KB into a + // negative length; real tables top out around 12 KB, so this has no effect on + // current data and costs nothing to get right. + const length = buffer.readUInt16LE(offset + 5) + offset += RECORD_HEADER_BYTES + + if (offset + length > buffer.length) { + throw new ClilocFormatError( + `Truncated record body at byte ${offset} (${entries.length} entries read)`, + 'TRUNCATED', + ) + } + entries.push({ number, flag, text: buffer.toString('utf8', offset, offset + length) }) + offset += length + } + + return entries +} + +// A delimited line splits on the FIRST separator only: cliloc text is full of +// commas ("a scroll of magery, unfinished") and splitting on all of them would +// truncate every such entry at its first comma. +const TEXT_SEPARATORS = ['\t', ',', ';'] + +/** Unwrap one CSV field: strip surrounding quotes and unescape doubled quotes. */ +function unquote(value) { + const trimmed = value.trim() + if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) { + return trimmed.slice(1, -1).replace(/""/g, '"') + } + return trimmed +} + +/** + * Parse a delimited text export: `numbertext` per line. + * + * Tolerant by design — this is whatever an operator's GUI tool produced, not a + * format we control. A header row, blank lines, `#` comments and a trailing + * flags column are all ignored. A line whose first field is not an integer is + * skipped rather than fatal, because that is exactly what a header row is. + * + * The one thing it will NOT do is return an empty table quietly: a file that + * yields no entries at all is a wrong file, not an empty one. + */ +function parseClilocText(text) { + const entries = [] + for (const line of String(text).split(/\r?\n/)) { + // The line is deliberately NOT trimmed before the separator search. Roughly + // half of a real cliloc table is empty strings (unused ids), which export as + // `1005008` — and trimming eats that trailing separator, leaving a bare + // number that then looks like a header row and is skipped. That silently + // dropped 55,994 of 123,490 entries. Individual FIELDS are trimmed instead, + // by `unquote`. + if (line.trim() === '' || line.trimStart().startsWith('#')) continue + + // Pick the separator that actually appears first, so a tab-delimited line + // whose text contains a comma still splits on the tab. + let cut = -1 + for (const sep of TEXT_SEPARATORS) { + const at = line.indexOf(sep) + if (at !== -1 && (cut === -1 || at < cut)) cut = at + } + if (cut === -1) continue + + // An EMPTY first field must not become id 0: `Number('')` is 0, not NaN, so + // a line that merely starts with a separator would otherwise import as a + // bogus cliloc 0 instead of being skipped. + const head = unquote(line.slice(0, cut)) + if (head === '') continue + const number = Number(head) + if (!Number.isInteger(number)) continue // header row, or a wrapped line + + let rest = line.slice(cut + 1) + // Some exports carry `number,flag,text`. A bare integer in the second field + // is a flag; anything else is the text itself (and a text field that IS just + // a number is indistinguishable, so it stays as the text — the safer miss). + let flag = 0 + for (const sep of TEXT_SEPARATORS) { + const at = rest.indexOf(sep) + if (at === -1) continue + const head = unquote(rest.slice(0, at)) + if (/^\d{1,3}$/.test(head) && rest.slice(at + 1).trim() !== '') { + flag = Number(head) + rest = rest.slice(at + 1) + } + break + } + + entries.push({ number, flag, text: unquote(rest) }) + } + + if (entries.length === 0) { + throw new ClilocFormatError('No cliloc entries found in the text export', 'EMPTY') + } + return entries +} + +/** + * Parse either supported shape, sniffing which one this is. + * + * The sniff is on the binary header rather than the file extension: operators + * name these things whatever they like, and an `.enu` that is really a TSV (or a + * `.txt` that is really binary) should still import. + */ +function parseCliloc(buffer) { + const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer) + + if (isCompressedCliloc(buf)) { + throw new ClilocFormatError( + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.', + 'COMPRESSED', + ) + } + + // The plain layout always opens with version 2 / language 1. Anything else is + // treated as text, which is the recoverable guess: a mis-sniffed text file + // yields "no entries found", while a mis-sniffed binary yields nonsense. + if (buf.length >= HEADER_BYTES && buf.readInt32LE(0) === 2 && buf.readUInt16LE(4) === 1) { + return parseClilocBinary(buf) + } + return parseClilocText(buf.toString('utf8')) +} + +// ── Display ──────────────────────────────────────────────────────────────── + +// Cliloc strings interpolate arguments the client supplies out of an item's +// property list: `~1_val~`, `~2_NAME~`, `~1_ITEM~`. We never have those — the +// bridge sends the id, not the packet — so a name carrying them must be reduced +// to what is actually knowable rather than shown with the raw tokens in it. +const PLACEHOLDER_RE = /~\d+_[^~]*~/g + +/** + * Reduce a raw cliloc string to something displayable. + * + * Placeholders are dropped and the leftover punctuation tidied, so + * `"[~1_stuff~]"` becomes `""` (correctly nothing — the whole string was the + * argument) and `"cold damage ~1_val~%"` becomes `"cold damage"`. + * + * **Punctuation is only tidied when a placeholder was actually removed.** The + * trailing `%` above is the unit belonging to the number we never had, and the + * brackets in `[~1_stuff~]` only ever wrapped the argument — but a string with + * no placeholder has no such debris, and trimming it anyway corrupts real names. + * A shard's `"Runic Gateway Sigil (v2)"` came back as `"(v2"` while this was + * unconditional. + * + * Returns `''` when nothing survives, which callers treat as "no name" and fall + * back to the item id — better than showing a bracket. + */ +const DEBRIS = /^[\s\-–—,.;:%[\]()]+|[\s\-–—,.;:%[\]()]+$/g + +function displayText(raw) { + if (raw == null) return '' + const source = String(raw) + const hadPlaceholder = PLACEHOLDER_RE.test(source) + PLACEHOLDER_RE.lastIndex = 0 // the regex is global; `test` advances it + + if (!hadPlaceholder) return source.replace(/\s+/g, ' ').trim() + + return source + .replace(PLACEHOLDER_RE, ' ') + .replace(/\s+/g, ' ') + .replace(/\s+([,.;:!?])/g, '$1') + .replace(DEBRIS, '') + .trim() +} + +/** True when a raw cliloc string is nothing but interpolated arguments. */ +const isPlaceholderOnly = (raw) => raw != null && String(raw).trim() !== '' && displayText(raw) === '' + +module.exports = { + ClilocFormatError, + PARSER_VERSION, + HEADER_BYTES, + isCompressedCliloc, + parseCliloc, + parseClilocBinary, + parseClilocText, + displayText, + isPlaceholderOnly, +} diff --git a/server/src/utils/clilocSource.js b/server/src/utils/clilocSource.js new file mode 100644 index 0000000..93ae061 --- /dev/null +++ b/server/src/utils/clilocSource.js @@ -0,0 +1,316 @@ +// Cliloc table — the filesystem layer. +// +// `clilocParse.js` holds the pure parsers; this module is the only thing that +// touches cliloc files on disk, and it is shared by both callers: +// +// - the server, which refreshes the table on boot (`shardClilocs.model.js`) +// - the admin panel, which can force a reimport without a restart +// +// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived +// from them is committed: the repo holds no string table, exactly as it holds no +// map snapshot and no artwork. That rule is why this module reads a configured +// path instead of a path inside the repo. +// +// ── Why this reads a SET of files, not one ──────────────────────────────── +// +// Shards edit items and add new ones. Those carry cliloc ids that a stock client +// table does not have — and forcing a 5 MB client re-export every time an +// operator adds one item would be miserable enough that the table would simply +// go stale, which is the exact failure the spawn atlas was redesigned to avoid. +// +// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted +// client file) plus every operator-maintained OVERLAY beside it, all re-read on +// every boot and hash-gated as a SET. Adding, editing or removing any overlay +// counts as drift and re-imports. Later sources win, so an overlay both adds new +// ids and overrides stock ones. +// +// Measured on a real shard: the script tree references 16,434 cliloc ids and only +// 37 are absent from the stock client table. Tens of entries against a 67k base +// is what makes the overlay the right shape rather than a second full table. +// +// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so +// the boot path hashes first and only parses when something actually changed. + +const crypto = require('crypto') +const fs = require('fs') +const path = require('path') + +const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse') + +/** + * Filenames looked for as the BASE table when the configured path is a directory. + * + * Ordered by how specific they are: an explicitly converted file wins over + * something that merely sits in a client folder, so an operator who dropped a + * `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one + * they made rather than the one that will be rejected. + * + * Matching is case-insensitive against the real directory listing, because the + * client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where + * a hardcoded lowercase open would simply miss. + */ +const CANDIDATE_NAMES = [ + 'clilocs.tsv', + 'clilocs.csv', + 'clilocs.plain', + 'cliloc.plain', + 'cliloc.plain.enu', + 'cliloc.enu.plain', + 'clilocs.txt', + 'cliloc.enu', +] + +/** + * Where shard-specific additions and overrides live: a `custom/` directory + * beside the base table. + * + * ServUO has **no server-side convention** for custom clilocs — they live in the + * patched client file a shard distributes to its players, and nothing in the + * tree declares them. There is therefore nothing to discover, and this is the + * one place in the cliloc pipeline that is a convention we chose rather than one + * the shard already has. It is a directory rather than a single file so an + * operator can keep additions grouped however they like (per system, per patch) + * without the site caring. + */ +const CUSTOM_DIR = 'custom' +const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain'] + +class ClilocSourceError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocSourceError' + this.code = code + } +} + +function sha256(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex') +} + +/** + * Resolve the configured path to `{ root, base }`. + * + * Accepts either a direct file path or a directory to search, because operators + * reasonably supply both — "here is the file" and "here is the folder I put it + * in" are equally natural answers to the admin panel's prompt. When it is a + * file, `root` is the directory CONTAINING it, so overlays work either way: an + * operator who pointed at a file should not have to re-point at its folder just + * to add a `custom/` directory next to it. + */ +function resolveBase(configured) { + if (!configured || String(configured).trim() === '') { + throw new ClilocSourceError('No cliloc path configured', 'NO_PATH') + } + const target = String(configured).trim() + + let stat + try { + stat = fs.statSync(target) + } catch { + throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND') + } + + if (stat.isFile()) return { root: path.dirname(target), base: target } + + if (!stat.isDirectory()) { + throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND') + } + + let listing + try { + listing = fs.readdirSync(target) + } catch { + throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND') + } + + const byLower = new Map(listing.map((name) => [name.toLowerCase(), name])) + for (const candidate of CANDIDATE_NAMES) { + const actual = byLower.get(candidate) + if (actual) return { root: target, base: path.join(target, actual) } + } + + throw new ClilocSourceError( + `No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`, + 'NO_FILE', + ) +} + +/** Overlay files under `/custom/`, sorted so precedence is deterministic. */ +function listCustom(root) { + const dir = path.join(root, CUSTOM_DIR) + let listing + try { + listing = fs.readdirSync(dir, { withFileTypes: true }) + } catch (err) { + // No overlay directory is the normal case, not an error. + if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return [] + throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE') + } + return listing + .filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase())) + .map((e) => e.name) + .sort() + .map((name) => path.join(dir, name)) +} + +function readFileOrThrow(file) { + try { + return fs.readFileSync(file) + } catch { + throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE') + } +} + +/** + * Read every cliloc source under the configured path. + * + * Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }` + * with the base first and overlays after, in the order they must be merged. + * + * Labels are root-relative and forward-slashed so a hash map compares equal + * across platforms — the same directory read on Windows and Linux must produce + * the same fingerprint, or every boot would look like a change. (The same + * reasoning, and the same bug, as `spawnAtlasSource.readSources`.) + */ +function readSources(configured) { + const { root, base } = resolveBase(configured) + + const describe = (file, kind) => { + const buffer = readFileOrThrow(file) + return { + label: path.relative(root, file).split(path.sep).join('/'), + kind, + file, + buffer, + sha256: sha256(buffer), + bytes: buffer.length, + compressed: isCompressedCliloc(buffer), + } + } + + const files = [describe(base, 'base')] + for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom')) + + return { root, files } +} + +/** + * A fingerprint of every source: `{ "