diff --git a/.gitignore b/.gitignore index 622c8c6..9d33164 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,17 @@ logs/ # See docs/website/SPAWN_ATLAS.md and db/data/spawnAtlas.art.example.json. server/db/data/spawnAtlas.art.json +# Operator-supplied cliloc table. UO's localization strings are EA's, extracted +# from the operator's own client and converted once (docs/website/CLILOCS.md); +# the repo ships no string table, for the same reason it ships no artwork and no +# map snapshot. This covers the conventional in-repo location — the supported +# arrangement is a path OUTSIDE the repo, set from Admin → Shard. +server/db/data/cliloc* +server/db/data/clilocs.* +# The build output of tools/cliloc-export (a throwaway helper, not a package). +server/tools/cliloc-export/bin/ +server/tools/cliloc-export/obj/ + # reference material (extracted from the provided archives) _reference/ diff --git a/client/src/components/CharacterSheet.jsx b/client/src/components/CharacterSheet.jsx index 1a27376..58c8fb4 100644 --- a/client/src/components/CharacterSheet.jsx +++ b/client/src/components/CharacterSheet.jsx @@ -10,21 +10,42 @@ import ShardAccountActions from './ShardAccountActions.jsx' const RESIST_LABELS = { phys: 'Physical', fire: 'Fire', cold: 'Cold', pois: 'Poison', energy: 'Energy' } +// What to call an equipped item. +// +// Items on the wire carry a `LabelNumber`, not a name, so this used to be able +// to show nothing but the layer and `id 12345`. The server now resolves the +// cliloc against its own table and attaches `clilocName` (see +// docs/website/CLILOCS.md); a shard with no cliloc file configured sends none, +// and the layer fallback below is exactly what the sheet did before. +// +// A player-given `name` outranks the resolved type name — "Bob's lucky axe" +// should not be relabelled "hatchet" — and the server applies the same +// precedence, so this only re-states it for a profile that arrived with both. +const itemName = (it) => it.name || it.clilocName || it.layer || 'Item' + // The char.profile `titles` block (Protocol 2.0). fameKarma/skill are already // computed display strings; reward entries may be a cliloc NUMBER-as-string or a -// literal string. Without a cliloc table on the site we can only show literals, so -// numeric reward entries are skipped rather than shown as a raw number. Returns a -// de-duped list of human-readable title chips. +// literal string. +// +// `rewardResolved` is the server's parallel array with the numeric entries turned +// into words (null where the cliloc table had nothing, or is not configured at +// all). Prefer it, and keep the literal-only path as the fallback for a profile +// served before the cliloc table existed — a numeric entry with no resolution is +// still skipped rather than shown as a raw number. function displayTitles(titles) { if (!titles) return [] const out = [] if (titles.fameKarma) out.push(titles.fameKarma) if (titles.skill) out.push(titles.skill) - const reward = Array.isArray(titles.reward) ? titles.reward : [] + const raw = Array.isArray(titles.reward) ? titles.reward : [] + const resolved = Array.isArray(titles.rewardResolved) ? titles.rewardResolved : null + const reward = raw.map((r, i) => resolved?.[i] ?? (/^\d+$/.test(String(r)) ? null : String(r))) const sel = typeof titles.selected === 'number' ? titles.selected : -1 - // Prefer the selected reward title; fall back to the first literal one. - const candidate = sel >= 0 && sel < reward.length ? reward[sel] : reward.find((r) => r && !/^\d+$/.test(String(r))) - if (candidate && !/^\d+$/.test(String(candidate))) out.push(String(candidate)) + // Prefer the selected reward title; fall back to the first one that resolved. + // The `??` matters: a selected title whose cliloc did not resolve must fall + // through to the fallback rather than suppress the chip entirely. + const candidate = (sel >= 0 && sel < reward.length ? reward[sel] : null) ?? reward.find(Boolean) + if (candidate) out.push(String(candidate)) return [...new Set(out.filter(Boolean))] } @@ -241,12 +262,18 @@ export default function CharacterSheet({ char, moderation = false }) {
Equipment
- {equipment.map((it) => ( + {equipment.map((it) => { + const label = itemName(it) + const layer = it.layer || 'Item' + // The layer only earns its own line once the headline is a real + // name; when it IS the headline, repeating it is just noise. + const detail = [label === layer ? null : layer, `id ${it.itemId}`, it.hue ? `hue ${it.hue}` : null] + return (
-
{it.layer || 'Item'}
-
id {it.itemId}{it.hue ? ` · hue ${it.hue}` : ''}
+
{label}
+
{detail.filter(Boolean).join(' · ')}
{it.mods && Object.keys(it.mods).length > 0 && (
@@ -256,7 +283,8 @@ export default function CharacterSheet({ char, moderation = false }) {
)}
- ))} + ) + })}
)} diff --git a/server/db/schema.sql b/server/db/schema.sql index 8241434..1c5df9f 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -1180,6 +1180,39 @@ CREATE TABLE IF NOT EXISTS shard_champion_spawns ( INDEX idx_shard_champion_spawns_facet (facet) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- UO's localization table: cliloc id -> display string. Items carry a +-- `LabelNumber` rather than a name, so without this the site can only render +-- `id 1023721` where the game shows "quarter staff". The shard has always sent +-- the id (char.profile's `cliloc`, and one per marketplace listing) — the number +-- was never the missing piece, the table was. +-- +-- Sourced from a file the OPERATOR converts once from their own UO client and +-- points the site at (docs/website/CLILOCS.md); nothing derived from the client +-- is committed, the same rule the spawn atlas and the creature art map follow. +-- A shard with no cliloc file configured simply renders item ids, which is what +-- it did before this table existed. +-- +-- `text` is TEXT, not VARCHAR: real tables top out around 12 KB for the long +-- property descriptions, and truncating them silently would be worse than +-- storing them. Item NAMES are all short — the index that matters for search is +-- on the denormalized `shard_vendor_items.display_name`, not here. +CREATE TABLE IF NOT EXISTS shard_clilocs ( + number INT NOT NULL PRIMARY KEY, + flag SMALLINT NOT NULL DEFAULT 0, + text TEXT NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Singleton (id = 1) describing the cliloc table currently loaded: the source +-- file, its sha256, the entry count and the parser version. The boot path +-- compares the stored hash against the file on disk and skips the parse when +-- they match, which is every restart that did not follow a client patch. +CREATE TABLE IF NOT EXISTS shard_cliloc_meta ( + id TINYINT NOT NULL PRIMARY KEY DEFAULT 1, + payload JSON NOT NULL, + imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_cliloc_meta_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- Singleton (id = 1) describing the artifact currently loaded: when it was -- built, its counts, and a sha256 per ServUO source file. The admin drift check -- compares this against db/data/spawnAtlas.meta.json to report when the database diff --git a/server/routes.guards.json b/server/routes.guards.json index 23fbad8..ccd273e 100644 --- a/server/routes.guards.json +++ b/server/routes.guards.json @@ -727,6 +727,37 @@ "validate" ] }, + { + "method": "GET", + "path": "/api/v1/admin/shard/clilocs", + "handlers": 2, + "gates": [ + "noindex", + "requireAuth" + ] + }, + { + "method": "POST", + "path": "/api/v1/admin/shard/clilocs/import", + "handlers": 4, + "gates": [ + "noindex", + "requireAuth", + "middleware", + "validate" + ] + }, + { + "method": "PUT", + "path": "/api/v1/admin/shard/clilocs/path", + "handlers": 4, + "gates": [ + "noindex", + "requireAuth", + "middleware", + "validate" + ] + }, { "method": "GET", "path": "/api/v1/admin/shard/houses", diff --git a/server/routes.manifest.json b/server/routes.manifest.json index f1e55e7..6c6544b 100644 --- a/server/routes.manifest.json +++ b/server/routes.manifest.json @@ -293,6 +293,18 @@ "method": "GET", "path": "/api/v1/admin/shard/char/:serial" }, + { + "method": "GET", + "path": "/api/v1/admin/shard/clilocs" + }, + { + "method": "POST", + "path": "/api/v1/admin/shard/clilocs/import" + }, + { + "method": "PUT", + "path": "/api/v1/admin/shard/clilocs/path" + }, { "method": "GET", "path": "/api/v1/admin/shard/houses" diff --git a/server/src/model/shardClilocs/shardClilocs.db.js b/server/src/model/shardClilocs/shardClilocs.db.js new file mode 100644 index 0000000..530ec4b --- /dev/null +++ b/server/src/model/shardClilocs/shardClilocs.db.js @@ -0,0 +1,106 @@ +const { pool, query } = require('../../utils/db') + +// Raw SQL for the cliloc table. `shard_clilocs` is IMPORT-OWNED: `replaceAll` +// empties and refills it inside one transaction, and nothing else in the +// codebase writes to it. No foreign keys, consistent with every other shard_* +// table. + +const BATCH = 1000 + +/** + * Replace the entire cliloc table in one transaction. + * + * All-or-nothing on purpose: a failed reload must leave the previous table + * intact rather than a half-loaded one, because a partially-imported cliloc + * table is indistinguishable from a complete one to anyone reading it — you + * would just see some items named and some not, which is also what "no table at + * all" looks like. + * + * `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly + * commits, which would defeat exactly that guarantee. (The same trap the spawn + * atlas import documents; at ~123k rows `DELETE` is still well under a second.) + */ +async function replaceAll(entries, meta) { + const conn = await pool.getConnection() + try { + await conn.beginTransaction() + await conn.query('DELETE FROM shard_clilocs') + + // Blank entries are dropped rather than stored. Roughly HALF of a real + // cliloc table is empty strings — ids the client reserves and never uses — + // and a row that resolves to no name is indistinguishable from no row at + // all to every caller. Dropping them halves the table (123,490 → ~67,500) + // and, more importantly, makes the binary and text imports converge on + // identical content: the binary format carries the blanks explicitly and a + // text export may or may not, depending on the tool. + // + // Later duplicates win. The plain format permits a repeated id and the + // client's own loader resolves it the same way (its dictionary assignment + // overwrites), so collapsing here keeps the batch insert from failing on a + // primary-key collision for a file the game itself would load. + const byNumber = new Map() + let blank = 0 + for (const entry of entries) { + if (!Number.isInteger(entry.number)) continue + if (String(entry.text ?? '').trim() === '') { + blank++ + continue + } + byNumber.set(entry.number, entry) + } + + const rows = [...byNumber.values()].map((e) => [e.number, e.flag ?? 0, e.text]) + for (let i = 0; i < rows.length; i += BATCH) { + await conn.batch('INSERT INTO shard_clilocs (number, flag, text) VALUES (?,?,?)', rows.slice(i, i + BATCH)) + } + + await conn.query( + 'INSERT INTO shard_cliloc_meta (id, payload) VALUES (1, ?) ' + + 'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP', + [JSON.stringify({ ...meta, count: rows.length })], + ) + + await conn.commit() + return { count: rows.length, blank, duplicates: entries.length - blank - rows.length } + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +async function getMeta() { + const rows = await query('SELECT payload, imported_at FROM shard_cliloc_meta WHERE id = 1') + if (rows.length === 0) return null + const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload + return { ...payload, importedAt: rows[0].imported_at } +} + +/** + * Look up a batch of ids. + * + * Batched rather than one-at-a-time because every caller has a LIST: a character + * sheet resolves a dozen equipment ids at once, and a page of marketplace + * listings resolves fifty. `IN (...)` with generated placeholders keeps it one + * round trip and one parameterized statement. + */ +async function lookup(numbers) { + if (!Array.isArray(numbers) || numbers.length === 0) return [] + const ids = [...new Set(numbers.filter((n) => Number.isInteger(n)))] + if (ids.length === 0) return [] + const placeholders = ids.map(() => '?').join(',') + return query(`SELECT number, text FROM shard_clilocs WHERE number IN (${placeholders})`, ids) +} + +async function count() { + const rows = await query('SELECT COUNT(*) AS n FROM shard_clilocs') + return Number(rows[0]?.n) || 0 +} + +module.exports = { + replaceAll, + getMeta, + lookup, + count, +} diff --git a/server/src/model/shardClilocs/shardClilocs.model.js b/server/src/model/shardClilocs/shardClilocs.model.js new file mode 100644 index 0000000..8b72efa --- /dev/null +++ b/server/src/model/shardClilocs/shardClilocs.model.js @@ -0,0 +1,307 @@ +const db = require('./shardClilocs.db') +const settings = require('../settings/settings.model') +const { displayText } = require('../../utils/clilocParse') +const { + ClilocFormatError, + ClilocSourceError, + PARSER_VERSION, + hashSource, + readCliloc, +} = require('../../utils/clilocSource') +const log = require('../../utils/logger')('shardClilocs') + +// The cliloc table — UO's id → display-string map, refreshed from a file the +// operator converts once from their own client. +// +// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a +// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace +// listing sends one too. Without the table the UI can only print `id 1023721` +// where the game prints "quarter staff". +// +// Two rules govern the boot path, both inherited from the spawn atlas: +// +// 1. **It never blocks startup.** No configured path, an unreadable file, a +// wrong-format file, a database error — all caught and logged. The site +// comes up either way, serving whatever table it already had (or none, in +// which case the UI falls back to item ids exactly as it did before). +// 2. **Nothing client-derived is committed.** The table is built from the +// operator's own file at a configured path. The repo ships no strings. +// +// Unlike the atlas there is no staged-approval flow, and the difference is +// deliberate: the atlas stages a refresh that would REMOVE a facet because a +// half-copied tree and a real map change look identical from here. A cliloc file +// is a single file with a single hash, and the realistic corruption — a partial +// copy — makes the parser fail on a truncated record rather than yield a +// plausible-but-short table. The failure mode the atlas has to guess about is +// one this parser can simply detect. + +const SETTING_KEY = 'cliloc_client_path' + +/** + * Where the converted cliloc file lives. + * + * The admin setting wins over the environment so an operator can repoint it + * without a redeploy, matching how the rest of the shard integration is + * admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the + * deploy-time default, since the path usually describes a mount the deployment + * sets up. + */ +async function getClientPath() { + try { + const configured = await settings.get(SETTING_KEY) + if (configured && String(configured).trim() !== '') return String(configured).trim() + } catch { + // Settings unavailable is not fatal — fall through to the env default. + } + const fromEnv = process.env.UO_CLIENT_PATH + return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : '' +} + +async function setClientPath(value, updatedBy = null) { + const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy) + invalidate() + return result +} + +// ── Refresh ──────────────────────────────────────────────────────────────── + +/** Was the loaded table built by THIS parser? */ +const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION + +/** + * Refresh the cliloc table from the configured file. + * + * Returns a result describing what happened rather than throwing, so the caller + * — including the boot path — can log it and move on: + * + * `skipped` no path configured + * `unavailable` path configured but missing / unreadable / not a cliloc file + * `unchanged` source hash matches the loaded table; nothing parsed + * `imported` parsed and applied + * `failed` parsed or applied and something went wrong + * + * `force` skips the hash check (an admin asking for a reimport). + */ +async function refresh({ force = false, path: pathOverride = '' } = {}) { + // An explicit override wins outright — a one-off "use this file", which must + // not be silently overruled by the configured path the way an env default is. + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() + if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' } + + let fingerprint + try { + fingerprint = hashSource(configured) + } catch (err) { + if (err instanceof ClilocSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: configured } + } + return { status: 'failed', reason: err.message, path: configured } + } + + const meta = await db.getMeta().catch(() => null) + + // Two things make a loaded table stale: the file changed, or the PARSER did. + // Only checking the file would strand an install whose client never patches on + // whatever an older build derived. + if (!force && meta?.sha256 === fingerprint.sha256 && currentParser(meta)) { + return { status: 'unchanged', path: configured, file: fingerprint.file, count: meta.count ?? null } + } + + let parsed + try { + parsed = readCliloc(configured) + } catch (err) { + if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: configured } + } + return { status: 'failed', reason: err.message, path: configured } + } + + try { + const applied = await db.replaceAll(parsed.entries, { ...parsed.source, mtime: fingerprint.mtime }) + invalidate() + return { + status: 'imported', + path: configured, + file: parsed.source.file, + count: applied.count, + parsed: parsed.entries.length, + blank: applied.blank, + duplicates: applied.duplicates, + } + } catch (err) { + return { status: 'failed', reason: err.message, path: configured } + } +} + +/** + * Boot hook. Best-effort by contract: it logs and returns, never throws, so a + * missing or malformed cliloc file can never stop the site coming up. + */ +async function refreshOnBoot() { + try { + const result = await refresh() + switch (result.status) { + case 'imported': + log.info('cliloc table refreshed', { file: result.file, count: result.count }) + break + case 'unavailable': + // Deliberately a warning, not an error: an operator who has not supplied + // a cliloc file is in a supported state (the UI shows item ids), and the + // most common cause — pointing at the client's own compressed file — + // needs the reason spelled out rather than a stack trace. + log.warn('cliloc source unavailable (item names will show as ids)', { + reason: result.reason, + code: result.code, + path: result.path, + }) + break + case 'failed': + log.warn('cliloc refresh failed', { reason: result.reason }) + break + default: + break + } + return result + } catch (err) { + log.warn('cliloc refresh errored', { error: err.message }) + return { status: 'failed', reason: err.message } + } +} + +/** Everything the admin panel needs to describe cliloc state. */ +async function status({ path: pathOverride = '' } = {}) { + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() + const meta = await db.getMeta().catch(() => null) + const loaded = await db.count().catch(() => 0) + + let fileReadable = false + let file = null + let drift = null + let problem = null + let code = null + if (configured !== '') { + try { + const fingerprint = hashSource(configured) + fileReadable = true + file = fingerprint.file + // A compressed file is readable but not importable, and the panel has to + // say so HERE — otherwise pointing at an unconverted client directory + // reports a healthy file with pending drift ("ready to import") and the + // operator only finds out when the import fails. `drift` stays null + // because comparing hashes with an unusable file answers nothing. + if (fingerprint.compressed) { + problem = + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.' + code = 'COMPRESSED' + } else { + drift = meta?.sha256 !== fingerprint.sha256 || !currentParser(meta) + } + } catch (err) { + fileReadable = false + problem = err.message + code = err.code ?? null + } + } + + return { + configured: configured !== '', + path: configured, + file, + fileReadable, + problem, + code, + drift, + count: loaded, + importedAt: meta?.importedAt ?? null, + sourceBytes: meta?.bytes ?? null, + } +} + +// ── Lookup ───────────────────────────────────────────────────────────────── +// +// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is +// ~123k rows and shipping it to a client would dwarf every page that uses it, +// and the Android app consumes the same JSON and would otherwise need its own +// copy. Callers get names, not ids-plus-a-table. + +// A small write-through cache in front of the table. Item ids repeat heavily — +// one page of listings is mostly the same few hundred clilocs, and a character +// sheet re-resolves the same gear on every view — so this turns the steady state +// into zero queries. Capped so a pathological caller cannot grow it without +// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry, +// which is cheap and correct for a table that only changes on reimport. +const CACHE_MAX = 20000 +let cache = new Map() + +function invalidate() { + cache = new Map() +} + +/** + * Resolve a batch of cliloc ids to display strings. + * + * Returns a `Map` holding only the ids that resolved to + * something displayable — an id with no row, or one whose text is nothing but + * interpolated arguments we do not have, is simply absent. Callers fall back to + * whatever they had (the item id), so "missing" and "unnamed" collapse into one + * branch at the call site. + * + * Never throws: a cliloc lookup is decoration on someone's character sheet, and + * a database blip must not fail the sheet. + */ +async function resolveMany(numbers) { + const out = new Map() + if (!Array.isArray(numbers)) return out + + const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))] + if (wanted.length === 0) return out + + const missing = [] + for (const number of wanted) { + if (cache.has(number)) { + const hit = cache.get(number) + if (hit !== '') out.set(number, hit) + } else { + missing.push(number) + } + } + + if (missing.length > 0) { + try { + const rows = await db.lookup(missing) + const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)])) + if (cache.size + missing.length > CACHE_MAX) invalidate() + for (const number of missing) { + // Cache the miss too ('' meaning "no usable name"), so an id absent from + // the table does not re-query on every page view. + const text = found.get(number) ?? '' + cache.set(number, text) + if (text !== '') out.set(number, text) + } + } catch (err) { + log.warn('cliloc lookup failed', { message: err.message }) + } + } + + return out +} + +/** Single-id convenience. Returns `null` when there is no usable name. */ +async function resolve(number) { + const found = await resolveMany([number]) + return found.get(number) ?? null +} + +module.exports = { + SETTING_KEY, + getClientPath, + setClientPath, + refresh, + refreshOnBoot, + status, + resolveMany, + resolve, + invalidate, +} diff --git a/server/src/router/v1/admin/shard.router.js b/server/src/router/v1/admin/shard.router.js index ed03ff7..4da5d7f 100644 --- a/server/src/router/v1/admin/shard.router.js +++ b/server/src/router/v1/admin/shard.router.js @@ -25,6 +25,7 @@ const { body, param } = require('express-validator') const shardOps = require('./shardOps.controller') const shardVisibility = require('./shardVisibility.controller') const shardAtlas = require('./shardAtlas.controller') +const shardClilocs = require('./shardClilocs.controller') const selfShard = require('../player/shard.controller') const { requireRole } = require('../../../utils/auth') const validate = require('../../../middleware/validate') @@ -304,6 +305,54 @@ shardRouter.put( shardAtlas.setPath, ) +// ── Cliloc table (admin only) ───────────────────────────────────────────── +// UO's id → display-string map, converted once by the operator from their own +// client (docs/website/CLILOCS.md). Sits beside the atlas for the same reason: +// it is static content derived from operator-supplied files rather than anything +// the sidecar sends, and operating it is shard administration. +// +// There is deliberately NO public counterpart. The table is never served as a +// table — 123k rows would dwarf any page that used it, and the Android client +// consumes the same already-resolved JSON. Names are applied server-side to the +// responses that need them. +shardRouter.get( + '/clilocs', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Cliloc table status: path, drift, entry count (admin only)' + // #swagger.description = 'Where the converted cliloc file is, whether it can be read, how many entries are loaded, and whether the file on disk has drifted from them. A shard with no cliloc file configured is a supported state — item names simply render as ids.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.responses[200] = { description: 'Cliloc status', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocStatus" } } } } */ + /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + adminOnly, + shardClilocs.getStatus, +) +shardRouter.post( + '/clilocs/import', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Re-import the cliloc table from the converted file (admin only)' + // #swagger.description = 'Applies a client patch without a restart. `force` reimports even when the source hash matches what is loaded. A missing file — or the common mistake of pointing at the client\'s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the file is unchanged." } } } } } } */ + /* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocRefreshResult" } } } } */ + adminOnly, + body('force').optional().isBoolean(), + validate, + shardClilocs.importClilocs, +) +shardRouter.put( + '/clilocs/path', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Set the cliloc file the site reads from (admin only)' + // #swagger.description = 'Accepts either the converted file itself or a directory to search. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." } } } } } } */ + /* #swagger.responses[200] = { description: 'Cliloc status after the change', content: { "application/json": { schema: { $ref: "#/components/schemas/ClilocStatus" } } } } */ + adminOnly, + body('path').isString().isLength({ max: 512 }), + validate, + shardClilocs.setPath, +) + // ── Feature visibility (admin only) ─────────────────────────────────── // Who can see which shard surface, and which sensitive fields within it. This // decides what ANONYMOUS visitors get, so it sits above the moderator tier. diff --git a/server/src/router/v1/admin/shardClilocs.controller.js b/server/src/router/v1/admin/shardClilocs.controller.js new file mode 100644 index 0000000..6004eae --- /dev/null +++ b/server/src/router/v1/admin/shardClilocs.controller.js @@ -0,0 +1,78 @@ +// ── Admin · Cliloc table ─────────────────────────────────────────────────── +// +// Operating the cliloc import: where the converted cliloc file is, whether it +// has drifted from what is loaded, and a forced reimport after a client patch +// (docs/website/CLILOCS.md). +// +// The policy lives in the model. This controller does three things and no more: +// it validates input, it maps a refresh RESULT onto an HTTP status, and it +// records the action in the admin activity log. +// +// **A refresh result is not an exception.** `shardClilocs.refresh()` reports +// `unavailable` / `failed` rather than throwing, because the boot path must never +// be stopped by a bad file. That contract is preserved here: a missing file, or +// the single most likely operator mistake — pointing at the client's own +// COMPRESSED `Cliloc.enu` — is a 200 carrying `status: 'unavailable'` and the +// reason, not a 500. A 500 would say only "something broke"; the operator needs +// to be told which file to convert. + +const clilocs = require('../../../model/shardClilocs/shardClilocs.model') +const activity = require('../../../model/activity/activity.model') + +const log = require('../../../utils/logger')('admin-shard-clilocs') + +// GET /admin/shard/clilocs — what is loaded, what the file looks like, whether +// they disagree. There is no public counterpart: the cliloc table is never +// served as a table, only applied to names the site already returns. +async function getStatus(req, res) { + try { + return res.json(await clilocs.status()) + } catch (err) { + log.error('getStatus', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +// POST /admin/shard/clilocs/import — reload after a client patch without a +// restart. `force` reimports even when the source hash matches what is loaded +// (the escape hatch for "the database is wrong but the file is not"). +async function importClilocs(req, res) { + try { + const force = !!req.body?.force + const result = await clilocs.refresh({ force }) + await activity.log({ + req, + action: 'shard.clilocs.import', + detail: { force, status: result.status, count: result.count ?? null }, + }) + return res.json(result) + } catch (err) { + log.error('importClilocs', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +// PUT /admin/shard/clilocs/path — point the site at a different cliloc file. +// +// Persisted as a setting, which wins over the UO_CLIENT_PATH env default so an +// operator can move the mount without a redeploy. Blank clears it, which turns +// resolution off (boot skips, the loaded table keeps serving) — a legitimate +// thing to want, so it is allowed rather than validated away. +// +// Deliberately does NOT import as a side effect, for the same reason the atlas +// path does not: changing where the table reads from and reloading it are +// separate decisions. The response carries the refreshed status so the panel can +// offer the import immediately. +async function setPath(req, res) { + try { + const value = String(req.body?.path ?? '').trim() + await clilocs.setClientPath(value, req.user?.id ?? null) + await activity.log({ req, action: 'shard.clilocs.path', detail: { path: value } }) + return res.json(await clilocs.status()) + } catch (err) { + log.error('setClilocPath', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +module.exports = { getStatus, importClilocs, setPath } diff --git a/server/src/router/v1/player/shard.controller.js b/server/src/router/v1/player/shard.controller.js index 1603000..9f7c5d3 100644 --- a/server/src/router/v1/player/shard.controller.js +++ b/server/src/router/v1/player/shard.controller.js @@ -10,6 +10,7 @@ const uoLinkClient = require('../../../utils/uoLinkClient') const shardLinks = require('../../../model/shardLinks/shardLinks.model') const shardState = require('../../../model/shardState/shardState.model') +const shardClilocs = require('../../../model/shardClilocs/shardClilocs.model') const settings = require('../../../model/settings/settings.model') const { salesForAccounts } = require('../../../utils/shardSales') const activity = require('../../../model/activity/activity.model') @@ -18,9 +19,62 @@ const log = require('../../../utils/logger')('player-shard') const SERIAL_RE = /^0x[0-9a-fA-F]+$/ +/** + * Resolve the cliloc ids on a profile into display names. + * + * Items on the wire carry a `LabelNumber`, not a name — `BridgeProfile.WriteItem` + * sends `cliloc` on every equipment entry and `name` only for the minority of + * items a player has renamed. Reward titles are the same shape: the shard sends + * a cliloc number as a string, which the sheet previously had to SKIP because it + * had no way to turn it into words. + * + * Resolution happens here rather than in the browser because the table is ~123k + * rows: shipping it to render a dozen names would dwarf the page, and the + * Android client consumes this same JSON and would otherwise need its own copy. + * + * A shard with no cliloc table configured resolves nothing and the sheet renders + * ids exactly as it did before — this is decoration, and it is applied in the + * same best-effort block as the guild/governor cross-links. + */ +async function resolveProfileClilocs(profile) { + const wanted = [] + + const equipment = Array.isArray(profile.equipment) ? profile.equipment : [] + for (const item of equipment) { + if (Number.isInteger(item?.cliloc)) wanted.push(item.cliloc) + } + + // Reward titles arrive as strings that may be either a literal ("Knight of + // Trinsic") or a cliloc number in string form. Only the numeric ones need us. + const reward = Array.isArray(profile.titles?.reward) ? profile.titles.reward : [] + const rewardNumbers = reward.map((r) => (/^\d+$/.test(String(r)) ? Number(r) : null)) + for (const n of rewardNumbers) if (n !== null) wanted.push(n) + + if (wanted.length === 0) return + + const names = await shardClilocs.resolveMany(wanted) + if (names.size === 0) return + + for (const item of equipment) { + // A player-given name always wins over the type name: an item called "Bob's + // lucky axe" should not be relabelled "hatchet". + if (item?.name) continue + const resolved = names.get(item?.cliloc) + if (resolved) item.clilocName = resolved + } + + if (rewardNumbers.some((n) => n !== null)) { + profile.titles.rewardResolved = reward.map((raw, i) => { + const n = rewardNumbers[i] + return n === null ? String(raw) : names.get(n) ?? null + }) + } +} + // Decorate a char.profile with cross-links from our own board data: the guild the -// character leads and any city governorship on its account. Best-effort — a -// failure here never fails the profile (it's a nicety, not the sheet). +// character leads and any city governorship on its account, plus resolved cliloc +// names. Best-effort — a failure here never fails the profile (it's a nicety, +// not the sheet). async function enrichCharProfile(profile) { if (!profile) return profile try { @@ -30,6 +84,7 @@ async function enrichCharProfile(profile) { const govs = await shardState.listGovernorshipsForAccounts([profile.acct]) if (govs.length) profile.governorOf = govs.map((g) => g.city) } + await resolveProfileClilocs(profile) } catch (err) { log.warn('enrichCharProfile failed', { serial: profile.serial, message: err.message }) } diff --git a/server/src/server.js b/server/src/server.js index ae06133..e2ecadd 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -15,6 +15,7 @@ const settings = require('./model/settings/settings.model') const revokedSessions = require('./model/revokedSessions/revokedSessions.model') const mobileAuthBridge = require('./model/mobileAuthBridge/mobileAuthBridge.model') const shardAtlas = require('./model/shardAtlas/shardAtlas.model') +const shardClilocs = require('./model/shardClilocs/shardClilocs.model') const createLogger = require('./utils/logger') const { evaluateBotInternalKey } = require('./utils/botInternalKey') const brand = require('./config/brand') @@ -89,6 +90,13 @@ async function start() { // REMOVE a facet is staged for admin approval instead of being applied. await shardAtlas.refreshOnBoot() + // Refresh the cliloc table (UO's id → display-string map) from the file the + // operator converted out of their own client. Same contract as the atlas: + // hash-gated so an unchanged file costs one read, and best-effort so a missing + // or wrong-format file never stops the site coming up — it just means item + // names render as ids, which is what they did before the table existed. + await shardClilocs.refreshOnBoot() + const mode = await settings.get('site_mode') log.info(`site mode: ${String(mode || 'live').toUpperCase()}`) diff --git a/server/src/utils/clilocParse.js b/server/src/utils/clilocParse.js new file mode 100644 index 0000000..37a1189 --- /dev/null +++ b/server/src/utils/clilocParse.js @@ -0,0 +1,283 @@ +// Cliloc parsing — the pure half. +// +// A "cliloc" is UO's localization table: an integer id mapped to a display +// string. Items carry a `LabelNumber` rather than a name, so without this table +// the site can only render `id 1023721` where the game shows "quarter staff". +// The shard already sends the id on every equipment entry (`char.profile`'s +// `cliloc` field) and will send one per marketplace listing — the *number* was +// never the missing piece, the *table* was. +// +// This module is fs-free on purpose, exactly like `spawnAtlasParse.js`: the +// suite runs in CI where there is no UO client, so every parser here is driven +// from inline fixtures. `clilocSource.js` is the only thing that touches disk. +// +// ── Two input formats, and why ───────────────────────────────────────────── +// +// The client's own `Cliloc.enu` is COMPRESSED (Mythic format) on any modern +// client, and decompressing it is a bit-level port of an inverse-BWT coder that +// nothing in this stack needs at runtime. ServUO's own bundled `Ultima.StringList` +// cannot read it either — which is why `VendorSearch.GetItemName` is already inert +// on such a shard and the plugin could not supply names even if we asked it to. +// +// So the operator converts once, from their own client, and points the site at +// the result (see docs/website/CLILOCS.md). Two shapes are accepted because +// different tools produce different things: +// +// • PLAIN BINARY — the pre-compression cliloc layout: a 6-byte header, then +// records of {int32 number, byte flag, uint16 length, UTF-8 bytes}. +// • DELIMITED TEXT — `numbertext` per line, which is what the common +// GUI exports emit. Quoted CSV fields and a header row are tolerated. +// +// Nothing derived from the client is ever committed: the converted file lives at +// an operator-supplied path and is gitignored, the same rule the spawn atlas art +// map already follows. + +/** Raised for a file we can identify but deliberately refuse to guess at. */ +class ClilocFormatError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocFormatError' + this.code = code + } +} + +/** + * Bumped when this parser produces DIFFERENT data from an IDENTICAL source file. + * + * Stored beside the source hash so the boot path can tell "same file, but the + * parser moved on" from "same file, nothing to do". Without it a corrected parse + * would ship and never reach an install whose cliloc file never changes — the + * trap `spawnAtlasSource.PARSER_VERSION` documents. + */ +const PARSER_VERSION = 1 + +// The plain layout's header is `02 00 00 00 01 00` — a 4-byte version and a +// 2-byte language marker. Only the size matters for parsing; the values are +// checked to sniff the format, not to validate it. +const HEADER_BYTES = 6 +const RECORD_HEADER_BYTES = 7 // int32 number + byte flag + uint16 length + +// Every compressed cliloc file the client ships begins with a DWORD whose high +// byte is 0x8E (the XOR key UOFiddler calls `HeaderXorKey`, 0x8E2C9A3D). That is +// the single cheapest way to tell an operator they exported the wrong file — +// without it, the plain parser happily reads compressed bytes as ~19k records of +// negative ids and 60 KB "strings" before dying somewhere in the middle, and the +// resulting error names the wrong problem. +const MYTHIC_HIGH_BYTE = 0x8e + +/** True when `buffer` is a Mythic-compressed cliloc rather than the plain layout. */ +function isCompressedCliloc(buffer) { + return buffer.length >= 4 && buffer[3] === MYTHIC_HIGH_BYTE +} + +/** + * Parse the plain binary cliloc layout. + * + * Strict about truncation, and that strictness is load-bearing: a half-copied or + * partly-written file is the realistic failure here, and it must fail loudly + * rather than import a silently short table that then renders half the world as + * `id 1023721`. A record that runs past the end of the buffer throws. + */ +function parseClilocBinary(buffer) { + if (!Buffer.isBuffer(buffer)) throw new ClilocFormatError('Not a buffer', 'NOT_BUFFER') + if (isCompressedCliloc(buffer)) { + throw new ClilocFormatError( + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.', + 'COMPRESSED', + ) + } + if (buffer.length < HEADER_BYTES) { + throw new ClilocFormatError('File is shorter than a cliloc header', 'TRUNCATED') + } + + const entries = [] + let offset = HEADER_BYTES + + while (offset < buffer.length) { + if (offset + RECORD_HEADER_BYTES > buffer.length) { + throw new ClilocFormatError( + `Truncated record header at byte ${offset} (${entries.length} entries read)`, + 'TRUNCATED', + ) + } + const number = buffer.readInt32LE(offset) + const flag = buffer.readUInt8(offset + 4) + // The length is written by the client as an unsigned 16-bit value. Reading it + // signed (as ServUO's own SDK does) turns any string over 32 KB into a + // negative length; real tables top out around 12 KB, so this has no effect on + // current data and costs nothing to get right. + const length = buffer.readUInt16LE(offset + 5) + offset += RECORD_HEADER_BYTES + + if (offset + length > buffer.length) { + throw new ClilocFormatError( + `Truncated record body at byte ${offset} (${entries.length} entries read)`, + 'TRUNCATED', + ) + } + entries.push({ number, flag, text: buffer.toString('utf8', offset, offset + length) }) + offset += length + } + + return entries +} + +// A delimited line splits on the FIRST separator only: cliloc text is full of +// commas ("a scroll of magery, unfinished") and splitting on all of them would +// truncate every such entry at its first comma. +const TEXT_SEPARATORS = ['\t', ',', ';'] + +/** Unwrap one CSV field: strip surrounding quotes and unescape doubled quotes. */ +function unquote(value) { + const trimmed = value.trim() + if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) { + return trimmed.slice(1, -1).replace(/""/g, '"') + } + return trimmed +} + +/** + * Parse a delimited text export: `numbertext` per line. + * + * Tolerant by design — this is whatever an operator's GUI tool produced, not a + * format we control. A header row, blank lines, `#` comments and a trailing + * flags column are all ignored. A line whose first field is not an integer is + * skipped rather than fatal, because that is exactly what a header row is. + * + * The one thing it will NOT do is return an empty table quietly: a file that + * yields no entries at all is a wrong file, not an empty one. + */ +function parseClilocText(text) { + const entries = [] + for (const line of String(text).split(/\r?\n/)) { + // The line is deliberately NOT trimmed before the separator search. Roughly + // half of a real cliloc table is empty strings (unused ids), which export as + // `1005008` — and trimming eats that trailing separator, leaving a bare + // number that then looks like a header row and is skipped. That silently + // dropped 55,994 of 123,490 entries. Individual FIELDS are trimmed instead, + // by `unquote`. + if (line.trim() === '' || line.trimStart().startsWith('#')) continue + + // Pick the separator that actually appears first, so a tab-delimited line + // whose text contains a comma still splits on the tab. + let cut = -1 + for (const sep of TEXT_SEPARATORS) { + const at = line.indexOf(sep) + if (at !== -1 && (cut === -1 || at < cut)) cut = at + } + if (cut === -1) continue + + // An EMPTY first field must not become id 0: `Number('')` is 0, not NaN, so + // a line that merely starts with a separator would otherwise import as a + // bogus cliloc 0 instead of being skipped. + const head = unquote(line.slice(0, cut)) + if (head === '') continue + const number = Number(head) + if (!Number.isInteger(number)) continue // header row, or a wrapped line + + let rest = line.slice(cut + 1) + // Some exports carry `number,flag,text`. A bare integer in the second field + // is a flag; anything else is the text itself (and a text field that IS just + // a number is indistinguishable, so it stays as the text — the safer miss). + let flag = 0 + for (const sep of TEXT_SEPARATORS) { + const at = rest.indexOf(sep) + if (at === -1) continue + const head = unquote(rest.slice(0, at)) + if (/^\d{1,3}$/.test(head) && rest.slice(at + 1).trim() !== '') { + flag = Number(head) + rest = rest.slice(at + 1) + } + break + } + + entries.push({ number, flag, text: unquote(rest) }) + } + + if (entries.length === 0) { + throw new ClilocFormatError('No cliloc entries found in the text export', 'EMPTY') + } + return entries +} + +/** + * Parse either supported shape, sniffing which one this is. + * + * The sniff is on the binary header rather than the file extension: operators + * name these things whatever they like, and an `.enu` that is really a TSV (or a + * `.txt` that is really binary) should still import. + */ +function parseCliloc(buffer) { + const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer) + + if (isCompressedCliloc(buf)) { + throw new ClilocFormatError( + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.', + 'COMPRESSED', + ) + } + + // The plain layout always opens with version 2 / language 1. Anything else is + // treated as text, which is the recoverable guess: a mis-sniffed text file + // yields "no entries found", while a mis-sniffed binary yields nonsense. + if (buf.length >= HEADER_BYTES && buf.readInt32LE(0) === 2 && buf.readUInt16LE(4) === 1) { + return parseClilocBinary(buf) + } + return parseClilocText(buf.toString('utf8')) +} + +// ── Display ──────────────────────────────────────────────────────────────── + +// Cliloc strings interpolate arguments the client supplies out of an item's +// property list: `~1_val~`, `~2_NAME~`, `~1_ITEM~`. We never have those — the +// bridge sends the id, not the packet — so a name carrying them must be reduced +// to what is actually knowable rather than shown with the raw tokens in it. +const PLACEHOLDER_RE = /~\d+_[^~]*~/g + +/** + * Reduce a raw cliloc string to something displayable. + * + * Placeholders are dropped and the leftover punctuation tidied, so + * `"[~1_stuff~]"` becomes `""` (correctly nothing — the whole string was the + * argument) and `"cold damage ~1_val~%"` becomes `"cold damage"`. + * + * The trailing `%` in that second example is only stripped BECAUSE a placeholder + * was removed — it is the unit belonging to the number we never had. Stripping + * `%` unconditionally would corrupt a string that legitimately ends in one. + * + * Returns `''` when nothing survives, which callers treat as "no name" and fall + * back to the item id — better than showing a bracket. + */ +function displayText(raw) { + if (raw == null) return '' + const source = String(raw) + const hadPlaceholder = PLACEHOLDER_RE.test(source) + PLACEHOLDER_RE.lastIndex = 0 // the regex is global; `test` advances it + + const trailing = hadPlaceholder ? /[\s\-–—,.;:%[\]()]+$/ : /[\s\-–—,.;:[\]()]+$/ + + return source + .replace(PLACEHOLDER_RE, ' ') + .replace(/\s+/g, ' ') + .replace(/\s+([,.;:!?])/g, '$1') + .replace(/^[\s\-–—,.;:[\]()]+/, '') + .replace(trailing, '') + .trim() +} + +/** True when a raw cliloc string is nothing but interpolated arguments. */ +const isPlaceholderOnly = (raw) => raw != null && String(raw).trim() !== '' && displayText(raw) === '' + +module.exports = { + ClilocFormatError, + PARSER_VERSION, + HEADER_BYTES, + isCompressedCliloc, + parseCliloc, + parseClilocBinary, + parseClilocText, + displayText, + isPlaceholderOnly, +} diff --git a/server/src/utils/clilocSource.js b/server/src/utils/clilocSource.js new file mode 100644 index 0000000..d26fda9 --- /dev/null +++ b/server/src/utils/clilocSource.js @@ -0,0 +1,184 @@ +// Cliloc table — the filesystem layer. +// +// `clilocParse.js` holds the pure parsers; this module is the only thing that +// touches the converted cliloc file on disk, and it is shared by both callers: +// +// - the server, which refreshes the table on boot (`shardClilocs.model.js`) +// - the admin panel, which can force a reimport without a restart +// +// The file is the OPERATOR'S, produced once from their own UO client (see +// docs/website/CLILOCS.md). Nothing derived from it is committed: the repo holds +// no string table, exactly as it holds no map snapshot and no artwork. That rule +// is why this module reads a configured path instead of a path inside the repo. +// +// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so +// the boot path hashes first and only parses when something actually changed. + +const crypto = require('crypto') +const fs = require('fs') +const path = require('path') + +const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse') + +/** + * Filenames looked for when the configured path is a DIRECTORY. + * + * Ordered by how specific they are: an explicitly converted file wins over + * something that merely sits in a client folder, so an operator who dropped a + * `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one + * they made rather than the one that will be rejected. + * + * Matching is case-insensitive against the real directory listing, because the + * client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where + * a hardcoded lowercase open would simply miss. + */ +const CANDIDATE_NAMES = [ + 'clilocs.tsv', + 'clilocs.csv', + 'cliloc.plain.enu', + 'cliloc.enu.plain', + 'clilocs.txt', + 'cliloc.enu', +] + +class ClilocSourceError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocSourceError' + this.code = code + } +} + +function sha256(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex') +} + +/** + * Resolve the configured path to an actual file. + * + * Accepts either a direct file path or a directory to search, because operators + * reasonably supply both — "here is the file" and "here is the folder I put it + * in" are equally natural answers to the admin panel's prompt. + */ +function resolveFile(configured) { + if (!configured || String(configured).trim() === '') { + throw new ClilocSourceError('No cliloc path configured', 'NO_PATH') + } + const target = String(configured).trim() + + let stat + try { + stat = fs.statSync(target) + } catch { + throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND') + } + + if (stat.isFile()) return target + + if (!stat.isDirectory()) { + throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND') + } + + let listing + try { + listing = fs.readdirSync(target) + } catch { + throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND') + } + + const byLower = new Map(listing.map((name) => [name.toLowerCase(), name])) + for (const candidate of CANDIDATE_NAMES) { + const actual = byLower.get(candidate) + if (actual) return path.join(target, actual) + } + + throw new ClilocSourceError( + `No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`, + 'NO_FILE', + ) +} + +/** + * A fingerprint of the source file: `{ file, sha256, bytes, mtime, compressed }`. + * + * The boot path compares the hash against what was last imported and skips the + * parse entirely when it matches — the normal case on every restart that did not + * follow a client patch. + * + * `compressed` is reported here rather than left to the parse because the admin + * panel calls this and NOT `readCliloc` (parsing 5 MB on every status poll would + * be wasteful). Without it, pointing the setting at an unconverted client + * directory reports a perfectly readable file with pending drift — "ready to + * import" — and the operator only learns otherwise when the import fails. The + * check is four bytes of a buffer already in hand. + */ +function hashSource(configured) { + const file = resolveFile(configured) + let buffer + try { + buffer = fs.readFileSync(file) + } catch { + throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE') + } + let mtime = null + try { + mtime = fs.statSync(file).mtime.toISOString() + } catch { + // A missing mtime is cosmetic (it is only shown in the admin panel). + } + return { + file, + sha256: sha256(buffer), + bytes: buffer.length, + mtime, + compressed: isCompressedCliloc(buffer), + } +} + +/** True when two source fingerprints describe the same file. */ +function sameSource(a, b) { + return !!a && !!b && a.sha256 === b.sha256 +} + +/** + * Read and parse the configured cliloc file. + * + * Returns `{ entries, source }`. Throws `ClilocSourceError` for anything about + * the path and `ClilocFormatError` for anything about the contents — the two are + * different problems for an operator (wrong place vs wrong file), and the admin + * panel says which. + */ +function readCliloc(configured) { + const file = resolveFile(configured) + + let buffer + try { + buffer = fs.readFileSync(file) + } catch { + throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE') + } + + const entries = parseCliloc(buffer) + + return { + entries, + source: { + file, + sha256: sha256(buffer), + bytes: buffer.length, + parserVersion: PARSER_VERSION, + count: entries.length, + }, + } +} + +module.exports = { + ClilocFormatError, + ClilocSourceError, + PARSER_VERSION, + CANDIDATE_NAMES, + resolveFile, + hashSource, + sameSource, + readCliloc, +} diff --git a/server/swagger/swagger-output.json b/server/swagger/swagger-output.json index f1f3c9f..02a11a5 100644 --- a/server/swagger/swagger-output.json +++ b/server/swagger/swagger-output.json @@ -4078,6 +4078,153 @@ ] } }, + "/api/v1/admin/shard/clilocs": { + "get": { + "tags": [ + "Admin · Shard" + ], + "summary": "Cliloc table status: path, drift, entry count (admin only)", + "description": "Where the converted cliloc file is, whether it can be read, how many entries are loaded, and whether the file on disk has drifted from them. A shard with no cliloc file configured is a supported state — item names simply render as ids.", + "responses": { + "200": { + "description": "Cliloc status", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClilocStatus" + } + } + } + }, + "403": { + "description": "Admin role required", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "500": { + "description": "Internal Server Error" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ] + } + }, + "/api/v1/admin/shard/clilocs/import": { + "post": { + "tags": [ + "Admin · Shard" + ], + "summary": "Re-import the cliloc table from the converted file (admin only)", + "description": "Applies a client patch without a restart. `force` reimports even when the source hash matches what is loaded. A missing file — or the common mistake of pointing at the client\\'s own COMPRESSED Cliloc.enu — answers 200 with status \"unavailable\" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.", + "responses": { + "200": { + "description": "What happened", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClilocRefreshResult" + } + } + } + }, + "400": { + "description": "Bad Request" + }, + "500": { + "description": "Internal Server Error" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "force": { + "type": "boolean", + "description": "Reimport even if the file is unchanged." + } + } + } + } + } + } + } + }, + "/api/v1/admin/shard/clilocs/path": { + "put": { + "tags": [ + "Admin · Shard" + ], + "summary": "Set the cliloc file the site reads from (admin only)", + "description": "Accepts either the converted file itself or a directory to search. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.", + "responses": { + "200": { + "description": "Cliloc status after the change", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClilocStatus" + } + } + } + }, + "400": { + "description": "Bad Request" + }, + "500": { + "description": "Internal Server Error" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string", + "description": "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." + } + } + } + } + } + } + } + }, "/api/v1/admin/shard/houses": { "get": { "tags": [ @@ -20312,6 +20459,358 @@ } } }, + "ClilocStatus": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "Admin view of cliloc state: where the converted file is, whether it is readable, how many entries are loaded, and whether the file has drifted from them. `configured: false` is a supported state — item names then render as ids." + }, + "properties": { + "type": "object", + "properties": { + "configured": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "example": { + "type": "boolean", + "example": true + } + } + }, + "path": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "example": { + "type": "string", + "example": "/srv/uo-client" + } + } + }, + "file": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The file actually resolved, when the path is a directory." + }, + "example": { + "type": "string", + "example": "/srv/uo-client/clilocs.tsv" + } + } + }, + "fileReadable": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "example": { + "type": "boolean", + "example": true + } + } + }, + "problem": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Why the file cannot be used, when it cannot. Set (with code COMPRESSED) for a readable-but-unconverted client file." + }, + "example": {} + } + }, + "code": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Machine-readable cause of `problem`." + }, + "enum": { + "type": "array", + "example": [ + "NO_PATH", + "NOT_FOUND", + "NO_FILE", + "UNREADABLE", + "COMPRESSED" + ], + "items": { + "type": "string" + } + } + } + }, + "drift": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "True when the file's hash differs from the loaded table. NULL when the file could not be read or is not usable." + }, + "example": { + "type": "boolean", + "example": false + } + } + }, + "count": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "Entries currently loaded." + }, + "example": { + "type": "number", + "example": 123490 + } + } + }, + "importedAt": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "format": { + "type": "string", + "example": "date-time" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "sourceBytes": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "example": { + "type": "number", + "example": 4973525 + } + } + } + } + } + } + }, + "ClilocRefreshResult": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "Outcome of a cliloc refresh. Reported rather than thrown, so a missing or compressed file is an answer and not a 500." + }, + "properties": { + "type": "object", + "properties": { + "status": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "enum": { + "type": "array", + "example": [ + "skipped", + "unavailable", + "unchanged", + "imported", + "failed" + ], + "items": { + "type": "string" + } + }, + "example": { + "type": "string", + "example": "imported" + } + } + }, + "reason": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "code": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Machine-readable cause. `COMPRESSED` means the client's own Cliloc.enu was supplied instead of a converted one." + }, + "enum": { + "type": "array", + "example": [ + "NO_PATH", + "NOT_FOUND", + "NO_FILE", + "UNREADABLE", + "COMPRESSED", + "TRUNCATED", + "EMPTY", + "NOT_BUFFER" + ], + "items": { + "type": "string" + } + } + } + }, + "path": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "file": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "count": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "example": { + "type": "number", + "example": 123490 + } + } + }, + "duplicates": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Repeated ids collapsed on import (last wins)." + }, + "example": { + "type": "number", + "example": 0 + } + } + } + } + } + } + }, "ShardLinkRequest": { "type": "object", "properties": { diff --git a/server/swagger/swagger.js b/server/swagger/swagger.js index 3f09282..5d55131 100644 --- a/server/swagger/swagger.js +++ b/server/swagger/swagger.js @@ -1161,6 +1161,46 @@ const doc = { removedFacets: { type: 'array', items: { type: 'string' } }, }, }, + ClilocStatus: { + type: 'object', + description: + 'Admin view of cliloc state: where the converted file is, whether it is readable, how many entries are loaded, and whether the file has drifted from them. `configured: false` is a supported state — item names then render as ids.', + properties: { + configured: { type: 'boolean', example: true }, + path: { type: 'string', example: '/srv/uo-client' }, + file: { type: 'string', nullable: true, description: 'The file actually resolved, when the path is a directory.', example: '/srv/uo-client/clilocs.tsv' }, + fileReadable: { type: 'boolean', example: true }, + problem: { type: 'string', nullable: true, description: 'Why the file cannot be used, when it cannot. Set (with code COMPRESSED) for a readable-but-unconverted client file.', example: null }, + code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED'] }, + drift: { type: 'boolean', nullable: true, description: 'True when the file\'s hash differs from the loaded table. NULL when the file could not be read or is not usable.', example: false }, + count: { type: 'integer', description: 'Entries currently loaded.', example: 123490 }, + importedAt: { type: 'string', format: 'date-time', nullable: true }, + sourceBytes: { type: 'integer', nullable: true, example: 4973525 }, + }, + }, + ClilocRefreshResult: { + type: 'object', + description: + 'Outcome of a cliloc refresh. Reported rather than thrown, so a missing or compressed file is an answer and not a 500.', + properties: { + status: { + type: 'string', + enum: ['skipped', 'unavailable', 'unchanged', 'imported', 'failed'], + example: 'imported', + }, + reason: { type: 'string', nullable: true }, + code: { + type: 'string', + nullable: true, + description: 'Machine-readable cause. `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.', + enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER'], + }, + path: { type: 'string', nullable: true }, + file: { type: 'string', nullable: true }, + count: { type: 'integer', nullable: true, example: 123490 }, + duplicates: { type: 'integer', nullable: true, description: 'Repeated ids collapsed on import (last wins).', example: 0 }, + }, + }, ShardLinkRequest: { type: 'object', required: ['code'], diff --git a/server/test/clilocParse.test.js b/server/test/clilocParse.test.js new file mode 100644 index 0000000..5db9c1f --- /dev/null +++ b/server/test/clilocParse.test.js @@ -0,0 +1,215 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') + +const { + ClilocFormatError, + parseCliloc, + parseClilocBinary, + parseClilocText, + isCompressedCliloc, + displayText, + isPlaceholderOnly, +} = require('../src/utils/clilocParse') + +// These parsers are pure and fs-free precisely so this suite can run in CI, +// where there is no UO client and no converted cliloc file. Every fixture below +// is built from the real layout, and the strings are verbatim entries from a +// real Cliloc.enu (123,490 entries) rather than invented ones. + +// ── Fixture builders ─────────────────────────────────────────────────────── + +/** Build a plain-format cliloc buffer: 6-byte header, then records. */ +function buildBinary(entries, { header1 = 2, header2 = 1 } = {}) { + const parts = [Buffer.alloc(6)] + parts[0].writeInt32LE(header1, 0) + parts[0].writeUInt16LE(header2, 4) + for (const e of entries) { + const text = Buffer.from(e.text, 'utf8') + const head = Buffer.alloc(7) + head.writeInt32LE(e.number, 0) + head.writeUInt8(e.flag ?? 0, 4) + head.writeUInt16LE(text.length, 5) + parts.push(head, text) + } + return Buffer.concat(parts) +} + +// ── Binary ───────────────────────────────────────────────────────────────── + +test('parseClilocBinary: reads a plain-format table', () => { + const buf = buildBinary([ + { number: 1015012, text: 'Greater Heal' }, + { number: 1023721, text: 'quarter staff' }, + { number: 1025913, flag: 1, text: 'bonnet' }, + ]) + assert.deepEqual(parseClilocBinary(buf), [ + { number: 1015012, flag: 0, text: 'Greater Heal' }, + { number: 1023721, flag: 0, text: 'quarter staff' }, + { number: 1025913, flag: 1, text: 'bonnet' }, + ]) +}) + +test('parseClilocBinary: length is UNSIGNED 16-bit', () => { + // ServUO's own SDK reads this field into a signed short, which turns any + // string over 32 KB into a negative length. Real tables top out around 12 KB + // so nothing is broken today, but the field is written unsigned and reading it + // that way costs nothing. + const text = 'x'.repeat(40000) + const [entry] = parseClilocBinary(buildBinary([{ number: 1000000, text }])) + assert.equal(entry.text.length, 40000) +}) + +test('parseClilocBinary: multi-byte UTF-8 survives (length is in BYTES)', () => { + const [entry] = parseClilocBinary(buildBinary([{ number: 1000000, text: 'Ilshenar — Ver Lor Reg' }])) + assert.equal(entry.text, 'Ilshenar — Ver Lor Reg') +}) + +test('parseClilocBinary: a truncated record body throws rather than importing short', () => { + // The realistic corruption is a half-copied file. It must fail loudly: a + // silently short table renders as "some items named, some not", which is + // indistinguishable from having no table at all. + const buf = buildBinary([{ number: 1023721, text: 'quarter staff' }]) + const truncated = buf.subarray(0, buf.length - 4) + assert.throws(() => parseClilocBinary(truncated), (err) => { + assert.ok(err instanceof ClilocFormatError) + assert.equal(err.code, 'TRUNCATED') + return true + }) +}) + +test('parseClilocBinary: a truncated record HEADER throws too', () => { + const buf = Buffer.concat([buildBinary([{ number: 1023721, text: 'quarter staff' }]), Buffer.alloc(3)]) + assert.throws(() => parseClilocBinary(buf), (err) => err.code === 'TRUNCATED') +}) + +test('parseClilocBinary: an empty table (header only) is valid', () => { + assert.deepEqual(parseClilocBinary(buildBinary([])), []) +}) + +// ── Compressed detection ─────────────────────────────────────────────────── + +test('isCompressedCliloc: recognises the Mythic marker', () => { + // Every cliloc the client ships opens with a DWORD whose high byte is 0x8E. + // Real first bytes of Cliloc.enu (e8 79 67 8e) and Cliloc.deu (99 5d 26 8e). + assert.equal(isCompressedCliloc(Buffer.from([0xe8, 0x79, 0x67, 0x8e])), true) + assert.equal(isCompressedCliloc(Buffer.from([0x99, 0x5d, 0x26, 0x8e])), true) + assert.equal(isCompressedCliloc(buildBinary([])), false) +}) + +test('parseCliloc: a compressed file is rejected by NAME, not parsed into nonsense', () => { + // This is the whole reason the marker check exists. Without it the plain + // parser reads compressed bytes as ~19k records of negative ids and 60 KB + // "strings" before dying somewhere in the middle — and the resulting error + // names truncation, which is the wrong problem to hand an operator. + const compressed = Buffer.concat([Buffer.from([0xe8, 0x79, 0x67, 0x8e]), Buffer.alloc(64, 0x41)]) + assert.throws(() => parseCliloc(compressed), (err) => { + assert.equal(err.code, 'COMPRESSED') + assert.match(err.message, /CLILOCS\.md/) + return true + }) +}) + +// ── Text ─────────────────────────────────────────────────────────────────── + +test('parseClilocText: tab-delimited, skipping a header row', () => { + const entries = parseClilocText('number\ttext\n1023721\tquarter staff\n1015012\tGreater Heal\n') + assert.deepEqual(entries, [ + { number: 1023721, flag: 0, text: 'quarter staff' }, + { number: 1015012, flag: 0, text: 'Greater Heal' }, + ]) +}) + +test('parseClilocText: splits on the FIRST separator only', () => { + // Cliloc text is full of commas. Splitting on all of them would truncate every + // such entry at its first one. + const [entry] = parseClilocText('1044000,a scroll of magery, unfinished\n') + assert.equal(entry.text, 'a scroll of magery, unfinished') +}) + +test('parseClilocText: unwraps quoted CSV fields and doubled quotes', () => { + const [entry] = parseClilocText('1023721,"a ""quarter"" staff, plain"\n') + assert.equal(entry.text, 'a "quarter" staff, plain') +}) + +test('parseClilocText: reads an optional flag column', () => { + const [entry] = parseClilocText('1025913\t1\tbonnet\n') + assert.deepEqual(entry, { number: 1025913, flag: 1, text: 'bonnet' }) +}) + +test('parseClilocText: text that is itself a number stays the text', () => { + // `number,text` where text is "100" is indistinguishable from `number,flag` + // with an empty text. Keeping it as the text is the safer miss — the other way + // silently deletes a real entry. + const [entry] = parseClilocText('1000000,100\n') + assert.equal(entry.text, '100') +}) + +test('parseClilocText: blank lines and # comments are ignored', () => { + const entries = parseClilocText('# exported by hand\n\n1023721\tquarter staff\n\n') + assert.equal(entries.length, 1) +}) + +test('parseClilocText: a file with no entries is an error, not an empty table', () => { + assert.throws(() => parseClilocText('nothing here\nnor here\n'), (err) => err.code === 'EMPTY') +}) + +test('parseClilocText: an empty leading field is skipped, not imported as id 0', () => { + // `Number('')` is 0, not NaN, so a line that merely starts with a separator + // would otherwise become a bogus cliloc 0. + assert.throws(() => parseClilocText('\tstray text\n,another\n'), (err) => err.code === 'EMPTY') +}) + +test('parseClilocText: keeps entries whose text is EMPTY', () => { + // About half of a real table is empty strings (unused ids). They must survive + // parsing — the import layer decides whether to store them, and both input + // formats have to agree on what the file contained. + const entries = parseClilocText('1005008\t\n1023721\tquarter staff\n') + assert.equal(entries.length, 2) + assert.deepEqual(entries[0], { number: 1005008, flag: 0, text: '' }) +}) + +// ── Sniffing ─────────────────────────────────────────────────────────────── + +test('parseCliloc: sniffs binary vs text from the header, not the extension', () => { + assert.equal(parseCliloc(buildBinary([{ number: 1023721, text: 'quarter staff' }]))[0].text, 'quarter staff') + assert.equal(parseCliloc(Buffer.from('1023721\tquarter staff\n'))[0].text, 'quarter staff') +}) + +test('parseCliloc: a binary-looking header that is not 2/1 falls through to text', () => { + // The recoverable guess: a mis-sniffed text file says "no entries found", + // while a mis-sniffed binary yields plausible nonsense. + assert.throws(() => parseCliloc(Buffer.from([9, 0, 0, 0, 9, 0, 65, 66])), (err) => err.code === 'EMPTY') +}) + +// ── Display ──────────────────────────────────────────────────────────────── + +test('displayText: drops interpolated arguments we never receive', () => { + // The bridge sends a cliloc id, never the property packet that carries the + // arguments, so a name containing them has to be reduced to what is knowable. + assert.equal(displayText('cold damage ~1_val~%'), 'cold damage') + assert.equal(displayText('~1_NAME~ the ~2_TITLE~'), 'the') +}) + +test('displayText: a string that is nothing but arguments resolves to nothing', () => { + assert.equal(displayText('[~1_stuff~]'), '') + assert.equal(isPlaceholderOnly('[~1_stuff~]'), true) + assert.equal(isPlaceholderOnly('quarter staff'), false) +}) + +test('displayText: a trailing % is only stripped when a placeholder was removed', () => { + // "cold damage ~1_val~%" loses its % because that % was the unit belonging to + // the number we never had. A string that genuinely ends in one keeps it. + assert.equal(displayText('50%'), '50%') + assert.equal(displayText('cold damage ~1_val~%'), 'cold damage') +}) + +test('displayText: ordinary names pass through untouched', () => { + assert.equal(displayText('quarter staff'), 'quarter staff') + assert.equal(displayText('a scroll of magery, unfinished'), 'a scroll of magery, unfinished') + assert.equal(displayText(' spiked collar '), 'spiked collar') +}) + +test('displayText: null and undefined are empty, not "null"', () => { + assert.equal(displayText(null), '') + assert.equal(displayText(undefined), '') +}) diff --git a/server/tools/cliloc-export/Program.cs b/server/tools/cliloc-export/Program.cs new file mode 100644 index 0000000..4a1dec3 --- /dev/null +++ b/server/tools/cliloc-export/Program.cs @@ -0,0 +1,140 @@ +// Cliloc export — converts a modern client's COMPRESSED Cliloc.enu into the +// plain format the website can read (docs/website/CLILOCS.md). +// +// Why this exists at all: every current UO client ships its cliloc files in the +// compressed "Mythic" format — the first DWORD's high byte is 0x8E — and the +// plain layout the website parses is what those files looked like before that +// change. Decompressing is a bit-level inverse-BWT coder that the site has no +// business carrying at runtime, and ServUO's own bundled `Ultima.StringList` +// cannot read it either (which is why `VendorSearch.GetItemName` is already +// inert on such a shard, and why the shard cannot supply names instead). +// +// So the conversion happens ONCE, here, against a decompressor that already +// exists and is maintained: UOFiddler's `Ultima.dll`. +// +// ── Why reflection rather than a project reference ──────────────────────── +// +// UOFiddler ships as net10.0. Referencing it from a project built by an older +// SDK fails at COMPILE time with CS1705 ("uses System.Runtime 10.0 which has a +// higher version than referenced assembly"). Loading it reflectively moves that +// question to run time, where `RollForward: LatestMajor` answers it — so this +// builds on whatever SDK an operator happens to have and runs on the newest +// runtime installed. +// +// ── Why not StringList.SaveStringList ──────────────────────────────────── +// +// It looks like exactly the right method and it is not: it RE-COMPRESSES on +// save, because its purpose is round-tripping a file back into the client. The +// output is byte-identical to the compressed input. The plain records below are +// written by hand for that reason. +// +// Usage: +// dotnet run -- [--tsv] +// +// Nothing produced by this tool is committed. See docs/website/CLILOCS.md. + +using System; +using System.Collections; +using System.IO; +using System.Reflection; +using System.Text; + +internal static class Program +{ + private static int Main(string[] args) + { + if (args.Length < 3) + { + Console.Error.WriteLine("usage: clilocexport [--tsv]"); + Console.Error.WriteLine(" Ultima.dll ships with UOFiddler (https://github.com/polserver/UOFiddler)."); + return 2; + } + + var (ultimaDll, input, output) = (args[0], args[1], args[2]); + var asTsv = Array.IndexOf(args, "--tsv") >= 0; + + // The language code only names the file when StringList resolves the path + // itself; here the path is explicit, so it is cosmetic. + var language = Path.GetExtension(input).TrimStart('.'); + if (string.IsNullOrWhiteSpace(language)) language = "enu"; + + var assembly = Assembly.LoadFrom(Path.GetFullPath(ultimaDll)); + var stringListType = assembly.GetType("Ultima.StringList") + ?? throw new InvalidOperationException("Ultima.StringList not found — is that really UOFiddler's Ultima.dll?"); + + // (language, path, decompress). `decompress: true` is the whole point; + // the loader falls back to a plain read on its own if the file turns out + // not to be compressed, so an already-converted file passes through. + var ctor = stringListType.GetConstructor(new[] { typeof(string), typeof(string), typeof(bool) }) + ?? throw new InvalidOperationException("Unexpected Ultima.StringList API — this tool targets UOFiddler 4.21+."); + + var stringList = ctor.Invoke(new object[] { language, Path.GetFullPath(input), true }); + + // A partial parse is reported rather than thrown. Surfacing it matters: + // the output would otherwise be a quietly short table, which is exactly + // the failure mode the website's parser refuses to import. + var warning = stringListType.GetProperty("LoadWarning")?.GetValue(stringList) as string; + if (!string.IsNullOrWhiteSpace(warning)) Console.Error.WriteLine("warning: " + warning); + + var entries = (IEnumerable)stringListType.GetProperty("Entries")!.GetValue(stringList)!; + var entryType = assembly.GetType("Ultima.StringEntry")!; + var numberProp = entryType.GetProperty("Number")!; + var textProp = entryType.GetProperty("Text")!; + var flagProp = entryType.GetProperty("Flag")!; + + int written = 0, skipped = 0, maxBytes = 0; + + if (asTsv) + { + using var writer = new StreamWriter(output, false, new UTF8Encoding(false)); + foreach (var entry in entries) + { + var number = (int)numberProp.GetValue(entry)!; + var text = (string?)textProp.GetValue(entry) ?? ""; + maxBytes = Math.Max(maxBytes, Encoding.UTF8.GetByteCount(text)); + // A tab or newline inside a cliloc string would break the row. + // Neither occurs in real tables, but silently emitting a broken + // file is worse than collapsing the whitespace. + writer.WriteLine($"{number}\t{text.Replace('\t', ' ').Replace('\r', ' ').Replace('\n', ' ')}"); + written++; + } + } + else + { + using var stream = new FileStream(output, FileMode.Create, FileAccess.Write); + using var binary = new BinaryWriter(stream); + binary.Write(2); // int32 — the plain-format version marker + binary.Write((short)1); // int16 — language marker + + foreach (var entry in entries) + { + var number = (int)numberProp.GetValue(entry)!; + var text = (string?)textProp.GetValue(entry) ?? ""; + var flag = Convert.ToByte(Convert.ToInt32(flagProp.GetValue(entry))); + + var utf8 = Encoding.UTF8.GetBytes(text); + maxBytes = Math.Max(maxBytes, utf8.Length); + + // The length field is 16 bits. Real tables peak around 12 KB, so + // this has never fired — but writing a truncated length would + // corrupt every record after it, so an oversize entry is dropped + // and counted instead. + if (utf8.Length > ushort.MaxValue) { skipped++; continue; } + + binary.Write(number); + binary.Write(flag); + binary.Write((ushort)utf8.Length); + binary.Write(utf8); + written++; + } + } + + Console.WriteLine($"wrote {written} entries to {output} (maxTextBytes={maxBytes}, skippedOversize={skipped})"); + if (written == 0) + { + Console.Error.WriteLine("no entries were written — is that a cliloc file?"); + return 1; + } + return 0; + } +} diff --git a/server/tools/cliloc-export/README.md b/server/tools/cliloc-export/README.md new file mode 100644 index 0000000..97efb1c --- /dev/null +++ b/server/tools/cliloc-export/README.md @@ -0,0 +1,64 @@ +# cliloc-export + +Converts a UO client's **compressed** `Cliloc.enu` into the plain format the +website can read. + +This is a one-off operator utility, not part of the website build. Nothing in the +Node application references it and CI never touches it. Full background — +including why the conversion is necessary at all — is in +[`docs/website/CLILOCS.md`](https://gitea.whitlocktech.com/RunicGateway/docs/src/branch/edge/website/CLILOCS.md). + +## The short version + +Every current UO client ships its cliloc files in the compressed "Mythic" +container (the first DWORD's high byte is `0x8E`). The website parses the plain +layout those files used before that change. Decompressing is an inverse-BWT coder +that the site has no business carrying at runtime — and ServUO's own bundled +`Ultima.StringList` cannot read it either, so the shard cannot supply item names +on our behalf. + +So: convert once, here, using a decompressor that already exists and is already +maintained — [UOFiddler](https://github.com/polserver/UOFiddler)'s `Ultima.dll`. + +## Usage + +```bash +dotnet build -c Release + +# plain binary (recommended — exact) +dotnet run -- "/Ultima.dll" "/Cliloc.enu" /srv/uo-data/clilocs.plain + +# tab-delimited text (convenient; does not preserve leading/trailing whitespace) +dotnet run -- "/Ultima.dll" "/Cliloc.enu" /srv/uo-data/clilocs.tsv --tsv +``` + +Then point the site at the output: **Admin → Shard → cliloc path**, or the +`UO_CLIENT_PATH` environment variable. The setting wins over the environment. + +Expected output for a stock English client: + +``` +wrote 123490 entries to /srv/uo-data/clilocs.plain (maxTextBytes=12150, skippedOversize=0) +``` + +The site stores ~67,500 of those — roughly half a cliloc table is empty strings +for ids the client reserves and never uses. + +## Two implementation notes worth keeping + +**`Ultima.dll` is loaded reflectively, not referenced.** UOFiddler ships as +net10.0; a project reference from an older SDK fails at *compile* time with +CS1705. Reflection moves that to run time, where `RollForward: LatestMajor` +answers it — so this builds on whatever SDK you have and runs on the newest +runtime installed. + +**`StringList.SaveStringList` is not the export path**, despite looking exactly +like it. It *re-compresses* on save, because its purpose is round-tripping a file +back into the client — its output is byte-identical to its input. The plain +records are written by hand for that reason. + +## Output is never committed + +UO's strings are EA's. `.gitignore` covers this project's build output and the +conventional in-repo output location, but the supported arrangement is a path +**outside** the repository entirely. diff --git a/server/tools/cliloc-export/clilocexport.csproj b/server/tools/cliloc-export/clilocexport.csproj new file mode 100644 index 0000000..06d6103 --- /dev/null +++ b/server/tools/cliloc-export/clilocexport.csproj @@ -0,0 +1,26 @@ + + + + + Exe + net8.0 + clilocexport + ClilocExport + enable + disable + LatestMajor + true + + +