feat(server): port the UO models, utils and schema fragment
The data half of the extraction: 8 model directories, 13 utils, the shard
stream catalog and the 27-table schema fragment with its purge.
server/core.js is what makes the port a one-line import change per file rather
than a signature change per function. Ported code requires its dependencies at
file scope -- `const { query } = require('../../core')` -- which runs before
register() has been called and before any ctx exists. So every member is a
stable function that resolves ctx when CALLED, and nothing may be destructured
off ctx at init either, because core is free to hand over a getter.
Two helpers are vendored rather than taken from ctx, and the line between them
is the point. utils/excerpt.js is core's deriveExcerpt -- nine lines of pure
text handling. Core's sanitiser next to it was NOT copied: a second copy of a
security control diverges silently the moment either is fixed. announceLinks.js
vendors legError and articleUrl the same way, but baseUrl could not be: core's
reads APP_BASE_URL, and §2.7 forbids a module reading core's environment, so it
comes off ctx.site.baseUrl.
The schema fragment is core's 27 shard_*/uo_link_* statements, verbs CREATE,
ALTER and UPDATE only, every CREATE TABLE guarded. Two of its tables carry a
foreign key INTO users, which is allowed and is why the replay order matters --
core's schema is in place before this runs. The reverse never occurs and must
not: it would make core unable to boot without a module installed.
One real port bug caught by the integration run, not by tests: the atlas art
map resolved `../../../db/data`, which pointed at core's tree when this file
lived there and points outside server/ now. A path that happens to resolve is
exactly what survives a green suite, because the absent-file branch returns {}
and looks like the normal case.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
110
server/model/shardClilocs/shardClilocs.db.js
Normal file
110
server/model/shardClilocs/shardClilocs.db.js
Normal file
@@ -0,0 +1,110 @@
|
||||
const core = require('../../core')
|
||||
|
||||
const { query } = core
|
||||
|
||||
// Raw SQL for the cliloc table. `shard_clilocs` is IMPORT-OWNED: `replaceAll`
|
||||
// empties and refills it inside one transaction, and nothing else in the
|
||||
// codebase writes to it. No foreign keys, consistent with every other shard_*
|
||||
// table.
|
||||
|
||||
const BATCH = 1000
|
||||
|
||||
/**
|
||||
* Replace the entire cliloc table in one transaction.
|
||||
*
|
||||
* All-or-nothing on purpose: a failed reload must leave the previous table
|
||||
* intact rather than a half-loaded one, because a partially-imported cliloc
|
||||
* table is indistinguishable from a complete one to anyone reading it — you
|
||||
* would just see some items named and some not, which is also what "no table at
|
||||
* all" looks like.
|
||||
*
|
||||
* `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly
|
||||
* commits, which would defeat exactly that guarantee. (The same trap the spawn
|
||||
* atlas import documents; at ~123k rows `DELETE` is still well under a second.)
|
||||
*/
|
||||
async function replaceAll(entries, meta) {
|
||||
const conn = await core.pool.getConnection()
|
||||
try {
|
||||
await conn.beginTransaction()
|
||||
await conn.query('DELETE FROM shard_clilocs')
|
||||
|
||||
// Blank entries are dropped rather than stored. Roughly HALF of a real
|
||||
// cliloc table is empty strings — ids the client reserves and never uses —
|
||||
// and a row that resolves to no name is indistinguishable from no row at
|
||||
// all to every caller. Dropping them halves the table (123,490 → ~67,500)
|
||||
// and, more importantly, makes the binary and text imports converge on
|
||||
// identical content: the binary format carries the blanks explicitly and a
|
||||
// text export may or may not, depending on the tool.
|
||||
//
|
||||
// Later duplicates win. Merging across sources already happened upstream in
|
||||
// `readCliloc`, so in practice this collapses nothing — it is kept because
|
||||
// the plain format permits a repeated id WITHIN one file and the client's
|
||||
// own loader resolves it the same way (its dictionary assignment
|
||||
// overwrites). Without it, a file the game itself would load happily would
|
||||
// fail the batch insert on a primary-key collision.
|
||||
const byNumber = new Map()
|
||||
let blank = 0
|
||||
for (const entry of entries) {
|
||||
if (!Number.isInteger(entry.number)) continue
|
||||
if (String(entry.text ?? '').trim() === '') {
|
||||
blank++
|
||||
continue
|
||||
}
|
||||
byNumber.set(entry.number, entry)
|
||||
}
|
||||
|
||||
const rows = [...byNumber.values()].map((e) => [e.number, e.flag ?? 0, e.text])
|
||||
for (let i = 0; i < rows.length; i += BATCH) {
|
||||
await conn.batch('INSERT INTO shard_clilocs (number, flag, text) VALUES (?,?,?)', rows.slice(i, i + BATCH))
|
||||
}
|
||||
|
||||
await conn.query(
|
||||
'INSERT INTO shard_cliloc_meta (id, payload) VALUES (1, ?) ' +
|
||||
'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP',
|
||||
[JSON.stringify({ ...meta, count: rows.length })],
|
||||
)
|
||||
|
||||
await conn.commit()
|
||||
return { count: rows.length, blank, duplicates: entries.length - blank - rows.length }
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {})
|
||||
throw err
|
||||
} finally {
|
||||
conn.release()
|
||||
}
|
||||
}
|
||||
|
||||
async function getMeta() {
|
||||
const rows = await query('SELECT payload, imported_at FROM shard_cliloc_meta WHERE id = 1')
|
||||
if (rows.length === 0) return null
|
||||
const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload
|
||||
return { ...payload, importedAt: rows[0].imported_at }
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up a batch of ids.
|
||||
*
|
||||
* Batched rather than one-at-a-time because every caller has a LIST: a character
|
||||
* sheet resolves a dozen equipment ids at once, and a page of marketplace
|
||||
* listings resolves fifty. `IN (...)` with generated placeholders keeps it one
|
||||
* round trip and one parameterized statement.
|
||||
*/
|
||||
async function lookup(numbers) {
|
||||
if (!Array.isArray(numbers) || numbers.length === 0) return []
|
||||
const ids = [...new Set(numbers.filter((n) => Number.isInteger(n)))]
|
||||
if (ids.length === 0) return []
|
||||
const placeholders = ids.map(() => '?').join(',')
|
||||
return query(`SELECT number, text FROM shard_clilocs WHERE number IN (${placeholders})`, ids)
|
||||
}
|
||||
|
||||
async function count() {
|
||||
const rows = await query('SELECT COUNT(*) AS n FROM shard_clilocs')
|
||||
return Number(rows[0]?.n) || 0
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
replaceAll,
|
||||
getMeta,
|
||||
lookup,
|
||||
count,
|
||||
}
|
||||
368
server/model/shardClilocs/shardClilocs.model.js
Normal file
368
server/model/shardClilocs/shardClilocs.model.js
Normal file
@@ -0,0 +1,368 @@
|
||||
const db = require('./shardClilocs.db')
|
||||
const { settings } = require('../../core')
|
||||
const { displayText } = require('../../utils/clilocParse')
|
||||
const {
|
||||
ClilocFormatError,
|
||||
ClilocSourceError,
|
||||
PARSER_VERSION,
|
||||
hashSources,
|
||||
sameSources,
|
||||
missingSources,
|
||||
readCliloc,
|
||||
} = require('../../utils/clilocSource')
|
||||
const log = require('../../core').logger('shardClilocs')
|
||||
|
||||
// The cliloc table — UO's id → display-string map, refreshed from a file the
|
||||
// operator converts once from their own client.
|
||||
//
|
||||
// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a
|
||||
// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace
|
||||
// listing sends one too. Without the table the UI can only print `id 1023721`
|
||||
// where the game prints "quarter staff".
|
||||
//
|
||||
// Two rules govern the boot path, both inherited from the spawn atlas:
|
||||
//
|
||||
// 1. **It never blocks startup.** No configured path, an unreadable file, a
|
||||
// wrong-format file, a database error — all caught and logged. The site
|
||||
// comes up either way, serving whatever table it already had (or none, in
|
||||
// which case the UI falls back to item ids exactly as it did before).
|
||||
// 2. **Nothing client-derived is committed.** The table is built from the
|
||||
// operator's own file at a configured path. The repo ships no strings.
|
||||
//
|
||||
// The table is built from a SET of sources — the converted client table plus
|
||||
// every operator-maintained overlay beside it — because shards edit items and
|
||||
// add new ones, and those carry cliloc ids no stock client table has. All of
|
||||
// them are re-read on every boot and hash-gated together, so adding one custom
|
||||
// item never means re-exporting a 5 MB client file. Later sources win.
|
||||
//
|
||||
// That set is also why this has the atlas's escalation, in a lighter form. A
|
||||
// single corrupt file fails the parse loudly, but a source that has simply
|
||||
// VANISHED parses perfectly and imports a table quietly missing everything it
|
||||
// contributed — the same ambiguity (real change vs half-copied mount) the atlas
|
||||
// stages a facet removal for. So a disappearing source is refused and reported
|
||||
// rather than applied.
|
||||
//
|
||||
// It is lighter than the atlas's because it needs to be: the atlas stores a
|
||||
// pending decision in its own table and adds approve/reject endpoints, whereas
|
||||
// here the decision is a single boolean an admin passes to the import they were
|
||||
// already going to run. Re-parsing at approval time — the property that makes
|
||||
// the atlas store only the decision — is automatic when there is nothing stored.
|
||||
|
||||
const SETTING_KEY = 'cliloc_client_path'
|
||||
|
||||
/**
|
||||
* Where the converted cliloc file lives.
|
||||
*
|
||||
* The admin setting wins over the environment so an operator can repoint it
|
||||
* without a redeploy, matching how the rest of the shard integration is
|
||||
* admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the
|
||||
* deploy-time default, since the path usually describes a mount the deployment
|
||||
* sets up.
|
||||
*/
|
||||
async function getClientPath() {
|
||||
try {
|
||||
const configured = await settings.get(SETTING_KEY)
|
||||
if (configured && String(configured).trim() !== '') return String(configured).trim()
|
||||
} catch {
|
||||
// Settings unavailable is not fatal — fall through to the env default.
|
||||
}
|
||||
const fromEnv = process.env.UO_CLIENT_PATH
|
||||
return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : ''
|
||||
}
|
||||
|
||||
async function setClientPath(value, updatedBy = null) {
|
||||
const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy)
|
||||
invalidate()
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Refresh ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Was the loaded table built by THIS parser? */
|
||||
const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION
|
||||
|
||||
/**
|
||||
* Refresh the cliloc table from the configured file.
|
||||
*
|
||||
* Returns a result describing what happened rather than throwing, so the caller
|
||||
* — including the boot path — can log it and move on:
|
||||
*
|
||||
* `skipped` no path configured
|
||||
* `unavailable` path configured but missing / unreadable / not a cliloc file
|
||||
* `unchanged` source hashes match the loaded table; nothing parsed
|
||||
* `imported` parsed and applied
|
||||
* `needsReview` a previously-present source has vanished; NOT applied
|
||||
* `failed` parsed or applied and something went wrong
|
||||
*
|
||||
* `force` skips the hash check (an admin asking for a reimport). `approve`
|
||||
* additionally accepts a vanished source.
|
||||
*/
|
||||
async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
|
||||
// An explicit override wins outright — a one-off "use this file", which must
|
||||
// not be silently overruled by the configured path the way an env default is.
|
||||
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
|
||||
if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' }
|
||||
|
||||
let fingerprint
|
||||
try {
|
||||
fingerprint = hashSources(configured)
|
||||
} catch (err) {
|
||||
if (err instanceof ClilocSourceError) {
|
||||
return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
|
||||
}
|
||||
return { status: 'failed', reason: err.message, path: configured }
|
||||
}
|
||||
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
|
||||
// Two things make a loaded table stale: any source changed, or the PARSER did.
|
||||
// Only checking the sources would strand an install whose client never patches
|
||||
// on whatever an older build derived.
|
||||
if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) {
|
||||
return {
|
||||
status: 'unchanged',
|
||||
path: configured,
|
||||
file: fingerprint.file,
|
||||
count: meta.count ?? null,
|
||||
customCount: fingerprint.customCount,
|
||||
}
|
||||
}
|
||||
|
||||
// A source that was there last import and is not there now is refused, not
|
||||
// applied — an unmounted volume and a deliberate deletion look identical from
|
||||
// here, and the wrong guess silently drops every name that file contributed.
|
||||
const gone = missingSources(fingerprint.hashes, meta?.hashes)
|
||||
if (gone.length > 0 && !approve) {
|
||||
return {
|
||||
status: 'needsReview',
|
||||
reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`,
|
||||
missingSources: gone,
|
||||
path: configured,
|
||||
file: fingerprint.file,
|
||||
}
|
||||
}
|
||||
|
||||
let parsed
|
||||
try {
|
||||
parsed = readCliloc(configured)
|
||||
} catch (err) {
|
||||
if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) {
|
||||
return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
|
||||
}
|
||||
return { status: 'failed', reason: err.message, path: configured }
|
||||
}
|
||||
|
||||
try {
|
||||
const applied = await db.replaceAll(parsed.entries, parsed.source)
|
||||
invalidate()
|
||||
return {
|
||||
status: 'imported',
|
||||
path: configured,
|
||||
file: parsed.source.file,
|
||||
count: applied.count,
|
||||
parsed: parsed.entries.length,
|
||||
blank: applied.blank,
|
||||
// Per-source breakdown: how many entries each file contributed and how
|
||||
// many of them overrode something already merged. An operator who adds an
|
||||
// overlay wants to see it took effect, and "overrode: 0" on a file meant
|
||||
// to re-label stock items says it did not.
|
||||
sources: parsed.source.sources,
|
||||
acceptedMissing: gone.length > 0 ? gone : undefined,
|
||||
}
|
||||
} catch (err) {
|
||||
return { status: 'failed', reason: err.message, path: configured }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Boot hook. Best-effort by contract: it logs and returns, never throws, so a
|
||||
* missing or malformed cliloc file can never stop the site coming up.
|
||||
*/
|
||||
async function refreshOnBoot() {
|
||||
try {
|
||||
const result = await refresh()
|
||||
switch (result.status) {
|
||||
case 'imported':
|
||||
log.info('cliloc table refreshed', {
|
||||
file: result.file,
|
||||
count: result.count,
|
||||
overlays: (result.sources || []).filter((s) => s.kind === 'custom').length,
|
||||
})
|
||||
break
|
||||
case 'needsReview':
|
||||
log.warn(
|
||||
'cliloc refresh staged for admin review — a previously-loaded source is missing; ' +
|
||||
'the existing table is unchanged',
|
||||
{ missing: result.missingSources },
|
||||
)
|
||||
break
|
||||
case 'unavailable':
|
||||
// Deliberately a warning, not an error: an operator who has not supplied
|
||||
// a cliloc file is in a supported state (the UI shows item ids), and the
|
||||
// most common cause — pointing at the client's own compressed file —
|
||||
// needs the reason spelled out rather than a stack trace.
|
||||
log.warn('cliloc source unavailable (item names will show as ids)', {
|
||||
reason: result.reason,
|
||||
code: result.code,
|
||||
path: result.path,
|
||||
})
|
||||
break
|
||||
case 'failed':
|
||||
log.warn('cliloc refresh failed', { reason: result.reason })
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
return result
|
||||
} catch (err) {
|
||||
log.warn('cliloc refresh errored', { error: err.message })
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
/** Everything the admin panel needs to describe cliloc state. */
|
||||
async function status({ path: pathOverride = '' } = {}) {
|
||||
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
const loaded = await db.count().catch(() => 0)
|
||||
|
||||
let fileReadable = false
|
||||
let file = null
|
||||
let drift = null
|
||||
let problem = null
|
||||
let code = null
|
||||
let sources = []
|
||||
let missing = []
|
||||
if (configured !== '') {
|
||||
try {
|
||||
const fingerprint = hashSources(configured)
|
||||
fileReadable = true
|
||||
file = fingerprint.file
|
||||
sources = Object.keys(fingerprint.hashes)
|
||||
missing = missingSources(fingerprint.hashes, meta?.hashes)
|
||||
// A compressed file is readable but not importable, and the panel has to
|
||||
// say so HERE — otherwise pointing at an unconverted client directory
|
||||
// reports a healthy file with pending drift ("ready to import") and the
|
||||
// operator only finds out when the import fails. `drift` stays null
|
||||
// because comparing hashes with an unusable file answers nothing.
|
||||
if (fingerprint.compressed) {
|
||||
problem =
|
||||
'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' +
|
||||
'Convert it to the plain format first — see docs/website/CLILOCS.md.'
|
||||
code = 'COMPRESSED'
|
||||
} else {
|
||||
drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta)
|
||||
}
|
||||
} catch (err) {
|
||||
fileReadable = false
|
||||
problem = err.message
|
||||
code = err.code ?? null
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
configured: configured !== '',
|
||||
path: configured,
|
||||
file,
|
||||
fileReadable,
|
||||
problem,
|
||||
code,
|
||||
drift,
|
||||
count: loaded,
|
||||
// Every source found now (base first, then overlays), what each contributed
|
||||
// at the last import, and any that have since vanished — which is the state
|
||||
// an import will refuse without `approve`.
|
||||
sources,
|
||||
loadedSources: meta?.sources ?? null,
|
||||
missingSources: missing,
|
||||
importedAt: meta?.importedAt ?? null,
|
||||
sourceBytes: meta?.bytes ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Lookup ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is
|
||||
// ~123k rows and shipping it to a client would dwarf every page that uses it,
|
||||
// and the Android app consumes the same JSON and would otherwise need its own
|
||||
// copy. Callers get names, not ids-plus-a-table.
|
||||
|
||||
// A small write-through cache in front of the table. Item ids repeat heavily —
|
||||
// one page of listings is mostly the same few hundred clilocs, and a character
|
||||
// sheet re-resolves the same gear on every view — so this turns the steady state
|
||||
// into zero queries. Capped so a pathological caller cannot grow it without
|
||||
// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry,
|
||||
// which is cheap and correct for a table that only changes on reimport.
|
||||
const CACHE_MAX = 20000
|
||||
let cache = new Map()
|
||||
|
||||
function invalidate() {
|
||||
cache = new Map()
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a batch of cliloc ids to display strings.
|
||||
*
|
||||
* Returns a `Map<number, string>` holding only the ids that resolved to
|
||||
* something displayable — an id with no row, or one whose text is nothing but
|
||||
* interpolated arguments we do not have, is simply absent. Callers fall back to
|
||||
* whatever they had (the item id), so "missing" and "unnamed" collapse into one
|
||||
* branch at the call site.
|
||||
*
|
||||
* Never throws: a cliloc lookup is decoration on someone's character sheet, and
|
||||
* a database blip must not fail the sheet.
|
||||
*/
|
||||
async function resolveMany(numbers) {
|
||||
const out = new Map()
|
||||
if (!Array.isArray(numbers)) return out
|
||||
|
||||
const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))]
|
||||
if (wanted.length === 0) return out
|
||||
|
||||
const missing = []
|
||||
for (const number of wanted) {
|
||||
if (cache.has(number)) {
|
||||
const hit = cache.get(number)
|
||||
if (hit !== '') out.set(number, hit)
|
||||
} else {
|
||||
missing.push(number)
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
try {
|
||||
const rows = await db.lookup(missing)
|
||||
const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)]))
|
||||
if (cache.size + missing.length > CACHE_MAX) invalidate()
|
||||
for (const number of missing) {
|
||||
// Cache the miss too ('' meaning "no usable name"), so an id absent from
|
||||
// the table does not re-query on every page view.
|
||||
const text = found.get(number) ?? ''
|
||||
cache.set(number, text)
|
||||
if (text !== '') out.set(number, text)
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn('cliloc lookup failed', { message: err.message })
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
/** Single-id convenience. Returns `null` when there is no usable name. */
|
||||
async function resolve(number) {
|
||||
const found = await resolveMany([number])
|
||||
return found.get(number) ?? null
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
SETTING_KEY,
|
||||
getClientPath,
|
||||
setClientPath,
|
||||
refresh,
|
||||
refreshOnBoot,
|
||||
status,
|
||||
resolveMany,
|
||||
resolve,
|
||||
invalidate,
|
||||
}
|
||||
Reference in New Issue
Block a user