feat(server): port the UO models, utils and schema fragment
The data half of the extraction: 8 model directories, 13 utils, the shard
stream catalog and the 27-table schema fragment with its purge.
server/core.js is what makes the port a one-line import change per file rather
than a signature change per function. Ported code requires its dependencies at
file scope -- `const { query } = require('../../core')` -- which runs before
register() has been called and before any ctx exists. So every member is a
stable function that resolves ctx when CALLED, and nothing may be destructured
off ctx at init either, because core is free to hand over a getter.
Two helpers are vendored rather than taken from ctx, and the line between them
is the point. utils/excerpt.js is core's deriveExcerpt -- nine lines of pure
text handling. Core's sanitiser next to it was NOT copied: a second copy of a
security control diverges silently the moment either is fixed. announceLinks.js
vendors legError and articleUrl the same way, but baseUrl could not be: core's
reads APP_BASE_URL, and §2.7 forbids a module reading core's environment, so it
comes off ctx.site.baseUrl.
The schema fragment is core's 27 shard_*/uo_link_* statements, verbs CREATE,
ALTER and UPDATE only, every CREATE TABLE guarded. Two of its tables carry a
foreign key INTO users, which is allowed and is why the replay order matters --
core's schema is in place before this runs. The reverse never occurs and must
not: it would make core unable to boot without a module installed.
One real port bug caught by the integration run, not by tests: the atlas art
map resolved `../../../db/data`, which pointed at core's tree when this file
lived there and points outside server/ now. A path that happens to resolve is
exactly what survives a green suite, because the absent-file branch returns {}
and looks like the normal case.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
368
server/model/shardClilocs/shardClilocs.model.js
Normal file
368
server/model/shardClilocs/shardClilocs.model.js
Normal file
@@ -0,0 +1,368 @@
|
||||
const db = require('./shardClilocs.db')
|
||||
const { settings } = require('../../core')
|
||||
const { displayText } = require('../../utils/clilocParse')
|
||||
const {
|
||||
ClilocFormatError,
|
||||
ClilocSourceError,
|
||||
PARSER_VERSION,
|
||||
hashSources,
|
||||
sameSources,
|
||||
missingSources,
|
||||
readCliloc,
|
||||
} = require('../../utils/clilocSource')
|
||||
const log = require('../../core').logger('shardClilocs')
|
||||
|
||||
// The cliloc table — UO's id → display-string map, refreshed from a file the
|
||||
// operator converts once from their own client.
|
||||
//
|
||||
// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a
|
||||
// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace
|
||||
// listing sends one too. Without the table the UI can only print `id 1023721`
|
||||
// where the game prints "quarter staff".
|
||||
//
|
||||
// Two rules govern the boot path, both inherited from the spawn atlas:
|
||||
//
|
||||
// 1. **It never blocks startup.** No configured path, an unreadable file, a
|
||||
// wrong-format file, a database error — all caught and logged. The site
|
||||
// comes up either way, serving whatever table it already had (or none, in
|
||||
// which case the UI falls back to item ids exactly as it did before).
|
||||
// 2. **Nothing client-derived is committed.** The table is built from the
|
||||
// operator's own file at a configured path. The repo ships no strings.
|
||||
//
|
||||
// The table is built from a SET of sources — the converted client table plus
|
||||
// every operator-maintained overlay beside it — because shards edit items and
|
||||
// add new ones, and those carry cliloc ids no stock client table has. All of
|
||||
// them are re-read on every boot and hash-gated together, so adding one custom
|
||||
// item never means re-exporting a 5 MB client file. Later sources win.
|
||||
//
|
||||
// That set is also why this has the atlas's escalation, in a lighter form. A
|
||||
// single corrupt file fails the parse loudly, but a source that has simply
|
||||
// VANISHED parses perfectly and imports a table quietly missing everything it
|
||||
// contributed — the same ambiguity (real change vs half-copied mount) the atlas
|
||||
// stages a facet removal for. So a disappearing source is refused and reported
|
||||
// rather than applied.
|
||||
//
|
||||
// It is lighter than the atlas's because it needs to be: the atlas stores a
|
||||
// pending decision in its own table and adds approve/reject endpoints, whereas
|
||||
// here the decision is a single boolean an admin passes to the import they were
|
||||
// already going to run. Re-parsing at approval time — the property that makes
|
||||
// the atlas store only the decision — is automatic when there is nothing stored.
|
||||
|
||||
const SETTING_KEY = 'cliloc_client_path'
|
||||
|
||||
/**
|
||||
* Where the converted cliloc file lives.
|
||||
*
|
||||
* The admin setting wins over the environment so an operator can repoint it
|
||||
* without a redeploy, matching how the rest of the shard integration is
|
||||
* admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the
|
||||
* deploy-time default, since the path usually describes a mount the deployment
|
||||
* sets up.
|
||||
*/
|
||||
async function getClientPath() {
|
||||
try {
|
||||
const configured = await settings.get(SETTING_KEY)
|
||||
if (configured && String(configured).trim() !== '') return String(configured).trim()
|
||||
} catch {
|
||||
// Settings unavailable is not fatal — fall through to the env default.
|
||||
}
|
||||
const fromEnv = process.env.UO_CLIENT_PATH
|
||||
return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : ''
|
||||
}
|
||||
|
||||
async function setClientPath(value, updatedBy = null) {
|
||||
const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy)
|
||||
invalidate()
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Refresh ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Was the loaded table built by THIS parser? */
|
||||
const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION
|
||||
|
||||
/**
|
||||
* Refresh the cliloc table from the configured file.
|
||||
*
|
||||
* Returns a result describing what happened rather than throwing, so the caller
|
||||
* — including the boot path — can log it and move on:
|
||||
*
|
||||
* `skipped` no path configured
|
||||
* `unavailable` path configured but missing / unreadable / not a cliloc file
|
||||
* `unchanged` source hashes match the loaded table; nothing parsed
|
||||
* `imported` parsed and applied
|
||||
* `needsReview` a previously-present source has vanished; NOT applied
|
||||
* `failed` parsed or applied and something went wrong
|
||||
*
|
||||
* `force` skips the hash check (an admin asking for a reimport). `approve`
|
||||
* additionally accepts a vanished source.
|
||||
*/
|
||||
async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
|
||||
// An explicit override wins outright — a one-off "use this file", which must
|
||||
// not be silently overruled by the configured path the way an env default is.
|
||||
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
|
||||
if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' }
|
||||
|
||||
let fingerprint
|
||||
try {
|
||||
fingerprint = hashSources(configured)
|
||||
} catch (err) {
|
||||
if (err instanceof ClilocSourceError) {
|
||||
return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
|
||||
}
|
||||
return { status: 'failed', reason: err.message, path: configured }
|
||||
}
|
||||
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
|
||||
// Two things make a loaded table stale: any source changed, or the PARSER did.
|
||||
// Only checking the sources would strand an install whose client never patches
|
||||
// on whatever an older build derived.
|
||||
if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) {
|
||||
return {
|
||||
status: 'unchanged',
|
||||
path: configured,
|
||||
file: fingerprint.file,
|
||||
count: meta.count ?? null,
|
||||
customCount: fingerprint.customCount,
|
||||
}
|
||||
}
|
||||
|
||||
// A source that was there last import and is not there now is refused, not
|
||||
// applied — an unmounted volume and a deliberate deletion look identical from
|
||||
// here, and the wrong guess silently drops every name that file contributed.
|
||||
const gone = missingSources(fingerprint.hashes, meta?.hashes)
|
||||
if (gone.length > 0 && !approve) {
|
||||
return {
|
||||
status: 'needsReview',
|
||||
reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`,
|
||||
missingSources: gone,
|
||||
path: configured,
|
||||
file: fingerprint.file,
|
||||
}
|
||||
}
|
||||
|
||||
let parsed
|
||||
try {
|
||||
parsed = readCliloc(configured)
|
||||
} catch (err) {
|
||||
if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) {
|
||||
return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
|
||||
}
|
||||
return { status: 'failed', reason: err.message, path: configured }
|
||||
}
|
||||
|
||||
try {
|
||||
const applied = await db.replaceAll(parsed.entries, parsed.source)
|
||||
invalidate()
|
||||
return {
|
||||
status: 'imported',
|
||||
path: configured,
|
||||
file: parsed.source.file,
|
||||
count: applied.count,
|
||||
parsed: parsed.entries.length,
|
||||
blank: applied.blank,
|
||||
// Per-source breakdown: how many entries each file contributed and how
|
||||
// many of them overrode something already merged. An operator who adds an
|
||||
// overlay wants to see it took effect, and "overrode: 0" on a file meant
|
||||
// to re-label stock items says it did not.
|
||||
sources: parsed.source.sources,
|
||||
acceptedMissing: gone.length > 0 ? gone : undefined,
|
||||
}
|
||||
} catch (err) {
|
||||
return { status: 'failed', reason: err.message, path: configured }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Boot hook. Best-effort by contract: it logs and returns, never throws, so a
|
||||
* missing or malformed cliloc file can never stop the site coming up.
|
||||
*/
|
||||
async function refreshOnBoot() {
|
||||
try {
|
||||
const result = await refresh()
|
||||
switch (result.status) {
|
||||
case 'imported':
|
||||
log.info('cliloc table refreshed', {
|
||||
file: result.file,
|
||||
count: result.count,
|
||||
overlays: (result.sources || []).filter((s) => s.kind === 'custom').length,
|
||||
})
|
||||
break
|
||||
case 'needsReview':
|
||||
log.warn(
|
||||
'cliloc refresh staged for admin review — a previously-loaded source is missing; ' +
|
||||
'the existing table is unchanged',
|
||||
{ missing: result.missingSources },
|
||||
)
|
||||
break
|
||||
case 'unavailable':
|
||||
// Deliberately a warning, not an error: an operator who has not supplied
|
||||
// a cliloc file is in a supported state (the UI shows item ids), and the
|
||||
// most common cause — pointing at the client's own compressed file —
|
||||
// needs the reason spelled out rather than a stack trace.
|
||||
log.warn('cliloc source unavailable (item names will show as ids)', {
|
||||
reason: result.reason,
|
||||
code: result.code,
|
||||
path: result.path,
|
||||
})
|
||||
break
|
||||
case 'failed':
|
||||
log.warn('cliloc refresh failed', { reason: result.reason })
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
return result
|
||||
} catch (err) {
|
||||
log.warn('cliloc refresh errored', { error: err.message })
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
/** Everything the admin panel needs to describe cliloc state. */
|
||||
async function status({ path: pathOverride = '' } = {}) {
|
||||
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
const loaded = await db.count().catch(() => 0)
|
||||
|
||||
let fileReadable = false
|
||||
let file = null
|
||||
let drift = null
|
||||
let problem = null
|
||||
let code = null
|
||||
let sources = []
|
||||
let missing = []
|
||||
if (configured !== '') {
|
||||
try {
|
||||
const fingerprint = hashSources(configured)
|
||||
fileReadable = true
|
||||
file = fingerprint.file
|
||||
sources = Object.keys(fingerprint.hashes)
|
||||
missing = missingSources(fingerprint.hashes, meta?.hashes)
|
||||
// A compressed file is readable but not importable, and the panel has to
|
||||
// say so HERE — otherwise pointing at an unconverted client directory
|
||||
// reports a healthy file with pending drift ("ready to import") and the
|
||||
// operator only finds out when the import fails. `drift` stays null
|
||||
// because comparing hashes with an unusable file answers nothing.
|
||||
if (fingerprint.compressed) {
|
||||
problem =
|
||||
'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' +
|
||||
'Convert it to the plain format first — see docs/website/CLILOCS.md.'
|
||||
code = 'COMPRESSED'
|
||||
} else {
|
||||
drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta)
|
||||
}
|
||||
} catch (err) {
|
||||
fileReadable = false
|
||||
problem = err.message
|
||||
code = err.code ?? null
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
configured: configured !== '',
|
||||
path: configured,
|
||||
file,
|
||||
fileReadable,
|
||||
problem,
|
||||
code,
|
||||
drift,
|
||||
count: loaded,
|
||||
// Every source found now (base first, then overlays), what each contributed
|
||||
// at the last import, and any that have since vanished — which is the state
|
||||
// an import will refuse without `approve`.
|
||||
sources,
|
||||
loadedSources: meta?.sources ?? null,
|
||||
missingSources: missing,
|
||||
importedAt: meta?.importedAt ?? null,
|
||||
sourceBytes: meta?.bytes ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Lookup ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is
|
||||
// ~123k rows and shipping it to a client would dwarf every page that uses it,
|
||||
// and the Android app consumes the same JSON and would otherwise need its own
|
||||
// copy. Callers get names, not ids-plus-a-table.
|
||||
|
||||
// A small write-through cache in front of the table. Item ids repeat heavily —
|
||||
// one page of listings is mostly the same few hundred clilocs, and a character
|
||||
// sheet re-resolves the same gear on every view — so this turns the steady state
|
||||
// into zero queries. Capped so a pathological caller cannot grow it without
|
||||
// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry,
|
||||
// which is cheap and correct for a table that only changes on reimport.
|
||||
const CACHE_MAX = 20000
|
||||
let cache = new Map()
|
||||
|
||||
function invalidate() {
|
||||
cache = new Map()
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a batch of cliloc ids to display strings.
|
||||
*
|
||||
* Returns a `Map<number, string>` holding only the ids that resolved to
|
||||
* something displayable — an id with no row, or one whose text is nothing but
|
||||
* interpolated arguments we do not have, is simply absent. Callers fall back to
|
||||
* whatever they had (the item id), so "missing" and "unnamed" collapse into one
|
||||
* branch at the call site.
|
||||
*
|
||||
* Never throws: a cliloc lookup is decoration on someone's character sheet, and
|
||||
* a database blip must not fail the sheet.
|
||||
*/
|
||||
async function resolveMany(numbers) {
|
||||
const out = new Map()
|
||||
if (!Array.isArray(numbers)) return out
|
||||
|
||||
const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))]
|
||||
if (wanted.length === 0) return out
|
||||
|
||||
const missing = []
|
||||
for (const number of wanted) {
|
||||
if (cache.has(number)) {
|
||||
const hit = cache.get(number)
|
||||
if (hit !== '') out.set(number, hit)
|
||||
} else {
|
||||
missing.push(number)
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
try {
|
||||
const rows = await db.lookup(missing)
|
||||
const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)]))
|
||||
if (cache.size + missing.length > CACHE_MAX) invalidate()
|
||||
for (const number of missing) {
|
||||
// Cache the miss too ('' meaning "no usable name"), so an id absent from
|
||||
// the table does not re-query on every page view.
|
||||
const text = found.get(number) ?? ''
|
||||
cache.set(number, text)
|
||||
if (text !== '') out.set(number, text)
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn('cliloc lookup failed', { message: err.message })
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
/** Single-id convenience. Returns `null` when there is no usable name. */
|
||||
async function resolve(number) {
|
||||
const found = await resolveMany([number])
|
||||
return found.get(number) ?? null
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
SETTING_KEY,
|
||||
getClientPath,
|
||||
setClientPath,
|
||||
refresh,
|
||||
refreshOnBoot,
|
||||
status,
|
||||
resolveMany,
|
||||
resolve,
|
||||
invalidate,
|
||||
}
|
||||
Reference in New Issue
Block a user