feat(server): port the UO models, utils and schema fragment
The data half of the extraction: 8 model directories, 13 utils, the shard
stream catalog and the 27-table schema fragment with its purge.
server/core.js is what makes the port a one-line import change per file rather
than a signature change per function. Ported code requires its dependencies at
file scope -- `const { query } = require('../../core')` -- which runs before
register() has been called and before any ctx exists. So every member is a
stable function that resolves ctx when CALLED, and nothing may be destructured
off ctx at init either, because core is free to hand over a getter.
Two helpers are vendored rather than taken from ctx, and the line between them
is the point. utils/excerpt.js is core's deriveExcerpt -- nine lines of pure
text handling. Core's sanitiser next to it was NOT copied: a second copy of a
security control diverges silently the moment either is fixed. announceLinks.js
vendors legError and articleUrl the same way, but baseUrl could not be: core's
reads APP_BASE_URL, and §2.7 forbids a module reading core's environment, so it
comes off ctx.site.baseUrl.
The schema fragment is core's 27 shard_*/uo_link_* statements, verbs CREATE,
ALTER and UPDATE only, every CREATE TABLE guarded. Two of its tables carry a
foreign key INTO users, which is allowed and is why the replay order matters --
core's schema is in place before this runs. The reverse never occurs and must
not: it would make core unable to boot without a module installed.
One real port bug caught by the integration run, not by tests: the atlas art
map resolved `../../../db/data`, which pointed at core's tree when this file
lived there and points outside server/ now. A path that happens to resolve is
exactly what survives a green suite, because the absent-file branch returns {}
and looks like the normal case.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
316
server/utils/clilocSource.js
Normal file
316
server/utils/clilocSource.js
Normal file
@@ -0,0 +1,316 @@
|
||||
// Cliloc table — the filesystem layer.
|
||||
//
|
||||
// `clilocParse.js` holds the pure parsers; this module is the only thing that
|
||||
// touches cliloc files on disk, and it is shared by both callers:
|
||||
//
|
||||
// - the server, which refreshes the table on boot (`shardClilocs.model.js`)
|
||||
// - the admin panel, which can force a reimport without a restart
|
||||
//
|
||||
// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived
|
||||
// from them is committed: the repo holds no string table, exactly as it holds no
|
||||
// map snapshot and no artwork. That rule is why this module reads a configured
|
||||
// path instead of a path inside the repo.
|
||||
//
|
||||
// ── Why this reads a SET of files, not one ────────────────────────────────
|
||||
//
|
||||
// Shards edit items and add new ones. Those carry cliloc ids that a stock client
|
||||
// table does not have — and forcing a 5 MB client re-export every time an
|
||||
// operator adds one item would be miserable enough that the table would simply
|
||||
// go stale, which is the exact failure the spawn atlas was redesigned to avoid.
|
||||
//
|
||||
// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted
|
||||
// client file) plus every operator-maintained OVERLAY beside it, all re-read on
|
||||
// every boot and hash-gated as a SET. Adding, editing or removing any overlay
|
||||
// counts as drift and re-imports. Later sources win, so an overlay both adds new
|
||||
// ids and overrides stock ones.
|
||||
//
|
||||
// Measured on a real shard: the script tree references 16,434 cliloc ids and only
|
||||
// 37 are absent from the stock client table. Tens of entries against a 67k base
|
||||
// is what makes the overlay the right shape rather than a second full table.
|
||||
//
|
||||
// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so
|
||||
// the boot path hashes first and only parses when something actually changed.
|
||||
|
||||
const crypto = require('crypto')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse')
|
||||
|
||||
/**
|
||||
* Filenames looked for as the BASE table when the configured path is a directory.
|
||||
*
|
||||
* Ordered by how specific they are: an explicitly converted file wins over
|
||||
* something that merely sits in a client folder, so an operator who dropped a
|
||||
* `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one
|
||||
* they made rather than the one that will be rejected.
|
||||
*
|
||||
* Matching is case-insensitive against the real directory listing, because the
|
||||
* client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where
|
||||
* a hardcoded lowercase open would simply miss.
|
||||
*/
|
||||
const CANDIDATE_NAMES = [
|
||||
'clilocs.tsv',
|
||||
'clilocs.csv',
|
||||
'clilocs.plain',
|
||||
'cliloc.plain',
|
||||
'cliloc.plain.enu',
|
||||
'cliloc.enu.plain',
|
||||
'clilocs.txt',
|
||||
'cliloc.enu',
|
||||
]
|
||||
|
||||
/**
|
||||
* Where shard-specific additions and overrides live: a `custom/` directory
|
||||
* beside the base table.
|
||||
*
|
||||
* ServUO has **no server-side convention** for custom clilocs — they live in the
|
||||
* patched client file a shard distributes to its players, and nothing in the
|
||||
* tree declares them. There is therefore nothing to discover, and this is the
|
||||
* one place in the cliloc pipeline that is a convention we chose rather than one
|
||||
* the shard already has. It is a directory rather than a single file so an
|
||||
* operator can keep additions grouped however they like (per system, per patch)
|
||||
* without the site caring.
|
||||
*/
|
||||
const CUSTOM_DIR = 'custom'
|
||||
const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain']
|
||||
|
||||
class ClilocSourceError extends Error {
|
||||
constructor(message, code) {
|
||||
super(message)
|
||||
this.name = 'ClilocSourceError'
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
function sha256(buffer) {
|
||||
return crypto.createHash('sha256').update(buffer).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the configured path to `{ root, base }`.
|
||||
*
|
||||
* Accepts either a direct file path or a directory to search, because operators
|
||||
* reasonably supply both — "here is the file" and "here is the folder I put it
|
||||
* in" are equally natural answers to the admin panel's prompt. When it is a
|
||||
* file, `root` is the directory CONTAINING it, so overlays work either way: an
|
||||
* operator who pointed at a file should not have to re-point at its folder just
|
||||
* to add a `custom/` directory next to it.
|
||||
*/
|
||||
function resolveBase(configured) {
|
||||
if (!configured || String(configured).trim() === '') {
|
||||
throw new ClilocSourceError('No cliloc path configured', 'NO_PATH')
|
||||
}
|
||||
const target = String(configured).trim()
|
||||
|
||||
let stat
|
||||
try {
|
||||
stat = fs.statSync(target)
|
||||
} catch {
|
||||
throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND')
|
||||
}
|
||||
|
||||
if (stat.isFile()) return { root: path.dirname(target), base: target }
|
||||
|
||||
if (!stat.isDirectory()) {
|
||||
throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND')
|
||||
}
|
||||
|
||||
let listing
|
||||
try {
|
||||
listing = fs.readdirSync(target)
|
||||
} catch {
|
||||
throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND')
|
||||
}
|
||||
|
||||
const byLower = new Map(listing.map((name) => [name.toLowerCase(), name]))
|
||||
for (const candidate of CANDIDATE_NAMES) {
|
||||
const actual = byLower.get(candidate)
|
||||
if (actual) return { root: target, base: path.join(target, actual) }
|
||||
}
|
||||
|
||||
throw new ClilocSourceError(
|
||||
`No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`,
|
||||
'NO_FILE',
|
||||
)
|
||||
}
|
||||
|
||||
/** Overlay files under `<root>/custom/`, sorted so precedence is deterministic. */
|
||||
function listCustom(root) {
|
||||
const dir = path.join(root, CUSTOM_DIR)
|
||||
let listing
|
||||
try {
|
||||
listing = fs.readdirSync(dir, { withFileTypes: true })
|
||||
} catch (err) {
|
||||
// No overlay directory is the normal case, not an error.
|
||||
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
|
||||
throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE')
|
||||
}
|
||||
return listing
|
||||
.filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase()))
|
||||
.map((e) => e.name)
|
||||
.sort()
|
||||
.map((name) => path.join(dir, name))
|
||||
}
|
||||
|
||||
function readFileOrThrow(file) {
|
||||
try {
|
||||
return fs.readFileSync(file)
|
||||
} catch {
|
||||
throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read every cliloc source under the configured path.
|
||||
*
|
||||
* Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }`
|
||||
* with the base first and overlays after, in the order they must be merged.
|
||||
*
|
||||
* Labels are root-relative and forward-slashed so a hash map compares equal
|
||||
* across platforms — the same directory read on Windows and Linux must produce
|
||||
* the same fingerprint, or every boot would look like a change. (The same
|
||||
* reasoning, and the same bug, as `spawnAtlasSource.readSources`.)
|
||||
*/
|
||||
function readSources(configured) {
|
||||
const { root, base } = resolveBase(configured)
|
||||
|
||||
const describe = (file, kind) => {
|
||||
const buffer = readFileOrThrow(file)
|
||||
return {
|
||||
label: path.relative(root, file).split(path.sep).join('/'),
|
||||
kind,
|
||||
file,
|
||||
buffer,
|
||||
sha256: sha256(buffer),
|
||||
bytes: buffer.length,
|
||||
compressed: isCompressedCliloc(buffer),
|
||||
}
|
||||
}
|
||||
|
||||
const files = [describe(base, 'base')]
|
||||
for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom'))
|
||||
|
||||
return { root, files }
|
||||
}
|
||||
|
||||
/**
|
||||
* A fingerprint of every source: `{ "<label>": "<sha256>" }`, plus the base's
|
||||
* details for the admin panel.
|
||||
*
|
||||
* `compressed` is reported here rather than left to the parse because the admin
|
||||
* panel calls this and NOT `readCliloc` (parsing 5 MB on every status poll would
|
||||
* be wasteful). Without it, pointing the setting at an unconverted client
|
||||
* directory reports a perfectly readable file with pending drift — "ready to
|
||||
* import" — and the operator only learns otherwise when the import fails. The
|
||||
* check is four bytes of a buffer already in hand.
|
||||
*/
|
||||
function hashSources(configured) {
|
||||
const { root, files } = readSources(configured)
|
||||
const hashes = {}
|
||||
for (const file of files) hashes[file.label] = file.sha256
|
||||
const base = files[0]
|
||||
return {
|
||||
root,
|
||||
hashes,
|
||||
file: base.file,
|
||||
bytes: base.bytes,
|
||||
compressed: files.some((f) => f.compressed),
|
||||
customCount: files.length - 1,
|
||||
}
|
||||
}
|
||||
|
||||
/** True when two source fingerprints describe the same set of files. */
|
||||
function sameSources(a, b) {
|
||||
if (!a || !b) return false
|
||||
const aKeys = Object.keys(a).sort()
|
||||
const bKeys = Object.keys(b).sort()
|
||||
if (aKeys.length !== bKeys.length) return false
|
||||
return aKeys.every((key, i) => key === bKeys[i] && a[key] === b[key])
|
||||
}
|
||||
|
||||
/**
|
||||
* Labels present in `loaded` that are absent from `current`.
|
||||
*
|
||||
* This is the multi-source hazard that a single file did not have. One corrupt
|
||||
* file fails the parse loudly, but a source that has simply VANISHED — an
|
||||
* unmounted volume, a half-copied deploy — parses perfectly and imports a table
|
||||
* quietly missing everything that file contributed. That is the same ambiguity
|
||||
* the spawn atlas escalates for a disappearing facet, so it is escalated here
|
||||
* too rather than applied.
|
||||
*/
|
||||
function missingSources(current, loaded) {
|
||||
if (!loaded) return []
|
||||
return Object.keys(loaded).filter((label) => !Object.hasOwn(current, label))
|
||||
}
|
||||
|
||||
/**
|
||||
* Read and parse every source, merged into one entry list.
|
||||
*
|
||||
* Later sources win: the base client table first, then each overlay in sorted
|
||||
* order, so an overlay both ADDS ids the client never had and OVERRIDES stock
|
||||
* ones the shard has re-purposed.
|
||||
*
|
||||
* Returns `{ entries, source }`. Throws `ClilocSourceError` for anything about
|
||||
* the paths and `ClilocFormatError` for anything about the contents — different
|
||||
* problems for an operator (wrong place vs wrong file), and the admin panel says
|
||||
* which. A format error names the file it came from, because "which of my six
|
||||
* overlay files is malformed" is otherwise a guessing game.
|
||||
*/
|
||||
function readCliloc(configured) {
|
||||
const { root, files } = readSources(configured)
|
||||
|
||||
const merged = new Map()
|
||||
const perSource = []
|
||||
|
||||
for (const file of files) {
|
||||
let entries
|
||||
try {
|
||||
entries = parseCliloc(file.buffer)
|
||||
} catch (err) {
|
||||
if (err instanceof ClilocFormatError) {
|
||||
throw new ClilocFormatError(`${file.label}: ${err.message}`, err.code)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
let added = 0
|
||||
let overrode = 0
|
||||
for (const entry of entries) {
|
||||
if (!Number.isInteger(entry.number)) continue
|
||||
if (merged.has(entry.number)) overrode++
|
||||
else added++
|
||||
merged.set(entry.number, entry)
|
||||
}
|
||||
perSource.push({ label: file.label, kind: file.kind, entries: entries.length, added, overrode })
|
||||
}
|
||||
|
||||
return {
|
||||
entries: [...merged.values()],
|
||||
source: {
|
||||
root,
|
||||
file: files[0].file,
|
||||
sha256: files[0].sha256,
|
||||
bytes: files[0].bytes,
|
||||
hashes: Object.fromEntries(files.map((f) => [f.label, f.sha256])),
|
||||
parserVersion: PARSER_VERSION,
|
||||
sources: perSource,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ClilocFormatError,
|
||||
ClilocSourceError,
|
||||
PARSER_VERSION,
|
||||
CANDIDATE_NAMES,
|
||||
CUSTOM_DIR,
|
||||
CUSTOM_EXTENSIONS,
|
||||
resolveBase,
|
||||
listCustom,
|
||||
readSources,
|
||||
hashSources,
|
||||
sameSources,
|
||||
missingSources,
|
||||
readCliloc,
|
||||
}
|
||||
Reference in New Issue
Block a user