RunicGateway/Module-uo
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(server): the whole server half (phase 3, slice 1) #3

Merged
whitlocktech merged 3 commits from feature/module-extract-server into main 2026-08-11 21:05:59 +00:00
Conversation 0 Commits 3 Files Changed 87 +7967 -3
40 changed files with 7967 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fe3251a543 - Show all commits

172
server/config/shardStreams.js Normal file
View File

@@ -0,0 +1,172 @@
// ── Shard-derived push streams + event → stream mapping ────────────────────
//
// MODULE-UO CONTENT, still living in core. MODULE_SYSTEM.md §1.8 named
// config/notificationStreams.js as one of the three genuinely entangled files:
// most of its catalog and all of `mapShardEvent` are shard-derived, and it reads
// `PUBLIC_KINDS` out of utils/shardBroadcast. PR 4 split it — core's one stream
// is config/coreStreams.js, and everything shard-shaped is here, in a file that
// moves to module-uo whole in Phase 3. Nothing in core imports it except
// modules/registries.js's registerCore(), which is the one line Phase 3 deletes.
//
// Two families:
// • public / opt-in — no linked game account required; delivered to every
// subscriber. Drawn ONLY from the SSE public allowlist
// (utils/shardBroadcast PUBLIC_KINDS) — a sensitive kind
// can never produce a public push.
// • personal / owner-keyed — require a linked game account; delivered ONLY to
// the owning user's devices (resolved from the event's
// game account via shardLinks), never fanned out publicly.
//
// The payload the relay ever carries is a CONTENT-FREE tickle ({ stream, ref });
// `ref` is an opaque hint (serial / city / timestamp) the app uses to pull the
// real, ownership-checked content over the authenticated API. So even a leaked
// ntfy topic reveals nothing (docs/android/PLAN.md §11).
const { PUBLIC_KINDS } = require('../utils/shardBroadcast')
const STREAMS = [
{
id: 'server.status',
label: 'Server up / down',
description: 'The shard comes online or goes offline.',
personal: false,
requiresLinkedAccount: false,
},
{
id: 'idoc.warning',
label: 'IDOC warnings',
description: 'A house falls into its final (IDOC) decay stage.',
personal: false,
requiresLinkedAccount: false,
},
{
id: 'champ.start',
label: 'Champion spawn starts',
description: 'A champion spawn becomes active.',
personal: false,
requiresLinkedAccount: false,
},
{
id: 'governor.election',
label: 'Governor elections',
description: 'A town elects a new governor.',
personal: false,
requiresLinkedAccount: false,
},
{
id: 'vendor.sale',
label: 'Your vendor sold an item',
description: 'One of your player vendors made a sale.',
personal: true,
requiresLinkedAccount: true,
},
{
id: 'house.idoc',
label: 'Your house entered IDOC',
description: 'One of your houses fell into its final decay stage.',
personal: true,
requiresLinkedAccount: true,
},
{
id: 'account.login',
label: 'A login to your account',
description: 'An authentication attempt against your game account.',
personal: true,
requiresLinkedAccount: true,
},
]
// The owner-keyed subset, needed by mapShardEvent's public-safety filter below.
// Derived from this file's own catalog rather than read back out of the registry:
// the filter is about THESE streams, and a module must not be able to weaken it
// by registering something that happens to share an id.
const PERSONAL_STREAMS = new Set(STREAMS.filter((s) => s.personal).map((s) => s.id))
// Per-process transition state so full-state upserts (champ.update / city.update
// are upserts, not discrete "started"/"elected" events — see docs/link
// PROTOCOL_2 §383) only fire once, on an actual transition. Injectable so tests
// pass a fresh tracker; a module-level default backs the live dispatcher.
function createTracker() {
return { champActive: new Map(), cityGovernor: new Map() }
}
const defaultTracker = createTracker()
// Per-kind mappers, each pushing 0+ targets onto `out` (and updating `tracker`
// for the upsert-transition kinds). Split out of mapShardEvent so that function
// stays a trivial dispatch + the public-safety filter.
const serverStatusUp = (event, tracker, out) =>
out.push({ streamId: 'server.status', ref: `up:${event.bootId || ''}` })
const serverStatusDown = (event, tracker, out) => out.push({ streamId: 'server.status', ref: 'down' })
const EVENT_MAPPERS = {
'server.hello': serverStatusUp,
'server.shutdown': serverStatusDown,
'server.crashed': serverStatusDown,
'house.decay': (event, tracker, out) => {
if (String(event.to).toUpperCase() !== 'IDOC') return
const ref = String(event.serial ?? '')
out.push({ streamId: 'idoc.warning', ref }) // public — location only
if (event.ownerAcct) {
out.push({ streamId: 'house.idoc', ref, ownerAccount: event.ownerAcct }) // personal
}
},
'champ.update': (event, tracker, out) => {
const { serial } = event
if (serial == null) return
const wasActive = tracker.champActive.get(serial) === true
const isActive = event.active === true
tracker.champActive.set(serial, isActive)
if (isActive && !wasActive) out.push({ streamId: 'champ.start', ref: String(serial) })
},
'champ.remove': (event, tracker) => {
if (event.serial != null) tracker.champActive.delete(event.serial)
},
'city.update': (event, tracker, out) => {
const { city } = event
if (!city) return
const gov = event.governor && event.governor.serial != null ? String(event.governor.serial) : null
const prev = tracker.cityGovernor.get(city)
tracker.cityGovernor.set(city, gov)
// Only a real transition to a new governor, and never on first sight
// (prev === undefined) so a reconnect snapshot isn't read as an election.
if (prev !== undefined && gov && gov !== prev) {
out.push({ streamId: 'governor.election', ref: String(city) })
}
},
'vendor.sale': (event, tracker, out) => {
if (event.ownerAcct) {
out.push({ streamId: 'vendor.sale', ref: String(event.t ?? ''), ownerAccount: event.ownerAcct })
}
},
'account.login.attempt': (event, tracker, out) => {
if (event.acct) {
out.push({ streamId: 'account.login', ref: String(event.t ?? ''), ownerAccount: event.acct })
}
},
}
// Map one shard event → an array of targets ({ streamId, ref, ownerAccount? }).
// May yield 0, 1, or 2 targets (an owner house.decay produces both the public
// idoc.warning and the personal house.idoc). Pure given `tracker`.
function mapShardEvent(event, tracker = defaultTracker) {
if (!event || typeof event.kind !== 'string') return []
const kind = event.kind
const out = []
const mapper = EVENT_MAPPERS[kind]
if (mapper) mapper(event, tracker, out)
// Defense in depth: a PUBLIC (non-personal) target may only ride a public-safe
// kind. Personal targets are owner-keyed and delivered solely to the owner, so
// they are exempt from the public allowlist (that is the whole point of the
// owner-keyed split). This guarantees a sensitive kind can never leak publicly
// even if a future mapping case is added carelessly.
//
// This filter, the kinds it reads and the streams it protects now all live in
// one file and move together — the reason PR 4 dropped the contract's
// `mapEvent` half rather than leaving the mapping in core and the catalog in a
// module (MODULE_API.md §2.4).
return out.filter((t) => (PERSONAL_STREAMS.has(t.streamId) ? true : PUBLIC_KINDS.has(kind)))
}
module.exports = { STREAMS, mapShardEvent, createTracker, PERSONAL_STREAMS }

112
server/core.js Normal file
View File

@@ -0,0 +1,112 @@
// ── Everything this module reaches in core ─────────────────────────────────
//
// `ctx` arrives once, as an argument to `register()` (MODULE_API.md §2.3). The
// code below it — models, utils, controllers — is ordinary Node that requires
// its dependencies at file scope, the way it did when it lived in core. This
// file is what lets both be true.
//
// **The shape is a lazy accessor, not a stored reference, and that is the whole
// point.** A ported file writes
//
// const { query } = require('../../core')
//
// at require time, which is before `register()` has been called and therefore
// before any `ctx` exists. Handing out `ctx.db.query` there would hand out
// `undefined`, permanently, and the failure would surface much later as a
// TypeError inside a model. So every export here is a stable function that
// resolves `ctx` when it is CALLED. Require order stops mattering, and the port
// stays a one-line import change per file rather than a signature change per
// function.
//
// The other half of the same rule: nothing here may be destructured off `ctx`
// at init time either, for the same reason in the other direction — core is
// free to hand over a getter (`ctx.site.baseUrl` is one), and a value captured
// once is a value that cannot change.
//
// If `ctx` is missing, every accessor throws with the same message. That is
// deliberate: the only way to reach one before `register()` is a require cycle
// or a test that forgot to call `init`, and both want naming, not `undefined`.
let ctx = null
function need() {
if (!ctx) {
throw new Error('module-uo: core accessed before register() — see server/core.js')
}
return ctx
}
/** Called once, first thing in `register()`. */
function init(value) {
ctx = value
}
/** Test seam. Nothing in the module calls this; there is no de-registration. */
function _reset() {
ctx = null
}
// A logger that can be taken at require time and used after `register()`.
//
// Ported files write `const log = require('../core').logger('shard-ingest')` at
// file scope — the same shape as core's `require('./logger')('…')` — so the
// object returned has to exist before `ctx` does. It is a façade whose four
// methods each resolve the real logger on call. Core namespaces it with the
// module id, so these come out as `[uo:shard-ingest]`.
function logger(namespace) {
const call = (level) => (message, meta) => need().log(namespace)[level](message, meta)
return { error: call('error'), warn: call('warn'), info: call('info'), debug: call('debug') }
}
module.exports = {
init,
_reset,
logger,
// Shared server dependencies. Core owns exactly one express, as it owns
// exactly one React on the client, and for the same reason: a second copy in
// the process is a second Router prototype and a second set of instanceof
// checks. A module lives outside core's `server/`, so it could not resolve
// these for itself even if it were allowed to (§7.2).
get express() { return need().express },
get validator() { return need().validator },
// Database. `query` is the one every `*.db.js` uses; `pool` is for the
// streamed atlas import, which needs a connection it can hold.
query: (...args) => need().db.query(...args),
get pool() { return need().db.pool },
// Core state a module may read or append to, each narrowed to what is
// actually used (§2.3).
settings: {
get: (...args) => need().settings.get(...args),
set: (...args) => need().settings.set(...args),
getInstanceName: (...args) => need().settings.getInstanceName(...args),
},
activity: { log: (...args) => need().activity.log(...args) },
users: { getById: (...args) => need().users.getById(...args) },
posts: {
listAll: (...args) => need().posts.listAll(...args),
getById: (...args) => need().posts.getById(...args),
linkAnnounceJob: (...args) => need().posts.linkAnnounceJob(...args),
markAnnounced: (...args) => need().posts.markAnnounced(...args),
},
auth: { getUserFromRequest: (...args) => need().auth.getUserFromRequest(...args) },
push: { publish: (...args) => need().push.publish(...args) },
secretBox: {
encrypt: (...args) => need().secretBox.encrypt(...args),
decrypt: (...args) => need().secretBox.decrypt(...args),
},
get uploads() { return need().uploads },
// Middleware. Taken as values rather than wrapped, because express stores the
// function reference at mount time — a wrapper would be what ends up in the
// stack, and `requireRole('admin')` returns a middleware rather than being
// one. Routers are built inside `register()`, so `ctx` is set by then.
get middleware() { return need().middleware },
// Deployment facts.
get baseUrl() { return need().site.baseUrl },
get moduleRoot() { return need().paths.moduleRoot },
get moduleId() { return need().moduleId },
}

24
server/data/spawnAtlas.art.example.json Normal file
View File

@@ -0,0 +1,24 @@
{
"_comment": [
"OPTIONAL operator-supplied creature art for the spawn atlas. Copy this file to",
"spawnAtlas.art.json (same directory) and edit it, then restart the server or run",
"`npm run atlas:import` — the art map is read on every atlas refresh.",
"",
"This project ships NO creature artwork and never will. UO sprites live in your",
"own client's .mul/.uop files and are yours to extract, not ours to redistribute.",
"If you want art on the atlas pages, export it yourself (UOFiddler, ClassicUO's",
"tooling, or any art extractor), drop the images under server/uploads/atlas/, and",
"map each creature slug to its file name here.",
"",
"Both spawnAtlas.art.json and server/uploads/ are gitignored, so neither the map",
"nor the images can be committed by accident.",
"",
"Keys are creature slugs, as reported by the atlas API and derived from the type",
"names in your own shard's Spawns/*.xml. Values are file names relative to",
"server/uploads/atlas/. Any creature with no entry here simply renders without",
"art — that is the default and fully supported state, not a degraded one."
],
"lizardman": "lizardman.png",
"orc": "orc.png",
"dragon": "dragon.png"
}

52
server/db/purge.sql Normal file
View File

@@ -0,0 +1,52 @@
-- ── module-uo's teardown ──────────────────────────────────────────────────
--
-- Destructive, and run ONLY by an explicit admin purge (MODULE_API.md §2.6).
-- Nothing on the boot path ever executes this file — uninstalling a module
-- leaves its data alone, and removing the data is a separate decision an
-- operator has to make on purpose.
--
-- It exists because `schema.sql` does. A module that can create tables and
-- cannot drop them leaves an operator with orphaned data and no supported way
-- to remove it, which is why core refuses to load a module that declares one
-- without the other.
--
-- **The order is the reverse of creation, and that is load-bearing**: two of
-- these tables carry a foreign key into core's `users`, and several reference
-- each other. Dropping a parent before its children fails on the constraint,
-- and a purge that fails halfway is worse than one that does not run — it
-- leaves exactly the orphaned data this file exists to remove. `IF EXISTS` on
-- every line so a partially-installed module still tears down cleanly.
--
-- What is NOT here, deliberately: rows this module wrote into core's tables.
-- `notification_subs` rows for `shard.*` streams and `announce_job_legs` rows
-- with leg `towncrier` belong to core's tables, and a module does not delete
-- from those — core prunes them when it drops the registrations, which it can
-- do because it knows which registrant owned what.
DROP TABLE IF EXISTS `shard_atlas_pending`;
DROP TABLE IF EXISTS `shard_atlas_meta`;
DROP TABLE IF EXISTS `shard_cliloc_meta`;
DROP TABLE IF EXISTS `shard_clilocs`;
DROP TABLE IF EXISTS `shard_champion_spawns`;
DROP TABLE IF EXISTS `shard_landmarks`;
DROP TABLE IF EXISTS `shard_regions`;
DROP TABLE IF EXISTS `shard_spawn_point_types`;
DROP TABLE IF EXISTS `shard_spawn_points`;
DROP TABLE IF EXISTS `shard_spawn_creatures`;
DROP TABLE IF EXISTS `shard_feature_visibility`;
DROP TABLE IF EXISTS `shard_vendor_items`;
DROP TABLE IF EXISTS `shard_vendors`;
DROP TABLE IF EXISTS `shard_points_boards`;
DROP TABLE IF EXISTS `shard_ruleset`;
DROP TABLE IF EXISTS `shard_presence`;
DROP TABLE IF EXISTS `shard_governor_terms`;
DROP TABLE IF EXISTS `shard_governors`;
DROP TABLE IF EXISTS `shard_guilds`;
DROP TABLE IF EXISTS `shard_pages`;
DROP TABLE IF EXISTS `shard_champs`;
DROP TABLE IF EXISTS `shard_account_links`;
DROP TABLE IF EXISTS `shard_houses`;
DROP TABLE IF EXISTS `shard_economy`;
DROP TABLE IF EXISTS `shard_online`;
DROP TABLE IF EXISTS `shard_events`;
DROP TABLE IF EXISTS `uo_link_config`;

617
server/db/schema.sql Normal file
View File

@@ -0,0 +1,617 @@
-- ── module-uo's schema fragment ───────────────────────────────────────────
--
-- Replayed by core on EVERY boot, after core's own schema.sql and before
-- seedDefaults (MODULE_API.md §2.6). Everything here is therefore idempotent:
-- every CREATE TABLE carries IF NOT EXISTS and every ALTER carries
-- IF NOT EXISTS, because a statement that succeeds once and fails afterwards
-- presents as a module that worked until the first restart.
--
-- Core validates this file at LOAD time, before anything mounts — statement by
-- statement, split by the same code that splits core's schema. The rules it
-- enforces and the reason each exists:
--
-- • Leading verbs are an allowlist: CREATE, ALTER, INSERT, UPDATE. Not a
-- DROP denylist — this file replays every boot, so TRUNCATE or DELETE would
-- empty a table on each restart.
-- • Every table is prefixed. `shard_*` and `uo_link_*` are grandfathered to
-- this module by name (loader.js LEGACY_TABLE_PREFIXES): they predate the
-- module system by two years, they hold live data, and renaming them would
-- be a migration this workstream deliberately does not do. Every module
-- written after this one prefixes with its own id.
-- • No table core declares may appear here, and no table another module has
-- claimed.
--
-- Two tables carry a foreign key INTO core (`users`), which is allowed and is
-- why the replay order matters: core's schema is already in place when this
-- runs, so `users` exists. The reverse — a core table referencing one of these
-- — does not occur and must not: it would make core's schema depend on a module
-- being installed.
--
-- Teardown is `purge.sql`, which is never run by a boot. See it for the drop
-- order, which is the reverse of the dependency order here.
-- ── uo-link sidecar ────────────────────────────────────────────────────────
-- Connection config for the uo-link sidecar (the HTTP + WebSocket bridge to the
-- ServUO shard). Singleton row (id = 1), mirroring bot_config/email_config: the
-- DB only ever holds the AES-256-GCM-encrypted shared-secret auth token, never
-- plaintext, and it is only decrypted server-side (to call the sidecar). It is
-- never returned to the admin UI — responses expose only `hasToken`. base_url is
-- the REST endpoint, ws_url the live-feed endpoint; both are configurable because
-- in production the sidecar runs on a different host from the website. `status`/
-- `plugin_connected`/`last_event_at`/`boot_id` mirror the sidecar's last-known
-- state for the admin panel between polls; `boot_id` tracks server.hello.bootId
-- so a shard restart can be detected (and caches dropped).
CREATE TABLE IF NOT EXISTS uo_link_config (
id INT PRIMARY KEY DEFAULT 1,
base_url VARCHAR(255) NULL,
ws_url VARCHAR(255) NULL,
auth_token_enc TEXT NULL,
protocol INT NOT NULL DEFAULT 3,
enabled TINYINT(1) NOT NULL DEFAULT 0,
status VARCHAR(20) NOT NULL DEFAULT 'disconnected',
status_detail VARCHAR(500) NULL,
plugin_connected TINYINT(1) NOT NULL DEFAULT 0,
last_event_at DATETIME NULL,
boot_id VARCHAR(64) NULL,
updated_by INT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT fk_uo_link_config_user FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL,
CONSTRAINT chk_uo_link_config_singleton CHECK (id = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Append-only log of notable shard events ingested from the uo-link WebSocket
-- feed. The site OWNS this data (it does not query the sidecar's SQLite): the WS
-- client writes here, and the public/admin read endpoints + live feeds read from
-- here. Only "notable" kinds are logged (sales, deaths, murders, mob.killed,
-- IDOC transitions, quests, skill.gain, fame/karma, audit.*, cheat.*, link.*,
-- server.*). High-frequency kinds (char.vitals, economy.supply) are NOT logged
-- here — they update shard_online / shard_economy instead, keeping the log lean.
-- dedupe_key = sha256(kind + t + stable-json(payload)) truncated to 40 hex chars
-- (fits CHAR(40)); with the UNIQUE index it makes INSERT IGNORE idempotent so
-- WS-reconnect backfill never double-inserts.
CREATE TABLE IF NOT EXISTS shard_events (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
kind VARCHAR(48) NOT NULL,
t BIGINT NOT NULL, -- event time, epoch ms (from the sidecar)
boot_id VARCHAR(64) NULL, -- shard boot id at ingest (server.hello.bootId)
payload JSON NOT NULL, -- the full event object
dedupe_key CHAR(40) NOT NULL UNIQUE,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_shard_events_kind_t (kind, t),
INDEX idx_shard_events_t (t)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Current online players. Upserted on mob.login, refreshed on char.vitals, and
-- removed on mob.logout. Cleared wholesale when the shard restarts (a new
-- server.hello.bootId). web_id is the linked website user id (present when the
-- account is linked), so the roster can be correlated to site accounts.
CREATE TABLE IF NOT EXISTS shard_online (
serial VARCHAR(20) NOT NULL PRIMARY KEY, -- mobile serial (opaque hex key)
name VARCHAR(120) NULL,
acct VARCHAR(120) NULL,
web_id INT NULL,
map VARCHAR(40) NULL,
x INT NULL,
y INT NULL,
z INT NULL,
hits INT NULL,
hits_max INT NULL,
mana INT NULL,
mana_max INT NULL,
stam INT NULL,
stam_max INT NULL,
str INT NULL,
dex INT NULL,
`int` INT NULL,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_shard_online_acct (acct)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Total-gold-supply time series (from the periodic economy.supply event). Kept
-- append-only so the public status page can render a supply-over-time sparkline.
CREATE TABLE IF NOT EXISTS shard_economy (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
accounts INT NULL, -- number of accounts included in the total
gold BIGINT NULL, -- total gold supply across all accounts
t BIGINT NOT NULL, -- sample time, epoch ms
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_shard_economy_t (t)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Current decay stage per house, upserted on house.decay. is_idoc is a derived
-- flag (stage == 'IDOC') so the public "houses in danger" list is a cheap
-- indexed lookup rather than a scan.
CREATE TABLE IF NOT EXISTS shard_houses (
serial VARCHAR(20) NOT NULL PRIMARY KEY,
stage VARCHAR(24) NULL, -- Somewhat | Fairly | Greatly | IDOC | Collapsed | ...
map VARCHAR(40) NULL,
x INT NULL,
y INT NULL,
z INT NULL,
region VARCHAR(120) NULL,
name VARCHAR(160) NULL,
owner_serial VARCHAR(20) NULL,
owner_acct VARCHAR(120) NULL,
built_on DATETIME NULL,
last_refreshed DATETIME NULL,
is_idoc TINYINT(1) NOT NULL DEFAULT 0,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_shard_houses_idoc (is_idoc)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- a single user may link several game accounts.
CREATE TABLE IF NOT EXISTS shard_account_links (
account VARCHAR(120) NOT NULL PRIMARY KEY,
user_id INT NOT NULL,
char_name VARCHAR(120) NULL,
linked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT fk_shard_links_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
INDEX idx_shard_links_user (user_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Current champion-spawn board, upserted on champ.update and removed on
-- champ.remove. Mirrors the sidecar's /champs projection into our own store so
-- the public Champions page (and its live deltas) survive a shard outage, the
-- same way shard_online / shard_houses do. Three families share one table, told
-- apart by `category` (champion | mini | sea); category-specific fields (level,
-- kills, boss, restartAt, hits, …) live in the JSON `payload` so the schema does
-- not have to model every variant.
CREATE TABLE IF NOT EXISTS shard_champs (
serial VARCHAR(20) NOT NULL PRIMARY KEY, -- controller/mobile serial (opaque hex)
category VARCHAR(16) NULL, -- champion | mini | sea
type VARCHAR(80) NULL,
name VARCHAR(120) NULL,
status VARCHAR(16) NULL, -- active | cooldown | dormant
active TINYINT(1) NOT NULL DEFAULT 0,
map VARCHAR(40) NULL,
x INT NULL,
y INT NULL,
z INT NULL,
boss_up TINYINT(1) NOT NULL DEFAULT 0,
payload JSON NOT NULL, -- the full champ.update object
t BIGINT NULL, -- event time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_shard_champs_category (category)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Current open help-page (support ticket) queue, upserted on page.new/page.updated
-- and removed on page.closed. Snapshotted authoritatively from the sidecar's
-- GET /pages on every (re)connect. page_id is the sender's serial (one page per
-- player). Staff-only data — served on the admin channel, never public.
CREATE TABLE IF NOT EXISTS shard_pages (
page_id VARCHAR(20) NOT NULL PRIMARY KEY, -- sender serial (one page per player)
type VARCHAR(40) NULL, -- Bug | Stuck | Account | Question | ...
sender_name VARCHAR(120) NULL,
sender_acct VARCHAR(120) NULL,
web_id INT NULL, -- linked website user id, if any
message TEXT NULL,
map VARCHAR(40) NULL,
x INT NULL,
y INT NULL,
z INT NULL,
sent_ms BIGINT NULL, -- when the page was opened, epoch ms
handled TINYINT(1) NOT NULL DEFAULT 0, -- a staffer claimed it in game
handler VARCHAR(120) NULL,
payload JSON NOT NULL, -- the full page.new/updated object
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_shard_pages_handled (handled)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Guild roster board (Protocol 2.0). Upserted on guild.update (a full-state
-- snapshot emitted only on change) and removed on guild.remove. The leader is an
-- actor object flattened into leader_* columns; the full event is kept in
-- `payload` for anything not hoisted. Mirrors the sidecar's GET /guilds
-- projection into our store so the public Guilds page survives a shard outage.
CREATE TABLE IF NOT EXISTS shard_guilds (
id INT NOT NULL PRIMARY KEY, -- in-game guild id
name VARCHAR(120) NULL,
abbr VARCHAR(24) NULL,
members INT NULL,
online INT NULL,
alliance VARCHAR(120) NULL,
leader_serial VARCHAR(20) NULL,
leader_name VARCHAR(120) NULL,
leader_acct VARCHAR(120) NULL,
leader_web_id INT NULL,
payload JSON NOT NULL, -- the full guild.update object
t BIGINT NULL, -- event time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_shard_guilds_name (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Town-governor board (Protocol 2.0, City Loyalty). One row per city, upserted on
-- city.update (full-state, emitted only on change; there is no remove event since
-- the set of cities is fixed). governor / governorElect are actor objects
-- flattened into columns; the full event is kept in `payload`. Empty on shards
-- that do not run the City Loyalty system.
CREATE TABLE IF NOT EXISTS shard_governors (
city VARCHAR(40) NOT NULL PRIMARY KEY, -- Britain | Moonglow | ...
governor_serial VARCHAR(20) NULL,
governor_name VARCHAR(120) NULL,
governor_acct VARCHAR(120) NULL,
governor_web_id INT NULL,
elect_serial VARCHAR(20) NULL,
elect_name VARCHAR(120) NULL,
elect_acct VARCHAR(120) NULL,
election_phase VARCHAR(16) NULL, -- none | nominate | vote | pending
candidates INT NULL,
auto_pick_at DATETIME NULL,
payload JSON NOT NULL, -- the full city.update object
t BIGINT NULL, -- event time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Governor term history — the "who governed when" ledger behind the Governors
-- board. Captured from day one (history cannot be backfilled) on every observed
-- governor CHANGE: the open term (ended_at IS NULL) is closed and a new one
-- opened. `votes` stays NULL — the city.update feed exposes only the candidate
-- COUNT and election phase, not per-candidate tallies, so we record who governed
-- and when (reliable) and never fabricate vote numbers. The look-back UI ("who
-- were all the governors of Britain?") reads this table.
CREATE TABLE IF NOT EXISTS shard_governor_terms (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
city VARCHAR(40) NOT NULL,
governor_serial VARCHAR(20) NULL,
governor_name VARCHAR(120) NULL,
governor_acct VARCHAR(120) NULL,
governor_web_id INT NULL,
started_at BIGINT NOT NULL, -- term start, epoch ms
ended_at BIGINT NULL, -- term end epoch ms (NULL = current)
votes INT NULL, -- not in the feed (reserved)
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_shard_gov_terms_city (city, started_at),
INDEX idx_shard_gov_terms_open (city, ended_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Online-population snapshot (Protocol 2.0). Singleton row (id = 1) holding the
-- latest presence.online aggregate: total count plus per-facet and per-region
-- breakdown maps (stored as JSON). Distinct from shard_online (per-player) — this
-- is the rolled-up headcount the public "Players Online" widget renders. The
-- time series, if ever needed, is available from GET /history?kind=presence.online.
CREATE TABLE IF NOT EXISTS shard_presence (
id INT PRIMARY KEY DEFAULT 1,
count INT NOT NULL DEFAULT 0,
by_facet JSON NULL, -- { "Felucca": 12, "Trammel": 30 }
by_region JSON NULL, -- { "Britain": 18, "Wilderness": 9 }
t BIGINT NULL, -- snapshot time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT chk_shard_presence_singleton CHECK (id = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- The shard's published ruleset (Protocol 3.0 world.ruleset). Singleton row
-- (id = 1) holding the latest frame: expansion, which optional systems are on,
-- skill/stat caps, account and house limits, champion scroll rules, the
-- save/restart schedule. The shard re-emits it on every sidecar connect, so this
-- row is simply overwritten; `rev` is the shard's own FNV-1a of the body, which
-- distinguishes "same ruleset, re-sent on reconnect" from "an operator changed a
-- .cfg". No row at all means the shard has never published one — served as null,
-- which the rules page renders differently from a published ruleset.
CREATE TABLE IF NOT EXISTS shard_ruleset (
id INT PRIMARY KEY DEFAULT 1,
rev VARCHAR(32) NULL,
expansion VARCHAR(16) NULL, -- hoisted for cheap display
payload JSON NOT NULL, -- the whole world.ruleset frame
t BIGINT NULL, -- frame time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT chk_shard_ruleset_singleton CHECK (id = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Points/loyalty leaderboards (Protocol 3.0 points.board). One row per point
-- system, keyed by the shard's own PointsType name. The shard publishes ~25 of
-- these (Queen's Loyalty, Void Pool, the nine city loyalties, …), each a standing
-- players accumulate over months.
--
-- The top-N list stays inside `payload` rather than being normalized into a
-- shard_points_entries table. It is a fixed-size list (10 by default) that is only
-- ever read whole, exactly like shard_governors.candidates — normalizing it would
-- buy nothing until something needs a per-character reverse lookup, and a
-- character's own standings already ride inside char.profile instead.
--
-- No delete path: the shard's set of systems is fixed at startup, so there is no
-- points.remove to mirror.
CREATE TABLE IF NOT EXISTS shard_points_boards (
system VARCHAR(48) PRIMARY KEY, -- PointsType name, e.g. QueensLoyalty
name VARCHAR(128) NULL, -- resolved display name, if the shard sent a literal
name_cliloc INT NULL, -- cliloc id when the name is a TextDefinition number
max_points BIGINT NULL,
players INT NULL, -- players actually holding points in this system
show_on_gump TINYINT(1) NOT NULL DEFAULT 1, -- the shard's own "is this player-facing?" flag
payload JSON NOT NULL, -- the whole points.board frame, incl. `top`
t BIGINT NULL, -- frame time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Player-vendor market index (Protocol 3.0 vendor.listing). One row per player
-- vendor and one per priced listing, so the site can offer the search the in-game
-- Vendor Search gump offers — from outside the game.
--
-- The shard sweeps vendors round-robin and emits one AUTHORITATIVE frame per
-- vendor, so ingest is delete-then-insert of that vendor's items inside one
-- transaction (see shardMarket.db.js). No foreign key from items to vendors, in
-- keeping with every other shard_* table: the ingest transaction is what keeps
-- them consistent, and an FK would turn a malformed frame into a failed write
-- rather than a dropped row.
--
-- Only vendors whose owner left the in-game Vendor Search flag ON are ever sent,
-- so a player who hid their shop in game is hidden here too — see BridgeMarket.cs.
CREATE TABLE IF NOT EXISTS shard_vendors (
serial VARCHAR(20) NOT NULL PRIMARY KEY, -- "0x40001234"
shop_name VARCHAR(160) NULL,
owner_serial VARCHAR(20) NULL,
owner_name VARCHAR(64) NULL,
map VARCHAR(40) NULL,
x INT NULL,
y INT NULL,
z INT NULL,
region VARCHAR(80) NULL,
house VARCHAR(160) NULL, -- the house SIGN's name, not the house type
item_count INT NOT NULL DEFAULT 0, -- listings published in the frame
item_total INT NOT NULL DEFAULT 0, -- listings the shop actually holds
truncated TINYINT(1) NOT NULL DEFAULT 0, -- item_total > item_count
t BIGINT NULL, -- frame time, epoch ms
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_shard_vendors_owner (owner_name),
INDEX idx_shard_vendors_map (map),
INDEX idx_shard_vendors_region (region),
-- The market page's staleness banner is MIN(updated_at) over this column: the
-- round-robin sweep means the oldest row is how far behind the index can be.
INDEX idx_shard_vendors_updated (updated_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- One priced listing. Unlike the points board's top-N — a fixed-size list read
-- whole — these are the searchable rows the whole feature exists for, so they are
-- normalized rather than left inside a payload column, and there is no payload
-- column on shard_vendors at all.
--
-- `display_name` is DENORMALIZED at ingest: the shard sends `cliloc` (the item's
-- LabelNumber) and, rarely, a literal `name`, and resolving 50 clilocs per page
-- at query time would make the cliloc table a join on the hot path AND make
-- search-by-name impossible. Resolving once on write buys the index. It is
-- re-resolved in bulk after a cliloc import, because the diff sweep will not
-- re-send an unchanged shop just because the site learned what its items are
-- called.
CREATE TABLE IF NOT EXISTS shard_vendor_items (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
vendor_serial VARCHAR(20) NOT NULL,
serial VARCHAR(20) NOT NULL,
item_id INT NOT NULL DEFAULT 0, -- ItemID (the art/graphic id)
hue INT NOT NULL DEFAULT 0,
amount INT NOT NULL DEFAULT 1,
price BIGINT NOT NULL DEFAULT 0,
name VARCHAR(160) NULL, -- the item's literal Name, null for most
cliloc INT NULL, -- LabelNumber, resolved against shard_clilocs
display_name VARCHAR(160) NULL, -- resolved at ingest; what search matches
child TINYINT(1) NOT NULL DEFAULT 0, -- priced by an enclosing container, not itself
INDEX idx_shard_vendor_items_vendor (vendor_serial),
INDEX idx_shard_vendor_items_price (price),
INDEX idx_shard_vendor_items_item (item_id),
INDEX idx_shard_vendor_items_name (display_name),
-- Search filters on name and sorts on price; the composite covers the common
-- "cheapest matching X" without a filesort over the whole table.
INDEX idx_shard_vendor_items_name_price (display_name, price)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Per-feature visibility for every shard-derived surface (Protocol 3.0). One row
-- per feature; an absent row means "use the compiled default", and the compiled
-- defaults reproduce the behavior that shipped before v3 — so an empty table is
-- a no-op. See utils/shardVisibility.js for the catalog and the ladder, and
-- docs/link/v3.md §3 for the contract.
--
-- audience the minimum rung on anonymous < logged_in < player < staff < admin
-- stream whether this feature's kinds fan out over SSE at all (the market
-- index ships with this off: no page needs a live firehose of
-- whole vendor inventories)
-- field_rules {"<field>": "<rung>"} for SENSITIVE fields only. `acct` and
-- `webId` are admin-only always and are rejected here — they are
-- not in-game visible and are deliberately not configurable.
CREATE TABLE IF NOT EXISTS shard_feature_visibility (
feature VARCHAR(48) NOT NULL PRIMARY KEY,
enabled TINYINT(1) NOT NULL DEFAULT 1,
audience VARCHAR(20) NOT NULL DEFAULT 'anonymous',
stream TINYINT(1) NOT NULL DEFAULT 1,
field_rules JSON NULL,
updated_by INT NULL,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- `facets` is a per-facet point count, so the facet filter and "where does this
-- live" both answer without touching shard_spawn_points.
CREATE TABLE IF NOT EXISTS shard_spawn_creatures (
slug VARCHAR(120) NOT NULL PRIMARY KEY, -- slugified class name; the /atlas/:slug key
name VARCHAR(120) NOT NULL, -- display spelling chosen by the build
total INT NOT NULL DEFAULT 0,
points INT NOT NULL DEFAULT 0,
facets JSON NULL, -- { "Felucca": 171, "Trammel": 160, ... }
-- Operator-supplied artwork, always NULL on a fresh import. The repo ships no
-- creature art: sprites live in the operator's own client .mul/.uop files and
-- are theirs to extract and place under uploads/atlas/. The UI renders without
-- art when this is NULL, which is the normal case.
art VARCHAR(255) NULL,
-- Plain INDEX, deliberately NOT FULLTEXT: ~800 rows makes a LIKE scan free,
-- and FULLTEXT's min-token-length would break searches for names like "orc".
INDEX idx_shard_spawn_creatures_name (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- One row per spawner. `region`/`landmark` are the resolved place name — the
-- point-in-rect transform that turns "5411,1234" into "Despise" — and `label` is
-- the resolved display string (region, else landmark, else 'Wilderness').
CREATE TABLE IF NOT EXISTS shard_spawn_points (
id INT AUTO_INCREMENT PRIMARY KEY,
facet VARCHAR(40) NOT NULL,
name VARCHAR(120) NULL, -- the ServUO spawner's own name
x INT NOT NULL,
y INT NOT NULL,
width INT NOT NULL DEFAULT 0,
height INT NOT NULL DEFAULT 0,
spawn_range INT NOT NULL DEFAULT 0, -- `range` is reserved in MariaDB
max_count INT NOT NULL DEFAULT 0,
min_delay INT NOT NULL DEFAULT 0,
max_delay INT NOT NULL DEFAULT 0,
tod_start INT NOT NULL DEFAULT 0, -- meaningless unless tod_mode <> 0
tod_end INT NOT NULL DEFAULT 0,
tod_mode INT NOT NULL DEFAULT 0,
region VARCHAR(120) NULL,
landmark VARCHAR(120) NULL,
label VARCHAR(120) NOT NULL DEFAULT 'Wilderness',
INDEX idx_shard_spawn_points_facet (facet),
INDEX idx_shard_spawn_points_label (label)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- The many-to-many between the two above: one spawner commonly carries several
-- types (a single Trammel point spawns six), each with its own max. This is how
-- /atlas/creatures/:slug finds the places a creature appears.
CREATE TABLE IF NOT EXISTS shard_spawn_point_types (
point_id INT NOT NULL,
slug VARCHAR(120) NOT NULL, -- → shard_spawn_creatures.slug (no FK)
max_count INT NOT NULL DEFAULT 1,
PRIMARY KEY (point_id, slug),
INDEX idx_shard_spawn_point_types_slug (slug)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Named regions from Data/Regions.xml, flattened out of their nesting. `rects`
-- holds the region's rectangles; `priority` and rect area are what resolved each
-- spawn point at build time, kept here so the admin drift check can re-derive.
CREATE TABLE IF NOT EXISTS shard_regions (
id INT AUTO_INCREMENT PRIMARY KEY,
facet VARCHAR(40) NOT NULL,
name VARCHAR(120) NOT NULL,
type VARCHAR(80) NULL, -- ServUO region class
priority INT NOT NULL DEFAULT 0,
parent VARCHAR(120) NULL, -- enclosing named region, if any
rects JSON NULL,
INDEX idx_shard_regions_facet (facet),
INDEX idx_shard_regions_name (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Points of interest from Data/Locations/*.xml. `grp` is the innermost enclosing
-- parent ("Covetous"), which is the label worth showing — "Covetous" reads
-- better than the individual marker "Level 1". (`group` is reserved in SQL.)
CREATE TABLE IF NOT EXISTS shard_landmarks (
id INT AUTO_INCREMENT PRIMARY KEY,
facet VARCHAR(40) NOT NULL,
name VARCHAR(120) NOT NULL,
grp VARCHAR(120) NULL,
x INT NOT NULL,
y INT NOT NULL,
z INT NOT NULL DEFAULT 0,
INDEX idx_shard_landmarks_facet (facet),
INDEX idx_shard_landmarks_name (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Configured champion altars from Config/ChampionSpawns.xml. This is static
-- roster data ("there is an Unholy Terror altar in Deceit") and is distinct from
-- the live champ.update feed in shard_champs ("it is on level 3 right now").
CREATE TABLE IF NOT EXISTS shard_champion_spawns (
slug VARCHAR(160) NOT NULL PRIMARY KEY, -- facet-name, e.g. "felucca-deceit"
name VARCHAR(120) NOT NULL,
grp VARCHAR(80) NULL, -- spawn group; one active per group
type VARCHAR(80) NULL, -- '' when randomised per activation
random_type TINYINT(1) NOT NULL DEFAULT 0,
facet VARCHAR(40) NOT NULL,
x INT NOT NULL,
y INT NOT NULL,
z INT NOT NULL DEFAULT 0,
radius INT NOT NULL DEFAULT 0,
label VARCHAR(120) NULL, -- resolved place name
INDEX idx_shard_champion_spawns_facet (facet)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- UO's localization table: cliloc id -> display string. Items carry a
-- `LabelNumber` rather than a name, so without this the site can only render
-- `id 1023721` where the game shows "quarter staff". The shard has always sent
-- the id (char.profile's `cliloc`, and one per marketplace listing) — the number
-- was never the missing piece, the table was.
--
-- Sourced from a file the OPERATOR converts once from their own UO client and
-- points the site at (docs/website/CLILOCS.md); nothing derived from the client
-- is committed, the same rule the spawn atlas and the creature art map follow.
-- A shard with no cliloc file configured simply renders item ids, which is what
-- it did before this table existed.
--
-- `text` is TEXT, not VARCHAR: real tables top out around 12 KB for the long
-- property descriptions, and truncating them silently would be worse than
-- storing them. Item NAMES are all short — the index that matters for search is
-- on the denormalized `shard_vendor_items.display_name`, not here.
CREATE TABLE IF NOT EXISTS shard_clilocs (
number INT NOT NULL PRIMARY KEY,
flag SMALLINT NOT NULL DEFAULT 0,
text TEXT NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Singleton (id = 1) describing the cliloc table currently loaded: the source
-- file, its sha256, the entry count and the parser version. The boot path
-- compares the stored hash against the file on disk and skips the parse when
-- they match, which is every restart that did not follow a client patch.
CREATE TABLE IF NOT EXISTS shard_cliloc_meta (
id TINYINT NOT NULL PRIMARY KEY DEFAULT 1,
payload JSON NOT NULL,
imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT chk_shard_cliloc_meta_singleton CHECK (id = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Singleton (id = 1) describing the artifact currently loaded: when it was
-- built, its counts, and a sha256 per ServUO source file. The admin drift check
-- compares this against db/data/spawnAtlas.meta.json to report when the database
-- is behind the committed artifact.
CREATE TABLE IF NOT EXISTS shard_atlas_meta (
id TINYINT NOT NULL PRIMARY KEY DEFAULT 1,
payload JSON NOT NULL,
imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT chk_shard_atlas_meta_singleton CHECK (id = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Singleton (id = 1) holding an atlas refresh that was parsed but deliberately
-- NOT applied, because it would remove a facet the site currently serves.
--
-- Losing a facet is the signature of a half-copied or mid-update ServUO tree as
-- much as of a real map change, and boot cannot tell the two apart — so the
-- refresh is staged here for a human instead of being applied. Startup is never
-- blocked by it: the site comes up serving the atlas it already had.
--
-- Only the DECISION is stored, not the parsed world: `payload` holds the source
-- hashes and the facet diff (a few KB), and approving re-parses the tree. That
-- keeps a multi-megabyte blob out of the database and guarantees the applied
-- atlas matches the tree as it is at approval time, not as it was at boot.
--
-- `rejected` is remembered against those exact source hashes so a declined
-- refresh does not re-prompt on every restart; changing the tree changes the
-- hashes and asks again.
CREATE TABLE IF NOT EXISTS shard_atlas_pending (
id TINYINT NOT NULL PRIMARY KEY DEFAULT 1,
status ENUM('pending','rejected') NOT NULL DEFAULT 'pending',
payload JSON NOT NULL, -- source hashes + facet diff
detected_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT chk_shard_atlas_pending_singleton CHECK (id = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- House registry (Protocol 2.0). The house.update full-state feed carries richer
-- fields than the house.decay transition feed shard_houses was built for. Rather
-- than a second table for one entity, extend shard_houses: house.update writes the
-- registry columns below (owner display name, co-owner/friend counts, placement
-- price, decay level name) while house.decay keeps owning `stage`/`is_idoc`. Each
-- upsert only touches its own columns, so the two feeds never clobber each other.
-- `price` is the placement value, NOT a "for sale" flag (stock ServUO has none).
ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS owner_name VARCHAR(120) NULL;
ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS co_owners INT NULL;
ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS friends INT NULL;
ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS price BIGINT NULL;
ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS decay VARCHAR(24) NULL;
-- Distinguishes a full registry row (seen via house.update) from a decay-only row,
-- so the public Houses browser can list registered houses without pulling in rows
-- we only ever saw an IDOC transition for.
ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS in_registry TINYINT(1) NOT NULL DEFAULT 0;
-- Protocol 3.0 cutover: this build speaks wire protocol 3 (world.ruleset,
-- points.board, vendor.listing), so the pinned version an existing install
-- carries has to move with it — a 2 against a v3 sidecar 409s every REST call
-- and closes the WS on ws.hello. MODIFY fixes the column default for installs
-- created before the bump (idempotent, like the other MODIFYs here).
ALTER TABLE uo_link_config MODIFY COLUMN protocol INT NOT NULL DEFAULT 3;
-- The row itself is admin-editable, and schema.sql runs on EVERY boot, so this
-- must be one-shot: an operator who deliberately pins an older sidecar in
-- Admin → Shard has to stay pinned. The marker row in `settings` is what makes
-- it fire once — written after the UPDATE, and on a fresh install (no
-- uo_link_config row yet) it is simply written with nothing to update.
UPDATE uo_link_config SET protocol = 3
WHERE id = 1 AND protocol < 3
AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_3_migrated');

392
server/model/shardAtlas/shardAtlas.db.js Normal file
View File

@@ -0,0 +1,392 @@
const core = require('../../core')
const { query } = core
// Raw SQL for the spawn atlas. Every table here is IMPORT-OWNED: `replaceAtlas`
// empties and refills all six inside one transaction, and nothing else in the
// codebase writes to them. There are no foreign keys, consistent with every
// other shard_* table.
const BATCH = 500
const ATLAS_TABLES = [
'shard_spawn_point_types',
'shard_spawn_points',
'shard_spawn_creatures',
'shard_regions',
'shard_landmarks',
'shard_champion_spawns',
]
async function insertBatched(conn, sql, rows) {
for (let i = 0; i < rows.length; i += BATCH) {
await conn.batch(sql, rows.slice(i, i + BATCH))
}
return rows.length
}
/**
* Replace the entire atlas in one transaction.
*
* All-or-nothing on purpose: a failed reload must leave the previous atlas
* intact rather than a half-loaded world, since a partially-imported atlas is
* indistinguishable from a real one to anyone reading it.
*
* `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly
* commits, which would defeat exactly that guarantee. At ~7k rows the cost of
* `DELETE` is irrelevant.
*/
async function replaceAtlas(atlas, art = {}) {
const conn = await core.pool.getConnection()
const counts = {}
try {
await conn.beginTransaction()
for (const table of ATLAS_TABLES) await conn.query(`DELETE FROM ${table}`)
counts.creatures = await insertBatched(
conn,
'INSERT INTO shard_spawn_creatures (slug, name, total, points, facets, art) VALUES (?,?,?,?,?,?)',
atlas.creatures.map((c) => [
c.slug,
c.name,
c.total ?? 0,
c.points ?? 0,
JSON.stringify(c.facets ?? {}),
art[c.slug] ?? null,
]),
)
counts.regions = await insertBatched(
conn,
'INSERT INTO shard_regions (facet, name, type, priority, parent, rects) VALUES (?,?,?,?,?,?)',
atlas.regions.map((r) => [
r.facet,
r.name,
r.type || null,
r.priority ?? 0,
r.parent || null,
JSON.stringify(r.rects ?? []),
]),
)
counts.landmarks = await insertBatched(
conn,
'INSERT INTO shard_landmarks (facet, name, grp, x, y, z) VALUES (?,?,?,?,?,?)',
atlas.landmarks.map((l) => [
l.facet,
l.name,
l.group || null,
l.x ?? 0,
l.y ?? 0,
l.z ?? 0,
]),
)
counts.champions = await insertBatched(
conn,
'INSERT INTO shard_champion_spawns ' +
'(slug, name, grp, type, random_type, facet, x, y, z, radius, label) ' +
'VALUES (?,?,?,?,?,?,?,?,?,?,?)',
atlas.champions.map((c) => [
c.slug,
c.name,
c.group || null,
c.type || null,
c.randomType ? 1 : 0,
c.facet,
c.x ?? 0,
c.y ?? 0,
c.z ?? 0,
c.radius ?? 0,
c.label || null,
]),
)
// Point ids are assigned explicitly rather than left to AUTO_INCREMENT: the
// join rows need to know them and `conn.batch()` reports no usable insertId
// for a multi-row insert. Safe because this transaction just emptied the
// table and nothing else writes to it.
counts.points = await insertBatched(
conn,
'INSERT INTO shard_spawn_points ' +
'(id, facet, name, x, y, width, height, spawn_range, max_count, min_delay, max_delay, ' +
'tod_start, tod_end, tod_mode, region, landmark, label) ' +
'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
atlas.points.map((p, i) => [
i + 1,
p.facet,
p.name,
p.x,
p.y,
p.width ?? 0,
p.height ?? 0,
p.range ?? 0,
p.maxCount ?? 0,
p.minDelay ?? 0,
p.maxDelay ?? 0,
p.todStart ?? 0,
p.todEnd ?? 0,
p.todMode ?? 0,
p.region,
p.landmark,
p.label || 'Wilderness',
]),
)
counts.pointTypes = await insertBatched(
conn,
'INSERT INTO shard_spawn_point_types (point_id, slug, max_count) VALUES (?,?,?)',
atlas.pointTypes,
)
await conn.query(
'INSERT INTO shard_atlas_meta (id, payload) VALUES (1, ?) ' +
'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP',
[JSON.stringify({ ...atlas.meta, importedCounts: counts })],
)
// A completed import answers whatever was pending.
await conn.query('DELETE FROM shard_atlas_pending')
await conn.commit()
return counts
} catch (err) {
await conn.rollback().catch(() => {})
throw err
} finally {
conn.release()
}
}
async function getMeta() {
const rows = await query('SELECT payload, imported_at FROM shard_atlas_meta WHERE id = 1')
if (rows.length === 0) return null
const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload
return { ...payload, importedAt: rows[0].imported_at }
}
/** Facet names currently loaded, used to detect a facet disappearing. */
async function getFacets() {
const rows = await query('SELECT DISTINCT facet FROM shard_spawn_points ORDER BY facet')
return rows.map((row) => row.facet)
}
// ── Pending review ─────────────────────────────────────────────────────────
async function getPending() {
const rows = await query('SELECT payload, status, detected_at FROM shard_atlas_pending WHERE id = 1')
if (rows.length === 0) return null
const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload
return { ...payload, status: rows[0].status, detectedAt: rows[0].detected_at }
}
async function setPending(payload, status = 'pending') {
return query(
'INSERT INTO shard_atlas_pending (id, status, payload) VALUES (1, ?, ?) ' +
'ON DUPLICATE KEY UPDATE status = VALUES(status), payload = VALUES(payload), ' +
'detected_at = CURRENT_TIMESTAMP',
[status, JSON.stringify(payload)],
)
}
async function clearPending() {
return query('DELETE FROM shard_atlas_pending')
}
// ── Reads (the public /atlas surface) ──────────────────────────────────────
//
// Every read here is a plain indexed query over ~7k rows and is served entirely
// from MariaDB: the atlas is static shard content, so nothing on this path
// touches the sidecar and nothing degrades when the shard is down.
//
// A facet filter is expressed as EXISTS over the points, never as a JSON path
// built from caller input. `shard_spawn_creatures.facets` is a JSON object keyed
// by facet name, and matching a key means either concatenating the name into a
// path or handing it to JSON_SEARCH — whose search string treats `%` and `_` as
// wildcards, so `?facet=%` would quietly match everything. The join is exact and
// uses the indexes that already exist.
const CREATURE_FACET_EXISTS = `EXISTS (
SELECT 1 FROM shard_spawn_point_types t
JOIN shard_spawn_points p ON p.id = t.point_id
WHERE t.slug = c.slug AND p.facet = ?
)`
// Build the WHERE for a creature search. `q` is a substring match on the display
// name — a LIKE scan, which is free at ~800 rows and, unlike FULLTEXT, has no
// minimum token length to break a search for "orc".
function creatureWhere({ q, facet }) {
const where = []
const params = []
if (q) {
where.push('c.name LIKE ?')
params.push(`%${q}%`)
}
if (facet) {
where.push(CREATURE_FACET_EXISTS)
params.push(facet)
}
return { sql: where.length ? `WHERE ${where.join(' AND ')}` : '', params }
}
async function countCreatures({ q = '', facet = '' } = {}) {
const { sql, params } = creatureWhere({ q, facet })
const rows = await query(`SELECT COUNT(*) AS n FROM shard_spawn_creatures c ${sql}`, params)
return rows[0] ? Number(rows[0].n) : 0
}
function listCreatures({ q = '', facet = '', limit = 50, offset = 0 } = {}) {
const { sql, params } = creatureWhere({ q, facet })
return query(
`SELECT c.slug, c.name, c.total, c.points, c.facets, c.art
FROM shard_spawn_creatures c
${sql}
ORDER BY c.total DESC, c.name ASC
LIMIT ? OFFSET ?`,
[...params, limit, offset],
)
}
async function getCreature(slug) {
const rows = await query(
'SELECT slug, name, total, points, facets, art FROM shard_spawn_creatures WHERE slug = ?',
[slug],
)
return rows[0] || null
}
/**
* Where a creature spawns, grouped by resolved place.
*
* This is the answer the atlas exists to give — "lizardman → Shrines,
* Isamu-Jima, Yew" — so it is aggregated in SQL rather than by summing 6,455
* point rows in Node.
*/
function listCreaturePlaces(slug, { facet = '' } = {}) {
const params = [slug]
let facetSql = ''
if (facet) {
facetSql = 'AND p.facet = ?'
params.push(facet)
}
return query(
`SELECT p.facet, p.label, COUNT(*) AS spawners, SUM(t.max_count) AS max_alive
FROM shard_spawn_point_types t
JOIN shard_spawn_points p ON p.id = t.point_id
WHERE t.slug = ? ${facetSql}
GROUP BY p.facet, p.label
ORDER BY spawners DESC, p.facet ASC, p.label ASC`,
params,
)
}
/** The individual spawners for a creature, newest-largest first. Bounded. */
function listCreaturePoints(slug, { facet = '', limit = 200 } = {}) {
const params = [slug]
let facetSql = ''
if (facet) {
facetSql = 'AND p.facet = ?'
params.push(facet)
}
params.push(limit)
return query(
`SELECT p.id, p.facet, p.name, p.x, p.y, p.width, p.height, p.spawn_range,
p.min_delay, p.max_delay, p.tod_start, p.tod_end, p.tod_mode,
p.region, p.landmark, p.label, t.max_count
FROM shard_spawn_point_types t
JOIN shard_spawn_points p ON p.id = t.point_id
WHERE t.slug = ? ${facetSql}
ORDER BY t.max_count DESC, p.facet ASC, p.label ASC, p.id ASC
LIMIT ?`,
params,
)
}
/** Every other creature sharing a spawner with this one. */
function listCreatureCompanions(slug, { limit = 24 } = {}) {
return query(
`SELECT o.slug, c.name, COUNT(*) AS shared
FROM shard_spawn_point_types t
JOIN shard_spawn_point_types o ON o.point_id = t.point_id AND o.slug <> t.slug
JOIN shard_spawn_creatures c ON c.slug = o.slug
WHERE t.slug = ?
GROUP BY o.slug, c.name
ORDER BY shared DESC, c.name ASC
LIMIT ?`,
[slug, limit],
)
}
function listRegions({ facet = '', q = '' } = {}) {
const where = []
const params = []
if (facet) {
where.push('facet = ?')
params.push(facet)
}
if (q) {
where.push('name LIKE ?')
params.push(`%${q}%`)
}
return query(
`SELECT facet, name, type, priority, parent, rects
FROM shard_regions
${where.length ? `WHERE ${where.join(' AND ')}` : ''}
ORDER BY facet ASC, name ASC`,
params,
)
}
function listLandmarks({ facet = '', q = '' } = {}) {
const where = []
const params = []
if (facet) {
where.push('facet = ?')
params.push(facet)
}
if (q) {
where.push('(name LIKE ? OR grp LIKE ?)')
params.push(`%${q}%`, `%${q}%`)
}
return query(
`SELECT facet, name, grp, x, y, z
FROM shard_landmarks
${where.length ? `WHERE ${where.join(' AND ')}` : ''}
ORDER BY facet ASC, grp ASC, name ASC`,
params,
)
}
function listChampions({ facet = '' } = {}) {
const params = []
let where = ''
if (facet) {
where = 'WHERE facet = ?'
params.push(facet)
}
return query(
`SELECT slug, name, grp, type, random_type, facet, x, y, z, radius, label
FROM shard_champion_spawns
${where}
ORDER BY facet ASC, name ASC`,
params,
)
}
module.exports = {
replaceAtlas,
getMeta,
getFacets,
getPending,
setPending,
clearPending,
countCreatures,
listCreatures,
getCreature,
listCreaturePlaces,
listCreaturePoints,
listCreatureCompanions,
listRegions,
listLandmarks,
listChampions,
}

491
server/model/shardAtlas/shardAtlas.model.js Normal file
View File

@@ -0,0 +1,491 @@
const fs = require('fs')
const path = require('path')
const db = require('./shardAtlas.db')
const core = require('../../core')
const { settings } = core
const { slugify } = require('../../utils/spawnAtlasParse')
const {
AtlasSourceError,
PARSER_VERSION,
buildAtlas,
hashSources,
sameSources,
} = require('../../utils/spawnAtlasSource')
const log = require('../../core').logger('shardAtlas')
// The spawn atlas, refreshed from the shard's own ServUO tree.
//
// The tree is the single source of truth. Nothing is precomputed and committed,
// because a shard's maps change over its lifetime — facets get added, replaced
// or renamed — and a snapshot in the repo would go stale against the world
// players actually see. So the atlas is re-derived on every boot.
//
// Two rules govern the boot path:
//
// 1. **It never blocks startup.** No configured path, an unreadable path, a
// malformed file, a database error — all of it is caught and logged. The
// site comes up either way, serving whatever atlas it already had.
// 2. **A facet disappearing is not applied automatically.** Losing a facet is
// the signature of a half-copied or mid-update tree as much as of a real
// map change, and the two are indistinguishable from here. The refresh is
// staged for a human instead, and an admin approves or rejects it.
//
// Everything else — new facets, renamed regions, changed spawns — applies
// straight away, because none of it can silently destroy data an operator would
// miss.
const SETTING_KEY = 'spawn_atlas_servuo_path'
/**
* Where the ServUO tree lives.
*
* The admin setting wins over the environment so an operator can point the
* atlas at a different tree without a redeploy, matching how the rest of the
* shard integration is admin-managed rather than env-configured. `SERVUO_PATH`
* remains as the deploy-time default, since the path usually describes a mount
* that the deployment sets up.
*/
async function getServuoPath() {
try {
const configured = await settings.get(SETTING_KEY)
if (configured && String(configured).trim() !== '') return String(configured).trim()
} catch {
// Settings unavailable is not fatal — fall through to the env default.
}
const fromEnv = process.env.SERVUO_PATH
return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : ''
}
async function setServuoPath(value, updatedBy = null) {
return settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy)
}
/**
* Optional operator-supplied art map, `{ "<slug>": "<file under uploads/atlas/>" }`.
*
* Never committed and never shipped — creature sprites come out of the
* operator's own client `.mul`/`.uop` files, which are theirs, not ours to
* redistribute. Absent (the normal case) every `art` stays NULL and the UI
* renders text-only.
*/
// Resolved from ctx.paths.moduleRoot rather than by walking up from __dirname.
// The ported default was `../../../db/data`, which pointed at core's tree when
// this file lived there and points OUTSIDE server/ now — a path that happens to
// resolve is exactly the kind of port bug that survives a green test suite,
// because the absent-file branch returns {} and looks like the normal case.
function loadArtMap(dir = path.join(core.moduleRoot, 'server', 'data')) {
try {
const file = path.join(dir, 'spawnAtlas.art.json')
if (!fs.existsSync(file)) return {}
const map = JSON.parse(fs.readFileSync(file, 'utf8'))
return map && typeof map === 'object' ? map : {}
} catch (err) {
log.warn('spawn atlas art map could not be read', { error: err.message })
return {}
}
}
/**
* Flatten each point's types into `shard_spawn_point_types` rows.
*
* A spawner may legitimately list the same type twice, and the primary key is
* (point_id, slug), so duplicates collapse to the larger max rather than
* failing the insert.
*/
function pointTypeRows(points) {
const rows = []
points.forEach((point, i) => {
const bySlug = new Map()
for (const entry of point.types ?? []) {
const slug = slugify(entry.type)
if (slug === '') continue
bySlug.set(slug, Math.max(bySlug.get(slug) ?? 0, entry.max ?? 1))
}
for (const [slug, max] of bySlug) rows.push([i + 1, slug, max])
})
return rows
}
async function applyAtlas(atlas) {
return db.replaceAtlas({ ...atlas, pointTypes: pointTypeRows(atlas.points) }, loadArtMap())
}
/**
* Refresh the atlas from the configured ServUO tree.
*
* Returns a result describing what happened rather than throwing, so the caller
* — including the boot path — can log it and move on:
*
* `skipped` no path configured
* `unavailable` path configured but unreadable / missing required files
* `unchanged` source hashes match the loaded atlas; nothing parsed
* `imported` parsed and applied
* `needsReview` parsed, but a facet would be lost; staged for an admin
* `failed` parsed or applied and something went wrong
*
* `force` skips the hash check (an admin asking for a reimport) and `approve`
* additionally accepts facet loss (an admin approving a staged refresh).
*/
/**
* Was the loaded atlas built by THIS parser?
*
* An atlas imported before `parserVersion` existed reports undefined, which is
* correctly "no" — those are exactly the ones carrying the old readings.
*/
const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION
async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
// An explicit override wins outright — it is a one-off "use this tree", and it
// must not be silently overruled by the configured path the way an env default
// would be.
const root = pathOverride.trim() !== '' ? pathOverride.trim() : await getServuoPath()
if (root === '') return { status: 'skipped', reason: 'no ServUO path configured' }
let hashes
try {
hashes = hashSources(root)
} catch (err) {
if (err instanceof AtlasSourceError) {
return { status: 'unavailable', reason: err.message, code: err.code, path: root }
}
return { status: 'failed', reason: err.message, path: root }
}
const meta = await db.getMeta().catch(() => null)
const loaded = meta?.source
? Object.fromEntries(Object.entries(meta.source).map(([label, v]) => [label, v.sha256]))
: null
// Two things make a loaded atlas stale: the tree changed, or the PARSER did.
// Only checking the tree would strand an install whose maps never change on
// whatever an older build derived — a corrected parse would ship and never
// reach the data.
if (!force && sameSources(hashes, loaded) && currentParser(meta)) {
return { status: 'unchanged', path: root }
}
// A rejected refresh must not re-prompt on every boot. It stays rejected until
// the tree changes again, at which point the hashes differ and it is a new
// decision.
const pending = await db.getPending().catch(() => null)
if (!approve && !force && pending?.status === 'rejected' && sameSources(hashes, pending.hashes)) {
return { status: 'unchanged', path: root, reason: 'refresh previously rejected' }
}
let atlas
try {
atlas = buildAtlas(root)
} catch (err) {
return { status: 'failed', reason: err.message, path: root }
}
const currentFacets = await db.getFacets().catch(() => [])
const incomingFacets = atlas.facets
const removedFacets = currentFacets.filter((facet) => !incomingFacets.includes(facet))
const addedFacets = incomingFacets.filter((facet) => !currentFacets.includes(facet))
// Losing a facet is indistinguishable here from a half-copied tree, so it is
// staged rather than applied — but startup is never blocked by it.
if (removedFacets.length > 0 && !approve) {
const summary = {
hashes,
path: root,
currentFacets,
incomingFacets,
removedFacets,
addedFacets,
counts: atlas.meta.counts,
}
await db.setPending(summary, 'pending').catch((err) => {
log.warn('could not stage spawn atlas refresh', { error: err.message })
})
return { status: 'needsReview', ...summary }
}
try {
const counts = await applyAtlas(atlas)
return { status: 'imported', path: root, counts, addedFacets, removedFacets }
} catch (err) {
return { status: 'failed', reason: err.message, path: root }
}
}
/** Admin approved a staged refresh: apply it, facet loss and all. */
async function approvePending(options = {}) {
return refresh({ ...options, approve: true, force: true })
}
/**
* Admin rejected a staged refresh: keep the current atlas and remember the
* decision against those exact source hashes, so it does not re-prompt every
* boot. A further change to the tree produces different hashes and asks again.
*/
async function rejectPending() {
const pending = await db.getPending()
if (!pending) return { status: 'none' }
await db.setPending({ ...pending, rejectedAt: new Date().toISOString() }, 'rejected')
return { status: 'rejected' }
}
/** Everything the admin panel needs to describe atlas state. */
async function status({ path: pathOverride = '' } = {}) {
const root = pathOverride.trim() !== '' ? pathOverride.trim() : await getServuoPath()
const [meta, pending, facets] = await Promise.all([
db.getMeta().catch(() => null),
db.getPending().catch(() => null),
db.getFacets().catch(() => []),
])
let treeReadable = false
let drift = null
if (root !== '') {
try {
const hashes = hashSources(root)
treeReadable = true
const loaded = meta?.source
? Object.fromEntries(Object.entries(meta.source).map(([l, v]) => [l, v.sha256]))
: null
// Same question `refresh` asks: an import picks something up when either
// the tree or the parser has moved on.
drift = !sameSources(hashes, loaded) || !currentParser(meta)
} catch {
treeReadable = false
}
}
return {
configured: root !== '',
path: root,
treeReadable,
drift,
facets,
importedAt: meta?.importedAt ?? null,
counts: meta?.counts ?? null,
pending,
}
}
/**
* Boot hook. Best-effort by contract: it logs and returns, never throws, so a
* missing tree or a bad file can never stop the site coming up.
*/
async function refreshOnBoot() {
try {
const result = await refresh()
switch (result.status) {
case 'imported':
log.info('spawn atlas refreshed from ServUO tree', {
...result.counts,
added: result.addedFacets,
})
break
case 'needsReview':
log.warn(
'spawn atlas refresh staged for admin review — a facet would be removed; ' +
'the existing atlas is unchanged',
{ removed: result.removedFacets, added: result.addedFacets },
)
break
case 'unavailable':
log.warn('spawn atlas source unavailable', { reason: result.reason, path: result.path })
break
case 'failed':
log.warn('spawn atlas refresh failed', { reason: result.reason })
break
default:
break
}
return result
} catch (err) {
log.warn('spawn atlas refresh errored', { error: err.message })
return { status: 'failed', reason: err.message }
}
}
// ── Reads ──────────────────────────────────────────────────────────────────
//
// The shapes the /public/atlas endpoints serve. Rows are camelCased here rather
// than in the controller, for the same reason shardState does it: the column
// names are an implementation detail of the import, and the browser contract
// should not move when a column is renamed.
const jsonOr = (value, fallback) => {
if (value == null) return fallback
if (typeof value !== 'string') return value
try {
return JSON.parse(value)
} catch {
return fallback
}
}
const shapeCreature = (row) => ({
slug: row.slug,
name: row.name,
// `total` is the summed MaxCount across every spawner (how many can be alive
// at once); `points` is how many spawners mention it. They answer different
// questions and the UI shows both.
total: row.total,
points: row.points,
facets: jsonOr(row.facets, {}),
art: row.art || null,
})
const shapePlace = (row) => ({
facet: row.facet,
label: row.label,
spawners: Number(row.spawners) || 0,
maxAlive: Number(row.max_alive) || 0,
})
const shapePoint = (row) => ({
id: row.id,
facet: row.facet,
name: row.name || null,
x: row.x,
y: row.y,
width: row.width,
height: row.height,
range: row.spawn_range,
maxCount: row.max_count,
minDelay: row.min_delay,
maxDelay: row.max_delay,
todStart: row.tod_start,
todEnd: row.tod_end,
todMode: row.tod_mode,
region: row.region || null,
landmark: row.landmark || null,
label: row.label,
})
/**
* Paginated creature search. Returns the page plus the unpaginated total, so
* the UI can say "showing 50 of 800" without a second round trip.
*/
async function searchCreatures({ q = '', facet = '', limit = 50, offset = 0 } = {}) {
const [rows, total] = await Promise.all([
db.listCreatures({ q, facet, limit, offset }),
db.countCreatures({ q, facet }),
])
return { total, limit, offset, creatures: rows.map(shapeCreature) }
}
/**
* One creature: its totals, the places it spawns (the aggregate the atlas
* exists for), the individual spawners, and what else shares those spawners.
*
* `null` when the slug is unknown — the controller turns that into a 404.
*/
async function getCreature(slug, { facet = '', points = 200 } = {}) {
const row = await db.getCreature(slug)
if (!row) return null
const [places, pointRows, alsoHere] = await Promise.all([
db.listCreaturePlaces(slug, { facet }),
db.listCreaturePoints(slug, { facet, limit: points }),
db.listCreatureCompanions(slug),
])
return {
...shapeCreature(row),
places: places.map(shapePlace),
// `spawners`, not `points`: shapeCreature already uses `points` for the
// COUNT of spawners, and reusing the key for the list of them would make the
// same field a number on the search route and an array here.
spawners: pointRows.map(shapePoint),
// Bounded by the query, so a creature on hundreds of spawners returns a page
// rather than the world.
spawnersTruncated: pointRows.length >= points,
alsoHere: alsoHere.map((r) => ({
slug: r.slug,
name: r.name,
shared: Number(r.shared) || 0,
})),
}
}
async function listRegions(opts = {}) {
const rows = await db.listRegions(opts)
return rows.map((r) => ({
facet: r.facet,
name: r.name,
type: r.type || null,
priority: r.priority,
parent: r.parent || null,
rects: jsonOr(r.rects, []),
}))
}
async function listLandmarks(opts = {}) {
const rows = await db.listLandmarks(opts)
return rows.map((r) => ({
facet: r.facet,
name: r.name,
group: r.grp || null,
x: r.x,
y: r.y,
z: r.z,
}))
}
async function listChampions(opts = {}) {
const rows = await db.listChampions(opts)
return rows.map((r) => ({
slug: r.slug,
name: r.name,
group: r.grp || null,
// '' on the wire means "randomised at activation"; `randomType` says so
// explicitly rather than making the client infer it from an empty string.
type: r.type || null,
randomType: !!r.random_type,
facet: r.facet,
x: r.x,
y: r.y,
z: r.z,
radius: r.radius,
label: r.label || null,
}))
}
/**
* What is loaded: the facet list, the counts, and when it was imported.
*
* Deliberately does NOT report the source path, the per-file hashes or whether
* a refresh is pending. Those describe the operator's filesystem, and this is a
* public endpoint; the admin status route carries them instead.
*/
async function publicMeta() {
const [meta, facets] = await Promise.all([
db.getMeta().catch(() => null),
db.getFacets().catch(() => []),
])
return {
importedAt: meta?.importedAt ?? null,
generatedAt: meta?.generatedAt ?? null,
// The parse counts, not the row counts: `unresolvedPoints` is what lets the
// page state its own placement accuracy instead of implying it is complete.
counts: meta?.counts ?? null,
facets,
}
}
const listFacets = () => db.getFacets()
module.exports = {
refresh,
refreshOnBoot,
approvePending,
rejectPending,
status,
getServuoPath,
setServuoPath,
pointTypeRows,
loadArtMap,
SETTING_KEY,
searchCreatures,
getCreature,
listRegions,
listLandmarks,
listChampions,
listFacets,
publicMeta,
}

110
server/model/shardClilocs/shardClilocs.db.js Normal file
View File

@@ -0,0 +1,110 @@
const core = require('../../core')
const { query } = core
// Raw SQL for the cliloc table. `shard_clilocs` is IMPORT-OWNED: `replaceAll`
// empties and refills it inside one transaction, and nothing else in the
// codebase writes to it. No foreign keys, consistent with every other shard_*
// table.
const BATCH = 1000
/**
* Replace the entire cliloc table in one transaction.
*
* All-or-nothing on purpose: a failed reload must leave the previous table
* intact rather than a half-loaded one, because a partially-imported cliloc
* table is indistinguishable from a complete one to anyone reading it — you
* would just see some items named and some not, which is also what "no table at
* all" looks like.
*
* `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly
* commits, which would defeat exactly that guarantee. (The same trap the spawn
* atlas import documents; at ~123k rows `DELETE` is still well under a second.)
*/
async function replaceAll(entries, meta) {
const conn = await core.pool.getConnection()
try {
await conn.beginTransaction()
await conn.query('DELETE FROM shard_clilocs')
// Blank entries are dropped rather than stored. Roughly HALF of a real
// cliloc table is empty strings — ids the client reserves and never uses —
// and a row that resolves to no name is indistinguishable from no row at
// all to every caller. Dropping them halves the table (123,490 → ~67,500)
// and, more importantly, makes the binary and text imports converge on
// identical content: the binary format carries the blanks explicitly and a
// text export may or may not, depending on the tool.
//
// Later duplicates win. Merging across sources already happened upstream in
// `readCliloc`, so in practice this collapses nothing — it is kept because
// the plain format permits a repeated id WITHIN one file and the client's
// own loader resolves it the same way (its dictionary assignment
// overwrites). Without it, a file the game itself would load happily would
// fail the batch insert on a primary-key collision.
const byNumber = new Map()
let blank = 0
for (const entry of entries) {
if (!Number.isInteger(entry.number)) continue
if (String(entry.text ?? '').trim() === '') {
blank++
continue
}
byNumber.set(entry.number, entry)
}
const rows = [...byNumber.values()].map((e) => [e.number, e.flag ?? 0, e.text])
for (let i = 0; i < rows.length; i += BATCH) {
await conn.batch('INSERT INTO shard_clilocs (number, flag, text) VALUES (?,?,?)', rows.slice(i, i + BATCH))
}
await conn.query(
'INSERT INTO shard_cliloc_meta (id, payload) VALUES (1, ?) ' +
'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP',
[JSON.stringify({ ...meta, count: rows.length })],
)
await conn.commit()
return { count: rows.length, blank, duplicates: entries.length - blank - rows.length }
} catch (err) {
await conn.rollback().catch(() => {})
throw err
} finally {
conn.release()
}
}
async function getMeta() {
const rows = await query('SELECT payload, imported_at FROM shard_cliloc_meta WHERE id = 1')
if (rows.length === 0) return null
const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload
return { ...payload, importedAt: rows[0].imported_at }
}
/**
* Look up a batch of ids.
*
* Batched rather than one-at-a-time because every caller has a LIST: a character
* sheet resolves a dozen equipment ids at once, and a page of marketplace
* listings resolves fifty. `IN (...)` with generated placeholders keeps it one
* round trip and one parameterized statement.
*/
async function lookup(numbers) {
if (!Array.isArray(numbers) || numbers.length === 0) return []
const ids = [...new Set(numbers.filter((n) => Number.isInteger(n)))]
if (ids.length === 0) return []
const placeholders = ids.map(() => '?').join(',')
return query(`SELECT number, text FROM shard_clilocs WHERE number IN (${placeholders})`, ids)
}
async function count() {
const rows = await query('SELECT COUNT(*) AS n FROM shard_clilocs')
return Number(rows[0]?.n) || 0
}
module.exports = {
replaceAll,
getMeta,
lookup,
count,
}

368
server/model/shardClilocs/shardClilocs.model.js Normal file
View File

@@ -0,0 +1,368 @@
const db = require('./shardClilocs.db')
const { settings } = require('../../core')
const { displayText } = require('../../utils/clilocParse')
const {
ClilocFormatError,
ClilocSourceError,
PARSER_VERSION,
hashSources,
sameSources,
missingSources,
readCliloc,
} = require('../../utils/clilocSource')
const log = require('../../core').logger('shardClilocs')
// The cliloc table — UO's id → display-string map, refreshed from a file the
// operator converts once from their own client.
//
// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a
// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace
// listing sends one too. Without the table the UI can only print `id 1023721`
// where the game prints "quarter staff".
//
// Two rules govern the boot path, both inherited from the spawn atlas:
//
// 1. **It never blocks startup.** No configured path, an unreadable file, a
// wrong-format file, a database error — all caught and logged. The site
// comes up either way, serving whatever table it already had (or none, in
// which case the UI falls back to item ids exactly as it did before).
// 2. **Nothing client-derived is committed.** The table is built from the
// operator's own file at a configured path. The repo ships no strings.
//
// The table is built from a SET of sources — the converted client table plus
// every operator-maintained overlay beside it — because shards edit items and
// add new ones, and those carry cliloc ids no stock client table has. All of
// them are re-read on every boot and hash-gated together, so adding one custom
// item never means re-exporting a 5 MB client file. Later sources win.
//
// That set is also why this has the atlas's escalation, in a lighter form. A
// single corrupt file fails the parse loudly, but a source that has simply
// VANISHED parses perfectly and imports a table quietly missing everything it
// contributed — the same ambiguity (real change vs half-copied mount) the atlas
// stages a facet removal for. So a disappearing source is refused and reported
// rather than applied.
//
// It is lighter than the atlas's because it needs to be: the atlas stores a
// pending decision in its own table and adds approve/reject endpoints, whereas
// here the decision is a single boolean an admin passes to the import they were
// already going to run. Re-parsing at approval time — the property that makes
// the atlas store only the decision — is automatic when there is nothing stored.
const SETTING_KEY = 'cliloc_client_path'
/**
* Where the converted cliloc file lives.
*
* The admin setting wins over the environment so an operator can repoint it
* without a redeploy, matching how the rest of the shard integration is
* admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the
* deploy-time default, since the path usually describes a mount the deployment
* sets up.
*/
async function getClientPath() {
try {
const configured = await settings.get(SETTING_KEY)
if (configured && String(configured).trim() !== '') return String(configured).trim()
} catch {
// Settings unavailable is not fatal — fall through to the env default.
}
const fromEnv = process.env.UO_CLIENT_PATH
return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : ''
}
async function setClientPath(value, updatedBy = null) {
const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy)
invalidate()
return result
}
// ── Refresh ────────────────────────────────────────────────────────────────
/** Was the loaded table built by THIS parser? */
const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION
/**
* Refresh the cliloc table from the configured file.
*
* Returns a result describing what happened rather than throwing, so the caller
* — including the boot path — can log it and move on:
*
* `skipped` no path configured
* `unavailable` path configured but missing / unreadable / not a cliloc file
* `unchanged` source hashes match the loaded table; nothing parsed
* `imported` parsed and applied
* `needsReview` a previously-present source has vanished; NOT applied
* `failed` parsed or applied and something went wrong
*
* `force` skips the hash check (an admin asking for a reimport). `approve`
* additionally accepts a vanished source.
*/
async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
// An explicit override wins outright — a one-off "use this file", which must
// not be silently overruled by the configured path the way an env default is.
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' }
let fingerprint
try {
fingerprint = hashSources(configured)
} catch (err) {
if (err instanceof ClilocSourceError) {
return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
}
return { status: 'failed', reason: err.message, path: configured }
}
const meta = await db.getMeta().catch(() => null)
// Two things make a loaded table stale: any source changed, or the PARSER did.
// Only checking the sources would strand an install whose client never patches
// on whatever an older build derived.
if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) {
return {
status: 'unchanged',
path: configured,
file: fingerprint.file,
count: meta.count ?? null,
customCount: fingerprint.customCount,
}
}
// A source that was there last import and is not there now is refused, not
// applied — an unmounted volume and a deliberate deletion look identical from
// here, and the wrong guess silently drops every name that file contributed.
const gone = missingSources(fingerprint.hashes, meta?.hashes)
if (gone.length > 0 && !approve) {
return {
status: 'needsReview',
reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`,
missingSources: gone,
path: configured,
file: fingerprint.file,
}
}
let parsed
try {
parsed = readCliloc(configured)
} catch (err) {
if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) {
return { status: 'unavailable', reason: err.message, code: err.code, path: configured }
}
return { status: 'failed', reason: err.message, path: configured }
}
try {
const applied = await db.replaceAll(parsed.entries, parsed.source)
invalidate()
return {
status: 'imported',
path: configured,
file: parsed.source.file,
count: applied.count,
parsed: parsed.entries.length,
blank: applied.blank,
// Per-source breakdown: how many entries each file contributed and how
// many of them overrode something already merged. An operator who adds an
// overlay wants to see it took effect, and "overrode: 0" on a file meant
// to re-label stock items says it did not.
sources: parsed.source.sources,
acceptedMissing: gone.length > 0 ? gone : undefined,
}
} catch (err) {
return { status: 'failed', reason: err.message, path: configured }
}
}
/**
* Boot hook. Best-effort by contract: it logs and returns, never throws, so a
* missing or malformed cliloc file can never stop the site coming up.
*/
async function refreshOnBoot() {
try {
const result = await refresh()
switch (result.status) {
case 'imported':
log.info('cliloc table refreshed', {
file: result.file,
count: result.count,
overlays: (result.sources || []).filter((s) => s.kind === 'custom').length,
})
break
case 'needsReview':
log.warn(
'cliloc refresh staged for admin review — a previously-loaded source is missing; ' +
'the existing table is unchanged',
{ missing: result.missingSources },
)
break
case 'unavailable':
// Deliberately a warning, not an error: an operator who has not supplied
// a cliloc file is in a supported state (the UI shows item ids), and the
// most common cause — pointing at the client's own compressed file —
// needs the reason spelled out rather than a stack trace.
log.warn('cliloc source unavailable (item names will show as ids)', {
reason: result.reason,
code: result.code,
path: result.path,
})
break
case 'failed':
log.warn('cliloc refresh failed', { reason: result.reason })
break
default:
break
}
return result
} catch (err) {
log.warn('cliloc refresh errored', { error: err.message })
return { status: 'failed', reason: err.message }
}
}
/** Everything the admin panel needs to describe cliloc state. */
async function status({ path: pathOverride = '' } = {}) {
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
const meta = await db.getMeta().catch(() => null)
const loaded = await db.count().catch(() => 0)
let fileReadable = false
let file = null
let drift = null
let problem = null
let code = null
let sources = []
let missing = []
if (configured !== '') {
try {
const fingerprint = hashSources(configured)
fileReadable = true
file = fingerprint.file
sources = Object.keys(fingerprint.hashes)
missing = missingSources(fingerprint.hashes, meta?.hashes)
// A compressed file is readable but not importable, and the panel has to
// say so HERE — otherwise pointing at an unconverted client directory
// reports a healthy file with pending drift ("ready to import") and the
// operator only finds out when the import fails. `drift` stays null
// because comparing hashes with an unusable file answers nothing.
if (fingerprint.compressed) {
problem =
'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' +
'Convert it to the plain format first — see docs/website/CLILOCS.md.'
code = 'COMPRESSED'
} else {
drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta)
}
} catch (err) {
fileReadable = false
problem = err.message
code = err.code ?? null
}
}
return {
configured: configured !== '',
path: configured,
file,
fileReadable,
problem,
code,
drift,
count: loaded,
// Every source found now (base first, then overlays), what each contributed
// at the last import, and any that have since vanished — which is the state
// an import will refuse without `approve`.
sources,
loadedSources: meta?.sources ?? null,
missingSources: missing,
importedAt: meta?.importedAt ?? null,
sourceBytes: meta?.bytes ?? null,
}
}
// ── Lookup ─────────────────────────────────────────────────────────────────
//
// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is
// ~123k rows and shipping it to a client would dwarf every page that uses it,
// and the Android app consumes the same JSON and would otherwise need its own
// copy. Callers get names, not ids-plus-a-table.
// A small write-through cache in front of the table. Item ids repeat heavily —
// one page of listings is mostly the same few hundred clilocs, and a character
// sheet re-resolves the same gear on every view — so this turns the steady state
// into zero queries. Capped so a pathological caller cannot grow it without
// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry,
// which is cheap and correct for a table that only changes on reimport.
const CACHE_MAX = 20000
let cache = new Map()
function invalidate() {
cache = new Map()
}
/**
* Resolve a batch of cliloc ids to display strings.
*
* Returns a `Map<number, string>` holding only the ids that resolved to
* something displayable — an id with no row, or one whose text is nothing but
* interpolated arguments we do not have, is simply absent. Callers fall back to
* whatever they had (the item id), so "missing" and "unnamed" collapse into one
* branch at the call site.
*
* Never throws: a cliloc lookup is decoration on someone's character sheet, and
* a database blip must not fail the sheet.
*/
async function resolveMany(numbers) {
const out = new Map()
if (!Array.isArray(numbers)) return out
const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))]
if (wanted.length === 0) return out
const missing = []
for (const number of wanted) {
if (cache.has(number)) {
const hit = cache.get(number)
if (hit !== '') out.set(number, hit)
} else {
missing.push(number)
}
}
if (missing.length > 0) {
try {
const rows = await db.lookup(missing)
const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)]))
if (cache.size + missing.length > CACHE_MAX) invalidate()
for (const number of missing) {
// Cache the miss too ('' meaning "no usable name"), so an id absent from
// the table does not re-query on every page view.
const text = found.get(number) ?? ''
cache.set(number, text)
if (text !== '') out.set(number, text)
}
} catch (err) {
log.warn('cliloc lookup failed', { message: err.message })
}
}
return out
}
/** Single-id convenience. Returns `null` when there is no usable name. */
async function resolve(number) {
const found = await resolveMany([number])
return found.get(number) ?? null
}
module.exports = {
SETTING_KEY,
getClientPath,
setClientPath,
refresh,
refreshOnBoot,
status,
resolveMany,
resolve,
invalidate,
}

46
server/model/shardEvents/shardEvents.db.js Normal file
View File

@@ -0,0 +1,46 @@
const { query } = require('../../core')
// INSERT IGNORE on the UNIQUE dedupe_key — a re-ingested event (WS-reconnect
// backfill overlap) is silently skipped rather than duplicated. Returns true if
// a new row was actually inserted.
async function insertIgnore({ kind, t, bootId, payload, dedupeKey }) {
const res = await query(
`INSERT IGNORE INTO shard_events (kind, t, boot_id, payload, dedupe_key)
VALUES (?, ?, ?, ?, ?)`,
[kind, t, bootId || null, JSON.stringify(payload), dedupeKey],
)
return res.affectedRows > 0
}
// Recent events, newest first. Filter by a single `kind`, or an allowlist of
// `kinds` (IN clause) — the public feed uses the allowlist so it can never leak
// staff/sensitive kinds. limit is clamped by the model.
async function list({ kind, kinds, limit }) {
// An allowlist that resolved to NOTHING means "serve nothing" — never "serve
// everything". Falling through to the unfiltered query below would have turned
// a fully-gated visibility config into a full dump of the event log, staff
// audit and cheat detections included.
if (kinds && kinds.length === 0) return []
if (kinds && kinds.length) {
const placeholders = kinds.map(() => '?').join(', ')
return query(
`SELECT id, kind, t, boot_id, payload, created_at
FROM shard_events WHERE kind IN (${placeholders}) ORDER BY t DESC LIMIT ?`,
[...kinds, limit],
)
}
if (kind) {
return query(
`SELECT id, kind, t, boot_id, payload, created_at
FROM shard_events WHERE kind = ? ORDER BY t DESC LIMIT ?`,
[kind, limit],
)
}
return query(
`SELECT id, kind, t, boot_id, payload, created_at
FROM shard_events ORDER BY t DESC LIMIT ?`,
[limit],
)
}
module.exports = { insertIgnore, list }

61
server/model/shardEvents/shardEvents.model.js Normal file
View File

@@ -0,0 +1,61 @@
// Append-only shard event log. The WS ingest dispatcher calls append() for the
// notable kinds; the public/admin read endpoints call list(). The DB layer only
// sees an already-computed dedupe_key so INSERT IGNORE is idempotent across
// WS-reconnect backfill.
const crypto = require('crypto')
const db = require('./shardEvents.db')
const MAX_LIMIT = 1000
const DEFAULT_LIMIT = 100
// Stable stringify — keys sorted — so the dedupe hash is independent of the
// property order the sidecar happened to serialize with.
function stableStringify(value) {
if (value === null || typeof value !== 'object') return JSON.stringify(value)
if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`
const keys = Object.keys(value).sort()
const entries = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`)
return `{${entries.join(',')}}`
}
// dedupe_key = sha256(kind + t + stable-json(payload)), truncated to 40 hex chars.
// This is a content fingerprint for idempotent INSERT IGNORE, not a security value,
// but we use SHA-256 rather than SHA-1 anyway; the truncation keeps it inside the
// CHAR(40) column (160 bits is ample collision resistance for dedupe). Two identical
// events (same kind, same timestamp, same body) collapse to one row.
function dedupeKey(kind, t, payload) {
return crypto
.createHash('sha256')
.update(`${kind}|${t}|${stableStringify(payload)}`)
.digest('hex')
.slice(0, 40)
}
// Append one event. Returns true if a new row was inserted (false = deduped).
async function append({ kind, t, bootId, payload }) {
return db.insertIgnore({ kind, t, bootId, payload, dedupeKey: dedupeKey(kind, t, payload) })
}
function normalizeLimit(limit) {
const n = Number(limit)
if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT
return Math.min(Math.floor(n), MAX_LIMIT)
}
// Recent events, newest first. Each row's JSON payload is parsed back to an
// object. `kinds` (array) restricts to an allowlist; `kind` filters a single kind.
async function list({ kind, kinds, limit } = {}) {
const rows = await db.list({ kind, kinds, limit: normalizeLimit(limit) })
return rows.map((row) => ({
id: row.id,
kind: row.kind,
t: row.t,
bootId: row.boot_id || null,
// mariadb returns JSON columns as strings on some versions; parse defensively.
payload: typeof row.payload === 'string' ? JSON.parse(row.payload) : row.payload,
createdAt: row.created_at,
}))
}
module.exports = { append, list, dedupeKey }

42
server/model/shardLinks/shardLinks.db.js Normal file
View File

@@ -0,0 +1,42 @@
const { query } = require('../../core')
const COLS = 'account, user_id, char_name, linked_at'
// Upsert a link. account is the PK, so a re-link moves the account to the new
// user (the sidecar already treats /link/confirm as authoritative).
async function upsert({ account, userId, charName }) {
await query(
`INSERT INTO shard_account_links (account, user_id, char_name)
VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE user_id = VALUES(user_id), char_name = VALUES(char_name)`,
[account, userId, charName || null],
)
return getByAccount(account)
}
async function getByAccount(account) {
const rows = await query(`SELECT ${COLS} FROM shard_account_links WHERE account = ? LIMIT 1`, [account])
return rows[0] || null
}
const listByUser = (userId) =>
query(`SELECT ${COLS} FROM shard_account_links WHERE user_id = ? ORDER BY linked_at DESC`, [userId])
async function isOwnedBy(account, userId) {
const rows = await query(
'SELECT 1 FROM shard_account_links WHERE account = ? AND user_id = ? LIMIT 1',
[account, userId],
)
return rows.length > 0
}
const remove = (account, userId) =>
query('DELETE FROM shard_account_links WHERE account = ? AND user_id = ?', [account, userId])
// Drop the mirror for an account regardless of which user held it — used to
// reconcile when the tie is severed at the source (an in-game [unlink →
// account.unlinked event, or a site-side DELETE /link/{account}).
const removeByAccount = (account) =>
query('DELETE FROM shard_account_links WHERE account = ?', [account])
module.exports = { upsert, getByAccount, listByUser, isOwnedBy, remove, removeByAccount }

37
server/model/shardLinks/shardLinks.model.js Normal file
View File

@@ -0,0 +1,37 @@
// Site-side mirror of in-game-account → website-user links. The sidecar owns the
// authoritative link (it tags the game account on /link/confirm); this model
// records it locally so the player portal can list links and enforce ownership.
const db = require('./shardLinks.db')
function toSafe(row) {
if (!row) return null
return {
account: row.account,
userId: row.user_id,
charName: row.char_name || null,
linkedAt: row.linked_at,
}
}
async function link({ account, userId, charName }) {
return toSafe(await db.upsert({ account, userId, charName }))
}
async function listForUser(userId) {
const rows = await db.listByUser(userId)
return rows.map(toSafe)
}
const ownsAccount = (account, userId) => db.isOwnedBy(account, userId)
async function getByAccount(account) {
return toSafe(await db.getByAccount(account))
}
const unlink = (account, userId) => db.remove(account, userId)
// Drop the local mirror for an account (source-of-truth severed elsewhere).
const removeByAccount = (account) => db.removeByAccount(account)
module.exports = { link, listForUser, ownsAccount, getByAccount, unlink, removeByAccount }

301
server/model/shardMarket/shardMarket.db.js Normal file
View File

@@ -0,0 +1,301 @@
const core = require('../../core')
const { query } = core
// Raw SQL for the player-vendor market index (Protocol 3.0 vendor.listing).
//
// Two tables, both INGEST-OWNED: `shard_vendors` (one row per shop) and
// `shard_vendor_items` (one row per priced listing). Nothing else in the codebase
// writes to either. No foreign keys, consistent with every other shard_* table.
// Insert batch size for one vendor's listings. A shop is capped at
// MarketMaxListings (250 by default) on the shard side, so in practice this is
// one batch — it exists for the operator who raised that cap.
const BATCH = 500
// LIKE wildcards in user input. `%` and `_` are not special to the parameterized
// query — they are special to LIKE itself — so a search for "50% off" would
// otherwise match everything containing "50" and a search for "_" would match
// every single-character name. Escaped with a backslash, which is MariaDB's
// default LIKE escape (no ESCAPE clause needed).
const likeTerm = (q) => `%${String(q).replace(/[\\%_]/g, (c) => `\\${c}`)}%`
/**
* Replace one vendor's whole row and listing set, in one transaction.
*
* Delete-then-insert rather than a diff, because the frame is AUTHORITATIVE for
* that vendor: the shard's sweep only emits a shop whose contents, prices or
* location moved, and when it does it sends the whole shop. Reconciling it item
* by item would be more code for the same result and would leave sold items
* behind on any path the reconciliation missed.
*
* All-or-nothing matters here for a specific reason: the two writes are "the
* shop" and "what is in it", and a failure between them leaves a shop advertising
* an inventory it no longer has (or none at all) — visibly wrong on the page, and
* indistinguishable from a genuinely empty shop.
*/
async function replaceVendor(vendor, items) {
const conn = await core.pool.getConnection()
try {
await conn.beginTransaction()
await conn.query(
`INSERT INTO shard_vendors
(serial, shop_name, owner_serial, owner_name, map, x, y, z, region, house,
item_count, item_total, truncated, t)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE shop_name = VALUES(shop_name), owner_serial = VALUES(owner_serial),
owner_name = VALUES(owner_name), map = VALUES(map), x = VALUES(x), y = VALUES(y),
z = VALUES(z), region = VALUES(region), house = VALUES(house),
item_count = VALUES(item_count), item_total = VALUES(item_total),
truncated = VALUES(truncated), t = VALUES(t),
-- Touched explicitly rather than left to ON UPDATE CURRENT_TIMESTAMP:
-- MariaDB does not fire that when every column is written back
-- unchanged, and a shop that is re-published identically is still
-- FRESHLY CONFIRMED. Without this the staleness banner would age a
-- perfectly current shop forever.
updated_at = CURRENT_TIMESTAMP`,
[
vendor.serial,
vendor.shopName ?? null,
vendor.ownerSerial ?? null,
vendor.ownerName ?? null,
vendor.map ?? null,
Number.isFinite(vendor.x) ? vendor.x : null,
Number.isFinite(vendor.y) ? vendor.y : null,
Number.isFinite(vendor.z) ? vendor.z : null,
vendor.region ?? null,
vendor.house ?? null,
items.length,
Number.isFinite(vendor.itemTotal) ? vendor.itemTotal : items.length,
vendor.truncated ? 1 : 0,
Number.isFinite(vendor.t) ? vendor.t : null,
],
)
await conn.query('DELETE FROM shard_vendor_items WHERE vendor_serial = ?', [vendor.serial])
const rows = items.map((i) => [
vendor.serial,
i.serial,
i.itemId,
i.hue,
i.amount,
i.price,
i.name,
i.cliloc,
i.displayName,
i.child ? 1 : 0,
])
for (let i = 0; i < rows.length; i += BATCH) {
await conn.batch(
`INSERT INTO shard_vendor_items
(vendor_serial, serial, item_id, hue, amount, price, name, cliloc, display_name, child)
VALUES (?,?,?,?,?,?,?,?,?,?)`,
rows.slice(i, i + BATCH),
)
}
await conn.commit()
return { items: rows.length }
} catch (err) {
await conn.rollback().catch(() => {})
throw err
} finally {
conn.release()
}
}
/** Drop one vendor and its listings (vendor.listing.remove). */
async function removeVendor(serial) {
const conn = await core.pool.getConnection()
try {
await conn.beginTransaction()
await conn.query('DELETE FROM shard_vendor_items WHERE vendor_serial = ?', [serial])
await conn.query('DELETE FROM shard_vendors WHERE serial = ?', [serial])
await conn.commit()
} catch (err) {
await conn.rollback().catch(() => {})
throw err
} finally {
conn.release()
}
}
// ── Search ─────────────────────────────────────────────────────────────────
//
// The unit of a search RESULT is a listing, not a vendor: "who sells a vanquishing
// kryss and for how much" is the question, and answering it per vendor would make
// the caller flatten the shops back out. The vendor's columns ride along on the
// join so a result row is self-contained.
function searchWhere({ q, minPrice, maxPrice, itemId, map, region }) {
const where = ['i.price > 0']
const params = []
if (q) {
// Both the resolved display name and the item's own literal, because an item
// with a player-set name (most of what is actually worth searching for on a
// player-run shard) may have a generic cliloc.
where.push('(i.display_name LIKE ? OR i.name LIKE ?)')
params.push(likeTerm(q), likeTerm(q))
}
if (Number.isFinite(minPrice)) {
where.push('i.price >= ?')
params.push(minPrice)
}
if (Number.isFinite(maxPrice)) {
where.push('i.price <= ?')
params.push(maxPrice)
}
if (Number.isFinite(itemId)) {
where.push('i.item_id = ?')
params.push(itemId)
}
if (map) {
where.push('v.map = ?')
params.push(map)
}
if (region) {
where.push('v.region = ?')
params.push(region)
}
return { sql: `WHERE ${where.join(' AND ')}`, params }
}
// Whitelisted, because this interpolates into the statement. `recent` sorts by
// the vendor's freshness, which is the only way to see what has just been listed
// on a shard whose sweep is minutes wide.
const SORTS = {
price_asc: 'i.price ASC, i.id ASC',
price_desc: 'i.price DESC, i.id ASC',
recent: 'v.updated_at DESC, i.id ASC',
}
async function searchListings({ q, minPrice, maxPrice, itemId, map, region, sort, limit, offset }) {
const { sql, params } = searchWhere({ q, minPrice, maxPrice, itemId, map, region })
const order = SORTS[sort] || SORTS.price_asc
const rows = await query(
`SELECT i.serial, i.item_id, i.hue, i.amount, i.price, i.name, i.cliloc, i.display_name, i.child,
v.serial AS vendor_serial, v.shop_name, v.owner_serial, v.owner_name,
v.map, v.x, v.y, v.z, v.region, v.house, v.updated_at
FROM shard_vendor_items i
JOIN shard_vendors v ON v.serial = i.vendor_serial
${sql}
ORDER BY ${order}
LIMIT ? OFFSET ?`,
[...params, limit, offset],
)
const counted = await query(
`SELECT COUNT(*) AS n
FROM shard_vendor_items i
JOIN shard_vendors v ON v.serial = i.vendor_serial
${sql}`,
params,
)
return { rows, total: Number(counted[0]?.n) || 0 }
}
async function getVendor(serial) {
const rows = await query(
`SELECT serial, shop_name, owner_serial, owner_name, map, x, y, z, region, house,
item_count, item_total, truncated, t, updated_at
FROM shard_vendors WHERE serial = ?`,
[serial],
)
return rows[0] || null
}
async function listVendorItems(serial, { limit, offset }) {
return query(
`SELECT serial, item_id, hue, amount, price, name, cliloc, display_name, child
FROM shard_vendor_items
WHERE vendor_serial = ?
ORDER BY price ASC, id ASC
LIMIT ? OFFSET ?`,
[serial, limit, offset],
)
}
/**
* What the market page's header needs: how big the index is, and how stale it may
* be. `staleAt` is the OLDEST vendor row — the round-robin sweep means a shop can
* be a full cycle behind, and the page says so rather than implying live prices.
*/
async function meta() {
const rows = await query(
`SELECT COUNT(*) AS vendors, MIN(updated_at) AS stale_at, MAX(updated_at) AS fresh_at
FROM shard_vendors`,
)
const items = await query('SELECT COUNT(*) AS n FROM shard_vendor_items')
return {
vendors: Number(rows[0]?.vendors) || 0,
items: Number(items[0]?.n) || 0,
staleAt: rows[0]?.stale_at || null,
freshAt: rows[0]?.fresh_at || null,
}
}
/** The distinct facets and regions holding vendors — drives the page's filters. */
async function listPlaces() {
const maps = await query(
'SELECT DISTINCT map FROM shard_vendors WHERE map IS NOT NULL ORDER BY map',
)
const regions = await query(
'SELECT DISTINCT region FROM shard_vendors WHERE region IS NOT NULL ORDER BY region',
)
return { maps: maps.map((r) => r.map), regions: regions.map((r) => r.region) }
}
// ── Cliloc re-resolution ───────────────────────────────────────────────────
/**
* One page of listings whose name still needs resolving, for the bulk pass that
* runs after a cliloc import.
*
* Keyed on `id > after` rather than OFFSET: the pass updates the very rows it is
* scanning, and an OFFSET walk over a table being rewritten skips rows. Every
* row with a cliloc is re-read, not just the unresolved ones, because an import
* can also CHANGE a name — a shard overlay relabelling a stock item is the whole
* reason overlays exist.
*/
async function listResolvableItems(after, limit) {
return query(
`SELECT id, cliloc, name, display_name
FROM shard_vendor_items
WHERE cliloc IS NOT NULL AND cliloc > 0 AND id > ?
ORDER BY id
LIMIT ?`,
[after, limit],
)
}
/** Write back a batch of re-resolved display names. */
async function updateDisplayNames(pairs) {
if (pairs.length === 0) return 0
const conn = await core.pool.getConnection()
try {
await conn.batch('UPDATE shard_vendor_items SET display_name = ? WHERE id = ?', pairs)
return pairs.length
} finally {
conn.release()
}
}
module.exports = {
replaceVendor,
removeVendor,
searchListings,
getVendor,
listVendorItems,
meta,
listPlaces,
listResolvableItems,
updateDisplayNames,
likeTerm,
}

329
server/model/shardMarket/shardMarket.model.js Normal file
View File

@@ -0,0 +1,329 @@
// ── Player-vendor market index (Protocol 3.0 vendor.listing) ───────────────
//
// The shard-wide shop index: what every player vendor is selling, for how much,
// and where it is standing. This is the website's half of the search the in-game
// Vendor Search gump offers — the same data, the same opt-out, reachable without
// logging in to the game.
//
// Ingest is per-vendor and authoritative: the shard's round-robin sweep emits one
// `vendor.listing` frame per shop whose contents, prices or location moved, and
// the frame is the whole shop (see docs/link/v3.md §8 and BridgeMarket.cs). This
// module normalizes it into shard_vendors + shard_vendor_items and, crucially,
// resolves each listing's cliloc to a DISPLAY NAME on the way in — a search for
// "kryss" is a search over names, and the shard only ever sends numbers.
const db = require('./shardMarket.db')
const clilocs = require('../shardClilocs/shardClilocs.model')
const log = require('../../core').logger('shard-market')
// Defense in depth on top of the shard's own MarketMaxListings cap. The shard is
// trusted, but it is a separately-versioned component: a frame from a plugin
// whose cap was raised (or a shard running modified scripts) must not be able to
// turn one ingest into an unbounded transaction.
const MAX_ITEMS_PER_VENDOR = 5000
// Column widths in schema.sql. Truncating here rather than letting MariaDB do it
// keeps the behavior the same in strict mode, where an over-length value is an
// ERROR and would fail the whole vendor rather than shortening one name.
const MAX_NAME = 160
const MAX_SHOP = 160
const MAX_OWNER = 64
const MAX_MAP = 40
const MAX_REGION = 80
const MAX_SERIAL = 20
const clip = (value, max) => {
if (value == null) return null
const s = String(value)
return s.length > max ? s.slice(0, max) : s
}
const int = (value, fallback = 0) => {
const n = Number(value)
return Number.isFinite(n) ? Math.trunc(n) : fallback
}
// ── Ingest ─────────────────────────────────────────────────────────────────
/**
* Flatten one `vendor.listing` frame into the row shapes the DB layer wants.
*
* `location` arrives as a nested object rather than flat map/x/y/region, and that
* shape is load-bearing rather than cosmetic: the visibility projection matches
* literal JSON keys, so ONE `market.location` rule can hide a vendor's
* whereabouts only if `location` is a single key on both the live frame and the
* stored read model. Flattening it here for storage and re-nesting it on read is
* what keeps that true on both paths.
*
* Exported for tests — it is the part with rules in it, and it is pure.
*/
function flattenFrame(ev) {
const loc = (ev && ev.location) || {}
return {
serial: clip(ev.serial, MAX_SERIAL),
shopName: clip(ev.shopName, MAX_SHOP),
ownerSerial: clip(ev.ownerSerial, MAX_SERIAL),
ownerName: clip(ev.ownerName, MAX_OWNER),
map: clip(loc.map, MAX_MAP),
x: Number.isFinite(loc.x) ? Math.trunc(loc.x) : null,
y: Number.isFinite(loc.y) ? Math.trunc(loc.y) : null,
z: Number.isFinite(loc.z) ? Math.trunc(loc.z) : null,
region: clip(loc.region, MAX_REGION),
house: clip(loc.house, MAX_SHOP),
// What the SHOP holds, which is not what the frame carries when it was
// truncated. Kept apart so the page can say "showing 250 of 3,104" rather
// than presenting a partial shop as a complete one.
itemTotal: int(ev.total, int(ev.count, 0)),
truncated: ev.truncated === true,
t: Number.isFinite(ev.t) ? ev.t : null,
}
}
/**
* Resolve each listing's display name.
*
* Order of preference is the item's own literal `name` first, then the cliloc.
* That is the opposite of what "resolve the id" suggests and it is right: a
* literal name only exists because a player set one ("Bob's vanquishing kryss"),
* and it is strictly more specific than the generic cliloc the item still
* carries.
*
* One batched lookup per frame rather than per item; `resolveMany` is cached and
* never throws, so a cliloc table that is missing entirely just leaves
* `displayName` null and the page renders item ids, exactly as it did before the
* table existed.
*/
async function shapeItems(ev) {
const raw = Array.isArray(ev.items) ? ev.items.slice(0, MAX_ITEMS_PER_VENDOR) : []
const wanted = raw
.map((i) => int(i && i.cliloc, 0))
.filter((n) => n > 0)
const names = await clilocs.resolveMany(wanted)
return raw
.filter((i) => i && i.serial)
.map((i) => {
const literal = clip(i.name, MAX_NAME)
const cliloc = int(i.cliloc, 0) || null
return {
serial: clip(i.serial, MAX_SERIAL),
itemId: int(i.itemId, 0),
hue: int(i.hue, 0),
amount: int(i.amount, 1),
price: int(i.price, 0),
name: literal,
cliloc,
displayName: literal || (cliloc ? clip(names.get(cliloc) ?? null, MAX_NAME) : null),
child: i.child === true,
}
})
// Unpriced rows are inventory, not listings. The shard already drops them;
// this is the same rule enforced where the table is written, so a plugin that
// stops enforcing it cannot put un-buyable rows on the market page.
.filter((i) => i.price > 0)
}
/** Ingest one `vendor.listing` frame. */
async function upsertVendor(ev) {
if (!ev || !ev.serial) return
const vendor = flattenFrame(ev)
const items = await shapeItems(ev)
await db.replaceVendor(vendor, items)
}
/** Ingest one `vendor.listing.remove` frame. */
async function removeVendor(serial) {
if (!serial) return
await db.removeVendor(String(serial).slice(0, MAX_SERIAL))
}
// ── Read models ────────────────────────────────────────────────────────────
//
// `location` is re-nested (see flattenFrame) so the stored read model and the
// live wire frame present the same keys to the visibility projection.
const place = (r) => ({
map: r.map,
x: r.x,
y: r.y,
z: r.z,
region: r.region,
house: r.house,
})
// A listing as the search returns it: the item, plus enough of its shop to be
// actionable without a second request. `displayName` falls back to nothing rather
// than to a fabricated "Item 3922" — the client decides how to render an
// unresolved id, and inventing a name here would make it indistinguishable from
// a real one.
const shapeListing = (r) => ({
serial: r.serial,
itemId: r.item_id,
hue: r.hue,
amount: r.amount,
price: Number(r.price),
name: r.name,
cliloc: r.cliloc,
displayName: r.display_name,
child: !!r.child,
vendor: {
serial: r.vendor_serial,
shopName: r.shop_name,
ownerSerial: r.owner_serial,
ownerName: r.owner_name,
location: place(r),
updatedAt: r.updated_at,
},
})
const shapeVendor = (r) => ({
serial: r.serial,
shopName: r.shop_name,
ownerSerial: r.owner_serial,
ownerName: r.owner_name,
location: place(r),
count: r.item_count,
total: r.item_total,
truncated: !!r.truncated,
updatedAt: r.updated_at,
})
const shapeItem = (r) => ({
serial: r.serial,
itemId: r.item_id,
hue: r.hue,
amount: r.amount,
price: Number(r.price),
name: r.name,
cliloc: r.cliloc,
displayName: r.display_name,
child: !!r.child,
})
/**
* Search the index. Returns a page of LISTINGS (not vendors) plus the
* unpaginated total and the staleness stamp the page's banner needs.
*/
async function search({
q = '',
minPrice,
maxPrice,
itemId,
map = '',
region = '',
sort = 'price_asc',
limit = 50,
offset = 0,
} = {}) {
const { rows, total } = await db.searchListings({
q: q.trim(),
minPrice: Number.isFinite(minPrice) ? minPrice : undefined,
maxPrice: Number.isFinite(maxPrice) ? maxPrice : undefined,
itemId: Number.isFinite(itemId) ? itemId : undefined,
map: map.trim(),
region: region.trim(),
sort,
limit,
offset,
})
const info = await db.meta()
return {
listings: rows.map(shapeListing),
total,
limit,
offset,
// Repeated on every search response rather than left to a separate /meta
// call: the banner that says how old these prices are must age with the
// results it labels, and a client that fetched it once would keep showing a
// stamp from before the page it is looking at.
staleAt: info.staleAt,
vendors: info.vendors,
}
}
/** One shop and its listings. `null` when the index has never seen that serial. */
async function getVendor(serial, { limit = 250, offset = 0 } = {}) {
const row = await db.getVendor(serial)
if (!row) return null
const items = await db.listVendorItems(serial, { limit, offset })
return { ...shapeVendor(row), items: items.map(shapeItem) }
}
/** Index size, staleness, and the facet/region filter options. */
async function meta() {
const [info, places] = await Promise.all([db.meta(), db.listPlaces()])
return { ...info, ...places }
}
// ── Cliloc re-resolution ───────────────────────────────────────────────────
// Batch size for the post-import pass. Big enough that a 40k-row table is ~40
// round trips, small enough that a single batch is not a long-held connection.
const RESOLVE_BATCH = 1000
/**
* Re-resolve every listing's display name against the current cliloc table.
*
* Called after a cliloc import, and it has to be: the market's diff sweep will
* NOT re-send an unchanged shop just because the site learned what its items are
* called, so without this an operator who configures clilocs after the first
* market sweep sees item ids until every shop happens to change. That is the same
* class of staleness the spawn atlas avoids by re-parsing on boot — here the
* source of truth for names moved, not the data.
*
* Never throws. It is a cosmetic backfill on a table that is already serving; a
* failure means names stay as they were, which is exactly the pre-import state.
*/
async function refreshDisplayNames() {
let after = 0
let scanned = 0
let changed = 0
try {
for (;;) {
const rows = await db.listResolvableItems(after, RESOLVE_BATCH)
if (rows.length === 0) break
after = rows[rows.length - 1].id
scanned += rows.length
const names = await clilocs.resolveMany(rows.map((r) => Number(r.cliloc)))
const pairs = []
for (const row of rows) {
// The literal name still wins, so a re-resolution never overwrites a
// player-set name with the generic cliloc behind it.
const next = row.name
? clip(row.name, MAX_NAME)
: clip(names.get(Number(row.cliloc)) ?? null, MAX_NAME)
if (next !== row.display_name) pairs.push([next, row.id])
}
changed += await db.updateDisplayNames(pairs)
}
if (changed > 0) log.info('market display names refreshed', { scanned, changed })
return { scanned, changed }
} catch (err) {
log.warn('market display-name refresh failed', { message: err.message, scanned, changed })
return { scanned, changed, error: err.message }
}
}
module.exports = {
upsertVendor,
removeVendor,
search,
getVendor,
meta,
refreshDisplayNames,
flattenFrame,
shapeItems,
shapeListing,
shapeVendor,
MAX_ITEMS_PER_VENDOR,
}

369
server/model/shardState/shardState.db.js Normal file
View File

@@ -0,0 +1,369 @@
const { query } = require('../../core')
// Shared upsert builder for the shard-state tables. Each is keyed on a single
// primary column (`pkCol` = pk); `fields` carries only the columns the model
// wants to write, so a partial refresh touches nothing else. `coalesce` keeps
// the prior column value when the incoming one is NULL (used by shard_online so a
// vitals frame that omits acct/name doesn't blank what mob.login set); otherwise
// the incoming value wins (VALUES()).
function upsertRow(table, pkCol, pk, fields, { coalesce = false } = {}) {
const cols = Object.keys(fields)
const allCols = [pkCol, ...cols]
const insertCols = allCols.map((c) => `\`${c}\``).join(', ')
const placeholders = allCols.map(() => '?').join(', ')
const rhs = coalesce
? (c) => `\`${c}\` = COALESCE(VALUES(\`${c}\`), \`${c}\`)`
: (c) => `\`${c}\` = VALUES(\`${c}\`)`
const updates = cols.map(rhs).join(', ')
return query(
`INSERT INTO ${table} (${insertCols}) VALUES (${placeholders})
ON DUPLICATE KEY UPDATE ${updates}`,
[pk, ...cols.map((c) => fields[c])],
)
}
// ── Online players ─────────────────────────────────────────────────────────
const ONLINE_COLS =
'serial, name, acct, web_id, map, x, y, z, hits, hits_max, mana, mana_max, stam, stam_max, str, dex, `int`, updated_at'
// Upsert one online player. `fields` already prepared by the model (only the
// columns it wants to write); serial is required and is the primary key.
// COALESCE variant: a char.vitals frame that omits acct/name must not blank what
// mob.login set, so an incoming NULL keeps the prior column value.
const upsertOnline = (serial, fields) =>
upsertRow('shard_online', 'serial', serial, fields, { coalesce: true })
const removeOnline = (serial) => query('DELETE FROM shard_online WHERE serial = ?', [serial])
const clearOnline = () => query('DELETE FROM shard_online')
async function countOnline() {
const rows = await query('SELECT COUNT(*) AS n FROM shard_online')
return rows[0] ? Number(rows[0].n) : 0
}
const listOnline = () =>
query(`SELECT ${ONLINE_COLS} FROM shard_online ORDER BY name ASC`)
// Online players on any of the given game accounts (admin: a user's linked
// accounts). Empty list short-circuits so we never emit `IN ()`.
const listOnlineByAccounts = (accounts) =>
accounts.length === 0
? Promise.resolve([])
: query(
`SELECT ${ONLINE_COLS} FROM shard_online
WHERE acct IN (${accounts.map(() => '?').join(', ')})
ORDER BY name ASC`,
accounts,
)
// Staff roles whose online presence is shown on the public Shard page. Players
// who link an account are NOT surfaced publicly — only staff opt into visibility
// by virtue of being staff.
const PUBLIC_ONLINE_ROLES = ['admin', 'editor', 'moderator']
// Online players whose game account is linked to a STAFF website user. Joined
// against shard_account_links (not the sidecar-supplied web_id) so a link takes
// effect immediately, regardless of whether the player has re-logged since
// linking, then through to users so only staff roles are surfaced publicly.
const listOnlineLinked = () => {
const cols = ONLINE_COLS.split(', ')
.map((c) => `o.${c}`)
.join(', ')
return query(
`SELECT ${cols}
FROM shard_online o
JOIN shard_account_links l ON l.account = o.acct
JOIN users u ON u.id = l.user_id
WHERE u.role IN (${PUBLIC_ONLINE_ROLES.map(() => '?').join(', ')})
ORDER BY o.name ASC`,
PUBLIC_ONLINE_ROLES,
)
}
// ── Economy supply series ────────────────────────────────────────────────
const insertEconomy = ({ accounts, gold, t }) =>
query('INSERT INTO shard_economy (accounts, gold, t) VALUES (?, ?, ?)', [
accounts ?? null,
gold ?? null,
t,
])
const listEconomy = (limit) =>
query('SELECT accounts, gold, t FROM shard_economy ORDER BY t DESC LIMIT ?', [limit])
async function latestEconomy() {
const rows = await query('SELECT accounts, gold, t FROM shard_economy ORDER BY t DESC LIMIT 1')
return rows[0] || null
}
// ── Houses / IDOC ────────────────────────────────────────────────────────
const HOUSE_COLS =
'serial, stage, map, x, y, z, region, name, owner_serial, owner_acct, built_on, last_refreshed, is_idoc, updated_at'
const upsertHouse = (serial, fields) => upsertRow('shard_houses', 'serial', serial, fields)
const listIdocHouses = () =>
query(`SELECT ${HOUSE_COLS} FROM shard_houses WHERE is_idoc = 1 ORDER BY updated_at DESC`)
// Houses owned by any of the given game accounts (admin: a user's linked
// accounts). IDOC houses first, then newest-refreshed. Empty list short-circuits.
const listHousesByAccounts = (accounts) =>
accounts.length === 0
? Promise.resolve([])
: query(
`SELECT ${HOUSE_REG_COLS} FROM shard_houses
WHERE owner_acct IN (${accounts.map(() => '?').join(', ')})
ORDER BY is_idoc DESC, updated_at DESC`,
accounts,
)
// ── House registry (Protocol 2.0 house.update / house.remove) ──────────────
// The registry columns extend HOUSE_COLS; a registry row is one we've seen via
// house.update (in_registry = 1), as opposed to a decay-only transition row.
const HOUSE_REG_COLS = `${HOUSE_COLS}, owner_name, co_owners, friends, price, decay, in_registry`
const removeHouse = (serial) => query('DELETE FROM shard_houses WHERE serial = ?', [serial])
// The full registered-house browser: every row we've seen via house.update.
const listRegistryHouses = () =>
query(`SELECT ${HOUSE_REG_COLS} FROM shard_houses WHERE in_registry = 1 ORDER BY name ASC`)
// ── Champion spawns ────────────────────────────────────────────────────────
const CHAMP_COLS =
'serial, category, type, name, status, active, map, x, y, z, boss_up, payload, t, updated_at'
const upsertChamp = (serial, fields) => upsertRow('shard_champs', 'serial', serial, fields)
const removeChamp = (serial) => query('DELETE FROM shard_champs WHERE serial = ?', [serial])
const clearChamps = () => query('DELETE FROM shard_champs')
// Ordered by name (matches the sidecar's /champs ordering).
const listChamps = () => query(`SELECT ${CHAMP_COLS} FROM shard_champs ORDER BY name ASC`)
// ── Help-page (support) queue ──────────────────────────────────────────────
const PAGE_COLS =
'page_id, type, sender_name, sender_acct, web_id, message, map, x, y, z, sent_ms, handled, handler, payload, updated_at'
async function upsertPage(pageId, fields) {
const cols = Object.keys(fields)
const allCols = ['page_id', ...cols]
const insertCols = allCols.map((c) => `\`${c}\``).join(', ')
const placeholders = allCols.map(() => '?').join(', ')
const updates = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(', ')
await query(
`INSERT INTO shard_pages (${insertCols}) VALUES (${placeholders})
ON DUPLICATE KEY UPDATE ${updates}`,
[pageId, ...cols.map((c) => fields[c])],
)
}
const removePage = (pageId) => query('DELETE FROM shard_pages WHERE page_id = ?', [pageId])
const clearPages = () => query('DELETE FROM shard_pages')
// Oldest-open first so the queue reads like a work list.
const listPages = () => query(`SELECT ${PAGE_COLS} FROM shard_pages ORDER BY sent_ms ASC`)
// ── Guild board (Protocol 2.0) ─────────────────────────────────────────────
const GUILD_COLS =
'id, name, abbr, members, online, alliance, leader_serial, leader_name, leader_acct, leader_web_id, payload, t, updated_at'
const upsertGuild = (id, fields) => upsertRow('shard_guilds', 'id', id, fields)
const removeGuild = (id) => query('DELETE FROM shard_guilds WHERE id = ?', [id])
const clearGuilds = () => query('DELETE FROM shard_guilds')
const listGuilds = () => query(`SELECT ${GUILD_COLS} FROM shard_guilds ORDER BY name ASC`)
// The guild an actor LEADS — matched on the current board (leader_serial or the
// linked leader_acct), so it reflects live state. Guild MEMBERSHIP for non-leaders
// is not modelled (the board carries only counts + leader), so we don't guess it.
const findGuildLedByActor = (serial, acct) =>
query(
`SELECT id, name, abbr, alliance, leader_name FROM shard_guilds
WHERE leader_serial = ? OR (leader_acct IS NOT NULL AND leader_acct = ?)
LIMIT 1`,
[serial ?? null, acct ?? null],
)
// Guilds led by any of the given game accounts (admin: a user's linked accounts).
const listGuildsLedByAccounts = (accounts) =>
accounts.length === 0
? Promise.resolve([])
: query(
`SELECT id, name, abbr, alliance, leader_name FROM shard_guilds
WHERE leader_acct IN (${accounts.map(() => '?').join(', ')})
ORDER BY name ASC`,
accounts,
)
// ── Governor board + term history (Protocol 2.0) ───────────────────────────
const GOV_COLS =
'city, governor_serial, governor_name, governor_acct, governor_web_id, elect_serial, elect_name, elect_acct, election_phase, candidates, auto_pick_at, payload, t, updated_at'
const upsertGovernor = (city, fields) => upsertRow('shard_governors', 'city', city, fields)
const listGovernors = () => query(`SELECT ${GOV_COLS} FROM shard_governors ORDER BY city ASC`)
// Cities whose current governor is one of the given game accounts (cross-link:
// does this user hold a governorship?). Empty list short-circuits.
const listGovernorshipsByAccounts = (accounts) =>
accounts.length === 0
? Promise.resolve([])
: query(
`SELECT ${GOV_COLS} FROM shard_governors
WHERE governor_acct IN (${accounts.map(() => '?').join(', ')})
ORDER BY city ASC`,
accounts,
)
// The single open term (ended_at IS NULL) for a city, if any.
async function currentGovernorTerm(city) {
const rows = await query(
'SELECT id, city, governor_serial, governor_name, governor_acct, governor_web_id, started_at, ended_at, votes FROM shard_governor_terms WHERE city = ? AND ended_at IS NULL ORDER BY started_at DESC LIMIT 1',
[city],
)
return rows[0] || null
}
const closeGovernorTerm = (id, endedAt) =>
query('UPDATE shard_governor_terms SET ended_at = ? WHERE id = ?', [endedAt, id])
const openGovernorTerm = ({ city, serial, name, acct, webId, startedAt }) =>
query(
`INSERT INTO shard_governor_terms
(city, governor_serial, governor_name, governor_acct, governor_web_id, started_at)
VALUES (?, ?, ?, ?, ?, ?)`,
[city, serial ?? null, name ?? null, acct ?? null, webId ?? null, startedAt],
)
const listGovernorTerms = (city, limit) =>
query(
'SELECT id, city, governor_serial, governor_name, governor_acct, governor_web_id, started_at, ended_at, votes FROM shard_governor_terms WHERE city = ? ORDER BY started_at DESC LIMIT ?',
[city, limit],
)
// ── Online-population snapshot (Protocol 2.0 presence.online) ───────────────
async function setPresence({ count, byFacet, byRegion, t }) {
await query(
`INSERT INTO shard_presence (id, count, by_facet, by_region, t) VALUES (1, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE count = VALUES(count), by_facet = VALUES(by_facet),
by_region = VALUES(by_region), t = VALUES(t)`,
[
Number.isFinite(count) ? count : 0,
byFacet ? JSON.stringify(byFacet) : null,
byRegion ? JSON.stringify(byRegion) : null,
Number.isFinite(t) ? t : null,
],
)
}
async function latestPresence() {
const rows = await query('SELECT count, by_facet, by_region, t FROM shard_presence WHERE id = 1')
return rows[0] || null
}
// ── Shard ruleset (Protocol 3.0 world.ruleset) ─────────────────────────────
// Singleton, same shape as shard_presence: the shard re-emits the whole frame on
// every connect, so there is nothing to merge — the latest one wins outright.
async function setRuleset({ rev, expansion, payload, t }) {
await query(
`INSERT INTO shard_ruleset (id, rev, expansion, payload, t) VALUES (1, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE rev = VALUES(rev), expansion = VALUES(expansion),
payload = VALUES(payload), t = VALUES(t)`,
[rev ?? null, expansion ?? null, payload, Number.isFinite(t) ? t : null],
)
}
async function getRuleset() {
const rows = await query(
'SELECT rev, expansion, payload, t, updated_at FROM shard_ruleset WHERE id = 1',
)
return rows[0] || null
}
// ── Points/loyalty boards (Protocol 3.0 points.board) ──────────────────────
// One row per point system. The shard only emits a system whose top N actually
// moved, so this is a sparse stream of overwrites; there is no delete, because
// the shard's set of systems is fixed at startup.
async function upsertPointsBoard({ system, name, nameCliloc, maxPoints, players, showOnGump, payload, t }) {
await query(
`INSERT INTO shard_points_boards
(system, name, name_cliloc, max_points, players, show_on_gump, payload, t)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE name = VALUES(name), name_cliloc = VALUES(name_cliloc),
max_points = VALUES(max_points), players = VALUES(players),
show_on_gump = VALUES(show_on_gump), payload = VALUES(payload), t = VALUES(t)`,
[
system,
name ?? null,
Number.isFinite(nameCliloc) ? nameCliloc : null,
Number.isFinite(maxPoints) ? maxPoints : null,
Number.isFinite(players) ? players : null,
showOnGump ? 1 : 0,
payload,
Number.isFinite(t) ? t : null,
],
)
}
// Ordered by display name, falling back to the system key for a board whose name
// arrived as a bare cliloc — otherwise every unresolved board would sort together
// under NULL.
async function listPointsBoards() {
return query(
`SELECT system, name, name_cliloc, max_points, players, show_on_gump, payload, t, updated_at
FROM shard_points_boards ORDER BY COALESCE(name, system), system`,
)
}
async function getPointsBoard(system) {
const rows = await query(
`SELECT system, name, name_cliloc, max_points, players, show_on_gump, payload, t, updated_at
FROM shard_points_boards WHERE system = ?`,
[system],
)
return rows[0] || null
}
module.exports = {
upsertOnline,
removeOnline,
clearOnline,
countOnline,
listOnline,
listOnlineLinked,
listOnlineByAccounts,
insertEconomy,
listEconomy,
latestEconomy,
upsertHouse,
listIdocHouses,
listHousesByAccounts,
removeHouse,
listRegistryHouses,
upsertGuild,
removeGuild,
clearGuilds,
listGuilds,
findGuildLedByActor,
listGuildsLedByAccounts,
upsertGovernor,
listGovernors,
listGovernorshipsByAccounts,
currentGovernorTerm,
closeGovernorTerm,
openGovernorTerm,
listGovernorTerms,
setPresence,
latestPresence,
setRuleset,
getRuleset,
upsertPointsBoard,
listPointsBoards,
getPointsBoard,
upsertChamp,
removeChamp,
clearChamps,
listChamps,
upsertPage,
removePage,
clearPages,
listPages,
}

638
server/model/shardState/shardState.model.js Normal file
View File

@@ -0,0 +1,638 @@
// Live shard state derived from the WS feed: who is online, the gold-supply
// series, and per-house decay stage. The ingest dispatcher calls the write
// methods; the public read endpoints call the list/count methods. Writes take
// camelCase semantic objects and map to the snake_case columns; only the keys
// present are written (so a char.vitals refresh doesn't clobber login fields).
const db = require('./shardState.db')
const MAX_ECONOMY = 1000
// Small coercion helpers, kept out of the upsert builders below so those stay
// flat (each inline `?? null` / ternary otherwise adds to cognitive complexity).
const orNull = (v) => v ?? null
const toDate = (v) => (v ? new Date(v) : null)
// Owner is an actor object (or null for an abandoned house); flatten to columns.
const ownerFields = (owner) => ({
owner_serial: orNull(owner?.serial),
owner_acct: orNull(owner?.acct),
owner_name: orNull(owner?.name),
})
// Map a camelCase online descriptor to DB columns, dropping undefined keys so a
// partial refresh only touches the fields it carries.
function onlineFields(data) {
const map = {
name: data.name,
acct: data.acct,
web_id: data.webId,
map: data.map,
x: data.x,
y: data.y,
z: data.z,
hits: data.hits,
hits_max: data.hitsMax,
mana: data.mana,
mana_max: data.manaMax,
stam: data.stam,
stam_max: data.stamMax,
str: data.str,
dex: data.dex,
int: data.int,
}
const fields = {}
for (const [k, v] of Object.entries(map)) if (v !== undefined) fields[k] = v
return fields
}
// Upsert an online player (mob.login) or refresh their vitals (char.vitals).
async function upsertOnline(data) {
if (!data || !data.serial) return
await db.upsertOnline(data.serial, onlineFields(data))
}
const setOffline = (serial) => db.removeOnline(serial)
const clearOnline = () => db.clearOnline()
const onlineCount = () => db.countOnline()
function shapeOnline(r) {
return {
serial: r.serial,
name: r.name,
acct: r.acct,
webId: r.web_id,
map: r.map,
x: r.x,
y: r.y,
z: r.z,
hits: r.hits,
hitsMax: r.hits_max,
mana: r.mana,
manaMax: r.mana_max,
stam: r.stam,
stamMax: r.stam_max,
str: r.str,
dex: r.dex,
int: r.int,
updatedAt: r.updated_at,
}
}
// Only players whose account is linked to a website user (opt-in visibility).
async function listOnlineLinked() {
const rows = await db.listOnlineLinked()
return rows.map(shapeOnline)
}
async function listOnline() {
const rows = await db.listOnline()
return rows.map(shapeOnline)
}
// Append a gold-supply sample (economy.supply).
async function addEconomySample({ accounts, gold, t }) {
await db.insertEconomy({ accounts, gold, t })
}
async function listEconomy(limit = 100) {
const n = Math.min(Math.max(Number(limit) || 100, 1), MAX_ECONOMY)
const rows = await db.listEconomy(n)
// Return oldest → newest for charting.
return rows
.map((r) => ({ accounts: r.accounts, gold: r.gold == null ? null : Number(r.gold), t: r.t }))
.reverse()
}
async function latestEconomy() {
const r = await db.latestEconomy()
return r ? { accounts: r.accounts, gold: r.gold == null ? null : Number(r.gold), t: r.t } : null
}
// Upsert a house's decay stage (house.decay). is_idoc is derived from the stage.
async function upsertHouse(data) {
if (!data || !data.serial) return
const fields = {
stage: data.stage ?? null,
map: data.map ?? null,
x: data.x ?? null,
y: data.y ?? null,
z: data.z ?? null,
region: data.region ?? null,
name: data.name ?? null,
owner_serial: data.ownerSerial ?? null,
owner_acct: data.ownerAcct ?? null,
built_on: data.builtOn ? new Date(data.builtOn) : null,
last_refreshed: data.lastRefreshed ? new Date(data.lastRefreshed) : null,
is_idoc: String(data.stage).toUpperCase() === 'IDOC' ? 1 : 0,
}
await db.upsertHouse(data.serial, fields)
}
function shapeHouse(r) {
return {
serial: r.serial,
stage: r.stage,
map: r.map,
x: r.x,
y: r.y,
z: r.z,
region: r.region,
name: r.name,
ownerSerial: r.owner_serial,
ownerAcct: r.owner_acct,
// Registry fields (Protocol 2.0 house.update); undefined on decay-only rows.
ownerName: r.owner_name,
coOwners: r.co_owners,
friends: r.friends,
price: r.price == null ? null : Number(r.price),
decay: r.decay,
inRegistry: r.in_registry == null ? undefined : Boolean(r.in_registry),
builtOn: r.built_on,
lastRefreshed: r.last_refreshed,
isIdoc: Boolean(r.is_idoc),
updatedAt: r.updated_at,
}
}
async function listIdoc() {
const rows = await db.listIdocHouses()
return rows.map(shapeHouse)
}
// Houses owned by the given game accounts (admin: a user's linked accounts).
async function listHousesForAccounts(accounts) {
const rows = await db.listHousesByAccounts(accounts)
return rows.map(shapeHouse)
}
// ── House registry (Protocol 2.0 house.update / house.remove) ──────────────
// Richer per-house snapshot than the decay-transition feed. Writes only the
// registry columns (+ shared location/owner fields); is_idoc/stage stay owned by
// the house.decay path, so the two feeds never clobber each other. owner is an
// actor object (or null for an abandoned house).
async function upsertHouseRegistry(data) {
if (!data || !data.serial) return
const fields = {
name: orNull(data.name),
...ownerFields(data.owner || null),
co_owners: orNull(data.coOwners),
friends: orNull(data.friends),
price: orNull(data.price),
decay: orNull(data.decay),
region: orNull(data.region),
map: orNull(data.map),
x: orNull(data.x),
y: orNull(data.y),
z: orNull(data.z),
built_on: toDate(data.builtOn),
last_refreshed: toDate(data.lastRefreshed),
in_registry: 1,
}
await db.upsertHouse(data.serial, fields)
}
const removeHouse = (serial) => (serial ? db.removeHouse(serial) : Promise.resolve())
async function listHouses() {
const rows = await db.listRegistryHouses()
return rows.map(shapeHouse)
}
// Online players on the given game accounts (admin: a user's linked accounts).
async function listOnlineForAccounts(accounts) {
const rows = await db.listOnlineByAccounts(accounts)
return rows.map(shapeOnline)
}
// ── Champion spawns ────────────────────────────────────────────────────────
// Upsert a champ spawn's state (champ.update). The full event is stored in
// `payload` for the category-specific fields; a few columns are hoisted out for
// querying/ordering. is-boss-up is derived from bossUp (sea bosses are always up).
async function upsertChamp(ev) {
if (!ev || !ev.serial) return
await db.upsertChamp(ev.serial, {
category: orNull(ev.category),
type: orNull(ev.type),
name: orNull(ev.name),
status: orNull(ev.status),
active: ev.active ? 1 : 0,
map: orNull(ev.map),
x: orNull(ev.x),
y: orNull(ev.y),
z: orNull(ev.z),
boss_up: ev.bossUp ? 1 : 0,
payload: JSON.stringify(ev),
t: Number.isFinite(ev.t) ? ev.t : null,
})
}
const removeChamp = (serial) => (serial ? db.removeChamp(serial) : Promise.resolve())
const clearChamps = () => db.clearChamps()
// Return the stored champ.update payload (the shape the sidecar/UI expect),
// falling back to the hoisted columns if an older row lacks a payload.
function shapeChamp(r) {
const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload
return payload || {
kind: 'champ.update',
serial: r.serial,
category: r.category,
type: r.type,
name: r.name,
status: r.status,
active: Boolean(r.active),
map: r.map,
x: r.x,
y: r.y,
z: r.z,
bossUp: Boolean(r.boss_up),
t: r.t,
}
}
async function listChamps() {
const rows = await db.listChamps()
return rows.map(shapeChamp)
}
// Replace the whole board with a fresh snapshot (sidecar GET /champs on connect).
async function replaceChamps(spawns) {
await db.clearChamps()
for (const ev of spawns || []) await upsertChamp(ev)
}
// ── Help-page (support) queue ──────────────────────────────────────────────
// Upsert a page (page.new / page.updated). The `sender` actor object carries the
// name/acct/webId; the rest are top-level fields.
async function upsertPage(ev) {
const pageId = ev && (ev.pageId || (ev.sender && ev.sender.serial))
if (!pageId) return
const sender = ev.sender || {}
await db.upsertPage(pageId, {
type: orNull(ev.type),
sender_name: orNull(sender.name),
sender_acct: orNull(sender.acct),
web_id: orNull(sender.webId),
message: orNull(ev.message),
map: orNull(ev.map),
x: orNull(ev.x),
y: orNull(ev.y),
z: orNull(ev.z),
sent_ms: Number.isFinite(ev.sentMs) ? ev.sentMs : null,
handled: ev.handled ? 1 : 0,
handler: orNull(ev.handler),
payload: JSON.stringify(ev),
})
}
const removePage = (pageId) => (pageId ? db.removePage(pageId) : Promise.resolve())
const clearPages = () => db.clearPages()
function shapePage(r) {
const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload
return {
pageId: r.page_id,
type: r.type,
sender: { serial: r.page_id, name: r.sender_name, acct: r.sender_acct, webId: r.web_id },
message: r.message,
map: r.map,
x: r.x,
y: r.y,
z: r.z,
sentMs: r.sent_ms == null ? null : Number(r.sent_ms),
handled: Boolean(r.handled),
handler: r.handler,
updatedAt: r.updated_at,
// Keep the raw payload available for any field not hoisted above.
payload: payload || undefined,
}
}
async function listPages() {
const rows = await db.listPages()
return rows.map(shapePage)
}
// Replace the whole queue with a fresh snapshot (sidecar GET /pages on connect).
async function replacePages(pages) {
await db.clearPages()
for (const ev of pages || []) await upsertPage(ev)
}
// ── Guild board (Protocol 2.0) ─────────────────────────────────────────────
// Upsert a guild's roster snapshot (guild.update). The leader is an actor object
// flattened into leader_* columns; the full event lives in `payload`.
async function upsertGuild(ev) {
if (!ev || ev.id == null) return
const leader = ev.leader || {}
await db.upsertGuild(ev.id, {
name: ev.name ?? null,
abbr: ev.abbr ?? null,
members: ev.members ?? null,
online: ev.online ?? null,
alliance: ev.alliance ?? null,
leader_serial: leader.serial ?? null,
leader_name: leader.name ?? null,
leader_acct: leader.acct ?? null,
leader_web_id: leader.webId ?? null,
payload: JSON.stringify(ev),
t: Number.isFinite(ev.t) ? ev.t : null,
})
}
const removeGuild = (id) => (id == null ? Promise.resolve() : db.removeGuild(id))
const clearGuilds = () => db.clearGuilds()
function shapeGuild(r) {
const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload
return payload || {
kind: 'guild.update',
id: r.id,
name: r.name,
abbr: r.abbr,
members: r.members,
online: r.online,
alliance: r.alliance,
leader: r.leader_serial
? { serial: r.leader_serial, name: r.leader_name, acct: r.leader_acct, webId: r.leader_web_id }
: null,
t: r.t,
}
}
async function listGuilds() {
const rows = await db.listGuilds()
return rows.map(shapeGuild)
}
// Replace the board with a fresh snapshot (sidecar GET /guilds on connect).
async function replaceGuilds(guilds) {
await db.clearGuilds()
for (const ev of guilds || []) await upsertGuild(ev)
}
// The guild an actor leads (cross-link on the character sheet). Leadership only —
// see the db note; membership for rank-and-file isn't in the feed, so we return
// null rather than show a possibly-stale guess.
async function findGuildForActor({ serial, acct }) {
const rows = await db.findGuildLedByActor(serial ?? null, acct ?? null)
const g = rows[0]
if (!g) return null
return { id: g.id, name: g.name, abbr: g.abbr, alliance: g.alliance, role: 'leader' }
}
// Guilds led by any of a user's linked accounts (admin user-detail cross-link).
async function listGuildsLedForAccounts(accounts) {
const rows = await db.listGuildsLedByAccounts(accounts)
return rows.map((g) => ({ id: g.id, name: g.name, abbr: g.abbr, alliance: g.alliance, leaderName: g.leader_name }))
}
// ── Town governors (Protocol 2.0) ──────────────────────────────────────────
// Upsert a city's governance snapshot (city.update) AND capture term history.
// Term capture runs first (it reads the CURRENT open term to decide whether the
// governor changed) and is idempotent: a repeat/backfill of the same governor is a
// no-op, so it's safe to call on the live feed and on reconnect snapshots alike.
async function upsertGovernor(ev) {
if (!ev || !ev.city) return
await recordGovernorTransition(ev)
const gov = ev.governor
const elect = ev.governorElect
await db.upsertGovernor(ev.city, {
governor_serial: orNull(gov?.serial),
governor_name: orNull(gov?.name),
governor_acct: orNull(gov?.acct),
governor_web_id: orNull(gov?.webId),
elect_serial: orNull(elect?.serial),
elect_name: orNull(elect?.name),
elect_acct: orNull(elect?.acct),
election_phase: orNull(ev.electionPhase),
candidates: orNull(ev.candidates),
auto_pick_at: toDate(ev.autoPickAt),
payload: JSON.stringify(ev),
t: Number.isFinite(ev.t) ? ev.t : null,
})
}
// Close the open term and open a new one when the governor CHANGES. Idempotent:
// same governor as the open term ⇒ nothing happens (so backfill/duplicate
// city.update events never spawn spurious terms).
async function recordGovernorTransition(ev) {
const gov = ev.governor || null
const newSerial = gov ? gov.serial ?? null : null
const t = Number.isFinite(ev.t) ? ev.t : Date.now()
const open = await db.currentGovernorTerm(ev.city)
const openSerial = open ? open.governor_serial : null
if (open && openSerial === newSerial) return // unchanged — nothing to record
if (open) await db.closeGovernorTerm(open.id, t) // governor changed or seat vacated
if (newSerial) {
await db.openGovernorTerm({
city: ev.city,
serial: newSerial,
name: gov.name ?? null,
acct: gov.acct ?? null,
webId: gov.webId ?? null,
startedAt: t,
})
}
}
function shapeGovernor(r) {
const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload
return payload || {
kind: 'city.update',
city: r.city,
governor: r.governor_serial
? { serial: r.governor_serial, name: r.governor_name, acct: r.governor_acct, webId: r.governor_web_id }
: null,
governorElect: r.elect_serial
? { serial: r.elect_serial, name: r.elect_name, acct: r.elect_acct }
: null,
electionPhase: r.election_phase,
candidates: r.candidates,
t: r.t,
}
}
async function listGovernors() {
const rows = await db.listGovernors()
return rows.map(shapeGovernor)
}
// Cities the given game accounts currently govern (cross-link badge).
async function listGovernorshipsForAccounts(accounts) {
const rows = await db.listGovernorshipsByAccounts(accounts)
return rows.map(shapeGovernor)
}
// Term history for a city (look-back), newest first.
async function listGovernorHistory(city, limit = 100) {
const n = Math.min(Math.max(Number(limit) || 100, 1), 500)
const rows = await db.listGovernorTerms(city, n)
return rows.map((r) => ({
city: r.city,
governor: r.governor_serial
? { serial: r.governor_serial, name: r.governor_name, acct: r.governor_acct, webId: r.governor_web_id }
: null,
startedAt: r.started_at == null ? null : Number(r.started_at),
endedAt: r.ended_at == null ? null : Number(r.ended_at),
votes: r.votes,
}))
}
// Upsert governors without clearing (cities are fixed, no remove event); term
// capture inside upsertGovernor stays idempotent across reconnect snapshots.
async function replaceGovernors(cities) {
for (const ev of cities || []) await upsertGovernor(ev)
}
// ── Online-population snapshot (Protocol 2.0 presence.online) ───────────────
async function setPresence(ev) {
if (!ev) return
await db.setPresence({
count: ev.count,
byFacet: ev.byFacet || null,
byRegion: ev.byRegion || null,
t: ev.t,
})
}
async function latestPresence() {
const r = await db.latestPresence()
if (!r) return { count: 0, byFacet: {}, byRegion: {}, t: null }
const parse = (v) => (typeof v === 'string' ? safeJson(v) || {} : v || {})
return {
count: Number(r.count) || 0,
byFacet: parse(r.by_facet),
byRegion: parse(r.by_region),
t: r.t == null ? null : Number(r.t),
}
}
// ── Shard ruleset (Protocol 3.0 world.ruleset) ─────────────────────────────
//
// The whole frame is stored in `payload` and served back whole. Nothing is
// normalized out of it: it is a flat description of config read as one page, and
// splitting it into columns would mean a schema change every time the shard grows
// a new block. `rev` and `expansion` are hoisted only because they are cheap to
// index/display, following shard_champs' payload-plus-hoisted-columns pattern.
async function setRuleset(ev) {
if (!ev) return
await db.setRuleset({
rev: ev.rev ?? null,
expansion: ev.expansion ?? null,
payload: JSON.stringify(ev),
t: ev.t,
})
}
// The stored ruleset, or null when the shard has never published one (an old
// plugin, or Bridge.RulesetEnabled=false). Null is a real answer here — the page
// says "not published yet" rather than rendering an empty ruleset as if the shard
// had no rules — so it is deliberately not smoothed into {}.
async function getRuleset() {
const r = await db.getRuleset()
if (!r) return null
const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload
if (!payload) return null
return { ...payload, updatedAt: r.updated_at }
}
// ── Points/loyalty boards (Protocol 3.0 points.board) ──────────────────────
//
// The whole frame is stored in `payload`; the columns beside it are hoisted for
// listing and ordering only. The top-N list deliberately stays inside the payload
// (see schema.sql) — it is a fixed-size list read whole, like the governor board's
// candidates.
async function upsertPointsBoard(ev) {
if (!ev || !ev.system) return
await db.upsertPointsBoard({
system: String(ev.system).slice(0, 48),
name: ev.nameString ?? null,
nameCliloc: ev.nameNumber,
maxPoints: ev.maxPoints,
players: ev.players,
showOnGump: ev.showOnGump !== false,
payload: JSON.stringify(ev),
t: ev.t,
})
}
// A stored frame plus the freshness stamp. `top` is normalized to an array so a
// caller never has to guard it — a board with nobody on it is a real state (a
// system nobody has scored in yet), distinct from a system that was never
// published at all, which is absent from the table entirely.
function shapePointsBoard(r) {
const payload = (typeof r.payload === 'string' ? safeJson(r.payload) : r.payload) || {}
return {
...payload,
system: r.system,
top: Array.isArray(payload.top) ? payload.top : [],
updatedAt: r.updated_at,
}
}
async function listPointsBoards() {
const rows = await db.listPointsBoards()
return rows.map(shapePointsBoard)
}
async function getPointsBoard(system) {
const r = await db.getPointsBoard(system)
return r ? shapePointsBoard(r) : null
}
function safeJson(s) {
try {
return JSON.parse(s)
} catch {
return null
}
}
module.exports = {
upsertOnline,
setOffline,
clearOnline,
onlineCount,
listOnline,
listOnlineLinked,
listOnlineForAccounts,
addEconomySample,
listEconomy,
latestEconomy,
upsertHouse,
listIdoc,
listHousesForAccounts,
upsertHouseRegistry,
removeHouse,
listHouses,
upsertChamp,
removeChamp,
clearChamps,
listChamps,
replaceChamps,
upsertPage,
removePage,
clearPages,
listPages,
replacePages,
upsertGuild,
removeGuild,
clearGuilds,
listGuilds,
replaceGuilds,
findGuildForActor,
listGuildsLedForAccounts,
upsertGovernor,
listGovernors,
listGovernorshipsForAccounts,
listGovernorHistory,
replaceGovernors,
setPresence,
latestPresence,
setRuleset,
getRuleset,
upsertPointsBoard,
listPointsBoards,
getPointsBoard,
}

37
server/model/shardVisibility/shardVisibility.db.js Normal file
View File

@@ -0,0 +1,37 @@
const { query } = require('../../core')
// One row per shard feature. Absent rows are fine — utils/shardVisibility.js
// compiles a default for every known feature and merges stored rows over it, so
// a fresh install with an empty table behaves exactly as the site did pre-v3.
const COLS = 'feature, enabled, audience, stream, field_rules, updated_by, updated_at'
const listAll = () => query(`SELECT ${COLS} FROM shard_feature_visibility`)
const getOne = (feature) =>
query(`SELECT ${COLS} FROM shard_feature_visibility WHERE feature = ?`, [feature])
// Upsert one feature's settings. `fieldRules` is stored as a JSON object of
// {field: rung}; the caller has already stripped locked fields and validated
// every rung against the ladder.
const upsert = ({ feature, enabled, audience, stream, fieldRules, updatedBy }) =>
query(
`INSERT INTO shard_feature_visibility (feature, enabled, audience, stream, field_rules, updated_by)
VALUES (?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
enabled = VALUES(enabled),
audience = VALUES(audience),
stream = VALUES(stream),
field_rules = VALUES(field_rules),
updated_by = VALUES(updated_by)`,
[
feature,
enabled ? 1 : 0,
audience,
stream ? 1 : 0,
fieldRules == null ? null : JSON.stringify(fieldRules),
updatedBy ?? null,
],
)
module.exports = { listAll, getOne, upsert }

44
server/model/shardVisibility/shardVisibility.model.js Normal file
View File

@@ -0,0 +1,44 @@
// ── Shard feature visibility (model) ───────────────────────────────────────
//
// Thin row-shaping layer over shardVisibility.db. The policy — the ladder, the
// feature catalog, the locked fields, the kind→feature map — lives in
// utils/shardVisibility.js; this file only reads and writes rows.
const db = require('./shardVisibility.db')
// The `field_rules` JSON column comes back as a string on the mariadb driver.
function parseRules(raw) {
if (raw == null) return {}
if (typeof raw === 'object') return raw
try {
const parsed = JSON.parse(raw)
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {}
} catch {
return {}
}
}
const toSafe = (row) =>
row && {
feature: row.feature,
enabled: !!row.enabled,
audience: row.audience,
stream: row.stream == null ? null : !!row.stream,
fieldRules: parseRules(row.field_rules),
updatedBy: row.updated_by,
updatedAt: row.updated_at,
}
async function listAll() {
const rows = await db.listAll()
return rows.map(toSafe)
}
async function getOne(feature) {
const rows = await db.getOne(feature)
return toSafe(rows[0])
}
const upsert = (entry) => db.upsert(entry)
module.exports = { listAll, getOne, upsert }

35
server/model/singletonConfigDb.js Normal file
View File

@@ -0,0 +1,35 @@
const { query } = require('../core')
// Factory for the singleton config tables (bot_config, email_config,
// uo_link_config). Each is a one-row table keyed on id = 1: `get()` returns the
// row (or null before the admin first saves it), and `upsert()` writes only the
// columns the model prepared, leaving the rest untouched. The three tables share
// this shape exactly, so the DB layer is generated rather than copy-pasted —
// only the table name and column list differ.
//
// `fields` are already prepared by the model (secrets pre-encrypted); ordering
// and encryption stay a model-layer concern.
function singletonConfigDb(table, cols) {
async function get() {
const rows = await query(`SELECT ${cols} FROM ${table} WHERE id = 1 LIMIT 1`)
return rows[0] || null
}
async function upsert(fields) {
const columns = Object.keys(fields)
const vals = columns.map((c) => fields[c])
const insertCols = ['id', ...columns].map((c) => `\`${c}\``).join(', ')
const placeholders = ['1', ...columns.map(() => '?')].join(', ')
const updates = columns.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(', ')
await query(
`INSERT INTO ${table} (${insertCols}) VALUES (${placeholders})
ON DUPLICATE KEY UPDATE ${updates}`,
vals,
)
return get()
}
return { get, upsert }
}
module.exports = singletonConfigDb

7
server/model/uoLinkConfig/uoLinkConfig.db.js Normal file
View File

@@ -0,0 +1,7 @@
const singletonConfigDb = require('../singletonConfigDb')
const COLS =
'id, base_url, ws_url, auth_token_enc, protocol, enabled, status, status_detail, plugin_connected, last_event_at, boot_id, updated_by, created_at, updated_at'
// Singleton row (id = 1). See ../singletonConfigDb for the get/upsert contract.
module.exports = singletonConfigDb('uo_link_config', COLS)

88
server/model/uoLinkConfig/uoLinkConfig.model.js Normal file
View File

@@ -0,0 +1,88 @@
// uo-link sidecar connection config store. Mirrors botConfig/emailConfig: the DB
// layer only ever sees ciphertext, and only getWithToken() (used server-side to
// call the sidecar over REST/WS) decrypts it. The admin-facing getSafe() never
// includes the token — it exposes only `hasToken`. A blank `token` on save means
// "leave the existing token unchanged" (same convention as the other configs).
const db = require('./uoLinkConfig.db')
const { secretBox } = require('../../core')
// The wire protocol this build speaks (link/sidecar/src/main.rs PROTOCOL_VERSION).
// Only used before an admin has saved anything — the stored row wins once it exists,
// and UOLINK_PROTOCOL still overrides for an operator running an older sidecar.
const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 3
function toSafe(row) {
if (!row) {
return {
baseUrl: process.env.UOLINK_BASE_URL || null,
wsUrl: process.env.UOLINK_WS_URL || null,
protocol: DEFAULT_PROTOCOL,
enabled: false,
hasToken: false,
status: 'disconnected',
statusDetail: null,
pluginConnected: false,
lastEventAt: null,
bootId: null,
}
}
return {
baseUrl: row.base_url || null,
wsUrl: row.ws_url || null,
protocol: row.protocol || DEFAULT_PROTOCOL,
enabled: Boolean(row.enabled),
hasToken: Boolean(row.auth_token_enc),
status: row.status || 'disconnected',
statusDetail: row.status_detail || null,
pluginConnected: Boolean(row.plugin_connected),
lastEventAt: row.last_event_at || null,
bootId: row.boot_id || null,
}
}
async function getSafe() {
return toSafe(await db.get())
}
// Decrypted token included — server-side only (calling the sidecar's REST/WS
// API). Returns null when nothing has been saved yet.
async function getWithToken() {
const row = await db.get()
if (!row) return null
return { ...toSafe(row), token: row.auth_token_enc ? secretBox.decrypt(row.auth_token_enc) : null }
}
// Save admin-supplied config. `token` undefined or '' means "leave the existing
// token unchanged" (same convention as botConfig.save).
async function save({ baseUrl, wsUrl, token, protocol, enabled, updatedBy }) {
const fields = {}
if (baseUrl !== undefined) fields.base_url = baseUrl
if (wsUrl !== undefined) fields.ws_url = wsUrl
if (token) fields.auth_token_enc = secretBox.encrypt(token)
if (protocol !== undefined) fields.protocol = protocol
if (enabled !== undefined) fields.enabled = enabled ? 1 : 0
if (updatedBy !== undefined) fields.updated_by = updatedBy
const row = await db.upsert(fields)
return toSafe(row)
}
// Mirror the sidecar's last-reported connection state into the DB so the admin
// panel has something to show between polls and the public status endpoint can
// read it without a live round-trip.
async function recordStatus({ status, statusDetail, pluginConnected, lastEventAt, bootId }) {
const fields = {}
if (status !== undefined) fields.status = status
if (statusDetail !== undefined) fields.status_detail = statusDetail
if (pluginConnected !== undefined) fields.plugin_connected = pluginConnected ? 1 : 0
// lastEventAt may arrive as an ISO string (e.g. "2026-07-10T22:08:27Z"); the
// mariadb DATETIME parser rejects the "T"/"Z", so hand it a real Date (same
// fix as botConfig.recordStatus's last_connected_at).
if (lastEventAt !== undefined) fields.last_event_at = lastEventAt ? new Date(lastEventAt) : null
if (bootId !== undefined) fields.boot_id = bootId
if (Object.keys(fields).length === 0) return getSafe()
const row = await db.upsert(fields)
return toSafe(row)
}
module.exports = { getSafe, getWithToken, save, recordStatus }

24
server/package-lock.json generated
View File

@@ -8,6 +8,9 @@
"name": "module-uo-server",
"version": "0.1.0",
"license": "GPL-3.0-or-later",
"dependencies": {
"ws": "^8.21.0"
},
"devDependencies": {
"express": "^4.19.2",
"express-validator": "^7.2.0"
@@ -927,6 +930,27 @@
"engines": {
"node": ">= 0.8"
}
},
"node_modules/ws": {
"version": "8.21.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
"integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
}
}
}

10
server/package.json
View File

@@ -6,15 +6,19 @@
"license": "GPL-3.0-or-later",
"main": "index.js",
"scripts": {
"test": "node --test",
"test": "node --test --require ./test/_setup.js",
"check:imports": "node scripts/checkImports.js"
},
"engines": {
"node": ">=20"
},
"//dependencies": "Deliberately none. Everything the shipped server half needs arrives on ctx (MODULE_API.md §2.3) — a module lives outside core's server/ and cannot resolve core's node_modules. The two below are devDependencies because test/_fakes.js builds a REAL express router: a fake Router would test the fake.",
"//dependencies": "The ONE runtime dependency, and it ships inside the release tarball: CI runs npm ci --omit=dev and packs server/node_modules, because an operator never builds (MODULE_SYSTEM.md 1.14). Node resolves it by walking up from modules/uo/server/. Everything else the shipped half needs arrives on ctx (MODULE_API.md 2.3) - express, express-validator, the database, the logger, the middleware and the rate-limit factory are all core-owned and handed over.",
"devDependencies": {
"express": "^4.19.2",
"express-validator": "^7.2.0"
}
},
"dependencies": {
"ws": "^8.21.0"
},
"//devDependencies": "Test-only. test/_fakes.js builds a REAL express router - a fake Router would test the fake."
}

127
server/scripts/importSpawnAtlas.js Normal file
View File

@@ -0,0 +1,127 @@
#!/usr/bin/env node
//
// Refresh the spawn atlas from a ServUO tree, from the command line.
//
// npm run atlas:import # use the configured path
// npm run atlas:import -- --servuo <path> # override it for this run
// npm run atlas:import -- --force # reimport even if unchanged
// npm run atlas:import -- --approve # apply a staged refresh
// npm run atlas:import -- --status # report without changing anything
//
// The server does this itself on every boot (see `shardAtlas.refreshOnBoot`), so
// this is for operators who want to apply a map change without a restart, and
// for approving a refresh that was staged because it would remove a facet.
//
// All the logic lives in `src/model/shardAtlas/shardAtlas.model.js`; this file
// is argument parsing and output formatting.
const db = () => require('../src/utils/db')
function parseArgs(argv) {
const args = {}
for (let i = 0; i < argv.length; i += 1) {
const flag = argv[i]
if (flag === '--servuo') args.servuo = argv[++i]
else if (flag === '--force') args.force = true
else if (flag === '--approve') args.approve = true
else if (flag === '--reject') args.reject = true
else if (flag === '--status') args.status = true
else if (flag === '--help' || flag === '-h') args.help = true
}
return args
}
const USAGE = `
Refresh the spawn atlas from a ServUO tree.
node scripts/importSpawnAtlas.js [options]
--servuo <path> Use this tree for this run instead of the configured path.
--force Reimport even when the source files are unchanged.
--approve Apply a refresh that was staged for removing a facet.
--reject Keep the current atlas and dismiss the staged refresh.
--status Report atlas and source state; change nothing.
With no options this imports only if the tree differs from what is loaded.
`
function describe(result) {
switch (result.status) {
case 'skipped':
return (
'No ServUO path configured — nothing to import.\n' +
'Set one with SERVUO_PATH, the admin panel, or --servuo <path>.\n'
)
case 'unavailable':
return `ServUO tree unavailable: ${result.reason}\n`
case 'unchanged':
return `Atlas is already up to date${result.reason ? ` (${result.reason})` : ''}.\n`
case 'needsReview': {
return (
'Refresh NOT applied — it would remove ' +
`${result.removedFacets.length} facet(s): ${result.removedFacets.join(', ')}.\n` +
'This is what a half-copied or mid-update tree looks like, so it has been\n' +
'staged for review. The current atlas is unchanged.\n' +
'Apply it with --approve, or dismiss it with --reject.\n'
)
}
case 'imported': {
const c = result.counts
const added = result.addedFacets?.length ? ` Added facets: ${result.addedFacets.join(', ')}.` : ''
const removed = result.removedFacets?.length
? ` Removed facets: ${result.removedFacets.join(', ')}.`
: ''
return (
`Atlas imported: ${c.points} points, ${c.creatures} creatures, ` +
`${c.pointTypes} point/type rows, ${c.regions} regions, ` +
`${c.landmarks} landmarks, ${c.champions} champion altars.${added}${removed}\n`
)
}
case 'failed':
return `Atlas refresh failed: ${result.reason}\n`
default:
return `${JSON.stringify(result, null, 2)}\n`
}
}
async function main() {
const args = parseArgs(process.argv.slice(2))
if (args.help) {
process.stdout.write(USAGE)
return
}
const shardAtlas = require('../src/model/shardAtlas/shardAtlas.model')
// `--servuo` is a per-run override and deliberately does NOT persist to the
// configured path; changing where the atlas permanently reads from is an
// admin action, not a side effect of a one-off import.
const override = { path: args.servuo ?? '' }
if (args.status) {
process.stdout.write(`${JSON.stringify(await shardAtlas.status(override), null, 2)}\n`)
return
}
if (args.reject) {
process.stdout.write(`${JSON.stringify(await shardAtlas.rejectPending(), null, 2)}\n`)
return
}
const result = args.approve
? await shardAtlas.approvePending(override)
: await shardAtlas.refresh({ ...override, force: Boolean(args.force) })
process.stdout.write(describe(result))
if (result.status === 'failed') process.exitCode = 1
}
if (require.main === module) {
main()
.catch((err) => {
process.stderr.write(`atlas:import failed: ${err.message}\n`)
process.exitCode = 1
})
.finally(() => db().close())
}
module.exports = { describe, parseArgs }

52
server/utils/announceLinks.js Normal file
View File

@@ -0,0 +1,52 @@
// The three helpers the town-crier leg needs from core's announce pipeline.
//
// Core owns `announce_jobs`, the worker that drains it and the retry policy;
// this module owns one leg of it (MODULE_API.md §2.4). These three lived in
// core's `announceJobs.logic` and are reproduced here rather than added to
// `ctx`, because each is a few lines of pure string handling with no state and
// no policy — the kind of thing a contract member would only make harder to
// change on both sides.
//
// The one that could NOT be vendored is `baseUrl`. Core's version reads
// `process.env.APP_BASE_URL`, and §2.7 forbids a module reading core's
// environment — it is core's deployment fact, not the module's. So it comes off
// `ctx.site.baseUrl` (API 1.1.0), read per call rather than captured, which also
// means a module built before an env change keeps agreeing with core after it.
// NOT destructured. `core.baseUrl` is a getter that resolves `ctx`, so pulling
// it out here would run at require time — before `register()` — and throw. Read
// it inside the function, where `ctx` exists.
const core = require('../core')
/** Where this deployment is reachable, without a trailing slash. */
function baseUrl() {
return core.baseUrl
}
/**
* The public link that goes in an announcement.
*
* News has no per-post route — core's SPA has only the list — so this links the
* list, matching what the pre-pipeline Discord announce did. It names a CORE
* route on purpose: the news list is core's page and stays core's through the
* whole extraction, so this is a module linking to its host, not a leftover.
*/
function articleUrl(base) {
return `${String(base || '').replace(/\/+$/, '')}/site/news`
}
/**
* Squeeze a leg client's `{ ok, status, data, error }` into the one line stored
* in `announce_job_legs.last_error` and shown in the admin panel.
*/
function legError(result) {
if (!result) return 'no response'
if (result.status) {
return result.data && result.data.message
? `${result.status}: ${result.data.message}`
: result.error || `status ${result.status}`
}
return result.error || 'request failed'
}
module.exports = { baseUrl, articleUrl, legError }

287
server/utils/clilocParse.js Normal file
View File

@@ -0,0 +1,287 @@
// Cliloc parsing — the pure half.
//
// A "cliloc" is UO's localization table: an integer id mapped to a display
// string. Items carry a `LabelNumber` rather than a name, so without this table
// the site can only render `id 1023721` where the game shows "quarter staff".
// The shard already sends the id on every equipment entry (`char.profile`'s
// `cliloc` field) and will send one per marketplace listing — the *number* was
// never the missing piece, the *table* was.
//
// This module is fs-free on purpose, exactly like `spawnAtlasParse.js`: the
// suite runs in CI where there is no UO client, so every parser here is driven
// from inline fixtures. `clilocSource.js` is the only thing that touches disk.
//
// ── Two input formats, and why ─────────────────────────────────────────────
//
// The client's own `Cliloc.enu` is COMPRESSED (Mythic format) on any modern
// client, and decompressing it is a bit-level port of an inverse-BWT coder that
// nothing in this stack needs at runtime. ServUO's own bundled `Ultima.StringList`
// cannot read it either — which is why `VendorSearch.GetItemName` is already inert
// on such a shard and the plugin could not supply names even if we asked it to.
//
// So the operator converts once, from their own client, and points the site at
// the result (see docs/website/CLILOCS.md). Two shapes are accepted because
// different tools produce different things:
//
// • PLAIN BINARY — the pre-compression cliloc layout: a 6-byte header, then
// records of {int32 number, byte flag, uint16 length, UTF-8 bytes}.
// • DELIMITED TEXT — `number<TAB|,|;>text` per line, which is what the common
// GUI exports emit. Quoted CSV fields and a header row are tolerated.
//
// Nothing derived from the client is ever committed: the converted file lives at
// an operator-supplied path and is gitignored, the same rule the spawn atlas art
// map already follows.
/** Raised for a file we can identify but deliberately refuse to guess at. */
class ClilocFormatError extends Error {
constructor(message, code) {
super(message)
this.name = 'ClilocFormatError'
this.code = code
}
}
/**
* Bumped when this parser produces DIFFERENT data from an IDENTICAL source file.
*
* Stored beside the source hash so the boot path can tell "same file, but the
* parser moved on" from "same file, nothing to do". Without it a corrected parse
* would ship and never reach an install whose cliloc file never changes — the
* trap `spawnAtlasSource.PARSER_VERSION` documents.
*/
const PARSER_VERSION = 1
// The plain layout's header is `02 00 00 00 01 00` — a 4-byte version and a
// 2-byte language marker. Only the size matters for parsing; the values are
// checked to sniff the format, not to validate it.
const HEADER_BYTES = 6
const RECORD_HEADER_BYTES = 7 // int32 number + byte flag + uint16 length
// Every compressed cliloc file the client ships begins with a DWORD whose high
// byte is 0x8E (the XOR key UOFiddler calls `HeaderXorKey`, 0x8E2C9A3D). That is
// the single cheapest way to tell an operator they exported the wrong file —
// without it, the plain parser happily reads compressed bytes as ~19k records of
// negative ids and 60 KB "strings" before dying somewhere in the middle, and the
// resulting error names the wrong problem.
const MYTHIC_HIGH_BYTE = 0x8e
/** True when `buffer` is a Mythic-compressed cliloc rather than the plain layout. */
function isCompressedCliloc(buffer) {
return buffer.length >= 4 && buffer[3] === MYTHIC_HIGH_BYTE
}
/**
* Parse the plain binary cliloc layout.
*
* Strict about truncation, and that strictness is load-bearing: a half-copied or
* partly-written file is the realistic failure here, and it must fail loudly
* rather than import a silently short table that then renders half the world as
* `id 1023721`. A record that runs past the end of the buffer throws.
*/
function parseClilocBinary(buffer) {
if (!Buffer.isBuffer(buffer)) throw new ClilocFormatError('Not a buffer', 'NOT_BUFFER')
if (isCompressedCliloc(buffer)) {
throw new ClilocFormatError(
'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' +
'Convert it to the plain format first — see docs/website/CLILOCS.md.',
'COMPRESSED',
)
}
if (buffer.length < HEADER_BYTES) {
throw new ClilocFormatError('File is shorter than a cliloc header', 'TRUNCATED')
}
const entries = []
let offset = HEADER_BYTES
while (offset < buffer.length) {
if (offset + RECORD_HEADER_BYTES > buffer.length) {
throw new ClilocFormatError(
`Truncated record header at byte ${offset} (${entries.length} entries read)`,
'TRUNCATED',
)
}
const number = buffer.readInt32LE(offset)
const flag = buffer.readUInt8(offset + 4)
// The length is written by the client as an unsigned 16-bit value. Reading it
// signed (as ServUO's own SDK does) turns any string over 32 KB into a
// negative length; real tables top out around 12 KB, so this has no effect on
// current data and costs nothing to get right.
const length = buffer.readUInt16LE(offset + 5)
offset += RECORD_HEADER_BYTES
if (offset + length > buffer.length) {
throw new ClilocFormatError(
`Truncated record body at byte ${offset} (${entries.length} entries read)`,
'TRUNCATED',
)
}
entries.push({ number, flag, text: buffer.toString('utf8', offset, offset + length) })
offset += length
}
return entries
}
// A delimited line splits on the FIRST separator only: cliloc text is full of
// commas ("a scroll of magery, unfinished") and splitting on all of them would
// truncate every such entry at its first comma.
const TEXT_SEPARATORS = ['\t', ',', ';']
/** Unwrap one CSV field: strip surrounding quotes and unescape doubled quotes. */
function unquote(value) {
const trimmed = value.trim()
if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
return trimmed.slice(1, -1).replace(/""/g, '"')
}
return trimmed
}
/**
* Parse a delimited text export: `number<sep>text` per line.
*
* Tolerant by design — this is whatever an operator's GUI tool produced, not a
* format we control. A header row, blank lines, `#` comments and a trailing
* flags column are all ignored. A line whose first field is not an integer is
* skipped rather than fatal, because that is exactly what a header row is.
*
* The one thing it will NOT do is return an empty table quietly: a file that
* yields no entries at all is a wrong file, not an empty one.
*/
function parseClilocText(text) {
const entries = []
for (const line of String(text).split(/\r?\n/)) {
// The line is deliberately NOT trimmed before the separator search. Roughly
// half of a real cliloc table is empty strings (unused ids), which export as
// `1005008<TAB>` — and trimming eats that trailing separator, leaving a bare
// number that then looks like a header row and is skipped. That silently
// dropped 55,994 of 123,490 entries. Individual FIELDS are trimmed instead,
// by `unquote`.
if (line.trim() === '' || line.trimStart().startsWith('#')) continue
// Pick the separator that actually appears first, so a tab-delimited line
// whose text contains a comma still splits on the tab.
let cut = -1
for (const sep of TEXT_SEPARATORS) {
const at = line.indexOf(sep)
if (at !== -1 && (cut === -1 || at < cut)) cut = at
}
if (cut === -1) continue
// An EMPTY first field must not become id 0: `Number('')` is 0, not NaN, so
// a line that merely starts with a separator would otherwise import as a
// bogus cliloc 0 instead of being skipped.
const head = unquote(line.slice(0, cut))
if (head === '') continue
const number = Number(head)
if (!Number.isInteger(number)) continue // header row, or a wrapped line
let rest = line.slice(cut + 1)
// Some exports carry `number,flag,text`. A bare integer in the second field
// is a flag; anything else is the text itself (and a text field that IS just
// a number is indistinguishable, so it stays as the text — the safer miss).
let flag = 0
for (const sep of TEXT_SEPARATORS) {
const at = rest.indexOf(sep)
if (at === -1) continue
const head = unquote(rest.slice(0, at))
if (/^\d{1,3}$/.test(head) && rest.slice(at + 1).trim() !== '') {
flag = Number(head)
rest = rest.slice(at + 1)
}
break
}
entries.push({ number, flag, text: unquote(rest) })
}
if (entries.length === 0) {
throw new ClilocFormatError('No cliloc entries found in the text export', 'EMPTY')
}
return entries
}
/**
* Parse either supported shape, sniffing which one this is.
*
* The sniff is on the binary header rather than the file extension: operators
* name these things whatever they like, and an `.enu` that is really a TSV (or a
* `.txt` that is really binary) should still import.
*/
function parseCliloc(buffer) {
const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer)
if (isCompressedCliloc(buf)) {
throw new ClilocFormatError(
'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' +
'Convert it to the plain format first — see docs/website/CLILOCS.md.',
'COMPRESSED',
)
}
// The plain layout always opens with version 2 / language 1. Anything else is
// treated as text, which is the recoverable guess: a mis-sniffed text file
// yields "no entries found", while a mis-sniffed binary yields nonsense.
if (buf.length >= HEADER_BYTES && buf.readInt32LE(0) === 2 && buf.readUInt16LE(4) === 1) {
return parseClilocBinary(buf)
}
return parseClilocText(buf.toString('utf8'))
}
// ── Display ────────────────────────────────────────────────────────────────
// Cliloc strings interpolate arguments the client supplies out of an item's
// property list: `~1_val~`, `~2_NAME~`, `~1_ITEM~`. We never have those — the
// bridge sends the id, not the packet — so a name carrying them must be reduced
// to what is actually knowable rather than shown with the raw tokens in it.
const PLACEHOLDER_RE = /~\d+_[^~]*~/g
/**
* Reduce a raw cliloc string to something displayable.
*
* Placeholders are dropped and the leftover punctuation tidied, so
* `"[~1_stuff~]"` becomes `""` (correctly nothing — the whole string was the
* argument) and `"cold damage ~1_val~%"` becomes `"cold damage"`.
*
* **Punctuation is only tidied when a placeholder was actually removed.** The
* trailing `%` above is the unit belonging to the number we never had, and the
* brackets in `[~1_stuff~]` only ever wrapped the argument — but a string with
* no placeholder has no such debris, and trimming it anyway corrupts real names.
* A shard's `"Runic Gateway Sigil (v2)"` came back as `"(v2"` while this was
* unconditional.
*
* Returns `''` when nothing survives, which callers treat as "no name" and fall
* back to the item id — better than showing a bracket.
*/
const DEBRIS = /^[\s\-–—,.;:%[\]()]+|[\s\-–—,.;:%[\]()]+$/g
function displayText(raw) {
if (raw == null) return ''
const source = String(raw)
const hadPlaceholder = PLACEHOLDER_RE.test(source)
PLACEHOLDER_RE.lastIndex = 0 // the regex is global; `test` advances it
if (!hadPlaceholder) return source.replace(/\s+/g, ' ').trim()
return source
.replace(PLACEHOLDER_RE, ' ')
.replace(/\s+/g, ' ')
.replace(/\s+([,.;:!?])/g, '$1')
.replace(DEBRIS, '')
.trim()
}
/** True when a raw cliloc string is nothing but interpolated arguments. */
const isPlaceholderOnly = (raw) => raw != null && String(raw).trim() !== '' && displayText(raw) === ''
module.exports = {
ClilocFormatError,
PARSER_VERSION,
HEADER_BYTES,
isCompressedCliloc,
parseCliloc,
parseClilocBinary,
parseClilocText,
displayText,
isPlaceholderOnly,
}

316
server/utils/clilocSource.js Normal file
View File

@@ -0,0 +1,316 @@
// Cliloc table — the filesystem layer.
//
// `clilocParse.js` holds the pure parsers; this module is the only thing that
// touches cliloc files on disk, and it is shared by both callers:
//
// - the server, which refreshes the table on boot (`shardClilocs.model.js`)
// - the admin panel, which can force a reimport without a restart
//
// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived
// from them is committed: the repo holds no string table, exactly as it holds no
// map snapshot and no artwork. That rule is why this module reads a configured
// path instead of a path inside the repo.
//
// ── Why this reads a SET of files, not one ────────────────────────────────
//
// Shards edit items and add new ones. Those carry cliloc ids that a stock client
// table does not have — and forcing a 5 MB client re-export every time an
// operator adds one item would be miserable enough that the table would simply
// go stale, which is the exact failure the spawn atlas was redesigned to avoid.
//
// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted
// client file) plus every operator-maintained OVERLAY beside it, all re-read on
// every boot and hash-gated as a SET. Adding, editing or removing any overlay
// counts as drift and re-imports. Later sources win, so an overlay both adds new
// ids and overrides stock ones.
//
// Measured on a real shard: the script tree references 16,434 cliloc ids and only
// 37 are absent from the stock client table. Tens of entries against a 67k base
// is what makes the overlay the right shape rather than a second full table.
//
// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so
// the boot path hashes first and only parses when something actually changed.
const crypto = require('crypto')
const fs = require('fs')
const path = require('path')
const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse')
/**
* Filenames looked for as the BASE table when the configured path is a directory.
*
* Ordered by how specific they are: an explicitly converted file wins over
* something that merely sits in a client folder, so an operator who dropped a
* `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one
* they made rather than the one that will be rejected.
*
* Matching is case-insensitive against the real directory listing, because the
* client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where
* a hardcoded lowercase open would simply miss.
*/
const CANDIDATE_NAMES = [
'clilocs.tsv',
'clilocs.csv',
'clilocs.plain',
'cliloc.plain',
'cliloc.plain.enu',
'cliloc.enu.plain',
'clilocs.txt',
'cliloc.enu',
]
/**
* Where shard-specific additions and overrides live: a `custom/` directory
* beside the base table.
*
* ServUO has **no server-side convention** for custom clilocs — they live in the
* patched client file a shard distributes to its players, and nothing in the
* tree declares them. There is therefore nothing to discover, and this is the
* one place in the cliloc pipeline that is a convention we chose rather than one
* the shard already has. It is a directory rather than a single file so an
* operator can keep additions grouped however they like (per system, per patch)
* without the site caring.
*/
const CUSTOM_DIR = 'custom'
const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain']
class ClilocSourceError extends Error {
constructor(message, code) {
super(message)
this.name = 'ClilocSourceError'
this.code = code
}
}
function sha256(buffer) {
return crypto.createHash('sha256').update(buffer).digest('hex')
}
/**
* Resolve the configured path to `{ root, base }`.
*
* Accepts either a direct file path or a directory to search, because operators
* reasonably supply both — "here is the file" and "here is the folder I put it
* in" are equally natural answers to the admin panel's prompt. When it is a
* file, `root` is the directory CONTAINING it, so overlays work either way: an
* operator who pointed at a file should not have to re-point at its folder just
* to add a `custom/` directory next to it.
*/
function resolveBase(configured) {
if (!configured || String(configured).trim() === '') {
throw new ClilocSourceError('No cliloc path configured', 'NO_PATH')
}
const target = String(configured).trim()
let stat
try {
stat = fs.statSync(target)
} catch {
throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND')
}
if (stat.isFile()) return { root: path.dirname(target), base: target }
if (!stat.isDirectory()) {
throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND')
}
let listing
try {
listing = fs.readdirSync(target)
} catch {
throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND')
}
const byLower = new Map(listing.map((name) => [name.toLowerCase(), name]))
for (const candidate of CANDIDATE_NAMES) {
const actual = byLower.get(candidate)
if (actual) return { root: target, base: path.join(target, actual) }
}
throw new ClilocSourceError(
`No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`,
'NO_FILE',
)
}
/** Overlay files under `<root>/custom/`, sorted so precedence is deterministic. */
function listCustom(root) {
const dir = path.join(root, CUSTOM_DIR)
let listing
try {
listing = fs.readdirSync(dir, { withFileTypes: true })
} catch (err) {
// No overlay directory is the normal case, not an error.
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE')
}
return listing
.filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase()))
.map((e) => e.name)
.sort()
.map((name) => path.join(dir, name))
}
function readFileOrThrow(file) {
try {
return fs.readFileSync(file)
} catch {
throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE')
}
}
/**
* Read every cliloc source under the configured path.
*
* Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }`
* with the base first and overlays after, in the order they must be merged.
*
* Labels are root-relative and forward-slashed so a hash map compares equal
* across platforms — the same directory read on Windows and Linux must produce
* the same fingerprint, or every boot would look like a change. (The same
* reasoning, and the same bug, as `spawnAtlasSource.readSources`.)
*/
function readSources(configured) {
const { root, base } = resolveBase(configured)
const describe = (file, kind) => {
const buffer = readFileOrThrow(file)
return {
label: path.relative(root, file).split(path.sep).join('/'),
kind,
file,
buffer,
sha256: sha256(buffer),
bytes: buffer.length,
compressed: isCompressedCliloc(buffer),
}
}
const files = [describe(base, 'base')]
for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom'))
return { root, files }
}
/**
* A fingerprint of every source: `{ "<label>": "<sha256>" }`, plus the base's
* details for the admin panel.
*
* `compressed` is reported here rather than left to the parse because the admin
* panel calls this and NOT `readCliloc` (parsing 5 MB on every status poll would
* be wasteful). Without it, pointing the setting at an unconverted client
* directory reports a perfectly readable file with pending drift — "ready to
* import" — and the operator only learns otherwise when the import fails. The
* check is four bytes of a buffer already in hand.
*/
function hashSources(configured) {
const { root, files } = readSources(configured)
const hashes = {}
for (const file of files) hashes[file.label] = file.sha256
const base = files[0]
return {
root,
hashes,
file: base.file,
bytes: base.bytes,
compressed: files.some((f) => f.compressed),
customCount: files.length - 1,
}
}
/** True when two source fingerprints describe the same set of files. */
function sameSources(a, b) {
if (!a || !b) return false
const aKeys = Object.keys(a).sort()
const bKeys = Object.keys(b).sort()
if (aKeys.length !== bKeys.length) return false
return aKeys.every((key, i) => key === bKeys[i] && a[key] === b[key])
}
/**
* Labels present in `loaded` that are absent from `current`.
*
* This is the multi-source hazard that a single file did not have. One corrupt
* file fails the parse loudly, but a source that has simply VANISHED — an
* unmounted volume, a half-copied deploy — parses perfectly and imports a table
* quietly missing everything that file contributed. That is the same ambiguity
* the spawn atlas escalates for a disappearing facet, so it is escalated here
* too rather than applied.
*/
function missingSources(current, loaded) {
if (!loaded) return []
return Object.keys(loaded).filter((label) => !Object.hasOwn(current, label))
}
/**
* Read and parse every source, merged into one entry list.
*
* Later sources win: the base client table first, then each overlay in sorted
* order, so an overlay both ADDS ids the client never had and OVERRIDES stock
* ones the shard has re-purposed.
*
* Returns `{ entries, source }`. Throws `ClilocSourceError` for anything about
* the paths and `ClilocFormatError` for anything about the contents — different
* problems for an operator (wrong place vs wrong file), and the admin panel says
* which. A format error names the file it came from, because "which of my six
* overlay files is malformed" is otherwise a guessing game.
*/
function readCliloc(configured) {
const { root, files } = readSources(configured)
const merged = new Map()
const perSource = []
for (const file of files) {
let entries
try {
entries = parseCliloc(file.buffer)
} catch (err) {
if (err instanceof ClilocFormatError) {
throw new ClilocFormatError(`${file.label}: ${err.message}`, err.code)
}
throw err
}
let added = 0
let overrode = 0
for (const entry of entries) {
if (!Number.isInteger(entry.number)) continue
if (merged.has(entry.number)) overrode++
else added++
merged.set(entry.number, entry)
}
perSource.push({ label: file.label, kind: file.kind, entries: entries.length, added, overrode })
}
return {
entries: [...merged.values()],
source: {
root,
file: files[0].file,
sha256: files[0].sha256,
bytes: files[0].bytes,
hashes: Object.fromEntries(files.map((f) => [f.label, f.sha256])),
parserVersion: PARSER_VERSION,
sources: perSource,
},
}
}
module.exports = {
ClilocFormatError,
ClilocSourceError,
PARSER_VERSION,
CANDIDATE_NAMES,
CUSTOM_DIR,
CUSTOM_EXTENSIONS,
resolveBase,
listCustom,
readSources,
hashSources,
sameSources,
missingSources,
readCliloc,
}

33
server/utils/excerpt.js Normal file
View File

@@ -0,0 +1,33 @@
// A plain-text excerpt of a post body, for the town crier and the news gump.
//
// **Vendored from core's `utils/sanitizeHtml.js`, deliberately, and it is worth
// being precise about what was and was not copied.** Core's file exports three
// things: `cleanBody` (the actual HTML sanitiser, backed by a dependency and a
// tag allowlist), `OPTIONS`, and this. Only this one came, because only this one
// is a pure function over a string with no security surface — it strips tags to
// get at the text, it does not decide what tags are safe to render.
//
// Copying the sanitiser would have been the wrong call for exactly the reason
// this comment exists: a second copy of a security control diverges from the
// first the moment either is fixed, and the divergence is silent. A module that
// needs to sanitise HTML for rendering should ask core for it. This one does
// not — its output goes into a game window and a chat message as text.
/**
* Flatten HTML to a single line of text, truncated with an ellipsis.
*
* @param {string|null} html
* @param {number} max characters, including the ellipsis
* @returns {string|null} null when there is nothing left after stripping
*/
function deriveExcerpt(html, max = 280) {
if (html == null) return null
const text = String(html)
.replace(/<[^>]+>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
if (!text) return null
return text.length > max ? `${text.slice(0, max - 3)}...` : text
}
module.exports = { deriveExcerpt }

123
server/utils/newsGump.js Normal file
View File

@@ -0,0 +1,123 @@
// ── Town Cryer News gump sync (Protocol 2.1) ───────────────────────────────
//
// Keeps the in-game Town Cryer *News* gump in sync with the site's published
// news posts. Distinct from the scrolling town-crier lines (that's a one-shot
// announce leg in announceWorker); this is a STATE SYNC — an article stays in the
// gump while its post is published news, and is pulled when the post is
// unpublished/deleted/re-categorised.
//
// The website is the source of truth. POST /news is idempotent (re-post replaces),
// so a refresh or a reconnect re-assert is safe. Every call is best-effort and
// never throws — a sidecar/shard hiccup must never break saving or deleting a
// post. Reliability comes from reassertAll() on every WS (re)connect
// (uoLinkSocket.backfill), which re-pushes the current published set silently and
// closes the gap if an earlier live push failed.
const { posts, settings } = require('../core')
const uoLinkClient = require('./uoLinkClient')
const { deriveExcerpt } = require('./excerpt')
const log = require('../core').logger('news-gump')
const MAX_TITLE = 120
const MAX_BODY = 900
function baseUrl() {
return (process.env.APP_BASE_URL || 'http://localhost:5173').replace(/\/+$/, '')
}
function clamp(value, max) {
const s = String(value == null ? '' : value).replace(/\s+/g, ' ').trim()
return s.length <= max ? s : `${s.slice(0, max - 1).trimEnd()}…`
}
// A post belongs in the gump exactly when it is published AND in the news category.
function inGump(post) {
return Boolean(post && post.published && post.category === 'news')
}
// Optional UO gump image id for news articles (a shard art id), from the
// `news_gump_image` setting. Omitted → the sidecar uses a neutral scroll.
async function gumpImage() {
try {
const raw = await settings.get('news_gump_image')
const n = Number(raw)
return Number.isInteger(n) && n > 0 ? n : undefined
} catch {
return undefined
}
}
// Build the in-game News article from a post. Body is a compact gump-HTML block
// (title centred + a plain-text excerpt) rather than the post's full rich HTML —
// the UO gump only supports a small HTML subset, so we keep it predictable. The
// "more info" URL is the public news list (news posts have no per-post route).
async function buildArticle(post, { announce = true } = {}) {
const title = clamp(post.title, MAX_TITLE)
const excerpt = clamp(post.excerpt || deriveExcerpt(post.body, MAX_BODY) || '', MAX_BODY)
const body = excerpt ? `<CENTER>${title}</CENTER><BR><BR>${excerpt}` : `<CENTER>${title}</CENTER>`
return {
id: String(post.id),
title,
body,
image: await gumpImage(),
url: `${baseUrl()}/site/news`,
announce,
}
}
// Push a post to the gump (only if it belongs there). announce=true has the criers
// proclaim the title; false is a silent refresh/re-assert.
async function pushPost(post, { announce = true } = {}) {
if (!inGump(post)) return { ok: false, skipped: true }
const res = await uoLinkClient.postNews(await buildArticle(post, { announce }))
if (!res.ok) log.warn('news gump push failed', { id: post.id, status: res.status, error: res.error })
return res
}
// Remove a post from the gump. A 404 (not present) is not an error worth noting.
async function removePost(id) {
const res = await uoLinkClient.deleteNews(String(id))
if (!res.ok && res.status !== 404) {
log.warn('news gump remove failed', { id, status: res.status, error: res.error })
}
return res
}
// Reconcile the gump after a post create/update/publish. `transition`
// ({ wasPublished, wasNews }) tells a fresh publish (announce) from an in-place
// edit (silent refresh) and catches a post leaving published-news (pull it).
async function syncPost(post, transition = {}) {
try {
if (inGump(post)) {
const wasInGump = Boolean(transition.wasPublished && transition.wasNews)
await pushPost(post, { announce: !wasInGump })
} else if (transition.wasPublished && transition.wasNews) {
await removePost(post.id)
}
} catch (err) {
log.warn('news gump sync failed', { id: post && post.id, message: err.message })
}
}
// Re-push every currently-published news post, silently — run on each WS
// (re)connect to reconcile the gump to our source of truth (also recovers any
// article whose original live push failed). Best-effort; never throws.
async function reassertAll() {
try {
const list = await posts.listAll('news')
const published = (list || []).filter((p) => p.published)
let pushed = 0
for (const p of published) {
const full = await posts.getById(p.id) // list projection may omit the body
if (full) {
await pushPost(full, { announce: false })
pushed += 1
}
}
if (pushed) log.info('re-asserted news gump articles', { count: pushed })
} catch (err) {
log.warn('news gump reassert failed', { message: err.message })
}
}
module.exports = { inGump, buildArticle, pushPost, removePost, syncPost, reassertAll }

77
server/utils/shardAnnounce.js Normal file
View File

@@ -0,0 +1,77 @@
// ── The in-game town-crier announce leg ────────────────────────────────────
//
// MODULE-UO CONTENT, still living in core. MODULE_SYSTEM.md §1.8's third
// entangled file: utils/announceWorker.js is core's news dispatcher, but one of
// its two delivery legs goes to the shard through uoLinkClient.postTownCrier.
// PR 4 turned the legs into registrations, and this file is what module-uo will
// register in Phase 3 — it moves whole, with `'core'` becoming `'uo'` and the
// leg id staying `towncrier` (grandfathered in registries.js: the id is a stored
// value in announce_job_legs.leg).
const uoLinkClient = require('./uoLinkClient')
const { deriveExcerpt } = require('./excerpt')
const { articleUrl, baseUrl, legError } = require('./announceLinks')
const TOWNCRIER_DURATION_SEC = Number(process.env.TOWNCRIER_DURATION_SEC) || 3600
// Sidecar town-crier caps, mirrored from the admin route validation
// (admin/uoLink.router.js: lines isArray({ max: 8 }), lines.* isLength({ max: 200 })).
// We pre-truncate to these so a published post never bounces with an error.
const MAX_LINES = 8
const MAX_LINE_LEN = 200
// Trim to a hard length, appending an ellipsis only when something was cut.
function clamp(value, max) {
const s = String(value == null ? '' : value)
.replace(/\s+/g, ' ')
.trim()
if (s.length <= max) return s
return `${s.slice(0, max - 1).trimEnd()}…`
}
// Build the town-crier lines: title, a one-line excerpt, then the URL. Each line
// is clamped to the sidecar's per-line cap and the whole thing to the line-count
// cap. Falls back to a stripped body excerpt when the post has no excerpt.
function buildTownCrierText(post, { baseUrl: base } = {}) {
const title = clamp(post.title, MAX_LINE_LEN)
const excerptSource = post.excerpt || deriveExcerpt(post.body, MAX_LINE_LEN) || ''
const lines = [title]
const excerpt = clamp(excerptSource, MAX_LINE_LEN)
if (excerpt) lines.push(excerpt)
const url = clamp(articleUrl(base), MAX_LINE_LEN)
if (url) lines.push(url)
return lines.filter(Boolean).slice(0, MAX_LINES)
}
async function dispatch(post) {
const lines = buildTownCrierText(post, { baseUrl: baseUrl() })
// Stable id: re-posting `post-<id>` REPLACES the prior town-crier entry rather
// than stacking a duplicate, so a retry after a partial failure is safe.
return uoLinkClient.postTownCrier({
id: `post-${post.id}`,
lines,
durationSec: TOWNCRIER_DURATION_SEC,
})
}
function classify(result) {
if (result && result.ok) return { outcome: 'done' }
const status = result ? result.status : 0
// 400 = over the line/duration caps (a data problem — do NOT retry).
// 401 = token mismatch, 409 = protocol mismatch (both config problems).
if (status === 400 || status === 401 || status === 409) {
return { outcome: 'terminal', error: legError(result) }
}
// 503 (shard not connected), 504 (shard timeout), 0 (network/timeout / not
// configured yet), and any other 5xx are transient — retry.
return { outcome: 'retry', error: legError(result) }
}
const leg = {
leg: 'towncrier',
label: 'In-game town crier',
dispatch,
classify,
}
module.exports = { leg, dispatch, classify, buildTownCrierText, MAX_LINES, MAX_LINE_LEN }

166
server/utils/shardBroadcast.js Normal file
View File

@@ -0,0 +1,166 @@
// ── Shard live-feed SSE broadcaster ────────────────────────────────────────
//
// The browser can't talk to the sidecar's WebSocket directly (the token must
// never reach it, and the WS may be on another host). Instead the server ingests
// the WS feed and re-broadcasts events to browsers over Server-Sent Events
// (plain HTTP — works through any reverse proxy).
//
// Since Protocol 3.0 the split is no longer "one public channel with a static
// allowlist plus one admin channel". Each subscriber carries the audience rung
// it resolved to at subscribe time, and every frame is
//
// 1. mapped kind → feature (an UNMAPPED kind reaches nobody below admin —
// fail closed; see utils/shardVisibility.js rule 2),
// 2. gated on that feature being enabled, streamed, and within the viewer's
// rung, and
// 3. passed through field projection, so `acct` / `webId` and any field an
// admin has re-gated are stripped per viewer.
//
// **This is the security boundary.** It used to be the PUBLIC_KINDS set in this
// file; it is now the kind map plus the visibility config. PUBLIC_KINDS still
// exists and is still exported, but it is now DERIVED from the kind map (see
// shardVisibility.js) so the two can no longer drift.
//
// shardIngest calls broadcast(event) for each ingested event; the public/admin
// SSE route handlers call subscribe(req, res, channel).
const visibility = require('./shardVisibility')
const log = require('../core').logger('shard-broadcast')
// Re-exported for back-compat: shardEvents `/feed` filtering and
// config/notificationStreams.js both ask "is this kind public-safe?".
const { PUBLIC_KINDS } = visibility
// Open streams. Each entry is { res, level }. The admin bucket is kept separate
// because it is unconditional and must not depend on a config read.
const clients = { public: new Set(), admin: new Set() }
const KEEPALIVE_MS = 25000
// Register an SSE stream on a channel. Sets the SSE headers, sends an initial
// comment, keeps the connection warm with periodic pings, and cleans up on close.
//
// The viewer's rung is resolved ONCE, here, and frozen for the life of the
// connection — a long-lived stream must not silently gain privilege because the
// caller's session changed underneath it. (Config changes, by contrast, DO take
// effect live: the config is read per broadcast, cached ~5s.)
async function subscribe(req, res, channel) {
const bucket = clients[channel]
if (!bucket) {
res.status(400).end()
return
}
let level = 'admin'
if (channel === 'public') {
try {
level = await visibility.viewerLevel(req)
} catch (err) {
// Fail closed: an unresolvable viewer is anonymous, not privileged.
log.warn('viewerLevel failed on subscribe; treating as anonymous', { message: err.message })
level = 'anonymous'
}
}
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache, no-transform',
Connection: 'keep-alive',
'X-Accel-Buffering': 'no', // disable proxy buffering so events flush immediately
})
res.write('retry: 5000\n\n') // tell EventSource to reconnect after 5s if dropped
res.write(': connected\n\n')
const client = { res, level, ping: null }
bucket.add(client)
client.ping = setInterval(() => {
try {
res.write(': ping\n\n')
} catch {
/* write after close — cleanup below handles it */
}
}, KEEPALIVE_MS)
const cleanup = () => drop(bucket, client)
req.on('close', cleanup)
res.on('error', cleanup)
}
// The ONLY way a client leaves a bucket. Clearing the keepalive here (rather
// than only in the close handler) matters: a client dropped because its write
// threw never fires `req.close`, so its interval would otherwise keep firing on
// a dead socket for the life of the process.
function drop(bucket, client) {
clearInterval(client.ping)
bucket.delete(client)
}
function writeTo(bucket, client, payload) {
try {
client.res.write(payload)
} catch (err) {
log.warn('sse write failed; dropping client', { message: err.message })
drop(bucket, client)
}
}
// Fan an ingested event out. The admin channel gets it verbatim, always. Public
// subscribers are filtered and projected per their own rung — so two viewers on
// the same channel can legitimately receive different versions of one frame, or
// one of them nothing at all.
async function broadcast(event) {
if (!event || !event.kind) return
if (clients.admin.size) {
const frame = `data: ${JSON.stringify(event)}\n\n`
for (const client of [...clients.admin]) writeTo(clients.admin, client, frame)
}
if (!clients.public.size) return
let config
try {
config = await visibility.getConfig()
} catch (err) {
// Fail closed: without a config we cannot prove a frame is safe to send.
log.error('visibility config unavailable; withholding public frame', err)
return
}
// Most frames land on one rung set, so cache the serialised payload per level
// instead of re-projecting and re-stringifying for every subscriber.
const byLevel = new Map()
for (const client of [...clients.public]) {
let frame = byLevel.get(client.level)
if (frame === undefined) {
frame = visibility.kindVisibleTo(event.kind, client.level, config)
? `data: ${JSON.stringify(visibility.projectFeature(visibility.KIND_FEATURE.get(event.kind), event, client.level, config))}\n\n`
: null
byLevel.set(client.level, frame)
}
if (frame) writeTo(clients.public, client, frame)
}
}
// Close every open stream (graceful shutdown). Clears each keepalive timer too —
// without that the intervals keep the event loop alive after the streams are
// gone, and the process won't exit.
function closeAll() {
for (const bucket of Object.values(clients)) {
for (const client of [...bucket]) {
drop(bucket, client)
try {
client.res.end()
} catch {
/* ignore */
}
}
}
}
function stats() {
return { publicClients: clients.public.size, adminClients: clients.admin.size }
}
module.exports = { subscribe, broadcast, closeAll, stats, PUBLIC_KINDS }

314
server/utils/shardIngest.js Normal file
View File

@@ -0,0 +1,314 @@
// ── Shard event ingest dispatcher ──────────────────────────────────────────
//
// The single entry point for every event that arrives on the uo-link WebSocket
// feed (and for backfilled /history events on reconnect). It routes by kind:
// • state-changing kinds update shard_online / shard_economy / shard_houses,
// • notable kinds are appended to the append-only shard_events log,
// • every kind is fanned out to the SSE broadcaster (which decides public vs
// admin visibility).
// High-frequency kinds (char.vitals, economy.supply) are deliberately NOT logged
// to shard_events — they only update state — keeping the event log lean.
//
// Dependencies are injected (defaulting to the real models) so the routing can
// be unit-tested with mocked writes.
const shardEventsModel = require('../model/shardEvents/shardEvents.model')
const shardStateModel = require('../model/shardState/shardState.model')
const shardLinksModel = require('../model/shardLinks/shardLinks.model')
const shardMarketModel = require('../model/shardMarket/shardMarket.model')
const uoLinkConfigModel = require('../model/uoLinkConfig/uoLinkConfig.model')
const { settings: settingsModel } = require('../core')
const broadcaster = require('./shardBroadcast')
const shardPush = require('./shardPush')
const defaultLog = require('../core').logger('shard-ingest')
// Notable kinds appended to the shard_events log. High-frequency/session kinds
// (char.vitals, economy.supply, mob.login/logout, account.login.attempt,
// gold.change, vendor.buy/sell) are excluded on purpose. house.decay is handled
// specially — logged only on the transition INTO IDOC.
const LOGGED_KINDS = new Set([
'vendor.sale',
'player.death',
'player.murdered',
'mob.killed',
'quest.complete',
'skill.gain',
'fame.change',
'karma.change',
'audit.set',
'audit.command',
'admin.audit',
'cheat.fastwalk',
'link.request',
'server.hello',
'server.shutdown',
'server.crashed',
// Protocol 2.0: a real-time guild join (the board itself is state, not logged).
'guild.join',
// Protocol 2.0 provisioning audit (admin channel only — not in PUBLIC_KINDS).
'account.audit',
'account.unlinked',
])
// Tracks the current shard boot id so a restart (changed bootId on server.hello)
// can be detected and stale online state dropped. Module-level so it survives
// across events within a process; reset() is exposed for tests.
const state = { bootId: null }
function reset() {
state.bootId = null
}
// Should this event be written to the append-only log?
function shouldLog(event) {
if (event.kind === 'house.decay') return String(event.to).toUpperCase() === 'IDOC'
return LOGGED_KINDS.has(event.kind)
}
// ServUO's stock Server.cfg name. An operator who never set one publishes this
// verbatim, so it carries no more information than a blank — matched
// case-insensitively and trim-tolerantly, but ONLY as an exact whole value: a
// shard genuinely called "My Shard Reborn" keeps its name.
const STOCK_SHARD_NAME = 'my shard'
/**
* The name to publish for the shard: its own, or this instance's when it has
* effectively not given one.
*
* Deliberately not a general "blank means brand" rule applied across the wire —
* it is scoped to this one field, where the two names denote the same thing.
*/
async function resolveShardName(shard, deps) {
const given = String(shard ?? '').trim()
if (given !== '' && given.toLowerCase() !== STOCK_SHARD_NAME) return given
try {
return (await deps.settings.getInstanceName()) || given
} catch {
// A ruleset that publishes the stock name is still better than one that
// fails to store because the settings read hiccuped.
return given
}
}
// Apply the state-change side effect for a kind (if any). Returns a promise.
async function applyStateChange(event, deps) {
const { shardState, uoLinkConfig, log } = deps
switch (event.kind) {
case 'server.hello': {
const incoming = event.bootId || null
if (incoming && state.bootId && incoming !== state.bootId) {
log.warn('shard restarted (bootId changed) — clearing online roster', {
from: state.bootId,
to: incoming,
})
await shardState.clearOnline()
}
if (incoming) state.bootId = incoming
await uoLinkConfig.recordStatus({ pluginConnected: true, bootId: incoming, lastEventAt: event.t })
return
}
case 'server.shutdown':
case 'server.crashed':
// Shard is going away — nobody is online anymore.
await shardState.clearOnline()
await uoLinkConfig.recordStatus({ pluginConnected: false })
return
case 'mob.login': {
const who = event.who || {}
await shardState.upsertOnline({
serial: who.serial,
name: who.name,
acct: who.acct,
webId: event.webId,
map: event.map,
x: event.x,
y: event.y,
z: event.z,
})
return
}
case 'mob.logout': {
const who = event.who || {}
if (who.serial) await shardState.setOffline(who.serial)
return
}
case 'char.vitals':
await shardState.upsertOnline({
serial: event.serial,
hits: event.hits,
hitsMax: event.hitsMax,
mana: event.mana,
manaMax: event.manaMax,
stam: event.stam,
stamMax: event.stamMax,
str: event.str,
dex: event.dex,
int: event.int,
map: event.map,
x: event.x,
y: event.y,
})
return
case 'economy.supply':
await shardState.addEconomySample({ accounts: event.accounts, gold: event.gold, t: event.t })
return
case 'house.decay':
await shardState.upsertHouse({
serial: event.serial,
stage: event.to,
map: event.map,
x: event.x,
y: event.y,
z: event.z,
region: event.region,
name: event.name,
ownerSerial: event.ownerSerial,
ownerAcct: event.ownerAcct,
builtOn: event.builtOn,
lastRefreshed: event.lastRefreshed,
})
return
case 'champ.update':
await shardState.upsertChamp(event)
return
case 'champ.remove':
await shardState.removeChamp(event.serial)
return
case 'page.new':
case 'page.updated':
await shardState.upsertPage(event)
return
case 'page.closed':
await shardState.removePage(event.pageId)
return
// ── Protocol 2.0 boards ──────────────────────────────────────────────
case 'guild.update':
await shardState.upsertGuild(event)
return
case 'guild.remove':
await shardState.removeGuild(event.id)
return
case 'city.update':
// Upserts the board AND captures term history (idempotent).
await shardState.upsertGovernor(event)
return
case 'presence.online':
await shardState.setPresence(event)
return
case 'house.update':
await shardState.upsertHouseRegistry(event)
return
case 'house.remove':
await shardState.removeHouse(event.serial)
return
// ── Protocol 3.0 ─────────────────────────────────────────────────────
// The shard re-emits its whole ruleset on every sidecar connect, so this is
// an overwrite, not an append — and deliberately NOT in LOGGED_KINDS: it
// would put a duplicate row in the event log on every reconnect, and
// server.hello already marks each of those.
case 'world.ruleset':
// A shard whose operator never edited Server.cfg publishes ServUO's stock
// "My Shard". That is the shard saying *unnamed*, not a name, so the site
// answers with its own — the rules page reading "My Shard" under a header
// reading UOMysticmoon is the shard failing to introduce itself.
//
// Normalized HERE rather than on read because the ruleset is also live: the
// same `event` object is handed to the SSE broadcast a few lines below, and
// a read-time fix would be undone by the next reconnect's frame.
event.shard = await resolveShardName(event.shard, deps)
await shardState.setRuleset(event)
return
// Board state, like guild.update — the newest frame for a system replaces the
// previous one, so it is NOT in LOGGED_KINDS. Logging would append a row every
// time anyone's score moved the top ten, which is a board, not an event.
case 'points.board':
await shardState.upsertPointsBoard(event)
return
// Player-vendor market index. Each frame is authoritative for one shop, so
// the model replaces that vendor's whole listing set rather than merging.
//
// NOT in LOGGED_KINDS, and this is the strongest case of the three v3 kinds:
// one frame carries up to 250 listings, the sweep re-emits a shop on any
// price change, and appending each of those to the event log would make
// shard_events mostly a price history nobody reads. The market IS the state.
case 'vendor.listing':
await deps.shardMarket.upsertVendor(event)
return
case 'vendor.listing.remove':
await deps.shardMarket.removeVendor(event.serial)
return
case 'account.unlinked':
// A player ran [unlink in game (or a site-side unlink echoed back) — drop
// our local link mirror so attribution stops immediately.
if (event.account) await deps.shardLinks.removeByAccount(event.account)
return
// guild.join / account.audit → logged; region.enter → broadcast-only.
default:
// No state side effect (e.g. vendor.sale, audit.*, cheat.*) — logging and
// broadcasting still happen in ingest().
}
}
// Ingest one event. Returns { logged, stored } for tests/stats. `fromBackfill`
// suppresses the SSE broadcast (a reconnect replay shouldn't re-animate the
// live ticker). Never throws — a bad single event must not kill the feed.
// Resolve the injectable dependencies to their live defaults (tests override a
// subset). Split out so ingest() isn't penalised for the fan of `|| default`s.
function resolveDeps(deps) {
return {
shardEvents: deps.shardEvents || shardEventsModel,
shardState: deps.shardState || shardStateModel,
shardLinks: deps.shardLinks || shardLinksModel,
shardMarket: deps.shardMarket || shardMarketModel,
uoLinkConfig: deps.uoLinkConfig || uoLinkConfigModel,
settings: deps.settings || settingsModel,
broadcast: deps.broadcast || broadcaster.broadcast,
pushDispatch: deps.pushDispatch || shardPush.fromShardEvent,
log: deps.log || defaultLog,
}
}
async function ingest(event, deps = {}) {
const d = resolveDeps(deps)
if (!event || typeof event.kind !== 'string') return { logged: false, stored: false }
// ws.hello / pong are transport frames, not game events.
if (event.kind === 'ws.hello' || event.kind === 'pong') return { logged: false, stored: false }
const t = Number.isFinite(event.t) ? event.t : Date.now()
let stored = false
let logged = false
try {
await applyStateChange(event, d)
} catch (err) {
d.log.warn('state-change write failed', { kind: event.kind, message: err.message })
}
if (shouldLog(event)) {
logged = true
try {
stored = await d.shardEvents.append({ kind: event.kind, t, bootId: state.bootId, payload: event })
} catch (err) {
d.log.warn('event log write failed', { kind: event.kind, message: err.message })
}
}
if (!deps.fromBackfill) {
// Broadcast is async since v3 (it reads the visibility config to decide what
// each subscriber may see). Fire-and-forget, like the push fan-out below: a
// slow config read must never delay or fail ingest.
Promise.resolve(d.broadcast(event)).catch((err) =>
d.log.warn('broadcast failed', { kind: event.kind, message: err.message }),
)
// Opt-in push fan-out, off the same event source as the SSE broadcast.
// Fire-and-forget (a slow/dead ntfy relay must never delay or fail ingest);
// fromShardEvent is self-guarding, but .catch() covers any lookup rejection.
Promise.resolve(d.pushDispatch(event, { shardLinks: d.shardLinks })).catch((err) =>
d.log.warn('push dispatch failed', { kind: event.kind, message: err.message }),
)
}
return { logged, stored }
}
module.exports = { ingest, shouldLog, reset, LOGGED_KINDS, state }

49
server/utils/shardPush.js Normal file
View File

@@ -0,0 +1,49 @@
// ── Shard event → push fan-out ─────────────────────────────────────────────
//
// MODULE-UO CONTENT, still living in core — the inverted half of
// MODULE_SYSTEM.md §1.8's second entangled file. `utils/pushDispatch.js` is core
// infrastructure, but its `fromShardEvent()` required the shardLinks model and
// the shard event mapper, which is a core file importing content. PR 4 inverted
// it: `publish()` stays core, and this — the thing that knows what a shard event
// is — moved out to call it. Phase 3 moves this file to module-uo whole, where it
// will reach `publish` through `ctx.push.publish` instead of a require.
//
// Owner resolution is the reason this cannot just be a mapper: a personal
// (owner-keyed) target names a GAME account, and turning that into a website user
// needs the shardLinks model. An unlinked account is simply nobody to notify.
const shardLinks = require('../model/shardLinks/shardLinks.model')
const { mapShardEvent } = require('../config/shardStreams')
const { push } = require('../core')
const { publish } = push
const log = require('../core').logger('shard-push')
// Fan a shard event out to push. Resolves personal (owner-keyed) targets to the
// owning website user via shardLinks (an unlinked account → nobody to notify).
// Never throws — a dead relay must never affect ingest.
async function fromShardEvent(event, deps = {}) {
const links = deps.shardLinks || shardLinks
const doPublish = deps.publish || publish
const targets = mapShardEvent(event, deps.tracker)
for (const t of targets) {
try {
if (t.ownerAccount) {
let owner = null
try {
owner = await links.getByAccount(t.ownerAccount)
} catch {
owner = null
}
if (!owner || owner.userId == null) continue
await doPublish(t.streamId, { ref: t.ref, ownerUserId: owner.userId }, deps)
} else {
await doPublish(t.streamId, { ref: t.ref }, deps)
}
} catch (err) {
log.warn('push dispatch target failed', { streamId: t.streamId, message: err.message })
}
}
}
module.exports = { fromShardEvent }

26
server/utils/shardSales.js Normal file
View File

@@ -0,0 +1,26 @@
// Recent player-vendor sales for a set of game accounts. Shared by the player
// self endpoint (the caller's linked accounts) and the admin user-detail
// endpoint (a target user's linked accounts). Reads the site's own ingested
// event log — no sidecar round-trip — and filters to sales whose owning account
// is in the set. Newest 50, already newest-first from shardEvents.list.
const shardEvents = require('../model/shardEvents/shardEvents.model')
async function salesForAccounts(accounts) {
const set = accounts instanceof Set ? accounts : new Set(accounts)
if (set.size === 0) return []
const events = await shardEvents.list({ kind: 'vendor.sale', limit: 500 })
return events
.filter((e) => e.payload && set.has(e.payload.ownerAcct))
.slice(0, 50)
.map((e) => ({
t: e.t,
itemType: e.payload.itemType,
amount: e.payload.amount,
price: e.payload.price,
commission: e.payload.commission,
ownerAcct: e.payload.ownerAcct,
}))
}
module.exports = { salesForAccounts }

435
server/utils/shardVisibility.js Normal file
View File

@@ -0,0 +1,435 @@
// ── Shard feature visibility ───────────────────────────────────────────────
//
// Admin-configurable, per-feature and per-field audience control over every
// shard-derived surface on the site. Replaces the hardcoded split that used to
// live in two places (the PUBLIC_KINDS allowlist in shardBroadcast.js, and the
// ad-hoc `canSeeStaffLocation` style checks in the public controllers).
//
// Design rules (docs/link/v3.md §3):
//
// • Visibility lives HERE, on the website — never in the sidecar. The sidecar
// is a dumb forwarder: it accepts frames, stores them, forwards them
// verbatim, and serves store-backed reads. It defines no audiences.
// • Every default reproduces the behavior that shipped before this module, so
// installing it changes nothing until an admin edits the config.
// • Two rules an admin CANNOT override:
// 1. `acct` / `webId` are admin-only, always. They are not in-game
// visible (unlike a character name) and are not configurable fields.
// 2. A kind absent from KIND_FEATURE is never broadcast below `admin`.
// Fail closed — this is what keeps the kind map a security boundary
// rather than a convenience filter.
//
// The audience ladder is ordered; each rung implies the ones below it.
const db = require('../model/shardVisibility/shardVisibility.model')
const shardLinks = require('../model/shardLinks/shardLinks.model')
const { auth } = require('../core')
const log = require('../core').logger('shard-visibility')
// ── The ladder ─────────────────────────────────────────────────────────────
const LADDER = ['anonymous', 'logged_in', 'player', 'staff', 'admin']
const RANK = new Map(LADDER.map((level, i) => [level, i]))
const isLevel = (level) => RANK.has(level)
// The two fallbacks are deliberately ASYMMETRIC, and the asymmetry is the whole
// point: an unrecognised value must always lose. A single shared fallback cannot
// do that — whichever direction it picks, it fails open on one side. So:
//
// • an unknown VIEWER level floors to the bottom rung (grants nothing), and
// • an unknown REQUIREMENT ceils to the top rung (satisfied by nobody but admin).
//
// With one `rank()` defaulting to admin, a viewer level that fell through (a
// typo, a future rung this build doesn't know, a value from a caller that
// skipped viewerLevel) would have been treated as an ADMIN and passed every gate.
const viewerRank = (level) => RANK.get(level) ?? 0
const requiredRank = (level) => RANK.get(level) ?? RANK.get('admin')
// True when a viewer at `viewer` satisfies a requirement of `required`.
const meets = (viewer, required) => viewerRank(viewer) >= requiredRank(required)
// Exported for tests/diagnostics; `meets` is what callers should use.
const rank = viewerRank
// ── Features ───────────────────────────────────────────────────────────────
//
// All ten shard surfaces: the six that shipped before v3 plus the four v3 adds.
// `fields` lists only the SENSITIVE fields — those an admin may re-gate. A field
// not listed here is visible whenever the feature itself is.
//
// LOCKED_FIELDS are exempt from configuration entirely (rule 1 above).
const LOCKED_FIELDS = { acct: 'admin', webId: 'admin' }
// Rule 1 matches on the FIELD'S MEANING, not on one exact spelling. The wire
// frames nest actors (`leader.acct`), but several read models flatten them
// instead (`shapeHouse` emits `ownerAcct`, `shapeGuild`'s fallback emits
// `leaderAcct`/`leaderWebId`), and an exact-key check silently missed every
// flattened one — which is how `GET /public/shard/idoc` served `ownerAcct` to
// anonymous callers while the same account name was correctly stripped from the
// live `house.decay` frame.
//
// So a key is locked when it IS `acct`/`webId` or ENDS in one, case-insensitively
// (`ownerAcct`, `leaderWebId`, `governorAcct`). Suffix matching is what makes this
// fail closed for shapes nobody has written yet.
const LOCKED_SUFFIXES = ['acct', 'webid']
const isLockedField = (key) => {
const k = String(key).toLowerCase()
return LOCKED_SUFFIXES.some((suffix) => k === suffix || k.endsWith(suffix))
}
const FEATURES = {
// ── Shipped before v3. Defaults reproduce the previous hardcoded behavior. ──
status: { audience: 'anonymous', fields: {} },
activity: { audience: 'anonymous', fields: {} },
champs: { audience: 'anonymous', fields: {} },
guilds: { audience: 'anonymous', fields: {} },
governors: { audience: 'anonymous', fields: {} },
// The public Houses page showed IDOC location only; owner/price were staff.
// `owner` is the actor object on the house.decay/house.update frames;
// `ownerName`/`ownerSerial` are the flattened spellings shapeHouse emits on the
// REST read models. Both are listed so one rule covers the wire and the read
// model — the flattened `ownerAcct` needs no entry, being locked by rule 1.
houses: {
audience: 'anonymous',
fields: { owner: 'staff', ownerName: 'staff', ownerSerial: 'staff', price: 'staff' },
},
// /public/shard/online listed linked staff to everyone but gated location to
// admin+moderator — which is exactly the `staff` rung.
presence: { audience: 'anonymous', fields: { location: 'staff' } },
// ── New in v3. ──
ruleset: { audience: 'anonymous', fields: { connect: 'anonymous' } },
atlas: { audience: 'anonymous', fields: {} },
// `name` is the ranked character's name inside points.board's `top` entries, and
// it is spelled the way the WIRE spells it, not the way v3.md §7.4 describes it
// ("characterName"). projectValue matches on the literal JSON key, so a rule
// named for the field's meaning rather than its key silently does nothing — the
// same failure §3.6.1 records for the flattened `ownerAcct` spelling. Within a
// leaderboards payload `name` can only be a character name: the board's own
// display name arrives as `nameString`/`nameNumber`.
leaderboards: { audience: 'anonymous', fields: { name: 'anonymous' } },
// Shop name, owner character name and vendor location are already globally
// visible in-game via the stock Vendor Search gump, so publishing them is not
// a new disclosure — but they stay configurable so an admin can tighten them.
//
// `ownerName` and `location` were pre-wired here by Part A, before the frame
// existed; both were re-checked against the real `vendor.listing` and both are
// genuine keys on it (unlike leaderboards' `characterName`, which was inert).
// `location` is a NESTED object on the wire and on the read model precisely so
// that one rule hides map, coordinates, region and house together — five flat
// keys would be five rules that drift apart.
//
// `ownerSerial` is listed alongside `ownerName` for the same reason `houses`
// lists both: an admin who hides the owner's name and is left with a serial
// that every other board resolves back to that name has not hidden anything.
market: {
audience: 'anonymous',
fields: { ownerName: 'anonymous', ownerSerial: 'anonymous', location: 'anonymous' },
},
}
const FEATURE_NAMES = Object.keys(FEATURES)
const isFeature = (name) => Object.hasOwn(FEATURES, name)
// ── Kind → feature ─────────────────────────────────────────────────────────
//
// Every event kind that may ever leave the admin channel must appear here.
// Anything else is admin-only by omission (rule 2). This map is seeded from
// what PUBLIC_KINDS listed before v3, so the public stream carries exactly the
// same kinds it did — now attributed to a feature that an admin can re-gate.
const KIND_FEATURE = new Map(
Object.entries({
// status / lifecycle
'server.hello': 'status',
'server.shutdown': 'status',
'server.crashed': 'status',
'economy.supply': 'status',
// activity feed
'player.death': 'activity',
'player.murdered': 'activity',
'mob.killed': 'activity',
'quest.complete': 'activity',
'skill.gain': 'activity',
'fame.change': 'activity',
'karma.change': 'activity',
'mob.login': 'activity',
'mob.logout': 'activity',
// boards
'champ.update': 'champs',
'champ.remove': 'champs',
'guild.update': 'guilds',
'guild.remove': 'guilds',
'guild.join': 'guilds',
'city.update': 'governors',
'presence.online': 'presence',
'region.enter': 'presence',
// house.decay is the IDOC signal the public Houses page renders. The full
// registry (house.update / house.remove — owner, price, co-owners) stays
// off the map deliberately, so it remains admin-only exactly as before.
'house.decay': 'houses',
// v3
'world.ruleset': 'ruleset',
'points.board': 'leaderboards',
// vendor.listing IS mapped, but the market feature ships with its stream
// disabled (see DEFAULT_STREAM_OFF): a live firehose of full vendor
// inventories would be the site's biggest bandwidth consumer and no page
// needs it live. An admin can turn it on.
'vendor.listing': 'market',
'vendor.listing.remove': 'market',
}),
)
// Features whose SSE fan-out is off unless an admin enables it. The REST reads
// are unaffected; only the live stream is suppressed.
const DEFAULT_STREAM_OFF = new Set(['market'])
// Back-compat: the set of kinds that reach an anonymous viewer under the default
// config. shardEvents `/feed` filtering and notificationStreams.js both consume
// this. Derived from the map above rather than hand-maintained, so the two can
// no longer drift.
const PUBLIC_KINDS = new Set(
[...KIND_FEATURE.entries()]
.filter(([, feature]) => {
if (DEFAULT_STREAM_OFF.has(feature)) return false
return FEATURES[feature].audience === 'anonymous'
})
.map(([kind]) => kind),
)
// ── Config (DB-backed, cached) ─────────────────────────────────────────────
const CONFIG_TTL_MS = 5000
let cache = null
let cachedAt = 0
// Merge a stored row over its compiled default. Unknown feature names in the DB
// are ignored (a stale row from a removed feature must not resurrect it), and an
// invalid rung falls back to the default rather than failing open.
function applyRow(name, row) {
const base = FEATURES[name]
const audience = isLevel(row?.audience) ? row.audience : base.audience
const fields = { ...base.fields }
for (const [field, level] of Object.entries(row?.fieldRules || {})) {
if (isLockedField(field)) continue // rule 1: not configurable
if (isLevel(level)) fields[field] = level
}
return {
enabled: row ? !!row.enabled : true,
audience,
fields,
stream: row?.stream == null ? !DEFAULT_STREAM_OFF.has(name) : !!row.stream,
}
}
function compileDefaults() {
const out = {}
for (const name of FEATURE_NAMES) out[name] = applyRow(name, null)
return out
}
// Read the config, cached briefly. Falls back to compiled defaults if the DB is
// unreachable — the defaults reproduce pre-v3 behavior, so a DB blip degrades to
// "what the site did before" rather than to "everything is public".
async function getConfig() {
const now = Date.now()
if (cache && now - cachedAt < CONFIG_TTL_MS) return cache
try {
const rows = await db.listAll()
const byName = new Map(rows.map((r) => [r.feature, r]))
const out = {}
for (const name of FEATURE_NAMES) out[name] = applyRow(name, byName.get(name))
cache = out
cachedAt = now
} catch (err) {
log.error('getConfig; falling back to defaults', err)
cache = cache || compileDefaults()
cachedAt = now
}
return cache
}
const invalidate = () => {
cache = null
cachedAt = 0
}
// ── Viewer level ───────────────────────────────────────────────────────────
//
// anonymous no session
// logged_in authenticated, no linked game account
// player authenticated with a linked game account
// staff admin | moderator — the same set as the existing `modAccess` gate.
// `editor` is a CONTENT role with no shard privilege today, so it
// resolves by link status like any other member; mapping it to staff
// here would silently widen what editors can see.
// admin admin
//
// Staff always satisfy the `player` rung (rank order guarantees it) even without
// a linked account, matching the existing rule that /player/* is role-agnostic
// self-service.
// Same TTL as the config cache: this decides a privilege rung, so an unlinked
// (or newly relinked) account must not keep the old answer for long. Anonymous,
// staff and admin callers short-circuit before this runs, so the lookup only
// costs a query on the logged-in-member path.
const LINK_TTL_MS = CONFIG_TTL_MS
const linkCache = new Map() // userId → { hasLink, at }
async function hasLinkedAccount(userId) {
const hit = linkCache.get(userId)
const now = Date.now()
if (hit && now - hit.at < LINK_TTL_MS) return hit.hasLink
let hasLink = false
try {
const links = await shardLinks.listForUser(userId)
hasLink = Array.isArray(links) && links.length > 0
} catch (err) {
log.warn('hasLinkedAccount failed; treating as unlinked', { message: err.message })
}
linkCache.set(userId, { hasLink, at: now })
return hasLink
}
// Drop a user's cached link status (called when a link is created or removed so
// the rung takes effect immediately rather than up to LINK_TTL_MS later).
const forgetUser = (userId) => linkCache.delete(userId)
async function viewerLevel(req) {
const viewer = req.user || auth.getUserFromRequest(req)
if (!viewer) return 'anonymous'
if (viewer.role === 'admin') return 'admin'
if (viewer.role === 'moderator') return 'staff'
return (await hasLinkedAccount(viewer.id)) ? 'player' : 'logged_in'
}
// ── Enforcement ────────────────────────────────────────────────────────────
// Route gate. 404 when the feature is disabled (do not leak that it exists);
// 403 when it exists but the viewer sits below its audience. Stashes the
// resolved level on the request so controllers can project without re-resolving.
function requireFeature(name) {
return async (req, res, next) => {
try {
const config = await getConfig()
const feature = config[name]
if (!feature || !feature.enabled) return res.status(404).json({ message: 'Not Found' })
const level = await viewerLevel(req)
req.viewerLevel = level
if (!meets(level, feature.audience)) return res.status(403).json({ message: 'Forbidden' })
return next()
} catch (err) {
log.error(`requireFeature(${name})`, err)
return res.status(500).json({ message: 'Internal Server Error' })
}
}
}
// Strip the fields a viewer at `level` may not see. Applies the locked rules
// first (so acct/webId can never survive below admin), then the feature's
// configured field rules. Recurses into arrays and nested objects because the
// sensitive fields sit inside actor sub-objects (guild.leader, city.governor).
// Only ARRAYS and PLAIN objects are walked. A Date, Buffer or other class
// instance is a value, not a bag of fields: rebuilding one key-by-key would
// return `{}` (a Date has no enumerable own properties), which is how the DB-
// backed read models — whose rows carry real Date columns — differ from the
// pure-JSON wire frames the projection was first written against.
const isPlainObject = (v) => {
if (v === null || typeof v !== 'object') return false
const proto = Object.getPrototypeOf(v)
return proto === Object.prototype || proto === null
}
function projectValue(value, rules, level) {
if (Array.isArray(value)) return value.map((v) => projectValue(v, rules, level))
if (!isPlainObject(value)) return value
const out = {}
for (const [key, v] of Object.entries(value)) {
// Locked fields are checked by meaning first, so no configured rule (and no
// flattened spelling) can widen them past `admin`.
const required = isLockedField(key) ? 'admin' : rules[key]
if (required && !meets(level, required)) continue
out[key] = projectValue(v, rules, level)
}
return out
}
// Project a payload for one feature. `level` defaults to admin-equivalent only
// when explicitly passed; callers should always pass a resolved level.
function projectFeature(name, payload, level, config) {
const feature = config?.[name]
const rules = { ...LOCKED_FIELDS, ...(feature ? feature.fields : {}) }
return projectValue(payload, rules, level)
}
// Convenience for controllers: resolve config once, project, return.
async function project(name, payload, req) {
const config = await getConfig()
const level = req.viewerLevel || (await viewerLevel(req))
return projectFeature(name, payload, level, config)
}
// Is this event kind allowed to reach a viewer at `level`? Fail closed on an
// unmapped kind (rule 2), and honour both the feature gate and its stream flag.
function kindVisibleTo(kind, level, config) {
if (level === 'admin') return true
const name = KIND_FEATURE.get(kind)
if (!name) return false // rule 2: unmapped ⇒ admin-only
const feature = config?.[name]
if (!feature || !feature.enabled || !feature.stream) return false
return meets(level, feature.audience)
}
// The event kinds a viewer at `level` may read under the CURRENT config. This is
// the live counterpart of PUBLIC_KINDS, which is a module-load constant derived
// from the compiled DEFAULTS and therefore cannot answer "may THIS viewer see
// this kind, given what the admin has configured?".
//
// Deliberately ignores the `stream` flag: that governs SSE fan-out only, so a
// feature whose live firehose is off (market) is still readable from the stored
// history. Unmapped kinds are absent by construction (rule 2).
function visibleKinds(level, config) {
return [...KIND_FEATURE.entries()]
.filter(([, name]) => {
const feature = config?.[name]
return !!feature && feature.enabled && meets(level, feature.audience)
})
.map(([kind]) => kind)
}
// The features a viewer at `level` can actually see — drives SPA nav so it never
// renders a link that would 403.
function visibleFeatures(level, config) {
return FEATURE_NAMES.filter((name) => {
const feature = config[name]
return feature.enabled && meets(level, feature.audience)
})
}
module.exports = {
LADDER,
FEATURES,
FEATURE_NAMES,
LOCKED_FIELDS,
KIND_FEATURE,
PUBLIC_KINDS,
DEFAULT_STREAM_OFF,
isLevel,
isFeature,
isLockedField,
rank,
meets,
getConfig,
invalidate,
compileDefaults,
viewerLevel,
forgetUser,
requireFeature,
projectFeature,
project,
kindVisibleTo,
visibleKinds,
visibleFeatures,
}

686
server/utils/spawnAtlasParse.js Normal file
View File

@@ -0,0 +1,686 @@
// Spawn atlas parsers — pure functions over strings, no `fs`, no dependencies.
//
// These back the CLI build script (`scripts/buildSpawnAtlas.js`), which is the
// only thing that reads a ServUO tree. Keeping every parser pure and fs-free is
// what lets the test suite cover them in CI, where no ServUO tree exists: the
// tests hand these functions literal XML strings.
//
// Four source shapes, two very different parsing strategies:
//
// Spawns/*.xml ~10.5 MB across 13 files, FLAT <Points> records
// → streaming regex, never a DOM. See parsePoints().
// Data/Regions.xml 129 KB, genuinely nested <region> inside <region>
// Data/Locations/*.xml nested <parent>/<child>
// Config/ChampionSpawns.xml 4.8 KB, <spawn>/<location>
// → the small recursive tokenizer below.
//
// The server has zero XML dependencies and this adds none. The tokenizer is
// deliberately a *subset* parser: it handles the constructs these four files
// actually use (elements, attributes, self-closing tags, comments, the XML
// declaration, CDATA, the five predefined entities plus numeric refs) and
// nothing else. It is not a general-purpose XML parser and must not be reused
// as one — no namespaces, no DTDs, no entity declarations.
// ── Entities ───────────────────────────────────────────────────────────────
const NAMED_ENTITIES = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
}
// Region and location names carry apostrophes ("Mondain's Legacy", "Wrong's
// Level 3"), so entity decoding is load-bearing here, not decorative.
function decodeEntities(text) {
if (!text.includes('&')) return text
return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, body) => {
if (body[0] === '#') {
const code =
body[1] === 'x' || body[1] === 'X'
? Number.parseInt(body.slice(2), 16)
: Number.parseInt(body.slice(1), 10)
return Number.isFinite(code) ? String.fromCodePoint(code) : match
}
const named = NAMED_ENTITIES[body.toLowerCase()]
return named === undefined ? match : named
})
}
// ── The tokenizer ──────────────────────────────────────────────────────────
const ATTR_RE = /([\w:.-]+)\s*=\s*("([^"]*)"|'([^']*)')/g
function parseAttrs(source) {
const attrs = {}
ATTR_RE.lastIndex = 0
let match
while ((match = ATTR_RE.exec(source)) !== null) {
const raw = match[3] !== undefined ? match[3] : match[4]
attrs[match[1]] = decodeEntities(raw)
}
return attrs
}
/**
* Parse a small nested XML document into `{ name, attrs, children, text }`.
*
* Intended for Regions.xml / Locations / ChampionSpawns.xml only — never for
* the multi-megabyte Spawns files. Returns the root element, or `null` for a
* document with no elements.
*
* Mismatched or stray closing tags are ignored rather than thrown on: these are
* hand-maintained shard config files, and one malformed region should degrade
* to a missing region, not abort a build that is otherwise fine.
*/
function parseXml(source) {
const text = String(source)
const root = { name: '#document', attrs: {}, children: [], text: '' }
const stack = [root]
let i = 0
while (i < text.length) {
const lt = text.indexOf('<', i)
if (lt === -1) {
appendText(stack[stack.length - 1], text.slice(i))
break
}
if (lt > i) appendText(stack[stack.length - 1], text.slice(i, lt))
// Comment, declaration/DOCTYPE, or CDATA — skipped wholesale.
if (text.startsWith('<!--', lt)) {
const end = text.indexOf('-->', lt + 4)
i = end === -1 ? text.length : end + 3
continue
}
if (text.startsWith('<![CDATA[', lt)) {
const end = text.indexOf(']]>', lt + 9)
const stop = end === -1 ? text.length : end
appendRawText(stack[stack.length - 1], text.slice(lt + 9, stop))
i = end === -1 ? text.length : end + 3
continue
}
if (text.startsWith('<?', lt)) {
const end = text.indexOf('?>', lt + 2)
i = end === -1 ? text.length : end + 2
continue
}
if (text.startsWith('<!', lt)) {
const end = text.indexOf('>', lt + 2)
i = end === -1 ? text.length : end + 1
continue
}
const gt = findTagEnd(text, lt)
if (gt === -1) {
// Unterminated tag: nothing sane is left to read.
break
}
const inner = text.slice(lt + 1, gt)
if (inner[0] === '/') {
const name = inner.slice(1).trim()
// Pop to the nearest matching open element. If there is no match the tag
// is stray and we drop it rather than unwinding the whole stack.
for (let depth = stack.length - 1; depth > 0; depth -= 1) {
if (stack[depth].name === name) {
stack.length = depth
break
}
}
i = gt + 1
continue
}
const selfClosing = inner.endsWith('/')
const body = selfClosing ? inner.slice(0, -1) : inner
const space = body.search(/\s/)
const name = (space === -1 ? body : body.slice(0, space)).trim()
const node = {
name,
attrs: space === -1 ? {} : parseAttrs(body.slice(space)),
children: [],
text: '',
}
stack[stack.length - 1].children.push(node)
if (!selfClosing) stack.push(node)
i = gt + 1
}
return root.children.length > 0 ? root.children[0] : null
}
// `>` inside a quoted attribute value must not end the tag.
function findTagEnd(text, from) {
let quote = null
for (let i = from + 1; i < text.length; i += 1) {
const ch = text[i]
if (quote) {
if (ch === quote) quote = null
} else if (ch === '"' || ch === "'") {
quote = ch
} else if (ch === '>') {
return i
}
}
return -1
}
function appendText(node, chunk) {
if (chunk.trim() === '') return
appendRawText(node, decodeEntities(chunk))
}
function appendRawText(node, chunk) {
node.text = node.text ? `${node.text}${chunk}` : chunk
}
function childrenNamed(node, name) {
if (!node || !node.children) return []
return node.children.filter((child) => child.name === name)
}
// ── Facet names ────────────────────────────────────────────────────────────
//
// Facets are NOT a fixed list. A shard may add facets, replace them wholesale,
// or rename them when its maps are updated, so nothing here may name Felucca,
// Trammel or any other stock facet. The facet set is whatever the shard's own
// files say it is, discovered at parse time.
//
// The complication is that the sources disagree about spelling for the SAME
// facet and nothing in the files reconciles them: `Spawns/*.xml` `<Map>` and
// `Regions.xml` `<Facet name>` say `TerMur`, while `Data/Locations/*.xml` spells
// it `Ter Mur` and calls Tokuno `Tokuno Islands`. Left unreconciled this fails
// silently — the landmark bucket is keyed differently from the points looking it
// up, so the fallback never fires and every unregioned spawn on those facets
// reads "Wilderness".
//
// Reconciliation is therefore done by MATCHING, not by a lookup table:
// `facetKey()` collapses spelling differences, and `resolveFacetName()` matches
// a loosely-spelled name against the canonical set discovered from the shard's
// own data. A facet nobody else mentions keeps its own name rather than being
// dropped.
/**
* Collapse a facet name to a comparison key: lowercase, alphanumerics only.
* `TerMur`, `Ter Mur` and `ter-mur` all key alike.
*/
function facetKey(value) {
return String(value ?? '')
.toLowerCase()
.replace(/[^a-z0-9]+/g, '')
}
/**
* Build a key → canonical-spelling lookup from the authoritative facet names.
*
* The authority is what the spawn records and region definitions actually say,
* since those are the names the atlas keys everything on. Later names do not
* overwrite earlier ones, so the first source wins consistently.
*/
function buildFacetIndex(names) {
const index = new Map()
for (const name of names) {
const key = facetKey(name)
if (key !== '' && !index.has(key)) index.set(key, String(name).trim())
}
return index
}
/**
* Resolve a loosely-spelled facet name against the discovered canonical set.
*
* Tried in order: exact key match (`Ter Mur` → `TerMur`), then a prefix match in
* either direction (`Tokuno Islands` → `Tokuno`), longest candidate first so a
* more specific facet wins over a shorter one that merely prefixes it.
*
* A name matching nothing is returned trimmed rather than dropped — on a shard
* with a custom facet that is a real facet the atlas simply has no spawns for
* yet, and inventing a match would be worse than leaving it alone.
*/
function resolveFacetName(value, index) {
const raw = String(value ?? '').trim()
const key = facetKey(raw)
if (key === '') return ''
if (index.has(key)) return index.get(key)
let best = null
for (const [candidateKey, canonical] of index) {
if (!key.startsWith(candidateKey) && !candidateKey.startsWith(key)) continue
if (best === null || candidateKey.length > facetKey(best).length) best = canonical
}
return best ?? raw
}
// ── Small coercions ────────────────────────────────────────────────────────
function toInt(value, fallback = 0) {
const n = Number.parseInt(value, 10)
return Number.isFinite(n) ? n : fallback
}
function toBool(value) {
return String(value).trim().toLowerCase() === 'true'
}
/**
* URL-safe slug used as the creature primary key and in `/atlas/:slug`.
* Spawn type tokens are C# class names, so they are already ASCII-ish; this
* mainly lowercases and collapses punctuation.
*/
function slugify(value) {
return String(value)
.trim()
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
}
// ── Objects2 ───────────────────────────────────────────────────────────────
/**
* Parse a `<Objects2>` value into `[{ type, max }]`.
*
* The format is one or more segments joined by `:OBJ=`, each segment being
* `Type:MX=n:SB=0:RT=0:...` — the type is the token before the first `:`, and
* every following token is a `KEY=value` pair. Verified against trammel.xml,
* where a single point carries six types:
*
* Giantserpent:MX=1:...:OBJ=Giantspider:MX=1:...:OBJ=Boar:MX=1:...
*
* Splitting on `:` alone would shred this, which is why the `:OBJ=` split comes
* first. `MX` is that type's own max count and is what the atlas displays;
* every other flag (spawn/trigger/refractory bookkeeping) is dropped.
*
* The type token itself may carry XmlSpawner directives appended to the class
* name — property assignments after `/` and an amount/argument list after `,`:
*
* Agralem/Name/Agralem alchemist/z/-50 Fairy,{RND,4,8}
* GargishRefugee/hue/34532 greatape,true GargishRouser,1
*
* Taken literally these produce creatures that do not exist ("alchemist/z/-50")
* AND split real ones in two, because `Fairy` and `Fairy,{RND,4,8}` slug apart —
* 71 of 845 entries were affected before this was stripped. Only the leading
* class name identifies the creature, so everything from the first `/` or `,`
* is dropped.
*/
/** Reduce an XmlSpawner type token to the bare class name. */
function stripSpawnerDirectives(token) {
const cut = String(token).search(/[/,]/)
return (cut === -1 ? String(token) : String(token).slice(0, cut)).trim()
}
function parseObjects2(value) {
const source = String(value ?? '').trim()
if (source === '') return []
return source
.split(':OBJ=')
.map((segment) => {
const tokens = segment.split(':')
const type = stripSpawnerDirectives(tokens.shift() ?? '')
if (type === '') return null
let max = 1
for (const token of tokens) {
const eq = token.indexOf('=')
if (eq === -1) continue
if (token.slice(0, eq).trim().toUpperCase() === 'MX') {
max = toInt(token.slice(eq + 1), 1)
}
}
return { type, max }
})
.filter((entry) => entry !== null)
}
// ── Spawns/*.xml ───────────────────────────────────────────────────────────
const POINT_RE = /<Points>([\s\S]*?)<\/Points>/g
function tagValue(block, name) {
const match = block.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`))
return match ? decodeEntities(match[1]).trim() : ''
}
/**
* Parse a `Spawns/<facet>.xml` file into spawn point records.
*
* Deliberately regex/streaming and NOT `parseXml` — these files total ~10.5 MB
* and putting them through a DOM builder would allocate a node per element for
* ~40 fields on every one of ~6,500 records to keep 14 of them. The records are
* flat, so a per-record regex sweep is both correct and cheap.
*
* Only the fields the site can actually show are kept. Everything to do with
* triggering, refractory windows, proximity, sequential spawning, sounds and
* `UniqueId` is dropped here rather than downstream — that is what holds the
* committed artifact under 1 MB.
*
* NOTE: the facet comes from each record's own `<Map>`, never from the file
* name. `Eodon.xml`, `GravewaterLake.xml` and the other named-area files all
* carry TerMur/Trammel points, so there are 13 files but only 6 facets.
*/
/**
* A spawner's respawn window, in seconds.
*
* `DelayInSec` decides the unit of `MinDelay`/`MaxDelay`; absent (older files)
* it is false, which is minutes — the same default XmlSpawner assumes.
*/
function delaySeconds(block) {
const scale = toBool(tagValue(block, 'DelayInSec')) ? 1 : 60
return {
minDelay: toInt(tagValue(block, 'MinDelay')) * scale,
maxDelay: toInt(tagValue(block, 'MaxDelay')) * scale,
}
}
function parsePoints(source) {
const text = String(source)
const points = []
POINT_RE.lastIndex = 0
let match
while ((match = POINT_RE.exec(text)) !== null) {
const block = match[1]
// Reported exactly as written. `<Map>` is the authority the rest of the
// atlas keys on, so it is never rewritten.
const facet = tagValue(block, 'Map')
if (facet === '') continue
points.push({
name: tagValue(block, 'Name'),
facet,
x: toInt(tagValue(block, 'X')),
y: toInt(tagValue(block, 'Y')),
width: toInt(tagValue(block, 'Width')),
height: toInt(tagValue(block, 'Height')),
range: toInt(tagValue(block, 'Range')),
maxCount: toInt(tagValue(block, 'MaxCount')),
// Normalised to SECONDS here, because the unit is per-record. XmlSpawner
// writes minutes by default and switches to seconds only when a spawner's
// delay does not divide into whole minutes, flagging that with
// `DelayInSec` (XmlSpawner2.cs:7462-7480, read back at :6345-6358). Taken
// literally the two are indistinguishable — a `5` means five minutes on
// one spawner and five seconds on the next — so a consumer that assumed
// either unit would be wrong about the other. Stock ServUO 57.4 has ~30
// second-flagged spawners, few enough to look like noise and quietly
// mislabel.
...delaySeconds(block),
// Time-of-day gating: TODMode 0 means "always", in which case the start
// and end values are meaningless and the site must not render them.
todStart: toInt(tagValue(block, 'TODStart')),
todEnd: toInt(tagValue(block, 'TODEnd')),
todMode: toInt(tagValue(block, 'TODMode')),
// A spawner switched off in-world spawns nothing; the build filters these
// out so the atlas describes what actually appears, not what is merely
// configured. Parsed here so the decision stays in the build script.
running: toBool(tagValue(block, 'IsRunning')),
types: parseObjects2(tagValue(block, 'Objects2')),
})
}
return points
}
// ── Data/Regions.xml ───────────────────────────────────────────────────────
/**
* Flatten `Data/Regions.xml` into `[{ facet, name, type, priority, parent, rects }]`.
*
* Regions nest: a `<region>` may contain further `<region>` elements, and the
* inner ones frequently omit `name` and `priority` (`<region type="CrystalField">`
* inside "Prism of Light"). Unnamed regions are skipped — they cannot label a
* spawn point — but their children are still walked, and a child that omits
* `priority` inherits its parent's rather than defaulting to 0, which would
* quietly sort it below every top-level region.
*/
function parseRegions(source) {
const root = parseXml(source)
const regions = []
if (!root) return regions
for (const facetNode of childrenNamed(root, 'Facet')) {
const facet = (facetNode.attrs.name || '').trim()
if (facet === '') continue
walkRegions(facetNode, facet, null, 0, regions)
}
return regions
}
function walkRegions(node, facet, parentName, parentPriority, out) {
for (const regionNode of childrenNamed(node, 'region')) {
const name = regionNode.attrs.name || ''
const priority = Object.hasOwn(regionNode.attrs, 'priority')
? toInt(regionNode.attrs.priority, parentPriority)
: parentPriority
if (name !== '') {
const rects = childrenNamed(regionNode, 'rect').map((rect) => ({
x: toInt(rect.attrs.x),
y: toInt(rect.attrs.y),
width: toInt(rect.attrs.width),
height: toInt(rect.attrs.height),
}))
// A named region with no rects (some exist purely to carry music or a
// `go` point) can never contain anything, so it is not worth indexing.
if (rects.length > 0) {
out.push({
facet,
name,
type: regionNode.attrs.type || '',
priority,
parent: parentName,
rects,
})
}
}
walkRegions(regionNode, facet, name === '' ? parentName : name, priority, out)
}
}
// ── Data/Locations/*.xml ───────────────────────────────────────────────────
/**
* Flatten a `Data/Locations/<facet>.xml` into landmark points.
*
* The file nests `<parent>` arbitrarily deep and puts coordinates only on
* `<child>`: Trammel → Dungeons → Covetous → "Level 1". The outermost parent is
* the facet itself and is dropped from `path`; `group` is the innermost
* enclosing parent ("Covetous"), which is the label worth showing — "Covetous"
* reads better than "Level 1" when naming where a spawn is.
*/
function parseLocations(source, facetHint = '') {
const root = parseXml(source)
const landmarks = []
if (!root) return landmarks
for (const top of childrenNamed(root, 'parent')) {
// The file name (`Data/Locations/termur.xml`) is the more reliable signal
// and is preferred over the display label inside the file, which is where
// the `Ter Mur` / `Tokuno Islands` drift lives. Both are carried so the
// build can fall back to matching the label if the file name resolves to
// nothing — a shard may well name its files differently from its facets.
landmarks.push(
...collectLocations(top, facetHint || top.attrs.name || '', top.attrs.name || ''),
)
}
return landmarks
}
function collectLocations(top, facet, label) {
const out = []
walkLocations(top, facet, [], out)
for (const landmark of out) landmark.facetLabel = label
return out
}
function walkLocations(node, facet, path, out) {
for (const child of childrenNamed(node, 'child')) {
const name = child.attrs.name || ''
if (name === '') continue
out.push({
facet,
name,
group: path.length > 0 ? path[path.length - 1] : name,
path: [...path],
x: toInt(child.attrs.x),
y: toInt(child.attrs.y),
z: toInt(child.attrs.z),
})
}
for (const parent of childrenNamed(node, 'parent')) {
const name = parent.attrs.name || ''
walkLocations(parent, facet, name === '' ? path : [...path, name], out)
}
}
// ── Config/ChampionSpawns.xml ──────────────────────────────────────────────
/**
* Parse `Config/ChampionSpawns.xml` into champion altar records.
*
* This is the shard's *configured* champion roster — which altars exist, where,
* and which type each is pinned to. It is static content and distinct from the
* live `champ.update` feed the bridge already carries: this says "there is an
* Unholy Terror altar in Deceit", the feed says "it is on level 3 right now".
*
* A spawn with no `type` is randomised on every activation, which the site must
* render as "random" rather than as an empty type.
*/
function parseChampions(source) {
const root = parseXml(source)
const champions = []
if (!root) return champions
for (const spawnNode of childrenNamed(root, 'spawn')) {
const location = childrenNamed(spawnNode, 'location')[0]
const attrs = location ? location.attrs : {}
champions.push({
name: spawnNode.attrs.name || '',
group: spawnNode.attrs.group || '',
type: spawnNode.attrs.type || '',
randomType: !spawnNode.attrs.type,
facet: (attrs.map || '').trim(),
x: toInt(attrs.x),
y: toInt(attrs.y),
z: toInt(attrs.z),
radius: toInt(attrs.radius),
})
}
return champions
}
// ── Placement ──────────────────────────────────────────────────────────────
const DEFAULT_LANDMARK_RADIUS = 200
function inRect(x, y, rect) {
return (
x >= rect.x && x < rect.x + rect.width && y >= rect.y && y < rect.y + rect.height
)
}
function rectArea(rect) {
return Math.max(1, rect.width) * Math.max(1, rect.height)
}
/**
* Group parsed regions and landmarks by facet once, so the per-point resolve
* below is a scan of one facet instead of the whole world. With ~6,500 points
* and a few thousand rects this stays comfortably sub-second; there is no need
* for a spatial index and none is worth the complexity.
*/
function buildPlacementIndex(regions, landmarks) {
const byFacet = new Map()
// Keyed on facetKey(), not the raw name, so two spellings of one facet cannot
// land in separate buckets — the failure that silently emptied the landmark
// bucket for Ter Mur and Tokuno.
const facet = (name) => {
const key = facetKey(name)
if (!byFacet.has(key)) byFacet.set(key, { regions: [], landmarks: [] })
return byFacet.get(key)
}
for (const region of regions) facet(region.facet).regions.push(region)
for (const landmark of landmarks) facet(landmark.facet).landmarks.push(landmark)
return byFacet
}
/**
* Turn a raw coordinate into a human place name.
*
* This is the transform the whole atlas exists for: it is what makes a row read
* "Lizardman — Despise, Felucca" instead of "Lizardman — 5411, 1234".
*
* Resolution order:
* 1. The highest-`priority` named region whose rect contains the point. Ties
* break toward the SMALLEST rect, so a specific room inside a dungeon wins
* over the dungeon-wide rect it sits in.
* 2. Otherwise the nearest landmark within `landmarkRadius` tiles, labelled by
* its group ("Covetous"), not the individual marker ("Level 1").
* 3. Otherwise "Wilderness". The radius cap is what keeps step 3 reachable —
* without it the nearest landmark is always *some* landmark, however far,
* and open countryside would get labelled with a dungeon on the far side
* of the map.
*/
function resolveRegion(x, y, facetName, index, options = {}) {
const radius = options.landmarkRadius ?? DEFAULT_LANDMARK_RADIUS
const bucket = index.get(facetKey(facetName))
const result = { region: null, landmark: null, label: 'Wilderness' }
if (!bucket) return result
let best = null
let bestPriority = -Infinity
let bestArea = Infinity
for (const region of bucket.regions) {
for (const rect of region.rects) {
if (!inRect(x, y, rect)) continue
const area = rectArea(rect)
if (region.priority > bestPriority || (region.priority === bestPriority && area < bestArea)) {
best = region
bestPriority = region.priority
bestArea = area
}
}
}
if (best) {
result.region = best.name
result.label = best.name
return result
}
let nearest = null
let nearestDistance = Infinity
const limit = radius * radius
for (const landmark of bucket.landmarks) {
const dx = landmark.x - x
const dy = landmark.y - y
const distance = dx * dx + dy * dy
if (distance < nearestDistance) {
nearest = landmark
nearestDistance = distance
}
}
if (nearest && nearestDistance <= limit) {
result.landmark = nearest.group || nearest.name
result.label = result.landmark
}
return result
}
module.exports = {
parseXml,
parseObjects2,
parsePoints,
parseRegions,
parseLocations,
parseChampions,
buildPlacementIndex,
resolveRegion,
facetKey,
buildFacetIndex,
resolveFacetName,
slugify,
decodeEntities,
DEFAULT_LANDMARK_RADIUS,
}

336
server/utils/spawnAtlasSource.js Normal file
View File

@@ -0,0 +1,336 @@
// Spawn atlas — the filesystem layer over a ServUO tree.
//
// `spawnAtlasParse.js` holds the pure parsers; this module is the only thing
// that touches a ServUO tree on disk, and it is shared by both callers:
//
// - the server, which refreshes the atlas on boot (`shardAtlas.model.js`)
// - the CLI (`scripts/importSpawnAtlas.js`)
//
// The shard's own files are the single source of truth. Nothing is precomputed
// and committed, because a shard's maps change over its lifetime — facets get
// added, replaced or renamed — and a snapshot in the repo would silently go
// stale against the world players actually see.
//
// Reading and hashing the whole tree costs ~120 ms and a full parse ~400 ms, so
// the boot path hashes first and only parses when something actually changed.
const crypto = require('crypto')
const fs = require('fs')
const path = require('path')
const {
parsePoints,
parseRegions,
parseLocations,
parseChampions,
buildPlacementIndex,
buildFacetIndex,
resolveFacetName,
resolveRegion,
facetKey,
slugify,
} = require('./spawnAtlasParse')
const REGIONS_FILE = path.join('Data', 'Regions.xml')
const LOCATIONS_DIR = path.join('Data', 'Locations')
const SPAWNS_DIR = 'Spawns'
const CHAMPIONS_FILE = path.join('Config', 'ChampionSpawns.xml')
class AtlasSourceError extends Error {
constructor(message, code) {
super(message)
this.name = 'AtlasSourceError'
this.code = code
}
}
// ── Reading ────────────────────────────────────────────────────────────────
function sha256(text) {
return crypto.createHash('sha256').update(text, 'utf8').digest('hex')
}
function listXml(dir) {
try {
return fs
.readdirSync(dir)
.filter((name) => name.toLowerCase().endsWith('.xml'))
.sort()
} catch (err) {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return []
throw err
}
}
function readIfPresent(file) {
try {
return fs.readFileSync(file, 'utf8')
} catch (err) {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null
throw err
}
}
/**
* Read every atlas source file under `root`.
*
* Returns `{ files: [{ label, text, sha256, bytes }] }`, labels being
* tree-relative and forward-slashed so a hash map compares equal across
* platforms — the same tree read on Windows and Linux must produce the same
* fingerprint or every boot would look like a change.
*/
function readSources(root) {
if (!root || String(root).trim() === '') {
throw new AtlasSourceError('No ServUO path configured', 'NO_PATH')
}
if (!fs.existsSync(root)) {
throw new AtlasSourceError(`ServUO path does not exist: ${root}`, 'NOT_FOUND')
}
const files = []
const push = (label, file) => {
const text = readIfPresent(file)
if (text === null) return false
files.push({ label, text, sha256: sha256(text), bytes: Buffer.byteLength(text, 'utf8') })
return true
}
if (!push('Data/Regions.xml', path.join(root, REGIONS_FILE))) {
throw new AtlasSourceError(`Missing required file: ${REGIONS_FILE}`, 'NO_REGIONS')
}
for (const name of listXml(path.join(root, LOCATIONS_DIR))) {
push(`Data/Locations/${name}`, path.join(root, LOCATIONS_DIR, name))
}
const spawnFiles = listXml(path.join(root, SPAWNS_DIR))
if (spawnFiles.length === 0) {
throw new AtlasSourceError(`No spawn files found in ${SPAWNS_DIR}`, 'NO_SPAWNS')
}
for (const name of spawnFiles) push(`Spawns/${name}`, path.join(root, SPAWNS_DIR, name))
push('Config/ChampionSpawns.xml', path.join(root, CHAMPIONS_FILE))
return { files }
}
/**
* A fingerprint of the tree: `{ "<label>": "<sha256>" }`.
*
* The boot path compares this against what was last imported and skips the
* parse entirely when it matches, which is the normal case on every restart
* that did not follow a map update.
*/
function hashSources(root) {
const { files } = readSources(root)
const hashes = {}
for (const file of files) hashes[file.label] = file.sha256
return hashes
}
/**
* Bumped whenever the parser produces DIFFERENT data from IDENTICAL source
* files — a fixed misreading, a new field, a changed unit.
*
* Without it the hash gate is a trap: an install whose tree has not changed
* would keep serving what an older parser derived, indefinitely, because the
* only thing the boot path compares is the tree. The version is stored beside
* the source hashes and a mismatch counts as drift, so a deploy that corrects
* the parse actually reaches the data.
*
* 2 — respawn delays normalised to seconds (they are per-record minutes OR
* seconds in the source, decided by `DelayInSec`).
*/
const PARSER_VERSION = 2
/** True when two source fingerprints describe the same tree. */
function sameSources(a, b) {
if (!a || !b) return false
const aKeys = Object.keys(a).sort()
const bKeys = Object.keys(b).sort()
if (aKeys.length !== bKeys.length) return false
return aKeys.every((key, i) => key === bKeys[i] && a[key] === b[key])
}
// ── Aggregation ────────────────────────────────────────────────────────────
/**
* Choose one display spelling for a creature.
*
* Spawn files are not consistent about case — the same creature is `Lizardman`
* in one file and `lizardman` in another. Slugging collapses them correctly, but
* the display name would otherwise depend on file read order. Most frequent
* spelling wins; ties break toward more capitals, then alphabetically.
*/
function displayName(spellings) {
const capitals = (value) => (value.match(/[A-Z]/g) || []).length
return [...spellings.entries()].sort((a, b) => {
if (b[1] !== a[1]) return b[1] - a[1]
const caps = capitals(b[0]) - capitals(a[0])
if (caps !== 0) return caps
return a[0].localeCompare(b[0])
})[0][0]
}
/**
* Roll spawn points up into per-type creature rows.
*
* `total` is the sum of each type's own max across every point that spawns it —
* how many of this creature the world holds at once. `facets` is a per-facet
* point count, so "where does this live" answers without touching the points.
*/
function aggregateCreatures(points) {
const creatures = new Map()
for (const point of points) {
for (const entry of point.types) {
const slug = slugify(entry.type)
if (slug === '') continue
let creature = creatures.get(slug)
if (!creature) {
creature = { slug, name: '', total: 0, points: 0, facets: {}, spellings: new Map() }
creatures.set(slug, creature)
}
creature.total += entry.max
creature.points += 1
creature.facets[point.facet] = (creature.facets[point.facet] || 0) + 1
creature.spellings.set(entry.type, (creature.spellings.get(entry.type) || 0) + 1)
}
}
return [...creatures.values()]
.map(({ spellings, ...creature }) => ({ ...creature, name: displayName(spellings) }))
.sort((a, b) => a.slug.localeCompare(b.slug))
}
// ── Build ──────────────────────────────────────────────────────────────────
/**
* Parse a ServUO tree into the full atlas.
*
* Pure with respect to the database — it reads files and returns data; nothing
* here writes. `shardAtlas.model.js` decides what to do with the result.
*/
function buildAtlas(root, options = {}) {
const { files } = readSources(root)
const byLabel = new Map(files.map((file) => [file.label, file]))
const source = {}
for (const file of files) source[file.label] = { bytes: file.bytes, sha256: file.sha256 }
const regions = parseRegions(byLabel.get('Data/Regions.xml').text)
const rawLandmarks = []
for (const file of files) {
if (!file.label.startsWith('Data/Locations/')) continue
const basename = path.basename(file.label, '.xml')
rawLandmarks.push(...parseLocations(file.text, basename))
}
const rawPoints = []
for (const file of files) {
if (!file.label.startsWith('Spawns/')) continue
rawPoints.push(...parsePoints(file.text))
}
// The facet set is whatever THIS tree declares — never a built-in list. A
// shard may add facets, replace them outright, or rename them when its maps
// are updated, and the atlas has to follow without a code change. Spawn
// records and region definitions are the authority, because those are the
// names everything else is keyed on.
const facetIndex = buildFacetIndex([
...rawPoints.map((point) => point.facet),
...regions.map((region) => region.facet),
])
// Landmark facets are then matched against that set, which is what absorbs the
// `Ter Mur` / `Tokuno Islands` spelling drift between Locations and <Map>.
const landmarks = rawLandmarks.map(({ facetLabel, ...landmark }) => {
const fromFile = resolveFacetName(landmark.facet, facetIndex)
const matchedFile = facetIndex.has(facetKey(fromFile))
const resolved = matchedFile ? fromFile : resolveFacetName(facetLabel, facetIndex)
return { ...landmark, facet: resolved || landmark.facet }
})
const placement = buildPlacementIndex(regions, landmarks)
const resolveOpts = options.landmarkRadius ? { landmarkRadius: options.landmarkRadius } : {}
const disabled = rawPoints.filter((point) => !point.running).length
const points = rawPoints
// A spawner switched off in-world produces nothing; advertising it would be
// a straight lie to a player planning a hunt.
.filter((point) => point.running)
// A spawner with no types is a placeholder — nothing to show.
.filter((point) => point.types.length > 0)
.map((point) => {
const place = resolveRegion(point.x, point.y, point.facet, placement, resolveOpts)
return {
name: point.name,
facet: point.facet,
x: point.x,
y: point.y,
width: point.width,
height: point.height,
range: point.range,
maxCount: point.maxCount,
minDelay: point.minDelay,
maxDelay: point.maxDelay,
todStart: point.todStart,
todEnd: point.todEnd,
todMode: point.todMode,
region: place.region,
landmark: place.landmark,
label: place.label,
types: point.types,
}
})
const championsFile = byLabel.get('Config/ChampionSpawns.xml')
const champions = (championsFile ? parseChampions(championsFile.text) : []).map((champ) => {
const facet = resolveFacetName(champ.facet, facetIndex) || champ.facet
return {
...champ,
facet,
slug: slugify(`${facet}-${champ.name}`),
label: resolveRegion(champ.x, champ.y, facet, placement, resolveOpts).label,
}
})
const creatures = aggregateCreatures(points)
const facets = [...new Set(points.map((point) => point.facet))].sort()
const unresolved = points.filter((point) => !point.region && !point.landmark).length
return {
meta: {
generatedAt: new Date().toISOString(),
parserVersion: PARSER_VERSION,
landmarkRadius: options.landmarkRadius ?? undefined,
counts: {
facets: facets.length,
points: points.length,
pointsDisabled: disabled,
creatures: creatures.length,
regions: regions.length,
landmarks: landmarks.length,
champions: champions.length,
unresolvedPoints: unresolved,
},
source,
},
facets,
creatures,
regions,
landmarks,
champions,
points,
}
}
module.exports = {
AtlasSourceError,
PARSER_VERSION,
readSources,
hashSources,
sameSources,
buildAtlas,
aggregateCreatures,
displayName,
}

224
server/utils/uoLinkClient.js Normal file
View File

@@ -0,0 +1,224 @@
// ── uo-link sidecar REST client ────────────────────────────────────────────
//
// Server-side HTTP client for the uo-link sidecar (the bridge to the ServUO
// shard). Same shape as botInternalClient: never throws — every call returns
// { ok, data, status, error } so an admin poll or a public page never 500s just
// because the sidecar/shard is down or restarting.
//
// The base URL + shared-secret token come from the DB-backed uoLinkConfig
// (admin-managed, encrypted at rest) — NOT env vars, and the token is NEVER sent
// to the browser. Every request carries `Authorization: Bearer <token>` and
// `X-UOLink-Version: <protocol>` so a protocol mismatch is caught (409) rather
// than mis-parsed. Config is cached for a few seconds to avoid decrypting the
// token on every call.
const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
const log = require('../core').logger('uo-link-client')
const TIMEOUT_MS = 12000 // sidecar waits up to 10s on the shard before 504
const CONFIG_TTL_MS = 5000
let cachedConfig = null
let cachedAt = 0
// Read (and briefly cache) the connection config incl. decrypted token.
async function resolveConfig() {
const now = Date.now()
if (cachedConfig && now - cachedAt < CONFIG_TTL_MS) return cachedConfig
cachedConfig = await uoLinkConfig.getWithToken()
cachedAt = now
return cachedConfig
}
// Drop the cache after a save so the next call picks up new URL/token immediately.
function invalidateConfig() {
cachedConfig = null
cachedAt = 0
}
// Core request. Returns { ok, data, status, error }. `ok` is true only on a 2xx
// with a parseable JSON body. Non-2xx responses still return their status + body
// so callers can distinguish 503 (shard restarting — transient) from 404.
async function call(path, { method = 'GET', body } = {}) {
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS)
// resolveConfig() decrypts the stored auth token, and decryption THROWS when the
// ciphertext can't be authenticated — SECRET_ENC_KEY was rotated, or a DB dump was
// restored into an environment keyed differently. It must stay INSIDE the try: out
// here it escaped `call()` entirely and 500'd every live-shard route (admin and
// player character/roster/vendor lookups, GET /admin/uo-link/config) instead of
// degrading to "shard unavailable". This module never throws — see the header.
let configResolved = false
try {
const config = await resolveConfig()
configResolved = true
if (!config || !config.baseUrl) {
return { ok: false, status: 0, error: 'uo-link is not configured' }
}
const headers = {
'Content-Type': 'application/json',
'X-UOLink-Version': String(config.protocol || 3),
}
if (config.token) headers.Authorization = `Bearer ${config.token}`
const res = await fetch(`${config.baseUrl}${path}`, {
method,
headers,
body: body ? JSON.stringify(body) : undefined,
signal: controller.signal,
})
let data = null
try {
data = await res.json()
} catch {
// Non-JSON (or empty) body — leave data null; status still reported.
}
if (!res.ok) {
if (res.status === 401) log.warn('uo-link rejected auth token (401)', { path })
if (res.status === 409) log.error('uo-link protocol mismatch (409)', { path, body: data })
return { ok: false, status: res.status, data, error: `sidecar responded ${res.status}` }
}
return { ok: true, status: res.status, data }
} catch (err) {
// A failure before the config resolved is a misconfiguration, not a flaky
// sidecar: log it loudly (and distinctly) so "the shard looks offline" doesn't
// silently mean "the token can no longer be decrypted".
if (!configResolved) {
log.error('uo-link config unreadable — is SECRET_ENC_KEY the key the stored token was encrypted with?', {
path,
message: err.message,
})
return { ok: false, status: 0, error: 'uo-link config unreadable' }
}
log.warn('uo-link call failed', { path, message: err.message })
return { ok: false, status: 0, error: err.message }
} finally {
clearTimeout(timeout)
}
}
// ── Read queries ───────────────────────────────────────────────────────────
// Liveness (no auth required by the sidecar, but we send it anyway).
const health = () => call('/health')
const getCharBySerial = (serial) => call(`/char/serial/${encodeURIComponent(serial)}`)
const getCharBySlot = (account, slot) =>
call(`/char/${encodeURIComponent(account)}/${encodeURIComponent(slot)}`)
const getRoster = (account) => call(`/roster/${encodeURIComponent(account)}`)
const getVendors = (account) => call(`/vendors/${encodeURIComponent(account)}`)
// History / economy series — used for WS-reconnect backfill and public feeds.
function getHistory({ kind, limit = 100 } = {}) {
const params = new URLSearchParams()
if (kind) params.set('kind', kind)
if (limit) params.set('limit', String(limit))
const qs = params.toString()
const suffix = qs ? `?${qs}` : ''
return call(`/history${suffix}`)
}
const getEconomy = (limit = 100) => call(`/economy?limit=${encodeURIComponent(limit)}`)
// Live board / queue projections — snapshotted on WS (re)connect and served from
// our own store thereafter.
const getChamps = () => call('/champs')
const getPages = () => call('/pages')
// Protocol 2.0 board projections — same snapshot-on-connect pattern.
const getGuilds = () => call('/guilds')
const getGovernors = () => call('/governors')
const getHouses = () => call('/houses')
const getPresence = () => call('/online') // aggregate population (count + byFacet/byRegion)
// Protocol 3.0: the shard's published ruleset. Object-shaped, not a board — the
// sidecar answers `{ ruleset: null }` until the shard has published one.
const getRuleset = () => call('/ruleset')
// Protocol 3.0: points/loyalty leaderboards. `/points` is board-shaped (an array
// under `boards`); the per-system read 404s for a system the shard never published.
const getPoints = () => call('/points')
const getPointsBoard = (system) => call(`/points/${encodeURIComponent(system)}`)
// Protocol 3.0: the player-vendor market index. The one PAGED sidecar read — a
// whole-world market does not fit in a response — so it answers with
// `{ vendors, total, limit, offset }` and the caller walks it (see uoLinkSocket).
const getMarket = ({ limit = 200, offset = 0 } = {}) =>
call(`/market?limit=${encodeURIComponent(limit)}&offset=${encodeURIComponent(offset)}`)
// ── Commands ──────────────────────────────────────────────────────────────
const confirmLink = (code, websiteUserId) =>
call('/link/confirm', { method: 'POST', body: { code, websiteUserId: String(websiteUserId) } })
const linkLookup = (account) => call(`/link/${encodeURIComponent(account)}`)
// Account provisioning (Protocol 2.0). createAccount provisions a game account and
// auto-links it to the website user in one step; `ip` is the END USER's browser IP
// (read from the request), which the shard needs for its per-IP account cap — the
// sidecar only sees our server. The password is hashed on the shard and never
// appears in any reply/event/log. unlinkAccount severs a game account's tie from
// the site side. `actor` is the staff/website id, recorded in the shard audit.
const createAccount = ({ actor, account, password, websiteUserId, ip }) =>
call('/accounts/create', {
method: 'POST',
body: { actor, account, password, websiteUserId: websiteUserId == null ? undefined : String(websiteUserId), ip },
})
const unlinkAccount = ({ actor, account }) =>
call(`/link/${encodeURIComponent(account)}`, { method: 'DELETE', body: { actor } })
const postTownCrier = ({ id, lines, durationSec }) =>
call('/towncrier', { method: 'POST', body: { id, lines, durationSec } })
const deleteTownCrier = (id) => call(`/towncrier/${encodeURIComponent(id)}`, { method: 'DELETE' })
// Town Cryer News gump (Protocol 2.1). A full article (title/HTML body/image/URL)
// in the in-game News window; re-posting the same id REPLACES it. `announce`
// (default true on the sidecar) controls whether the criers proclaim the title.
const postNews = ({ id, title, body, image, url, announce }) =>
call('/news', { method: 'POST', body: { id: String(id), title, body, image, url, announce } })
const deleteNews = (id) => call(`/news/${encodeURIComponent(id)}`, { method: 'DELETE' })
// ── Staff write plane (§6) ─────────────────────────────────────────────────
// Every call carries `actor` — the website username of the staff member — set by
// the controller from the session, NEVER from the browser. The shard records it
// for attribution and echoes an admin.audit event back over the WS feed.
const adminKick = ({ actor, account, serial }) =>
call('/admin/kick', { method: 'POST', body: { actor, account, serial } })
const adminBan = ({ actor, account, serial, durationSec, reason }) =>
call('/admin/ban', { method: 'POST', body: { actor, account, serial, durationSec, reason } })
const adminUnban = ({ actor, account }) =>
call('/admin/unban', { method: 'POST', body: { actor, account } })
const adminBroadcast = ({ actor, text, hue }) =>
call('/admin/broadcast', { method: 'POST', body: { actor, text, hue } })
// ── Help-page (support) queue commands (§6) ────────────────────────────────
const respondPage = (pageId, { message, close }) =>
call(`/pages/${encodeURIComponent(pageId)}/respond`, { method: 'POST', body: { message, close } })
const closePage = (pageId) => call(`/pages/${encodeURIComponent(pageId)}/close`, { method: 'POST' })
module.exports = {
invalidateConfig,
health,
getCharBySerial,
getCharBySlot,
getRoster,
getVendors,
getHistory,
getEconomy,
getChamps,
getPages,
getGuilds,
getGovernors,
getHouses,
getPresence,
getRuleset,
getPoints,
getPointsBoard,
getMarket,
confirmLink,
linkLookup,
createAccount,
unlinkAccount,
postTownCrier,
deleteTownCrier,
postNews,
deleteNews,
adminKick,
adminBan,
adminUnban,
adminBroadcast,
respondPage,
closePage,
}

313
server/utils/uoLinkSocket.js Normal file
View File

@@ -0,0 +1,313 @@
// ── uo-link WebSocket ingest client ────────────────────────────────────────
//
// Long-lived client that connects to the sidecar's push-only WS feed and pumps
// every frame through the ingest dispatcher. This is the server's first
// outbound WebSocket. Lifecycle:
// • start() — connect if the config is enabled and has a token; verify the
// ws.hello protocol; backfill missed events via /history on every
// (re)connect (INSERT IGNORE dedupes the overlap); reconnect with
// capped backoff.
// • stop() — close the socket and stop reconnecting (graceful shutdown).
// Connection state is mirrored into uo_link_config (plugin_connected / status /
// last_event_at) so the admin panel and public status endpoint have live data.
const WebSocket = require('ws')
const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
const uoLinkClient = require('./uoLinkClient')
const shardIngest = require('./shardIngest')
const shardState = require('../model/shardState/shardState.model')
const newsGump = require('./newsGump')
const log = require('../core').logger('uo-link-socket')
const BACKOFF_MIN_MS = 1000
const BACKOFF_MAX_MS = 30000
const BACKFILL_LIMIT = 500
let ws = null
let reconnectTimer = null
let backoff = BACKOFF_MIN_MS
let running = false // set by start()/stop(); guards auto-reconnect
let helloSeen = false
const state = {
connected: false,
lastEventAt: null,
lastConnectedAt: null,
reconnects: 0,
protocol: null,
}
function buildUrl(wsUrl, token) {
const sep = wsUrl.includes('?') ? '&' : '?'
return token ? `${wsUrl}${sep}token=${encodeURIComponent(token)}` : wsUrl
}
// One guarded board snapshot: fetch, verify `data[key]` is an array, hand it to
// `apply`, and (when given) log `label` with the row count. Isolated so a
// failed/absent board never aborts the rest of backfill — and so backfill()
// stays a flat sequence rather than nine repetitions of the same guard.
async function snapshot(fetchFn, key, apply, label) {
const res = await fetchFn()
if (!res.ok || !res.data || !Array.isArray(res.data[key])) return
await apply(res.data[key])
if (label) log.info(label, { count: res.data[key].length })
}
// Replay events through the dispatcher oldest-first (history/economy arrive
// newest-first) so latest-wins state settles correctly.
async function ingestReversed(events) {
for (const ev of [...events].reverse()) await shardIngest.ingest(ev, { fromBackfill: true })
}
async function ingestEach(events) {
for (const ev of events) await shardIngest.ingest(ev, { fromBackfill: true })
}
// ── Market backfill ────────────────────────────────────────────────────────
//
// The market is the only board that does not fit in one response, so /market is
// paged and this walks it. Two bounds, both deliberate:
//
// • MARKET_SNAPSHOT_MAX caps the walk. A pathological world (or a sidecar whose
// store was never pruned) must not be able to hang startup — backfill runs
// before the site is serving the live feed, so an unbounded loop here is
// downtime, not slowness.
// • The loop stops on a SHORT page as well as on `total`, because a concurrent
// sweep can shrink the index underneath the walk and paging to a stale total
// would spin.
//
// Vendors are upserted, never reconciled-by-replacement. A vendor absent from the
// snapshot is absent because the sidecar dropped it on vendor.listing.remove —
// which our own ingest already processed — so clearing the table first would only
// create a window where the market page is empty.
const MARKET_SNAPSHOT_MAX = 5000
const MARKET_PAGE = 200
async function backfillMarket() {
let offset = 0
let seen = 0
for (;;) {
const res = await uoLinkClient.getMarket({ limit: MARKET_PAGE, offset })
if (!res.ok || !res.data || !Array.isArray(res.data.vendors)) return
const page = res.data.vendors
if (page.length === 0) break
await ingestEach(page)
seen += page.length
offset += page.length
if (page.length < MARKET_PAGE) break
if (seen >= MARKET_SNAPSHOT_MAX) {
log.warn('market snapshot truncated at the safety cap', {
cap: MARKET_SNAPSHOT_MAX,
total: res.data.total,
})
break
}
if (Number.isFinite(res.data.total) && offset >= res.data.total) break
}
if (seen > 0) log.info('snapshotted player-vendor market from /market', { count: seen })
}
// Pull recent events from the sidecar's own store and replay them through the
// dispatcher (fromBackfill = no SSE re-broadcast). dedupe_key + INSERT IGNORE
// make this idempotent, so overlap with what we already stored is harmless.
async function backfill() {
try {
await snapshot(() => uoLinkClient.getHistory({ limit: BACKFILL_LIMIT }), 'events', ingestReversed, 'backfilled events from /history')
await snapshot(() => uoLinkClient.getEconomy(200), 'series', ingestReversed)
// Champ board + help-page queue have no replay stream — snapshot the
// authoritative current state directly (the sidecar guide's advice for both),
// reconciling our tables to it so a stale row from before a disconnect can't
// linger. Live champ.*/page.* deltas keep them fresh thereafter.
await snapshot(() => uoLinkClient.getChamps(), 'spawns', (s) => shardState.replaceChamps(s), 'snapshotted champ board from /champs')
await snapshot(() => uoLinkClient.getPages(), 'pages', (p) => shardState.replacePages(p), 'snapshotted help-page queue from /pages')
// ── Protocol 2.0 boards ──────────────────────────────────────────────
// Same as champs/pages: snapshot the authoritative current state and
// reconcile our tables to it. Each call is independently guarded so a
// failed/absent board (e.g. no City Loyalty → empty /governors) never wipes
// another. Governors are NOT cleared before upsert (cities are fixed and the
// term-capture is idempotent, so a reconnect can't spawn spurious terms).
await snapshot(() => uoLinkClient.getGuilds(), 'guilds', (g) => shardState.replaceGuilds(g), 'snapshotted guild board from /guilds')
await snapshot(() => uoLinkClient.getGovernors(), 'cities', (c) => shardState.replaceGovernors(c), 'snapshotted governor board from /governors')
await snapshot(() => uoLinkClient.getHouses(), 'houses', ingestEach, 'snapshotted house registry from /houses')
// ── Protocol 3.0 ─────────────────────────────────────────────────────
// The ruleset is object-shaped, not a board, so it can't go through
// snapshot() (which asserts an array under `key`). The shard also re-emits
// world.ruleset on its own connect — this covers the other order, where the
// sidecar was already up and holding the ruleset when WE reconnected.
//
// Routed through the dispatcher rather than straight to shardState, exactly as
// ingestEach does for the array-shaped boards: the two orders must produce the
// same stored frame, and calling setRuleset directly here made this a second
// write path that silently skipped the shard-name normalization the live frame
// gets. One writer, one set of rules.
const ruleset = await uoLinkClient.getRuleset()
if (ruleset.ok && ruleset.data && ruleset.data.ruleset) {
await shardIngest.ingest(ruleset.data.ruleset, { fromBackfill: true })
log.info('snapshotted shard ruleset from /ruleset', { rev: ruleset.data.ruleset.rev })
}
// Points boards ARE array-shaped, so they go through snapshot() — but with
// ingestEach rather than a replace*: there is no points.remove and the shard's
// system set is fixed, so upserting is the whole reconciliation. A system the
// operator has since excluded keeps its last-known board rather than vanishing,
// which is the right answer for a month-scale standing.
await snapshot(() => uoLinkClient.getPoints(), 'boards', ingestEach, 'snapshotted points boards from /points')
await backfillMarket()
const presence = await uoLinkClient.getPresence()
if (presence.ok && presence.data && typeof presence.data.count === 'number') {
await shardState.setPresence(presence.data)
log.info('snapshotted online population from /online', { count: presence.data.count })
}
// Re-assert our published news into the in-game Town Cryer News gump. The
// website is the source of truth; this reconciles the gump on every
// (re)connect (and recovers any article whose original live push failed).
// Silent (announce:false) so a reconnect never re-proclaims old news.
await newsGump.reassertAll()
} catch (err) {
log.warn('backfill failed (continuing on live feed)', { message: err.message })
}
}
function scheduleReconnect() {
if (!running) return
clearTimeout(reconnectTimer)
reconnectTimer = setTimeout(connect, backoff)
log.info(`reconnecting in ${backoff}ms`)
backoff = Math.min(backoff * 2, BACKOFF_MAX_MS)
}
async function connect() {
if (!running) return
let config
try {
config = await uoLinkConfig.getWithToken()
} catch (err) {
log.error('could not read uo-link config', err)
scheduleReconnect()
return
}
if (!config || !config.enabled || !config.wsUrl || !config.token) {
log.info('uo-link WS not started (disabled or missing url/token)')
running = false
return
}
state.protocol = config.protocol || 3
helloSeen = false
const url = buildUrl(config.wsUrl, config.token)
try {
ws = new WebSocket(url)
} catch (err) {
log.error('failed to open WS', err)
scheduleReconnect()
return
}
ws.on('open', handleOpen)
ws.on('message', handleMessage)
ws.on('close', handleClose)
ws.on('error', (err) => {
log.warn('uo-link WS error', { message: err.message })
// 'close' fires after 'error'; reconnect is scheduled there.
})
}
// WS lifecycle handlers, split out of connect() so it stays a flat setup path.
async function handleOpen() {
log.info('uo-link WS connected')
state.connected = true
state.lastConnectedAt = Date.now()
backoff = BACKOFF_MIN_MS
await uoLinkConfig.recordStatus({ status: 'connected', statusDetail: null, pluginConnected: true }).catch(() => {})
await backfill()
}
// A ws.hello frame: mark it seen and, on a protocol mismatch, record the error
// and close (we won't run against an incompatible sidecar).
async function handleHello(event) {
helloSeen = true
if (!event.protocol || event.protocol === state.protocol) return
log.error('uo-link protocol mismatch on ws.hello — closing', { expected: state.protocol, got: event.protocol })
await uoLinkConfig
.recordStatus({ status: 'error', statusDetail: `protocol mismatch: expected ${state.protocol}, got ${event.protocol}` })
.catch(() => {})
running = false
try {
ws.close()
} catch {
/* ignore */
}
}
async function handleMessage(raw) {
let event
try {
event = JSON.parse(raw.toString())
} catch {
log.warn('dropping non-JSON WS frame')
return
}
if (event.kind === 'ws.hello') return handleHello(event)
if (event.kind === 'pong') return // sidecar heartbeat — ignore
state.lastEventAt = Number.isFinite(event.t) ? event.t : Date.now()
await shardIngest.ingest(event)
}
async function handleClose() {
state.connected = false
if (running) state.reconnects += 1
log.warn('uo-link WS closed')
await uoLinkConfig
.recordStatus({ status: running ? 'reconnecting' : 'disconnected', pluginConnected: false })
.catch(() => {})
ws = null
scheduleReconnect()
}
// Begin (or restart) the WS client. Idempotent — a running client is stopped
// first so a config save can re-point it at a new URL/token.
async function start() {
stop()
running = true
backoff = BACKOFF_MIN_MS
await connect()
}
// Stop the client and cancel any pending reconnect. Called on shutdown and
// before a restart.
function stop() {
running = false
clearTimeout(reconnectTimer)
reconnectTimer = null
if (ws) {
try {
ws.removeAllListeners()
ws.close()
} catch {
/* ignore */
}
ws = null
}
state.connected = false
}
// Ingestion stats for the admin panel.
function getState() {
return { ...state, running }
}
module.exports = { start, stop, getState }