diff --git a/server/config/shardStreams.js b/server/config/shardStreams.js new file mode 100644 index 0000000..35add2d --- /dev/null +++ b/server/config/shardStreams.js @@ -0,0 +1,172 @@ +// ── Shard-derived push streams + event → stream mapping ──────────────────── +// +// MODULE-UO CONTENT, still living in core. MODULE_SYSTEM.md §1.8 named +// config/notificationStreams.js as one of the three genuinely entangled files: +// most of its catalog and all of `mapShardEvent` are shard-derived, and it reads +// `PUBLIC_KINDS` out of utils/shardBroadcast. PR 4 split it — core's one stream +// is config/coreStreams.js, and everything shard-shaped is here, in a file that +// moves to module-uo whole in Phase 3. Nothing in core imports it except +// modules/registries.js's registerCore(), which is the one line Phase 3 deletes. +// +// Two families: +// • public / opt-in — no linked game account required; delivered to every +// subscriber. Drawn ONLY from the SSE public allowlist +// (utils/shardBroadcast PUBLIC_KINDS) — a sensitive kind +// can never produce a public push. +// • personal / owner-keyed — require a linked game account; delivered ONLY to +// the owning user's devices (resolved from the event's +// game account via shardLinks), never fanned out publicly. +// +// The payload the relay ever carries is a CONTENT-FREE tickle ({ stream, ref }); +// `ref` is an opaque hint (serial / city / timestamp) the app uses to pull the +// real, ownership-checked content over the authenticated API. So even a leaked +// ntfy topic reveals nothing (docs/android/PLAN.md §11). + +const { PUBLIC_KINDS } = require('../utils/shardBroadcast') + +const STREAMS = [ + { + id: 'server.status', + label: 'Server up / down', + description: 'The shard comes online or goes offline.', + personal: false, + requiresLinkedAccount: false, + }, + { + id: 'idoc.warning', + label: 'IDOC warnings', + description: 'A house falls into its final (IDOC) decay stage.', + personal: false, + requiresLinkedAccount: false, + }, + { + id: 'champ.start', + label: 'Champion spawn starts', + description: 'A champion spawn becomes active.', + personal: false, + requiresLinkedAccount: false, + }, + { + id: 'governor.election', + label: 'Governor elections', + description: 'A town elects a new governor.', + personal: false, + requiresLinkedAccount: false, + }, + { + id: 'vendor.sale', + label: 'Your vendor sold an item', + description: 'One of your player vendors made a sale.', + personal: true, + requiresLinkedAccount: true, + }, + { + id: 'house.idoc', + label: 'Your house entered IDOC', + description: 'One of your houses fell into its final decay stage.', + personal: true, + requiresLinkedAccount: true, + }, + { + id: 'account.login', + label: 'A login to your account', + description: 'An authentication attempt against your game account.', + personal: true, + requiresLinkedAccount: true, + }, +] + +// The owner-keyed subset, needed by mapShardEvent's public-safety filter below. +// Derived from this file's own catalog rather than read back out of the registry: +// the filter is about THESE streams, and a module must not be able to weaken it +// by registering something that happens to share an id. +const PERSONAL_STREAMS = new Set(STREAMS.filter((s) => s.personal).map((s) => s.id)) + +// Per-process transition state so full-state upserts (champ.update / city.update +// are upserts, not discrete "started"/"elected" events — see docs/link +// PROTOCOL_2 §383) only fire once, on an actual transition. Injectable so tests +// pass a fresh tracker; a module-level default backs the live dispatcher. +function createTracker() { + return { champActive: new Map(), cityGovernor: new Map() } +} +const defaultTracker = createTracker() + +// Per-kind mappers, each pushing 0+ targets onto `out` (and updating `tracker` +// for the upsert-transition kinds). Split out of mapShardEvent so that function +// stays a trivial dispatch + the public-safety filter. +const serverStatusUp = (event, tracker, out) => + out.push({ streamId: 'server.status', ref: `up:${event.bootId || ''}` }) +const serverStatusDown = (event, tracker, out) => out.push({ streamId: 'server.status', ref: 'down' }) + +const EVENT_MAPPERS = { + 'server.hello': serverStatusUp, + 'server.shutdown': serverStatusDown, + 'server.crashed': serverStatusDown, + 'house.decay': (event, tracker, out) => { + if (String(event.to).toUpperCase() !== 'IDOC') return + const ref = String(event.serial ?? '') + out.push({ streamId: 'idoc.warning', ref }) // public — location only + if (event.ownerAcct) { + out.push({ streamId: 'house.idoc', ref, ownerAccount: event.ownerAcct }) // personal + } + }, + 'champ.update': (event, tracker, out) => { + const { serial } = event + if (serial == null) return + const wasActive = tracker.champActive.get(serial) === true + const isActive = event.active === true + tracker.champActive.set(serial, isActive) + if (isActive && !wasActive) out.push({ streamId: 'champ.start', ref: String(serial) }) + }, + 'champ.remove': (event, tracker) => { + if (event.serial != null) tracker.champActive.delete(event.serial) + }, + 'city.update': (event, tracker, out) => { + const { city } = event + if (!city) return + const gov = event.governor && event.governor.serial != null ? String(event.governor.serial) : null + const prev = tracker.cityGovernor.get(city) + tracker.cityGovernor.set(city, gov) + // Only a real transition to a new governor, and never on first sight + // (prev === undefined) so a reconnect snapshot isn't read as an election. + if (prev !== undefined && gov && gov !== prev) { + out.push({ streamId: 'governor.election', ref: String(city) }) + } + }, + 'vendor.sale': (event, tracker, out) => { + if (event.ownerAcct) { + out.push({ streamId: 'vendor.sale', ref: String(event.t ?? ''), ownerAccount: event.ownerAcct }) + } + }, + 'account.login.attempt': (event, tracker, out) => { + if (event.acct) { + out.push({ streamId: 'account.login', ref: String(event.t ?? ''), ownerAccount: event.acct }) + } + }, +} + +// Map one shard event → an array of targets ({ streamId, ref, ownerAccount? }). +// May yield 0, 1, or 2 targets (an owner house.decay produces both the public +// idoc.warning and the personal house.idoc). Pure given `tracker`. +function mapShardEvent(event, tracker = defaultTracker) { + if (!event || typeof event.kind !== 'string') return [] + const kind = event.kind + const out = [] + + const mapper = EVENT_MAPPERS[kind] + if (mapper) mapper(event, tracker, out) + + // Defense in depth: a PUBLIC (non-personal) target may only ride a public-safe + // kind. Personal targets are owner-keyed and delivered solely to the owner, so + // they are exempt from the public allowlist (that is the whole point of the + // owner-keyed split). This guarantees a sensitive kind can never leak publicly + // even if a future mapping case is added carelessly. + // + // This filter, the kinds it reads and the streams it protects now all live in + // one file and move together — the reason PR 4 dropped the contract's + // `mapEvent` half rather than leaving the mapping in core and the catalog in a + // module (MODULE_API.md §2.4). + return out.filter((t) => (PERSONAL_STREAMS.has(t.streamId) ? true : PUBLIC_KINDS.has(kind))) +} + +module.exports = { STREAMS, mapShardEvent, createTracker, PERSONAL_STREAMS } diff --git a/server/core.js b/server/core.js new file mode 100644 index 0000000..076a05d --- /dev/null +++ b/server/core.js @@ -0,0 +1,112 @@ +// ── Everything this module reaches in core ───────────────────────────────── +// +// `ctx` arrives once, as an argument to `register()` (MODULE_API.md §2.3). The +// code below it — models, utils, controllers — is ordinary Node that requires +// its dependencies at file scope, the way it did when it lived in core. This +// file is what lets both be true. +// +// **The shape is a lazy accessor, not a stored reference, and that is the whole +// point.** A ported file writes +// +// const { query } = require('../../core') +// +// at require time, which is before `register()` has been called and therefore +// before any `ctx` exists. Handing out `ctx.db.query` there would hand out +// `undefined`, permanently, and the failure would surface much later as a +// TypeError inside a model. So every export here is a stable function that +// resolves `ctx` when it is CALLED. Require order stops mattering, and the port +// stays a one-line import change per file rather than a signature change per +// function. +// +// The other half of the same rule: nothing here may be destructured off `ctx` +// at init time either, for the same reason in the other direction — core is +// free to hand over a getter (`ctx.site.baseUrl` is one), and a value captured +// once is a value that cannot change. +// +// If `ctx` is missing, every accessor throws with the same message. That is +// deliberate: the only way to reach one before `register()` is a require cycle +// or a test that forgot to call `init`, and both want naming, not `undefined`. + +let ctx = null + +function need() { + if (!ctx) { + throw new Error('module-uo: core accessed before register() — see server/core.js') + } + return ctx +} + +/** Called once, first thing in `register()`. */ +function init(value) { + ctx = value +} + +/** Test seam. Nothing in the module calls this; there is no de-registration. */ +function _reset() { + ctx = null +} + +// A logger that can be taken at require time and used after `register()`. +// +// Ported files write `const log = require('../core').logger('shard-ingest')` at +// file scope — the same shape as core's `require('./logger')('…')` — so the +// object returned has to exist before `ctx` does. It is a façade whose four +// methods each resolve the real logger on call. Core namespaces it with the +// module id, so these come out as `[uo:shard-ingest]`. +function logger(namespace) { + const call = (level) => (message, meta) => need().log(namespace)[level](message, meta) + return { error: call('error'), warn: call('warn'), info: call('info'), debug: call('debug') } +} + +module.exports = { + init, + _reset, + logger, + + // Shared server dependencies. Core owns exactly one express, as it owns + // exactly one React on the client, and for the same reason: a second copy in + // the process is a second Router prototype and a second set of instanceof + // checks. A module lives outside core's `server/`, so it could not resolve + // these for itself even if it were allowed to (§7.2). + get express() { return need().express }, + get validator() { return need().validator }, + + // Database. `query` is the one every `*.db.js` uses; `pool` is for the + // streamed atlas import, which needs a connection it can hold. + query: (...args) => need().db.query(...args), + get pool() { return need().db.pool }, + + // Core state a module may read or append to, each narrowed to what is + // actually used (§2.3). + settings: { + get: (...args) => need().settings.get(...args), + set: (...args) => need().settings.set(...args), + getInstanceName: (...args) => need().settings.getInstanceName(...args), + }, + activity: { log: (...args) => need().activity.log(...args) }, + users: { getById: (...args) => need().users.getById(...args) }, + posts: { + listAll: (...args) => need().posts.listAll(...args), + getById: (...args) => need().posts.getById(...args), + linkAnnounceJob: (...args) => need().posts.linkAnnounceJob(...args), + markAnnounced: (...args) => need().posts.markAnnounced(...args), + }, + auth: { getUserFromRequest: (...args) => need().auth.getUserFromRequest(...args) }, + push: { publish: (...args) => need().push.publish(...args) }, + secretBox: { + encrypt: (...args) => need().secretBox.encrypt(...args), + decrypt: (...args) => need().secretBox.decrypt(...args), + }, + get uploads() { return need().uploads }, + + // Middleware. Taken as values rather than wrapped, because express stores the + // function reference at mount time — a wrapper would be what ends up in the + // stack, and `requireRole('admin')` returns a middleware rather than being + // one. Routers are built inside `register()`, so `ctx` is set by then. + get middleware() { return need().middleware }, + + // Deployment facts. + get baseUrl() { return need().site.baseUrl }, + get moduleRoot() { return need().paths.moduleRoot }, + get moduleId() { return need().moduleId }, +} diff --git a/server/data/spawnAtlas.art.example.json b/server/data/spawnAtlas.art.example.json new file mode 100644 index 0000000..309108a --- /dev/null +++ b/server/data/spawnAtlas.art.example.json @@ -0,0 +1,24 @@ +{ + "_comment": [ + "OPTIONAL operator-supplied creature art for the spawn atlas. Copy this file to", + "spawnAtlas.art.json (same directory) and edit it, then restart the server or run", + "`npm run atlas:import` — the art map is read on every atlas refresh.", + "", + "This project ships NO creature artwork and never will. UO sprites live in your", + "own client's .mul/.uop files and are yours to extract, not ours to redistribute.", + "If you want art on the atlas pages, export it yourself (UOFiddler, ClassicUO's", + "tooling, or any art extractor), drop the images under server/uploads/atlas/, and", + "map each creature slug to its file name here.", + "", + "Both spawnAtlas.art.json and server/uploads/ are gitignored, so neither the map", + "nor the images can be committed by accident.", + "", + "Keys are creature slugs, as reported by the atlas API and derived from the type", + "names in your own shard's Spawns/*.xml. Values are file names relative to", + "server/uploads/atlas/. Any creature with no entry here simply renders without", + "art — that is the default and fully supported state, not a degraded one." + ], + "lizardman": "lizardman.png", + "orc": "orc.png", + "dragon": "dragon.png" +} diff --git a/server/db/purge.sql b/server/db/purge.sql new file mode 100644 index 0000000..7215d90 --- /dev/null +++ b/server/db/purge.sql @@ -0,0 +1,52 @@ +-- ── module-uo's teardown ────────────────────────────────────────────────── +-- +-- Destructive, and run ONLY by an explicit admin purge (MODULE_API.md §2.6). +-- Nothing on the boot path ever executes this file — uninstalling a module +-- leaves its data alone, and removing the data is a separate decision an +-- operator has to make on purpose. +-- +-- It exists because `schema.sql` does. A module that can create tables and +-- cannot drop them leaves an operator with orphaned data and no supported way +-- to remove it, which is why core refuses to load a module that declares one +-- without the other. +-- +-- **The order is the reverse of creation, and that is load-bearing**: two of +-- these tables carry a foreign key into core's `users`, and several reference +-- each other. Dropping a parent before its children fails on the constraint, +-- and a purge that fails halfway is worse than one that does not run — it +-- leaves exactly the orphaned data this file exists to remove. `IF EXISTS` on +-- every line so a partially-installed module still tears down cleanly. +-- +-- What is NOT here, deliberately: rows this module wrote into core's tables. +-- `notification_subs` rows for `shard.*` streams and `announce_job_legs` rows +-- with leg `towncrier` belong to core's tables, and a module does not delete +-- from those — core prunes them when it drops the registrations, which it can +-- do because it knows which registrant owned what. + +DROP TABLE IF EXISTS `shard_atlas_pending`; +DROP TABLE IF EXISTS `shard_atlas_meta`; +DROP TABLE IF EXISTS `shard_cliloc_meta`; +DROP TABLE IF EXISTS `shard_clilocs`; +DROP TABLE IF EXISTS `shard_champion_spawns`; +DROP TABLE IF EXISTS `shard_landmarks`; +DROP TABLE IF EXISTS `shard_regions`; +DROP TABLE IF EXISTS `shard_spawn_point_types`; +DROP TABLE IF EXISTS `shard_spawn_points`; +DROP TABLE IF EXISTS `shard_spawn_creatures`; +DROP TABLE IF EXISTS `shard_feature_visibility`; +DROP TABLE IF EXISTS `shard_vendor_items`; +DROP TABLE IF EXISTS `shard_vendors`; +DROP TABLE IF EXISTS `shard_points_boards`; +DROP TABLE IF EXISTS `shard_ruleset`; +DROP TABLE IF EXISTS `shard_presence`; +DROP TABLE IF EXISTS `shard_governor_terms`; +DROP TABLE IF EXISTS `shard_governors`; +DROP TABLE IF EXISTS `shard_guilds`; +DROP TABLE IF EXISTS `shard_pages`; +DROP TABLE IF EXISTS `shard_champs`; +DROP TABLE IF EXISTS `shard_account_links`; +DROP TABLE IF EXISTS `shard_houses`; +DROP TABLE IF EXISTS `shard_economy`; +DROP TABLE IF EXISTS `shard_online`; +DROP TABLE IF EXISTS `shard_events`; +DROP TABLE IF EXISTS `uo_link_config`; diff --git a/server/db/schema.sql b/server/db/schema.sql new file mode 100644 index 0000000..89d4bf5 --- /dev/null +++ b/server/db/schema.sql @@ -0,0 +1,617 @@ +-- ── module-uo's schema fragment ─────────────────────────────────────────── +-- +-- Replayed by core on EVERY boot, after core's own schema.sql and before +-- seedDefaults (MODULE_API.md §2.6). Everything here is therefore idempotent: +-- every CREATE TABLE carries IF NOT EXISTS and every ALTER carries +-- IF NOT EXISTS, because a statement that succeeds once and fails afterwards +-- presents as a module that worked until the first restart. +-- +-- Core validates this file at LOAD time, before anything mounts — statement by +-- statement, split by the same code that splits core's schema. The rules it +-- enforces and the reason each exists: +-- +-- • Leading verbs are an allowlist: CREATE, ALTER, INSERT, UPDATE. Not a +-- DROP denylist — this file replays every boot, so TRUNCATE or DELETE would +-- empty a table on each restart. +-- • Every table is prefixed. `shard_*` and `uo_link_*` are grandfathered to +-- this module by name (loader.js LEGACY_TABLE_PREFIXES): they predate the +-- module system by two years, they hold live data, and renaming them would +-- be a migration this workstream deliberately does not do. Every module +-- written after this one prefixes with its own id. +-- • No table core declares may appear here, and no table another module has +-- claimed. +-- +-- Two tables carry a foreign key INTO core (`users`), which is allowed and is +-- why the replay order matters: core's schema is already in place when this +-- runs, so `users` exists. The reverse — a core table referencing one of these +-- — does not occur and must not: it would make core's schema depend on a module +-- being installed. +-- +-- Teardown is `purge.sql`, which is never run by a boot. See it for the drop +-- order, which is the reverse of the dependency order here. + + +-- ── uo-link sidecar ──────────────────────────────────────────────────────── +-- Connection config for the uo-link sidecar (the HTTP + WebSocket bridge to the +-- ServUO shard). Singleton row (id = 1), mirroring bot_config/email_config: the +-- DB only ever holds the AES-256-GCM-encrypted shared-secret auth token, never +-- plaintext, and it is only decrypted server-side (to call the sidecar). It is +-- never returned to the admin UI — responses expose only `hasToken`. base_url is +-- the REST endpoint, ws_url the live-feed endpoint; both are configurable because +-- in production the sidecar runs on a different host from the website. `status`/ +-- `plugin_connected`/`last_event_at`/`boot_id` mirror the sidecar's last-known +-- state for the admin panel between polls; `boot_id` tracks server.hello.bootId +-- so a shard restart can be detected (and caches dropped). +CREATE TABLE IF NOT EXISTS uo_link_config ( + id INT PRIMARY KEY DEFAULT 1, + base_url VARCHAR(255) NULL, + ws_url VARCHAR(255) NULL, + auth_token_enc TEXT NULL, + protocol INT NOT NULL DEFAULT 3, + enabled TINYINT(1) NOT NULL DEFAULT 0, + status VARCHAR(20) NOT NULL DEFAULT 'disconnected', + status_detail VARCHAR(500) NULL, + plugin_connected TINYINT(1) NOT NULL DEFAULT 0, + last_event_at DATETIME NULL, + boot_id VARCHAR(64) NULL, + updated_by INT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT fk_uo_link_config_user FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + CONSTRAINT chk_uo_link_config_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Append-only log of notable shard events ingested from the uo-link WebSocket +-- feed. The site OWNS this data (it does not query the sidecar's SQLite): the WS +-- client writes here, and the public/admin read endpoints + live feeds read from +-- here. Only "notable" kinds are logged (sales, deaths, murders, mob.killed, +-- IDOC transitions, quests, skill.gain, fame/karma, audit.*, cheat.*, link.*, +-- server.*). High-frequency kinds (char.vitals, economy.supply) are NOT logged +-- here — they update shard_online / shard_economy instead, keeping the log lean. +-- dedupe_key = sha256(kind + t + stable-json(payload)) truncated to 40 hex chars +-- (fits CHAR(40)); with the UNIQUE index it makes INSERT IGNORE idempotent so +-- WS-reconnect backfill never double-inserts. +CREATE TABLE IF NOT EXISTS shard_events ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + kind VARCHAR(48) NOT NULL, + t BIGINT NOT NULL, -- event time, epoch ms (from the sidecar) + boot_id VARCHAR(64) NULL, -- shard boot id at ingest (server.hello.bootId) + payload JSON NOT NULL, -- the full event object + dedupe_key CHAR(40) NOT NULL UNIQUE, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + INDEX idx_shard_events_kind_t (kind, t), + INDEX idx_shard_events_t (t) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Current online players. Upserted on mob.login, refreshed on char.vitals, and +-- removed on mob.logout. Cleared wholesale when the shard restarts (a new +-- server.hello.bootId). web_id is the linked website user id (present when the +-- account is linked), so the roster can be correlated to site accounts. +CREATE TABLE IF NOT EXISTS shard_online ( + serial VARCHAR(20) NOT NULL PRIMARY KEY, -- mobile serial (opaque hex key) + name VARCHAR(120) NULL, + acct VARCHAR(120) NULL, + web_id INT NULL, + map VARCHAR(40) NULL, + x INT NULL, + y INT NULL, + z INT NULL, + hits INT NULL, + hits_max INT NULL, + mana INT NULL, + mana_max INT NULL, + stam INT NULL, + stam_max INT NULL, + str INT NULL, + dex INT NULL, + `int` INT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_shard_online_acct (acct) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Total-gold-supply time series (from the periodic economy.supply event). Kept +-- append-only so the public status page can render a supply-over-time sparkline. +CREATE TABLE IF NOT EXISTS shard_economy ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + accounts INT NULL, -- number of accounts included in the total + gold BIGINT NULL, -- total gold supply across all accounts + t BIGINT NOT NULL, -- sample time, epoch ms + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + INDEX idx_shard_economy_t (t) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Current decay stage per house, upserted on house.decay. is_idoc is a derived +-- flag (stage == 'IDOC') so the public "houses in danger" list is a cheap +-- indexed lookup rather than a scan. +CREATE TABLE IF NOT EXISTS shard_houses ( + serial VARCHAR(20) NOT NULL PRIMARY KEY, + stage VARCHAR(24) NULL, -- Somewhat | Fairly | Greatly | IDOC | Collapsed | ... + map VARCHAR(40) NULL, + x INT NULL, + y INT NULL, + z INT NULL, + region VARCHAR(120) NULL, + name VARCHAR(160) NULL, + owner_serial VARCHAR(20) NULL, + owner_acct VARCHAR(120) NULL, + built_on DATETIME NULL, + last_refreshed DATETIME NULL, + is_idoc TINYINT(1) NOT NULL DEFAULT 0, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_shard_houses_idoc (is_idoc) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- a single user may link several game accounts. +CREATE TABLE IF NOT EXISTS shard_account_links ( + account VARCHAR(120) NOT NULL PRIMARY KEY, + user_id INT NOT NULL, + char_name VARCHAR(120) NULL, + linked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT fk_shard_links_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_shard_links_user (user_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Current champion-spawn board, upserted on champ.update and removed on +-- champ.remove. Mirrors the sidecar's /champs projection into our own store so +-- the public Champions page (and its live deltas) survive a shard outage, the +-- same way shard_online / shard_houses do. Three families share one table, told +-- apart by `category` (champion | mini | sea); category-specific fields (level, +-- kills, boss, restartAt, hits, …) live in the JSON `payload` so the schema does +-- not have to model every variant. +CREATE TABLE IF NOT EXISTS shard_champs ( + serial VARCHAR(20) NOT NULL PRIMARY KEY, -- controller/mobile serial (opaque hex) + category VARCHAR(16) NULL, -- champion | mini | sea + type VARCHAR(80) NULL, + name VARCHAR(120) NULL, + status VARCHAR(16) NULL, -- active | cooldown | dormant + active TINYINT(1) NOT NULL DEFAULT 0, + map VARCHAR(40) NULL, + x INT NULL, + y INT NULL, + z INT NULL, + boss_up TINYINT(1) NOT NULL DEFAULT 0, + payload JSON NOT NULL, -- the full champ.update object + t BIGINT NULL, -- event time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_shard_champs_category (category) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Current open help-page (support ticket) queue, upserted on page.new/page.updated +-- and removed on page.closed. Snapshotted authoritatively from the sidecar's +-- GET /pages on every (re)connect. page_id is the sender's serial (one page per +-- player). Staff-only data — served on the admin channel, never public. +CREATE TABLE IF NOT EXISTS shard_pages ( + page_id VARCHAR(20) NOT NULL PRIMARY KEY, -- sender serial (one page per player) + type VARCHAR(40) NULL, -- Bug | Stuck | Account | Question | ... + sender_name VARCHAR(120) NULL, + sender_acct VARCHAR(120) NULL, + web_id INT NULL, -- linked website user id, if any + message TEXT NULL, + map VARCHAR(40) NULL, + x INT NULL, + y INT NULL, + z INT NULL, + sent_ms BIGINT NULL, -- when the page was opened, epoch ms + handled TINYINT(1) NOT NULL DEFAULT 0, -- a staffer claimed it in game + handler VARCHAR(120) NULL, + payload JSON NOT NULL, -- the full page.new/updated object + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_shard_pages_handled (handled) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Guild roster board (Protocol 2.0). Upserted on guild.update (a full-state +-- snapshot emitted only on change) and removed on guild.remove. The leader is an +-- actor object flattened into leader_* columns; the full event is kept in +-- `payload` for anything not hoisted. Mirrors the sidecar's GET /guilds +-- projection into our store so the public Guilds page survives a shard outage. +CREATE TABLE IF NOT EXISTS shard_guilds ( + id INT NOT NULL PRIMARY KEY, -- in-game guild id + name VARCHAR(120) NULL, + abbr VARCHAR(24) NULL, + members INT NULL, + online INT NULL, + alliance VARCHAR(120) NULL, + leader_serial VARCHAR(20) NULL, + leader_name VARCHAR(120) NULL, + leader_acct VARCHAR(120) NULL, + leader_web_id INT NULL, + payload JSON NOT NULL, -- the full guild.update object + t BIGINT NULL, -- event time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_shard_guilds_name (name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Town-governor board (Protocol 2.0, City Loyalty). One row per city, upserted on +-- city.update (full-state, emitted only on change; there is no remove event since +-- the set of cities is fixed). governor / governorElect are actor objects +-- flattened into columns; the full event is kept in `payload`. Empty on shards +-- that do not run the City Loyalty system. +CREATE TABLE IF NOT EXISTS shard_governors ( + city VARCHAR(40) NOT NULL PRIMARY KEY, -- Britain | Moonglow | ... + governor_serial VARCHAR(20) NULL, + governor_name VARCHAR(120) NULL, + governor_acct VARCHAR(120) NULL, + governor_web_id INT NULL, + elect_serial VARCHAR(20) NULL, + elect_name VARCHAR(120) NULL, + elect_acct VARCHAR(120) NULL, + election_phase VARCHAR(16) NULL, -- none | nominate | vote | pending + candidates INT NULL, + auto_pick_at DATETIME NULL, + payload JSON NOT NULL, -- the full city.update object + t BIGINT NULL, -- event time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Governor term history — the "who governed when" ledger behind the Governors +-- board. Captured from day one (history cannot be backfilled) on every observed +-- governor CHANGE: the open term (ended_at IS NULL) is closed and a new one +-- opened. `votes` stays NULL — the city.update feed exposes only the candidate +-- COUNT and election phase, not per-candidate tallies, so we record who governed +-- and when (reliable) and never fabricate vote numbers. The look-back UI ("who +-- were all the governors of Britain?") reads this table. +CREATE TABLE IF NOT EXISTS shard_governor_terms ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + city VARCHAR(40) NOT NULL, + governor_serial VARCHAR(20) NULL, + governor_name VARCHAR(120) NULL, + governor_acct VARCHAR(120) NULL, + governor_web_id INT NULL, + started_at BIGINT NOT NULL, -- term start, epoch ms + ended_at BIGINT NULL, -- term end epoch ms (NULL = current) + votes INT NULL, -- not in the feed (reserved) + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + INDEX idx_shard_gov_terms_city (city, started_at), + INDEX idx_shard_gov_terms_open (city, ended_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Online-population snapshot (Protocol 2.0). Singleton row (id = 1) holding the +-- latest presence.online aggregate: total count plus per-facet and per-region +-- breakdown maps (stored as JSON). Distinct from shard_online (per-player) — this +-- is the rolled-up headcount the public "Players Online" widget renders. The +-- time series, if ever needed, is available from GET /history?kind=presence.online. +CREATE TABLE IF NOT EXISTS shard_presence ( + id INT PRIMARY KEY DEFAULT 1, + count INT NOT NULL DEFAULT 0, + by_facet JSON NULL, -- { "Felucca": 12, "Trammel": 30 } + by_region JSON NULL, -- { "Britain": 18, "Wilderness": 9 } + t BIGINT NULL, -- snapshot time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_presence_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- The shard's published ruleset (Protocol 3.0 world.ruleset). Singleton row +-- (id = 1) holding the latest frame: expansion, which optional systems are on, +-- skill/stat caps, account and house limits, champion scroll rules, the +-- save/restart schedule. The shard re-emits it on every sidecar connect, so this +-- row is simply overwritten; `rev` is the shard's own FNV-1a of the body, which +-- distinguishes "same ruleset, re-sent on reconnect" from "an operator changed a +-- .cfg". No row at all means the shard has never published one — served as null, +-- which the rules page renders differently from a published ruleset. +CREATE TABLE IF NOT EXISTS shard_ruleset ( + id INT PRIMARY KEY DEFAULT 1, + rev VARCHAR(32) NULL, + expansion VARCHAR(16) NULL, -- hoisted for cheap display + payload JSON NOT NULL, -- the whole world.ruleset frame + t BIGINT NULL, -- frame time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_ruleset_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Points/loyalty leaderboards (Protocol 3.0 points.board). One row per point +-- system, keyed by the shard's own PointsType name. The shard publishes ~25 of +-- these (Queen's Loyalty, Void Pool, the nine city loyalties, …), each a standing +-- players accumulate over months. +-- +-- The top-N list stays inside `payload` rather than being normalized into a +-- shard_points_entries table. It is a fixed-size list (10 by default) that is only +-- ever read whole, exactly like shard_governors.candidates — normalizing it would +-- buy nothing until something needs a per-character reverse lookup, and a +-- character's own standings already ride inside char.profile instead. +-- +-- No delete path: the shard's set of systems is fixed at startup, so there is no +-- points.remove to mirror. +CREATE TABLE IF NOT EXISTS shard_points_boards ( + system VARCHAR(48) PRIMARY KEY, -- PointsType name, e.g. QueensLoyalty + name VARCHAR(128) NULL, -- resolved display name, if the shard sent a literal + name_cliloc INT NULL, -- cliloc id when the name is a TextDefinition number + max_points BIGINT NULL, + players INT NULL, -- players actually holding points in this system + show_on_gump TINYINT(1) NOT NULL DEFAULT 1, -- the shard's own "is this player-facing?" flag + payload JSON NOT NULL, -- the whole points.board frame, incl. `top` + t BIGINT NULL, -- frame time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Player-vendor market index (Protocol 3.0 vendor.listing). One row per player +-- vendor and one per priced listing, so the site can offer the search the in-game +-- Vendor Search gump offers — from outside the game. +-- +-- The shard sweeps vendors round-robin and emits one AUTHORITATIVE frame per +-- vendor, so ingest is delete-then-insert of that vendor's items inside one +-- transaction (see shardMarket.db.js). No foreign key from items to vendors, in +-- keeping with every other shard_* table: the ingest transaction is what keeps +-- them consistent, and an FK would turn a malformed frame into a failed write +-- rather than a dropped row. +-- +-- Only vendors whose owner left the in-game Vendor Search flag ON are ever sent, +-- so a player who hid their shop in game is hidden here too — see BridgeMarket.cs. +CREATE TABLE IF NOT EXISTS shard_vendors ( + serial VARCHAR(20) NOT NULL PRIMARY KEY, -- "0x40001234" + shop_name VARCHAR(160) NULL, + owner_serial VARCHAR(20) NULL, + owner_name VARCHAR(64) NULL, + map VARCHAR(40) NULL, + x INT NULL, + y INT NULL, + z INT NULL, + region VARCHAR(80) NULL, + house VARCHAR(160) NULL, -- the house SIGN's name, not the house type + item_count INT NOT NULL DEFAULT 0, -- listings published in the frame + item_total INT NOT NULL DEFAULT 0, -- listings the shop actually holds + truncated TINYINT(1) NOT NULL DEFAULT 0, -- item_total > item_count + t BIGINT NULL, -- frame time, epoch ms + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_shard_vendors_owner (owner_name), + INDEX idx_shard_vendors_map (map), + INDEX idx_shard_vendors_region (region), + -- The market page's staleness banner is MIN(updated_at) over this column: the + -- round-robin sweep means the oldest row is how far behind the index can be. + INDEX idx_shard_vendors_updated (updated_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- One priced listing. Unlike the points board's top-N — a fixed-size list read +-- whole — these are the searchable rows the whole feature exists for, so they are +-- normalized rather than left inside a payload column, and there is no payload +-- column on shard_vendors at all. +-- +-- `display_name` is DENORMALIZED at ingest: the shard sends `cliloc` (the item's +-- LabelNumber) and, rarely, a literal `name`, and resolving 50 clilocs per page +-- at query time would make the cliloc table a join on the hot path AND make +-- search-by-name impossible. Resolving once on write buys the index. It is +-- re-resolved in bulk after a cliloc import, because the diff sweep will not +-- re-send an unchanged shop just because the site learned what its items are +-- called. +CREATE TABLE IF NOT EXISTS shard_vendor_items ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + vendor_serial VARCHAR(20) NOT NULL, + serial VARCHAR(20) NOT NULL, + item_id INT NOT NULL DEFAULT 0, -- ItemID (the art/graphic id) + hue INT NOT NULL DEFAULT 0, + amount INT NOT NULL DEFAULT 1, + price BIGINT NOT NULL DEFAULT 0, + name VARCHAR(160) NULL, -- the item's literal Name, null for most + cliloc INT NULL, -- LabelNumber, resolved against shard_clilocs + display_name VARCHAR(160) NULL, -- resolved at ingest; what search matches + child TINYINT(1) NOT NULL DEFAULT 0, -- priced by an enclosing container, not itself + INDEX idx_shard_vendor_items_vendor (vendor_serial), + INDEX idx_shard_vendor_items_price (price), + INDEX idx_shard_vendor_items_item (item_id), + INDEX idx_shard_vendor_items_name (display_name), + -- Search filters on name and sorts on price; the composite covers the common + -- "cheapest matching X" without a filesort over the whole table. + INDEX idx_shard_vendor_items_name_price (display_name, price) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Per-feature visibility for every shard-derived surface (Protocol 3.0). One row +-- per feature; an absent row means "use the compiled default", and the compiled +-- defaults reproduce the behavior that shipped before v3 — so an empty table is +-- a no-op. See utils/shardVisibility.js for the catalog and the ladder, and +-- docs/link/v3.md §3 for the contract. +-- +-- audience the minimum rung on anonymous < logged_in < player < staff < admin +-- stream whether this feature's kinds fan out over SSE at all (the market +-- index ships with this off: no page needs a live firehose of +-- whole vendor inventories) +-- field_rules {"": ""} for SENSITIVE fields only. `acct` and +-- `webId` are admin-only always and are rejected here — they are +-- not in-game visible and are deliberately not configurable. +CREATE TABLE IF NOT EXISTS shard_feature_visibility ( + feature VARCHAR(48) NOT NULL PRIMARY KEY, + enabled TINYINT(1) NOT NULL DEFAULT 1, + audience VARCHAR(20) NOT NULL DEFAULT 'anonymous', + stream TINYINT(1) NOT NULL DEFAULT 1, + field_rules JSON NULL, + updated_by INT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- `facets` is a per-facet point count, so the facet filter and "where does this +-- live" both answer without touching shard_spawn_points. +CREATE TABLE IF NOT EXISTS shard_spawn_creatures ( + slug VARCHAR(120) NOT NULL PRIMARY KEY, -- slugified class name; the /atlas/:slug key + name VARCHAR(120) NOT NULL, -- display spelling chosen by the build + total INT NOT NULL DEFAULT 0, + points INT NOT NULL DEFAULT 0, + facets JSON NULL, -- { "Felucca": 171, "Trammel": 160, ... } + -- Operator-supplied artwork, always NULL on a fresh import. The repo ships no + -- creature art: sprites live in the operator's own client .mul/.uop files and + -- are theirs to extract and place under uploads/atlas/. The UI renders without + -- art when this is NULL, which is the normal case. + art VARCHAR(255) NULL, + -- Plain INDEX, deliberately NOT FULLTEXT: ~800 rows makes a LIKE scan free, + -- and FULLTEXT's min-token-length would break searches for names like "orc". + INDEX idx_shard_spawn_creatures_name (name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- One row per spawner. `region`/`landmark` are the resolved place name — the +-- point-in-rect transform that turns "5411,1234" into "Despise" — and `label` is +-- the resolved display string (region, else landmark, else 'Wilderness'). +CREATE TABLE IF NOT EXISTS shard_spawn_points ( + id INT AUTO_INCREMENT PRIMARY KEY, + facet VARCHAR(40) NOT NULL, + name VARCHAR(120) NULL, -- the ServUO spawner's own name + x INT NOT NULL, + y INT NOT NULL, + width INT NOT NULL DEFAULT 0, + height INT NOT NULL DEFAULT 0, + spawn_range INT NOT NULL DEFAULT 0, -- `range` is reserved in MariaDB + max_count INT NOT NULL DEFAULT 0, + min_delay INT NOT NULL DEFAULT 0, + max_delay INT NOT NULL DEFAULT 0, + tod_start INT NOT NULL DEFAULT 0, -- meaningless unless tod_mode <> 0 + tod_end INT NOT NULL DEFAULT 0, + tod_mode INT NOT NULL DEFAULT 0, + region VARCHAR(120) NULL, + landmark VARCHAR(120) NULL, + label VARCHAR(120) NOT NULL DEFAULT 'Wilderness', + INDEX idx_shard_spawn_points_facet (facet), + INDEX idx_shard_spawn_points_label (label) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- The many-to-many between the two above: one spawner commonly carries several +-- types (a single Trammel point spawns six), each with its own max. This is how +-- /atlas/creatures/:slug finds the places a creature appears. +CREATE TABLE IF NOT EXISTS shard_spawn_point_types ( + point_id INT NOT NULL, + slug VARCHAR(120) NOT NULL, -- → shard_spawn_creatures.slug (no FK) + max_count INT NOT NULL DEFAULT 1, + PRIMARY KEY (point_id, slug), + INDEX idx_shard_spawn_point_types_slug (slug) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Named regions from Data/Regions.xml, flattened out of their nesting. `rects` +-- holds the region's rectangles; `priority` and rect area are what resolved each +-- spawn point at build time, kept here so the admin drift check can re-derive. +CREATE TABLE IF NOT EXISTS shard_regions ( + id INT AUTO_INCREMENT PRIMARY KEY, + facet VARCHAR(40) NOT NULL, + name VARCHAR(120) NOT NULL, + type VARCHAR(80) NULL, -- ServUO region class + priority INT NOT NULL DEFAULT 0, + parent VARCHAR(120) NULL, -- enclosing named region, if any + rects JSON NULL, + INDEX idx_shard_regions_facet (facet), + INDEX idx_shard_regions_name (name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Points of interest from Data/Locations/*.xml. `grp` is the innermost enclosing +-- parent ("Covetous"), which is the label worth showing — "Covetous" reads +-- better than the individual marker "Level 1". (`group` is reserved in SQL.) +CREATE TABLE IF NOT EXISTS shard_landmarks ( + id INT AUTO_INCREMENT PRIMARY KEY, + facet VARCHAR(40) NOT NULL, + name VARCHAR(120) NOT NULL, + grp VARCHAR(120) NULL, + x INT NOT NULL, + y INT NOT NULL, + z INT NOT NULL DEFAULT 0, + INDEX idx_shard_landmarks_facet (facet), + INDEX idx_shard_landmarks_name (name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Configured champion altars from Config/ChampionSpawns.xml. This is static +-- roster data ("there is an Unholy Terror altar in Deceit") and is distinct from +-- the live champ.update feed in shard_champs ("it is on level 3 right now"). +CREATE TABLE IF NOT EXISTS shard_champion_spawns ( + slug VARCHAR(160) NOT NULL PRIMARY KEY, -- facet-name, e.g. "felucca-deceit" + name VARCHAR(120) NOT NULL, + grp VARCHAR(80) NULL, -- spawn group; one active per group + type VARCHAR(80) NULL, -- '' when randomised per activation + random_type TINYINT(1) NOT NULL DEFAULT 0, + facet VARCHAR(40) NOT NULL, + x INT NOT NULL, + y INT NOT NULL, + z INT NOT NULL DEFAULT 0, + radius INT NOT NULL DEFAULT 0, + label VARCHAR(120) NULL, -- resolved place name + INDEX idx_shard_champion_spawns_facet (facet) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- UO's localization table: cliloc id -> display string. Items carry a +-- `LabelNumber` rather than a name, so without this the site can only render +-- `id 1023721` where the game shows "quarter staff". The shard has always sent +-- the id (char.profile's `cliloc`, and one per marketplace listing) — the number +-- was never the missing piece, the table was. +-- +-- Sourced from a file the OPERATOR converts once from their own UO client and +-- points the site at (docs/website/CLILOCS.md); nothing derived from the client +-- is committed, the same rule the spawn atlas and the creature art map follow. +-- A shard with no cliloc file configured simply renders item ids, which is what +-- it did before this table existed. +-- +-- `text` is TEXT, not VARCHAR: real tables top out around 12 KB for the long +-- property descriptions, and truncating them silently would be worse than +-- storing them. Item NAMES are all short — the index that matters for search is +-- on the denormalized `shard_vendor_items.display_name`, not here. +CREATE TABLE IF NOT EXISTS shard_clilocs ( + number INT NOT NULL PRIMARY KEY, + flag SMALLINT NOT NULL DEFAULT 0, + text TEXT NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Singleton (id = 1) describing the cliloc table currently loaded: the source +-- file, its sha256, the entry count and the parser version. The boot path +-- compares the stored hash against the file on disk and skips the parse when +-- they match, which is every restart that did not follow a client patch. +CREATE TABLE IF NOT EXISTS shard_cliloc_meta ( + id TINYINT NOT NULL PRIMARY KEY DEFAULT 1, + payload JSON NOT NULL, + imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_cliloc_meta_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Singleton (id = 1) describing the artifact currently loaded: when it was +-- built, its counts, and a sha256 per ServUO source file. The admin drift check +-- compares this against db/data/spawnAtlas.meta.json to report when the database +-- is behind the committed artifact. +CREATE TABLE IF NOT EXISTS shard_atlas_meta ( + id TINYINT NOT NULL PRIMARY KEY DEFAULT 1, + payload JSON NOT NULL, + imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_atlas_meta_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Singleton (id = 1) holding an atlas refresh that was parsed but deliberately +-- NOT applied, because it would remove a facet the site currently serves. +-- +-- Losing a facet is the signature of a half-copied or mid-update ServUO tree as +-- much as of a real map change, and boot cannot tell the two apart — so the +-- refresh is staged here for a human instead of being applied. Startup is never +-- blocked by it: the site comes up serving the atlas it already had. +-- +-- Only the DECISION is stored, not the parsed world: `payload` holds the source +-- hashes and the facet diff (a few KB), and approving re-parses the tree. That +-- keeps a multi-megabyte blob out of the database and guarantees the applied +-- atlas matches the tree as it is at approval time, not as it was at boot. +-- +-- `rejected` is remembered against those exact source hashes so a declined +-- refresh does not re-prompt on every restart; changing the tree changes the +-- hashes and asks again. +CREATE TABLE IF NOT EXISTS shard_atlas_pending ( + id TINYINT NOT NULL PRIMARY KEY DEFAULT 1, + status ENUM('pending','rejected') NOT NULL DEFAULT 'pending', + payload JSON NOT NULL, -- source hashes + facet diff + detected_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_shard_atlas_pending_singleton CHECK (id = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- House registry (Protocol 2.0). The house.update full-state feed carries richer +-- fields than the house.decay transition feed shard_houses was built for. Rather +-- than a second table for one entity, extend shard_houses: house.update writes the +-- registry columns below (owner display name, co-owner/friend counts, placement +-- price, decay level name) while house.decay keeps owning `stage`/`is_idoc`. Each +-- upsert only touches its own columns, so the two feeds never clobber each other. +-- `price` is the placement value, NOT a "for sale" flag (stock ServUO has none). +ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS owner_name VARCHAR(120) NULL; +ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS co_owners INT NULL; +ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS friends INT NULL; +ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS price BIGINT NULL; +ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS decay VARCHAR(24) NULL; +-- Distinguishes a full registry row (seen via house.update) from a decay-only row, +-- so the public Houses browser can list registered houses without pulling in rows +-- we only ever saw an IDOC transition for. +ALTER TABLE shard_houses ADD COLUMN IF NOT EXISTS in_registry TINYINT(1) NOT NULL DEFAULT 0; + +-- Protocol 3.0 cutover: this build speaks wire protocol 3 (world.ruleset, +-- points.board, vendor.listing), so the pinned version an existing install +-- carries has to move with it — a 2 against a v3 sidecar 409s every REST call +-- and closes the WS on ws.hello. MODIFY fixes the column default for installs +-- created before the bump (idempotent, like the other MODIFYs here). +ALTER TABLE uo_link_config MODIFY COLUMN protocol INT NOT NULL DEFAULT 3; +-- The row itself is admin-editable, and schema.sql runs on EVERY boot, so this +-- must be one-shot: an operator who deliberately pins an older sidecar in +-- Admin → Shard has to stay pinned. The marker row in `settings` is what makes +-- it fire once — written after the UPDATE, and on a fresh install (no +-- uo_link_config row yet) it is simply written with nothing to update. +UPDATE uo_link_config SET protocol = 3 + WHERE id = 1 AND protocol < 3 + AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_3_migrated'); \ No newline at end of file diff --git a/server/model/shardAtlas/shardAtlas.db.js b/server/model/shardAtlas/shardAtlas.db.js new file mode 100644 index 0000000..c3db3e5 --- /dev/null +++ b/server/model/shardAtlas/shardAtlas.db.js @@ -0,0 +1,392 @@ +const core = require('../../core') + +const { query } = core + +// Raw SQL for the spawn atlas. Every table here is IMPORT-OWNED: `replaceAtlas` +// empties and refills all six inside one transaction, and nothing else in the +// codebase writes to them. There are no foreign keys, consistent with every +// other shard_* table. + +const BATCH = 500 + +const ATLAS_TABLES = [ + 'shard_spawn_point_types', + 'shard_spawn_points', + 'shard_spawn_creatures', + 'shard_regions', + 'shard_landmarks', + 'shard_champion_spawns', +] + +async function insertBatched(conn, sql, rows) { + for (let i = 0; i < rows.length; i += BATCH) { + await conn.batch(sql, rows.slice(i, i + BATCH)) + } + return rows.length +} + +/** + * Replace the entire atlas in one transaction. + * + * All-or-nothing on purpose: a failed reload must leave the previous atlas + * intact rather than a half-loaded world, since a partially-imported atlas is + * indistinguishable from a real one to anyone reading it. + * + * `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly + * commits, which would defeat exactly that guarantee. At ~7k rows the cost of + * `DELETE` is irrelevant. + */ +async function replaceAtlas(atlas, art = {}) { + const conn = await core.pool.getConnection() + const counts = {} + try { + await conn.beginTransaction() + + for (const table of ATLAS_TABLES) await conn.query(`DELETE FROM ${table}`) + + counts.creatures = await insertBatched( + conn, + 'INSERT INTO shard_spawn_creatures (slug, name, total, points, facets, art) VALUES (?,?,?,?,?,?)', + atlas.creatures.map((c) => [ + c.slug, + c.name, + c.total ?? 0, + c.points ?? 0, + JSON.stringify(c.facets ?? {}), + art[c.slug] ?? null, + ]), + ) + + counts.regions = await insertBatched( + conn, + 'INSERT INTO shard_regions (facet, name, type, priority, parent, rects) VALUES (?,?,?,?,?,?)', + atlas.regions.map((r) => [ + r.facet, + r.name, + r.type || null, + r.priority ?? 0, + r.parent || null, + JSON.stringify(r.rects ?? []), + ]), + ) + + counts.landmarks = await insertBatched( + conn, + 'INSERT INTO shard_landmarks (facet, name, grp, x, y, z) VALUES (?,?,?,?,?,?)', + atlas.landmarks.map((l) => [ + l.facet, + l.name, + l.group || null, + l.x ?? 0, + l.y ?? 0, + l.z ?? 0, + ]), + ) + + counts.champions = await insertBatched( + conn, + 'INSERT INTO shard_champion_spawns ' + + '(slug, name, grp, type, random_type, facet, x, y, z, radius, label) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?)', + atlas.champions.map((c) => [ + c.slug, + c.name, + c.group || null, + c.type || null, + c.randomType ? 1 : 0, + c.facet, + c.x ?? 0, + c.y ?? 0, + c.z ?? 0, + c.radius ?? 0, + c.label || null, + ]), + ) + + // Point ids are assigned explicitly rather than left to AUTO_INCREMENT: the + // join rows need to know them and `conn.batch()` reports no usable insertId + // for a multi-row insert. Safe because this transaction just emptied the + // table and nothing else writes to it. + counts.points = await insertBatched( + conn, + 'INSERT INTO shard_spawn_points ' + + '(id, facet, name, x, y, width, height, spawn_range, max_count, min_delay, max_delay, ' + + 'tod_start, tod_end, tod_mode, region, landmark, label) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', + atlas.points.map((p, i) => [ + i + 1, + p.facet, + p.name, + p.x, + p.y, + p.width ?? 0, + p.height ?? 0, + p.range ?? 0, + p.maxCount ?? 0, + p.minDelay ?? 0, + p.maxDelay ?? 0, + p.todStart ?? 0, + p.todEnd ?? 0, + p.todMode ?? 0, + p.region, + p.landmark, + p.label || 'Wilderness', + ]), + ) + + counts.pointTypes = await insertBatched( + conn, + 'INSERT INTO shard_spawn_point_types (point_id, slug, max_count) VALUES (?,?,?)', + atlas.pointTypes, + ) + + await conn.query( + 'INSERT INTO shard_atlas_meta (id, payload) VALUES (1, ?) ' + + 'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP', + [JSON.stringify({ ...atlas.meta, importedCounts: counts })], + ) + + // A completed import answers whatever was pending. + await conn.query('DELETE FROM shard_atlas_pending') + + await conn.commit() + return counts + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +async function getMeta() { + const rows = await query('SELECT payload, imported_at FROM shard_atlas_meta WHERE id = 1') + if (rows.length === 0) return null + const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload + return { ...payload, importedAt: rows[0].imported_at } +} + +/** Facet names currently loaded, used to detect a facet disappearing. */ +async function getFacets() { + const rows = await query('SELECT DISTINCT facet FROM shard_spawn_points ORDER BY facet') + return rows.map((row) => row.facet) +} + +// ── Pending review ───────────────────────────────────────────────────────── + +async function getPending() { + const rows = await query('SELECT payload, status, detected_at FROM shard_atlas_pending WHERE id = 1') + if (rows.length === 0) return null + const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload + return { ...payload, status: rows[0].status, detectedAt: rows[0].detected_at } +} + +async function setPending(payload, status = 'pending') { + return query( + 'INSERT INTO shard_atlas_pending (id, status, payload) VALUES (1, ?, ?) ' + + 'ON DUPLICATE KEY UPDATE status = VALUES(status), payload = VALUES(payload), ' + + 'detected_at = CURRENT_TIMESTAMP', + [status, JSON.stringify(payload)], + ) +} + +async function clearPending() { + return query('DELETE FROM shard_atlas_pending') +} + +// ── Reads (the public /atlas surface) ────────────────────────────────────── +// +// Every read here is a plain indexed query over ~7k rows and is served entirely +// from MariaDB: the atlas is static shard content, so nothing on this path +// touches the sidecar and nothing degrades when the shard is down. +// +// A facet filter is expressed as EXISTS over the points, never as a JSON path +// built from caller input. `shard_spawn_creatures.facets` is a JSON object keyed +// by facet name, and matching a key means either concatenating the name into a +// path or handing it to JSON_SEARCH — whose search string treats `%` and `_` as +// wildcards, so `?facet=%` would quietly match everything. The join is exact and +// uses the indexes that already exist. +const CREATURE_FACET_EXISTS = `EXISTS ( + SELECT 1 FROM shard_spawn_point_types t + JOIN shard_spawn_points p ON p.id = t.point_id + WHERE t.slug = c.slug AND p.facet = ? +)` + +// Build the WHERE for a creature search. `q` is a substring match on the display +// name — a LIKE scan, which is free at ~800 rows and, unlike FULLTEXT, has no +// minimum token length to break a search for "orc". +function creatureWhere({ q, facet }) { + const where = [] + const params = [] + if (q) { + where.push('c.name LIKE ?') + params.push(`%${q}%`) + } + if (facet) { + where.push(CREATURE_FACET_EXISTS) + params.push(facet) + } + return { sql: where.length ? `WHERE ${where.join(' AND ')}` : '', params } +} + +async function countCreatures({ q = '', facet = '' } = {}) { + const { sql, params } = creatureWhere({ q, facet }) + const rows = await query(`SELECT COUNT(*) AS n FROM shard_spawn_creatures c ${sql}`, params) + return rows[0] ? Number(rows[0].n) : 0 +} + +function listCreatures({ q = '', facet = '', limit = 50, offset = 0 } = {}) { + const { sql, params } = creatureWhere({ q, facet }) + return query( + `SELECT c.slug, c.name, c.total, c.points, c.facets, c.art + FROM shard_spawn_creatures c + ${sql} + ORDER BY c.total DESC, c.name ASC + LIMIT ? OFFSET ?`, + [...params, limit, offset], + ) +} + +async function getCreature(slug) { + const rows = await query( + 'SELECT slug, name, total, points, facets, art FROM shard_spawn_creatures WHERE slug = ?', + [slug], + ) + return rows[0] || null +} + +/** + * Where a creature spawns, grouped by resolved place. + * + * This is the answer the atlas exists to give — "lizardman → Shrines, + * Isamu-Jima, Yew" — so it is aggregated in SQL rather than by summing 6,455 + * point rows in Node. + */ +function listCreaturePlaces(slug, { facet = '' } = {}) { + const params = [slug] + let facetSql = '' + if (facet) { + facetSql = 'AND p.facet = ?' + params.push(facet) + } + return query( + `SELECT p.facet, p.label, COUNT(*) AS spawners, SUM(t.max_count) AS max_alive + FROM shard_spawn_point_types t + JOIN shard_spawn_points p ON p.id = t.point_id + WHERE t.slug = ? ${facetSql} + GROUP BY p.facet, p.label + ORDER BY spawners DESC, p.facet ASC, p.label ASC`, + params, + ) +} + +/** The individual spawners for a creature, newest-largest first. Bounded. */ +function listCreaturePoints(slug, { facet = '', limit = 200 } = {}) { + const params = [slug] + let facetSql = '' + if (facet) { + facetSql = 'AND p.facet = ?' + params.push(facet) + } + params.push(limit) + return query( + `SELECT p.id, p.facet, p.name, p.x, p.y, p.width, p.height, p.spawn_range, + p.min_delay, p.max_delay, p.tod_start, p.tod_end, p.tod_mode, + p.region, p.landmark, p.label, t.max_count + FROM shard_spawn_point_types t + JOIN shard_spawn_points p ON p.id = t.point_id + WHERE t.slug = ? ${facetSql} + ORDER BY t.max_count DESC, p.facet ASC, p.label ASC, p.id ASC + LIMIT ?`, + params, + ) +} + +/** Every other creature sharing a spawner with this one. */ +function listCreatureCompanions(slug, { limit = 24 } = {}) { + return query( + `SELECT o.slug, c.name, COUNT(*) AS shared + FROM shard_spawn_point_types t + JOIN shard_spawn_point_types o ON o.point_id = t.point_id AND o.slug <> t.slug + JOIN shard_spawn_creatures c ON c.slug = o.slug + WHERE t.slug = ? + GROUP BY o.slug, c.name + ORDER BY shared DESC, c.name ASC + LIMIT ?`, + [slug, limit], + ) +} + +function listRegions({ facet = '', q = '' } = {}) { + const where = [] + const params = [] + if (facet) { + where.push('facet = ?') + params.push(facet) + } + if (q) { + where.push('name LIKE ?') + params.push(`%${q}%`) + } + return query( + `SELECT facet, name, type, priority, parent, rects + FROM shard_regions + ${where.length ? `WHERE ${where.join(' AND ')}` : ''} + ORDER BY facet ASC, name ASC`, + params, + ) +} + +function listLandmarks({ facet = '', q = '' } = {}) { + const where = [] + const params = [] + if (facet) { + where.push('facet = ?') + params.push(facet) + } + if (q) { + where.push('(name LIKE ? OR grp LIKE ?)') + params.push(`%${q}%`, `%${q}%`) + } + return query( + `SELECT facet, name, grp, x, y, z + FROM shard_landmarks + ${where.length ? `WHERE ${where.join(' AND ')}` : ''} + ORDER BY facet ASC, grp ASC, name ASC`, + params, + ) +} + +function listChampions({ facet = '' } = {}) { + const params = [] + let where = '' + if (facet) { + where = 'WHERE facet = ?' + params.push(facet) + } + return query( + `SELECT slug, name, grp, type, random_type, facet, x, y, z, radius, label + FROM shard_champion_spawns + ${where} + ORDER BY facet ASC, name ASC`, + params, + ) +} + +module.exports = { + replaceAtlas, + getMeta, + getFacets, + getPending, + setPending, + clearPending, + countCreatures, + listCreatures, + getCreature, + listCreaturePlaces, + listCreaturePoints, + listCreatureCompanions, + listRegions, + listLandmarks, + listChampions, +} diff --git a/server/model/shardAtlas/shardAtlas.model.js b/server/model/shardAtlas/shardAtlas.model.js new file mode 100644 index 0000000..083edd1 --- /dev/null +++ b/server/model/shardAtlas/shardAtlas.model.js @@ -0,0 +1,491 @@ +const fs = require('fs') +const path = require('path') + +const db = require('./shardAtlas.db') +const core = require('../../core') +const { settings } = core +const { slugify } = require('../../utils/spawnAtlasParse') +const { + AtlasSourceError, + PARSER_VERSION, + buildAtlas, + hashSources, + sameSources, +} = require('../../utils/spawnAtlasSource') +const log = require('../../core').logger('shardAtlas') + +// The spawn atlas, refreshed from the shard's own ServUO tree. +// +// The tree is the single source of truth. Nothing is precomputed and committed, +// because a shard's maps change over its lifetime — facets get added, replaced +// or renamed — and a snapshot in the repo would go stale against the world +// players actually see. So the atlas is re-derived on every boot. +// +// Two rules govern the boot path: +// +// 1. **It never blocks startup.** No configured path, an unreadable path, a +// malformed file, a database error — all of it is caught and logged. The +// site comes up either way, serving whatever atlas it already had. +// 2. **A facet disappearing is not applied automatically.** Losing a facet is +// the signature of a half-copied or mid-update tree as much as of a real +// map change, and the two are indistinguishable from here. The refresh is +// staged for a human instead, and an admin approves or rejects it. +// +// Everything else — new facets, renamed regions, changed spawns — applies +// straight away, because none of it can silently destroy data an operator would +// miss. + +const SETTING_KEY = 'spawn_atlas_servuo_path' + +/** + * Where the ServUO tree lives. + * + * The admin setting wins over the environment so an operator can point the + * atlas at a different tree without a redeploy, matching how the rest of the + * shard integration is admin-managed rather than env-configured. `SERVUO_PATH` + * remains as the deploy-time default, since the path usually describes a mount + * that the deployment sets up. + */ +async function getServuoPath() { + try { + const configured = await settings.get(SETTING_KEY) + if (configured && String(configured).trim() !== '') return String(configured).trim() + } catch { + // Settings unavailable is not fatal — fall through to the env default. + } + const fromEnv = process.env.SERVUO_PATH + return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : '' +} + +async function setServuoPath(value, updatedBy = null) { + return settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy) +} + +/** + * Optional operator-supplied art map, `{ "": "" }`. + * + * Never committed and never shipped — creature sprites come out of the + * operator's own client `.mul`/`.uop` files, which are theirs, not ours to + * redistribute. Absent (the normal case) every `art` stays NULL and the UI + * renders text-only. + */ +// Resolved from ctx.paths.moduleRoot rather than by walking up from __dirname. +// The ported default was `../../../db/data`, which pointed at core's tree when +// this file lived there and points OUTSIDE server/ now — a path that happens to +// resolve is exactly the kind of port bug that survives a green test suite, +// because the absent-file branch returns {} and looks like the normal case. +function loadArtMap(dir = path.join(core.moduleRoot, 'server', 'data')) { + try { + const file = path.join(dir, 'spawnAtlas.art.json') + if (!fs.existsSync(file)) return {} + const map = JSON.parse(fs.readFileSync(file, 'utf8')) + return map && typeof map === 'object' ? map : {} + } catch (err) { + log.warn('spawn atlas art map could not be read', { error: err.message }) + return {} + } +} + +/** + * Flatten each point's types into `shard_spawn_point_types` rows. + * + * A spawner may legitimately list the same type twice, and the primary key is + * (point_id, slug), so duplicates collapse to the larger max rather than + * failing the insert. + */ +function pointTypeRows(points) { + const rows = [] + points.forEach((point, i) => { + const bySlug = new Map() + for (const entry of point.types ?? []) { + const slug = slugify(entry.type) + if (slug === '') continue + bySlug.set(slug, Math.max(bySlug.get(slug) ?? 0, entry.max ?? 1)) + } + for (const [slug, max] of bySlug) rows.push([i + 1, slug, max]) + }) + return rows +} + +async function applyAtlas(atlas) { + return db.replaceAtlas({ ...atlas, pointTypes: pointTypeRows(atlas.points) }, loadArtMap()) +} + +/** + * Refresh the atlas from the configured ServUO tree. + * + * Returns a result describing what happened rather than throwing, so the caller + * — including the boot path — can log it and move on: + * + * `skipped` no path configured + * `unavailable` path configured but unreadable / missing required files + * `unchanged` source hashes match the loaded atlas; nothing parsed + * `imported` parsed and applied + * `needsReview` parsed, but a facet would be lost; staged for an admin + * `failed` parsed or applied and something went wrong + * + * `force` skips the hash check (an admin asking for a reimport) and `approve` + * additionally accepts facet loss (an admin approving a staged refresh). + */ +/** + * Was the loaded atlas built by THIS parser? + * + * An atlas imported before `parserVersion` existed reports undefined, which is + * correctly "no" — those are exactly the ones carrying the old readings. + */ +const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION + +async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) { + // An explicit override wins outright — it is a one-off "use this tree", and it + // must not be silently overruled by the configured path the way an env default + // would be. + const root = pathOverride.trim() !== '' ? pathOverride.trim() : await getServuoPath() + if (root === '') return { status: 'skipped', reason: 'no ServUO path configured' } + + let hashes + try { + hashes = hashSources(root) + } catch (err) { + if (err instanceof AtlasSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: root } + } + return { status: 'failed', reason: err.message, path: root } + } + + const meta = await db.getMeta().catch(() => null) + const loaded = meta?.source + ? Object.fromEntries(Object.entries(meta.source).map(([label, v]) => [label, v.sha256])) + : null + + // Two things make a loaded atlas stale: the tree changed, or the PARSER did. + // Only checking the tree would strand an install whose maps never change on + // whatever an older build derived — a corrected parse would ship and never + // reach the data. + if (!force && sameSources(hashes, loaded) && currentParser(meta)) { + return { status: 'unchanged', path: root } + } + + // A rejected refresh must not re-prompt on every boot. It stays rejected until + // the tree changes again, at which point the hashes differ and it is a new + // decision. + const pending = await db.getPending().catch(() => null) + if (!approve && !force && pending?.status === 'rejected' && sameSources(hashes, pending.hashes)) { + return { status: 'unchanged', path: root, reason: 'refresh previously rejected' } + } + + let atlas + try { + atlas = buildAtlas(root) + } catch (err) { + return { status: 'failed', reason: err.message, path: root } + } + + const currentFacets = await db.getFacets().catch(() => []) + const incomingFacets = atlas.facets + const removedFacets = currentFacets.filter((facet) => !incomingFacets.includes(facet)) + const addedFacets = incomingFacets.filter((facet) => !currentFacets.includes(facet)) + + // Losing a facet is indistinguishable here from a half-copied tree, so it is + // staged rather than applied — but startup is never blocked by it. + if (removedFacets.length > 0 && !approve) { + const summary = { + hashes, + path: root, + currentFacets, + incomingFacets, + removedFacets, + addedFacets, + counts: atlas.meta.counts, + } + await db.setPending(summary, 'pending').catch((err) => { + log.warn('could not stage spawn atlas refresh', { error: err.message }) + }) + return { status: 'needsReview', ...summary } + } + + try { + const counts = await applyAtlas(atlas) + return { status: 'imported', path: root, counts, addedFacets, removedFacets } + } catch (err) { + return { status: 'failed', reason: err.message, path: root } + } +} + +/** Admin approved a staged refresh: apply it, facet loss and all. */ +async function approvePending(options = {}) { + return refresh({ ...options, approve: true, force: true }) +} + +/** + * Admin rejected a staged refresh: keep the current atlas and remember the + * decision against those exact source hashes, so it does not re-prompt every + * boot. A further change to the tree produces different hashes and asks again. + */ +async function rejectPending() { + const pending = await db.getPending() + if (!pending) return { status: 'none' } + await db.setPending({ ...pending, rejectedAt: new Date().toISOString() }, 'rejected') + return { status: 'rejected' } +} + +/** Everything the admin panel needs to describe atlas state. */ +async function status({ path: pathOverride = '' } = {}) { + const root = pathOverride.trim() !== '' ? pathOverride.trim() : await getServuoPath() + const [meta, pending, facets] = await Promise.all([ + db.getMeta().catch(() => null), + db.getPending().catch(() => null), + db.getFacets().catch(() => []), + ]) + + let treeReadable = false + let drift = null + if (root !== '') { + try { + const hashes = hashSources(root) + treeReadable = true + const loaded = meta?.source + ? Object.fromEntries(Object.entries(meta.source).map(([l, v]) => [l, v.sha256])) + : null + // Same question `refresh` asks: an import picks something up when either + // the tree or the parser has moved on. + drift = !sameSources(hashes, loaded) || !currentParser(meta) + } catch { + treeReadable = false + } + } + + return { + configured: root !== '', + path: root, + treeReadable, + drift, + facets, + importedAt: meta?.importedAt ?? null, + counts: meta?.counts ?? null, + pending, + } +} + +/** + * Boot hook. Best-effort by contract: it logs and returns, never throws, so a + * missing tree or a bad file can never stop the site coming up. + */ +async function refreshOnBoot() { + try { + const result = await refresh() + switch (result.status) { + case 'imported': + log.info('spawn atlas refreshed from ServUO tree', { + ...result.counts, + added: result.addedFacets, + }) + break + case 'needsReview': + log.warn( + 'spawn atlas refresh staged for admin review — a facet would be removed; ' + + 'the existing atlas is unchanged', + { removed: result.removedFacets, added: result.addedFacets }, + ) + break + case 'unavailable': + log.warn('spawn atlas source unavailable', { reason: result.reason, path: result.path }) + break + case 'failed': + log.warn('spawn atlas refresh failed', { reason: result.reason }) + break + default: + break + } + return result + } catch (err) { + log.warn('spawn atlas refresh errored', { error: err.message }) + return { status: 'failed', reason: err.message } + } +} + +// ── Reads ────────────────────────────────────────────────────────────────── +// +// The shapes the /public/atlas endpoints serve. Rows are camelCased here rather +// than in the controller, for the same reason shardState does it: the column +// names are an implementation detail of the import, and the browser contract +// should not move when a column is renamed. + +const jsonOr = (value, fallback) => { + if (value == null) return fallback + if (typeof value !== 'string') return value + try { + return JSON.parse(value) + } catch { + return fallback + } +} + +const shapeCreature = (row) => ({ + slug: row.slug, + name: row.name, + // `total` is the summed MaxCount across every spawner (how many can be alive + // at once); `points` is how many spawners mention it. They answer different + // questions and the UI shows both. + total: row.total, + points: row.points, + facets: jsonOr(row.facets, {}), + art: row.art || null, +}) + +const shapePlace = (row) => ({ + facet: row.facet, + label: row.label, + spawners: Number(row.spawners) || 0, + maxAlive: Number(row.max_alive) || 0, +}) + +const shapePoint = (row) => ({ + id: row.id, + facet: row.facet, + name: row.name || null, + x: row.x, + y: row.y, + width: row.width, + height: row.height, + range: row.spawn_range, + maxCount: row.max_count, + minDelay: row.min_delay, + maxDelay: row.max_delay, + todStart: row.tod_start, + todEnd: row.tod_end, + todMode: row.tod_mode, + region: row.region || null, + landmark: row.landmark || null, + label: row.label, +}) + +/** + * Paginated creature search. Returns the page plus the unpaginated total, so + * the UI can say "showing 50 of 800" without a second round trip. + */ +async function searchCreatures({ q = '', facet = '', limit = 50, offset = 0 } = {}) { + const [rows, total] = await Promise.all([ + db.listCreatures({ q, facet, limit, offset }), + db.countCreatures({ q, facet }), + ]) + return { total, limit, offset, creatures: rows.map(shapeCreature) } +} + +/** + * One creature: its totals, the places it spawns (the aggregate the atlas + * exists for), the individual spawners, and what else shares those spawners. + * + * `null` when the slug is unknown — the controller turns that into a 404. + */ +async function getCreature(slug, { facet = '', points = 200 } = {}) { + const row = await db.getCreature(slug) + if (!row) return null + const [places, pointRows, alsoHere] = await Promise.all([ + db.listCreaturePlaces(slug, { facet }), + db.listCreaturePoints(slug, { facet, limit: points }), + db.listCreatureCompanions(slug), + ]) + return { + ...shapeCreature(row), + places: places.map(shapePlace), + // `spawners`, not `points`: shapeCreature already uses `points` for the + // COUNT of spawners, and reusing the key for the list of them would make the + // same field a number on the search route and an array here. + spawners: pointRows.map(shapePoint), + // Bounded by the query, so a creature on hundreds of spawners returns a page + // rather than the world. + spawnersTruncated: pointRows.length >= points, + alsoHere: alsoHere.map((r) => ({ + slug: r.slug, + name: r.name, + shared: Number(r.shared) || 0, + })), + } +} + +async function listRegions(opts = {}) { + const rows = await db.listRegions(opts) + return rows.map((r) => ({ + facet: r.facet, + name: r.name, + type: r.type || null, + priority: r.priority, + parent: r.parent || null, + rects: jsonOr(r.rects, []), + })) +} + +async function listLandmarks(opts = {}) { + const rows = await db.listLandmarks(opts) + return rows.map((r) => ({ + facet: r.facet, + name: r.name, + group: r.grp || null, + x: r.x, + y: r.y, + z: r.z, + })) +} + +async function listChampions(opts = {}) { + const rows = await db.listChampions(opts) + return rows.map((r) => ({ + slug: r.slug, + name: r.name, + group: r.grp || null, + // '' on the wire means "randomised at activation"; `randomType` says so + // explicitly rather than making the client infer it from an empty string. + type: r.type || null, + randomType: !!r.random_type, + facet: r.facet, + x: r.x, + y: r.y, + z: r.z, + radius: r.radius, + label: r.label || null, + })) +} + +/** + * What is loaded: the facet list, the counts, and when it was imported. + * + * Deliberately does NOT report the source path, the per-file hashes or whether + * a refresh is pending. Those describe the operator's filesystem, and this is a + * public endpoint; the admin status route carries them instead. + */ +async function publicMeta() { + const [meta, facets] = await Promise.all([ + db.getMeta().catch(() => null), + db.getFacets().catch(() => []), + ]) + return { + importedAt: meta?.importedAt ?? null, + generatedAt: meta?.generatedAt ?? null, + // The parse counts, not the row counts: `unresolvedPoints` is what lets the + // page state its own placement accuracy instead of implying it is complete. + counts: meta?.counts ?? null, + facets, + } +} + +const listFacets = () => db.getFacets() + +module.exports = { + refresh, + refreshOnBoot, + approvePending, + rejectPending, + status, + getServuoPath, + setServuoPath, + pointTypeRows, + loadArtMap, + SETTING_KEY, + searchCreatures, + getCreature, + listRegions, + listLandmarks, + listChampions, + listFacets, + publicMeta, +} diff --git a/server/model/shardClilocs/shardClilocs.db.js b/server/model/shardClilocs/shardClilocs.db.js new file mode 100644 index 0000000..b1291ce --- /dev/null +++ b/server/model/shardClilocs/shardClilocs.db.js @@ -0,0 +1,110 @@ +const core = require('../../core') + +const { query } = core + +// Raw SQL for the cliloc table. `shard_clilocs` is IMPORT-OWNED: `replaceAll` +// empties and refills it inside one transaction, and nothing else in the +// codebase writes to it. No foreign keys, consistent with every other shard_* +// table. + +const BATCH = 1000 + +/** + * Replace the entire cliloc table in one transaction. + * + * All-or-nothing on purpose: a failed reload must leave the previous table + * intact rather than a half-loaded one, because a partially-imported cliloc + * table is indistinguishable from a complete one to anyone reading it — you + * would just see some items named and some not, which is also what "no table at + * all" looks like. + * + * `DELETE`, not `TRUNCATE` — `TRUNCATE` is DDL in MariaDB and implicitly + * commits, which would defeat exactly that guarantee. (The same trap the spawn + * atlas import documents; at ~123k rows `DELETE` is still well under a second.) + */ +async function replaceAll(entries, meta) { + const conn = await core.pool.getConnection() + try { + await conn.beginTransaction() + await conn.query('DELETE FROM shard_clilocs') + + // Blank entries are dropped rather than stored. Roughly HALF of a real + // cliloc table is empty strings — ids the client reserves and never uses — + // and a row that resolves to no name is indistinguishable from no row at + // all to every caller. Dropping them halves the table (123,490 → ~67,500) + // and, more importantly, makes the binary and text imports converge on + // identical content: the binary format carries the blanks explicitly and a + // text export may or may not, depending on the tool. + // + // Later duplicates win. Merging across sources already happened upstream in + // `readCliloc`, so in practice this collapses nothing — it is kept because + // the plain format permits a repeated id WITHIN one file and the client's + // own loader resolves it the same way (its dictionary assignment + // overwrites). Without it, a file the game itself would load happily would + // fail the batch insert on a primary-key collision. + const byNumber = new Map() + let blank = 0 + for (const entry of entries) { + if (!Number.isInteger(entry.number)) continue + if (String(entry.text ?? '').trim() === '') { + blank++ + continue + } + byNumber.set(entry.number, entry) + } + + const rows = [...byNumber.values()].map((e) => [e.number, e.flag ?? 0, e.text]) + for (let i = 0; i < rows.length; i += BATCH) { + await conn.batch('INSERT INTO shard_clilocs (number, flag, text) VALUES (?,?,?)', rows.slice(i, i + BATCH)) + } + + await conn.query( + 'INSERT INTO shard_cliloc_meta (id, payload) VALUES (1, ?) ' + + 'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP', + [JSON.stringify({ ...meta, count: rows.length })], + ) + + await conn.commit() + return { count: rows.length, blank, duplicates: entries.length - blank - rows.length } + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +async function getMeta() { + const rows = await query('SELECT payload, imported_at FROM shard_cliloc_meta WHERE id = 1') + if (rows.length === 0) return null + const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload + return { ...payload, importedAt: rows[0].imported_at } +} + +/** + * Look up a batch of ids. + * + * Batched rather than one-at-a-time because every caller has a LIST: a character + * sheet resolves a dozen equipment ids at once, and a page of marketplace + * listings resolves fifty. `IN (...)` with generated placeholders keeps it one + * round trip and one parameterized statement. + */ +async function lookup(numbers) { + if (!Array.isArray(numbers) || numbers.length === 0) return [] + const ids = [...new Set(numbers.filter((n) => Number.isInteger(n)))] + if (ids.length === 0) return [] + const placeholders = ids.map(() => '?').join(',') + return query(`SELECT number, text FROM shard_clilocs WHERE number IN (${placeholders})`, ids) +} + +async function count() { + const rows = await query('SELECT COUNT(*) AS n FROM shard_clilocs') + return Number(rows[0]?.n) || 0 +} + +module.exports = { + replaceAll, + getMeta, + lookup, + count, +} diff --git a/server/model/shardClilocs/shardClilocs.model.js b/server/model/shardClilocs/shardClilocs.model.js new file mode 100644 index 0000000..b4fd3ea --- /dev/null +++ b/server/model/shardClilocs/shardClilocs.model.js @@ -0,0 +1,368 @@ +const db = require('./shardClilocs.db') +const { settings } = require('../../core') +const { displayText } = require('../../utils/clilocParse') +const { + ClilocFormatError, + ClilocSourceError, + PARSER_VERSION, + hashSources, + sameSources, + missingSources, + readCliloc, +} = require('../../utils/clilocSource') +const log = require('../../core').logger('shardClilocs') + +// The cliloc table — UO's id → display-string map, refreshed from a file the +// operator converts once from their own client. +// +// Why the site holds this at all: items on the wire carry a `LabelNumber`, not a +// name. `char.profile.equipment` has always sent `cliloc`, and every marketplace +// listing sends one too. Without the table the UI can only print `id 1023721` +// where the game prints "quarter staff". +// +// Two rules govern the boot path, both inherited from the spawn atlas: +// +// 1. **It never blocks startup.** No configured path, an unreadable file, a +// wrong-format file, a database error — all caught and logged. The site +// comes up either way, serving whatever table it already had (or none, in +// which case the UI falls back to item ids exactly as it did before). +// 2. **Nothing client-derived is committed.** The table is built from the +// operator's own file at a configured path. The repo ships no strings. +// +// The table is built from a SET of sources — the converted client table plus +// every operator-maintained overlay beside it — because shards edit items and +// add new ones, and those carry cliloc ids no stock client table has. All of +// them are re-read on every boot and hash-gated together, so adding one custom +// item never means re-exporting a 5 MB client file. Later sources win. +// +// That set is also why this has the atlas's escalation, in a lighter form. A +// single corrupt file fails the parse loudly, but a source that has simply +// VANISHED parses perfectly and imports a table quietly missing everything it +// contributed — the same ambiguity (real change vs half-copied mount) the atlas +// stages a facet removal for. So a disappearing source is refused and reported +// rather than applied. +// +// It is lighter than the atlas's because it needs to be: the atlas stores a +// pending decision in its own table and adds approve/reject endpoints, whereas +// here the decision is a single boolean an admin passes to the import they were +// already going to run. Re-parsing at approval time — the property that makes +// the atlas store only the decision — is automatic when there is nothing stored. + +const SETTING_KEY = 'cliloc_client_path' + +/** + * Where the converted cliloc file lives. + * + * The admin setting wins over the environment so an operator can repoint it + * without a redeploy, matching how the rest of the shard integration is + * admin-managed rather than env-configured. `UO_CLIENT_PATH` remains as the + * deploy-time default, since the path usually describes a mount the deployment + * sets up. + */ +async function getClientPath() { + try { + const configured = await settings.get(SETTING_KEY) + if (configured && String(configured).trim() !== '') return String(configured).trim() + } catch { + // Settings unavailable is not fatal — fall through to the env default. + } + const fromEnv = process.env.UO_CLIENT_PATH + return fromEnv && fromEnv.trim() !== '' ? fromEnv.trim() : '' +} + +async function setClientPath(value, updatedBy = null) { + const result = await settings.set(SETTING_KEY, String(value ?? '').trim(), updatedBy) + invalidate() + return result +} + +// ── Refresh ──────────────────────────────────────────────────────────────── + +/** Was the loaded table built by THIS parser? */ +const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION + +/** + * Refresh the cliloc table from the configured file. + * + * Returns a result describing what happened rather than throwing, so the caller + * — including the boot path — can log it and move on: + * + * `skipped` no path configured + * `unavailable` path configured but missing / unreadable / not a cliloc file + * `unchanged` source hashes match the loaded table; nothing parsed + * `imported` parsed and applied + * `needsReview` a previously-present source has vanished; NOT applied + * `failed` parsed or applied and something went wrong + * + * `force` skips the hash check (an admin asking for a reimport). `approve` + * additionally accepts a vanished source. + */ +async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) { + // An explicit override wins outright — a one-off "use this file", which must + // not be silently overruled by the configured path the way an env default is. + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() + if (configured === '') return { status: 'skipped', reason: 'no cliloc path configured' } + + let fingerprint + try { + fingerprint = hashSources(configured) + } catch (err) { + if (err instanceof ClilocSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: configured } + } + return { status: 'failed', reason: err.message, path: configured } + } + + const meta = await db.getMeta().catch(() => null) + + // Two things make a loaded table stale: any source changed, or the PARSER did. + // Only checking the sources would strand an install whose client never patches + // on whatever an older build derived. + if (!force && sameSources(fingerprint.hashes, meta?.hashes) && currentParser(meta)) { + return { + status: 'unchanged', + path: configured, + file: fingerprint.file, + count: meta.count ?? null, + customCount: fingerprint.customCount, + } + } + + // A source that was there last import and is not there now is refused, not + // applied — an unmounted volume and a deliberate deletion look identical from + // here, and the wrong guess silently drops every name that file contributed. + const gone = missingSources(fingerprint.hashes, meta?.hashes) + if (gone.length > 0 && !approve) { + return { + status: 'needsReview', + reason: `${gone.length} previously-loaded cliloc source(s) are missing; the existing table is unchanged`, + missingSources: gone, + path: configured, + file: fingerprint.file, + } + } + + let parsed + try { + parsed = readCliloc(configured) + } catch (err) { + if (err instanceof ClilocFormatError || err instanceof ClilocSourceError) { + return { status: 'unavailable', reason: err.message, code: err.code, path: configured } + } + return { status: 'failed', reason: err.message, path: configured } + } + + try { + const applied = await db.replaceAll(parsed.entries, parsed.source) + invalidate() + return { + status: 'imported', + path: configured, + file: parsed.source.file, + count: applied.count, + parsed: parsed.entries.length, + blank: applied.blank, + // Per-source breakdown: how many entries each file contributed and how + // many of them overrode something already merged. An operator who adds an + // overlay wants to see it took effect, and "overrode: 0" on a file meant + // to re-label stock items says it did not. + sources: parsed.source.sources, + acceptedMissing: gone.length > 0 ? gone : undefined, + } + } catch (err) { + return { status: 'failed', reason: err.message, path: configured } + } +} + +/** + * Boot hook. Best-effort by contract: it logs and returns, never throws, so a + * missing or malformed cliloc file can never stop the site coming up. + */ +async function refreshOnBoot() { + try { + const result = await refresh() + switch (result.status) { + case 'imported': + log.info('cliloc table refreshed', { + file: result.file, + count: result.count, + overlays: (result.sources || []).filter((s) => s.kind === 'custom').length, + }) + break + case 'needsReview': + log.warn( + 'cliloc refresh staged for admin review — a previously-loaded source is missing; ' + + 'the existing table is unchanged', + { missing: result.missingSources }, + ) + break + case 'unavailable': + // Deliberately a warning, not an error: an operator who has not supplied + // a cliloc file is in a supported state (the UI shows item ids), and the + // most common cause — pointing at the client's own compressed file — + // needs the reason spelled out rather than a stack trace. + log.warn('cliloc source unavailable (item names will show as ids)', { + reason: result.reason, + code: result.code, + path: result.path, + }) + break + case 'failed': + log.warn('cliloc refresh failed', { reason: result.reason }) + break + default: + break + } + return result + } catch (err) { + log.warn('cliloc refresh errored', { error: err.message }) + return { status: 'failed', reason: err.message } + } +} + +/** Everything the admin panel needs to describe cliloc state. */ +async function status({ path: pathOverride = '' } = {}) { + const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath() + const meta = await db.getMeta().catch(() => null) + const loaded = await db.count().catch(() => 0) + + let fileReadable = false + let file = null + let drift = null + let problem = null + let code = null + let sources = [] + let missing = [] + if (configured !== '') { + try { + const fingerprint = hashSources(configured) + fileReadable = true + file = fingerprint.file + sources = Object.keys(fingerprint.hashes) + missing = missingSources(fingerprint.hashes, meta?.hashes) + // A compressed file is readable but not importable, and the panel has to + // say so HERE — otherwise pointing at an unconverted client directory + // reports a healthy file with pending drift ("ready to import") and the + // operator only finds out when the import fails. `drift` stays null + // because comparing hashes with an unusable file answers nothing. + if (fingerprint.compressed) { + problem = + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.' + code = 'COMPRESSED' + } else { + drift = !sameSources(fingerprint.hashes, meta?.hashes) || !currentParser(meta) + } + } catch (err) { + fileReadable = false + problem = err.message + code = err.code ?? null + } + } + + return { + configured: configured !== '', + path: configured, + file, + fileReadable, + problem, + code, + drift, + count: loaded, + // Every source found now (base first, then overlays), what each contributed + // at the last import, and any that have since vanished — which is the state + // an import will refuse without `approve`. + sources, + loadedSources: meta?.sources ?? null, + missingSources: missing, + importedAt: meta?.importedAt ?? null, + sourceBytes: meta?.bytes ?? null, + } +} + +// ── Lookup ───────────────────────────────────────────────────────────────── +// +// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is +// ~123k rows and shipping it to a client would dwarf every page that uses it, +// and the Android app consumes the same JSON and would otherwise need its own +// copy. Callers get names, not ids-plus-a-table. + +// A small write-through cache in front of the table. Item ids repeat heavily — +// one page of listings is mostly the same few hundred clilocs, and a character +// sheet re-resolves the same gear on every view — so this turns the steady state +// into zero queries. Capped so a pathological caller cannot grow it without +// bound; on overflow it is dropped wholesale rather than evicted entry-by-entry, +// which is cheap and correct for a table that only changes on reimport. +const CACHE_MAX = 20000 +let cache = new Map() + +function invalidate() { + cache = new Map() +} + +/** + * Resolve a batch of cliloc ids to display strings. + * + * Returns a `Map` holding only the ids that resolved to + * something displayable — an id with no row, or one whose text is nothing but + * interpolated arguments we do not have, is simply absent. Callers fall back to + * whatever they had (the item id), so "missing" and "unnamed" collapse into one + * branch at the call site. + * + * Never throws: a cliloc lookup is decoration on someone's character sheet, and + * a database blip must not fail the sheet. + */ +async function resolveMany(numbers) { + const out = new Map() + if (!Array.isArray(numbers)) return out + + const wanted = [...new Set(numbers.filter((n) => Number.isInteger(n) && n > 0))] + if (wanted.length === 0) return out + + const missing = [] + for (const number of wanted) { + if (cache.has(number)) { + const hit = cache.get(number) + if (hit !== '') out.set(number, hit) + } else { + missing.push(number) + } + } + + if (missing.length > 0) { + try { + const rows = await db.lookup(missing) + const found = new Map(rows.map((r) => [Number(r.number), displayText(r.text)])) + if (cache.size + missing.length > CACHE_MAX) invalidate() + for (const number of missing) { + // Cache the miss too ('' meaning "no usable name"), so an id absent from + // the table does not re-query on every page view. + const text = found.get(number) ?? '' + cache.set(number, text) + if (text !== '') out.set(number, text) + } + } catch (err) { + log.warn('cliloc lookup failed', { message: err.message }) + } + } + + return out +} + +/** Single-id convenience. Returns `null` when there is no usable name. */ +async function resolve(number) { + const found = await resolveMany([number]) + return found.get(number) ?? null +} + +module.exports = { + SETTING_KEY, + getClientPath, + setClientPath, + refresh, + refreshOnBoot, + status, + resolveMany, + resolve, + invalidate, +} diff --git a/server/model/shardEvents/shardEvents.db.js b/server/model/shardEvents/shardEvents.db.js new file mode 100644 index 0000000..221fb52 --- /dev/null +++ b/server/model/shardEvents/shardEvents.db.js @@ -0,0 +1,46 @@ +const { query } = require('../../core') + +// INSERT IGNORE on the UNIQUE dedupe_key — a re-ingested event (WS-reconnect +// backfill overlap) is silently skipped rather than duplicated. Returns true if +// a new row was actually inserted. +async function insertIgnore({ kind, t, bootId, payload, dedupeKey }) { + const res = await query( + `INSERT IGNORE INTO shard_events (kind, t, boot_id, payload, dedupe_key) + VALUES (?, ?, ?, ?, ?)`, + [kind, t, bootId || null, JSON.stringify(payload), dedupeKey], + ) + return res.affectedRows > 0 +} + +// Recent events, newest first. Filter by a single `kind`, or an allowlist of +// `kinds` (IN clause) — the public feed uses the allowlist so it can never leak +// staff/sensitive kinds. limit is clamped by the model. +async function list({ kind, kinds, limit }) { + // An allowlist that resolved to NOTHING means "serve nothing" — never "serve + // everything". Falling through to the unfiltered query below would have turned + // a fully-gated visibility config into a full dump of the event log, staff + // audit and cheat detections included. + if (kinds && kinds.length === 0) return [] + if (kinds && kinds.length) { + const placeholders = kinds.map(() => '?').join(', ') + return query( + `SELECT id, kind, t, boot_id, payload, created_at + FROM shard_events WHERE kind IN (${placeholders}) ORDER BY t DESC LIMIT ?`, + [...kinds, limit], + ) + } + if (kind) { + return query( + `SELECT id, kind, t, boot_id, payload, created_at + FROM shard_events WHERE kind = ? ORDER BY t DESC LIMIT ?`, + [kind, limit], + ) + } + return query( + `SELECT id, kind, t, boot_id, payload, created_at + FROM shard_events ORDER BY t DESC LIMIT ?`, + [limit], + ) +} + +module.exports = { insertIgnore, list } diff --git a/server/model/shardEvents/shardEvents.model.js b/server/model/shardEvents/shardEvents.model.js new file mode 100644 index 0000000..0674d75 --- /dev/null +++ b/server/model/shardEvents/shardEvents.model.js @@ -0,0 +1,61 @@ +// Append-only shard event log. The WS ingest dispatcher calls append() for the +// notable kinds; the public/admin read endpoints call list(). The DB layer only +// sees an already-computed dedupe_key so INSERT IGNORE is idempotent across +// WS-reconnect backfill. + +const crypto = require('crypto') +const db = require('./shardEvents.db') + +const MAX_LIMIT = 1000 +const DEFAULT_LIMIT = 100 + +// Stable stringify — keys sorted — so the dedupe hash is independent of the +// property order the sidecar happened to serialize with. +function stableStringify(value) { + if (value === null || typeof value !== 'object') return JSON.stringify(value) + if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` + const keys = Object.keys(value).sort() + const entries = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`) + return `{${entries.join(',')}}` +} + +// dedupe_key = sha256(kind + t + stable-json(payload)), truncated to 40 hex chars. +// This is a content fingerprint for idempotent INSERT IGNORE, not a security value, +// but we use SHA-256 rather than SHA-1 anyway; the truncation keeps it inside the +// CHAR(40) column (160 bits is ample collision resistance for dedupe). Two identical +// events (same kind, same timestamp, same body) collapse to one row. +function dedupeKey(kind, t, payload) { + return crypto + .createHash('sha256') + .update(`${kind}|${t}|${stableStringify(payload)}`) + .digest('hex') + .slice(0, 40) +} + +// Append one event. Returns true if a new row was inserted (false = deduped). +async function append({ kind, t, bootId, payload }) { + return db.insertIgnore({ kind, t, bootId, payload, dedupeKey: dedupeKey(kind, t, payload) }) +} + +function normalizeLimit(limit) { + const n = Number(limit) + if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT + return Math.min(Math.floor(n), MAX_LIMIT) +} + +// Recent events, newest first. Each row's JSON payload is parsed back to an +// object. `kinds` (array) restricts to an allowlist; `kind` filters a single kind. +async function list({ kind, kinds, limit } = {}) { + const rows = await db.list({ kind, kinds, limit: normalizeLimit(limit) }) + return rows.map((row) => ({ + id: row.id, + kind: row.kind, + t: row.t, + bootId: row.boot_id || null, + // mariadb returns JSON columns as strings on some versions; parse defensively. + payload: typeof row.payload === 'string' ? JSON.parse(row.payload) : row.payload, + createdAt: row.created_at, + })) +} + +module.exports = { append, list, dedupeKey } diff --git a/server/model/shardLinks/shardLinks.db.js b/server/model/shardLinks/shardLinks.db.js new file mode 100644 index 0000000..f9a00e9 --- /dev/null +++ b/server/model/shardLinks/shardLinks.db.js @@ -0,0 +1,42 @@ +const { query } = require('../../core') + +const COLS = 'account, user_id, char_name, linked_at' + +// Upsert a link. account is the PK, so a re-link moves the account to the new +// user (the sidecar already treats /link/confirm as authoritative). +async function upsert({ account, userId, charName }) { + await query( + `INSERT INTO shard_account_links (account, user_id, char_name) + VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE user_id = VALUES(user_id), char_name = VALUES(char_name)`, + [account, userId, charName || null], + ) + return getByAccount(account) +} + +async function getByAccount(account) { + const rows = await query(`SELECT ${COLS} FROM shard_account_links WHERE account = ? LIMIT 1`, [account]) + return rows[0] || null +} + +const listByUser = (userId) => + query(`SELECT ${COLS} FROM shard_account_links WHERE user_id = ? ORDER BY linked_at DESC`, [userId]) + +async function isOwnedBy(account, userId) { + const rows = await query( + 'SELECT 1 FROM shard_account_links WHERE account = ? AND user_id = ? LIMIT 1', + [account, userId], + ) + return rows.length > 0 +} + +const remove = (account, userId) => + query('DELETE FROM shard_account_links WHERE account = ? AND user_id = ?', [account, userId]) + +// Drop the mirror for an account regardless of which user held it — used to +// reconcile when the tie is severed at the source (an in-game [unlink → +// account.unlinked event, or a site-side DELETE /link/{account}). +const removeByAccount = (account) => + query('DELETE FROM shard_account_links WHERE account = ?', [account]) + +module.exports = { upsert, getByAccount, listByUser, isOwnedBy, remove, removeByAccount } diff --git a/server/model/shardLinks/shardLinks.model.js b/server/model/shardLinks/shardLinks.model.js new file mode 100644 index 0000000..3d0f907 --- /dev/null +++ b/server/model/shardLinks/shardLinks.model.js @@ -0,0 +1,37 @@ +// Site-side mirror of in-game-account → website-user links. The sidecar owns the +// authoritative link (it tags the game account on /link/confirm); this model +// records it locally so the player portal can list links and enforce ownership. + +const db = require('./shardLinks.db') + +function toSafe(row) { + if (!row) return null + return { + account: row.account, + userId: row.user_id, + charName: row.char_name || null, + linkedAt: row.linked_at, + } +} + +async function link({ account, userId, charName }) { + return toSafe(await db.upsert({ account, userId, charName })) +} + +async function listForUser(userId) { + const rows = await db.listByUser(userId) + return rows.map(toSafe) +} + +const ownsAccount = (account, userId) => db.isOwnedBy(account, userId) + +async function getByAccount(account) { + return toSafe(await db.getByAccount(account)) +} + +const unlink = (account, userId) => db.remove(account, userId) + +// Drop the local mirror for an account (source-of-truth severed elsewhere). +const removeByAccount = (account) => db.removeByAccount(account) + +module.exports = { link, listForUser, ownsAccount, getByAccount, unlink, removeByAccount } diff --git a/server/model/shardMarket/shardMarket.db.js b/server/model/shardMarket/shardMarket.db.js new file mode 100644 index 0000000..7e33578 --- /dev/null +++ b/server/model/shardMarket/shardMarket.db.js @@ -0,0 +1,301 @@ +const core = require('../../core') + +const { query } = core + +// Raw SQL for the player-vendor market index (Protocol 3.0 vendor.listing). +// +// Two tables, both INGEST-OWNED: `shard_vendors` (one row per shop) and +// `shard_vendor_items` (one row per priced listing). Nothing else in the codebase +// writes to either. No foreign keys, consistent with every other shard_* table. + +// Insert batch size for one vendor's listings. A shop is capped at +// MarketMaxListings (250 by default) on the shard side, so in practice this is +// one batch — it exists for the operator who raised that cap. +const BATCH = 500 + +// LIKE wildcards in user input. `%` and `_` are not special to the parameterized +// query — they are special to LIKE itself — so a search for "50% off" would +// otherwise match everything containing "50" and a search for "_" would match +// every single-character name. Escaped with a backslash, which is MariaDB's +// default LIKE escape (no ESCAPE clause needed). +const likeTerm = (q) => `%${String(q).replace(/[\\%_]/g, (c) => `\\${c}`)}%` + +/** + * Replace one vendor's whole row and listing set, in one transaction. + * + * Delete-then-insert rather than a diff, because the frame is AUTHORITATIVE for + * that vendor: the shard's sweep only emits a shop whose contents, prices or + * location moved, and when it does it sends the whole shop. Reconciling it item + * by item would be more code for the same result and would leave sold items + * behind on any path the reconciliation missed. + * + * All-or-nothing matters here for a specific reason: the two writes are "the + * shop" and "what is in it", and a failure between them leaves a shop advertising + * an inventory it no longer has (or none at all) — visibly wrong on the page, and + * indistinguishable from a genuinely empty shop. + */ +async function replaceVendor(vendor, items) { + const conn = await core.pool.getConnection() + try { + await conn.beginTransaction() + + await conn.query( + `INSERT INTO shard_vendors + (serial, shop_name, owner_serial, owner_name, map, x, y, z, region, house, + item_count, item_total, truncated, t) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON DUPLICATE KEY UPDATE shop_name = VALUES(shop_name), owner_serial = VALUES(owner_serial), + owner_name = VALUES(owner_name), map = VALUES(map), x = VALUES(x), y = VALUES(y), + z = VALUES(z), region = VALUES(region), house = VALUES(house), + item_count = VALUES(item_count), item_total = VALUES(item_total), + truncated = VALUES(truncated), t = VALUES(t), + -- Touched explicitly rather than left to ON UPDATE CURRENT_TIMESTAMP: + -- MariaDB does not fire that when every column is written back + -- unchanged, and a shop that is re-published identically is still + -- FRESHLY CONFIRMED. Without this the staleness banner would age a + -- perfectly current shop forever. + updated_at = CURRENT_TIMESTAMP`, + [ + vendor.serial, + vendor.shopName ?? null, + vendor.ownerSerial ?? null, + vendor.ownerName ?? null, + vendor.map ?? null, + Number.isFinite(vendor.x) ? vendor.x : null, + Number.isFinite(vendor.y) ? vendor.y : null, + Number.isFinite(vendor.z) ? vendor.z : null, + vendor.region ?? null, + vendor.house ?? null, + items.length, + Number.isFinite(vendor.itemTotal) ? vendor.itemTotal : items.length, + vendor.truncated ? 1 : 0, + Number.isFinite(vendor.t) ? vendor.t : null, + ], + ) + + await conn.query('DELETE FROM shard_vendor_items WHERE vendor_serial = ?', [vendor.serial]) + + const rows = items.map((i) => [ + vendor.serial, + i.serial, + i.itemId, + i.hue, + i.amount, + i.price, + i.name, + i.cliloc, + i.displayName, + i.child ? 1 : 0, + ]) + + for (let i = 0; i < rows.length; i += BATCH) { + await conn.batch( + `INSERT INTO shard_vendor_items + (vendor_serial, serial, item_id, hue, amount, price, name, cliloc, display_name, child) + VALUES (?,?,?,?,?,?,?,?,?,?)`, + rows.slice(i, i + BATCH), + ) + } + + await conn.commit() + return { items: rows.length } + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +/** Drop one vendor and its listings (vendor.listing.remove). */ +async function removeVendor(serial) { + const conn = await core.pool.getConnection() + try { + await conn.beginTransaction() + await conn.query('DELETE FROM shard_vendor_items WHERE vendor_serial = ?', [serial]) + await conn.query('DELETE FROM shard_vendors WHERE serial = ?', [serial]) + await conn.commit() + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +// ── Search ───────────────────────────────────────────────────────────────── +// +// The unit of a search RESULT is a listing, not a vendor: "who sells a vanquishing +// kryss and for how much" is the question, and answering it per vendor would make +// the caller flatten the shops back out. The vendor's columns ride along on the +// join so a result row is self-contained. + +function searchWhere({ q, minPrice, maxPrice, itemId, map, region }) { + const where = ['i.price > 0'] + const params = [] + + if (q) { + // Both the resolved display name and the item's own literal, because an item + // with a player-set name (most of what is actually worth searching for on a + // player-run shard) may have a generic cliloc. + where.push('(i.display_name LIKE ? OR i.name LIKE ?)') + params.push(likeTerm(q), likeTerm(q)) + } + if (Number.isFinite(minPrice)) { + where.push('i.price >= ?') + params.push(minPrice) + } + if (Number.isFinite(maxPrice)) { + where.push('i.price <= ?') + params.push(maxPrice) + } + if (Number.isFinite(itemId)) { + where.push('i.item_id = ?') + params.push(itemId) + } + if (map) { + where.push('v.map = ?') + params.push(map) + } + if (region) { + where.push('v.region = ?') + params.push(region) + } + + return { sql: `WHERE ${where.join(' AND ')}`, params } +} + +// Whitelisted, because this interpolates into the statement. `recent` sorts by +// the vendor's freshness, which is the only way to see what has just been listed +// on a shard whose sweep is minutes wide. +const SORTS = { + price_asc: 'i.price ASC, i.id ASC', + price_desc: 'i.price DESC, i.id ASC', + recent: 'v.updated_at DESC, i.id ASC', +} + +async function searchListings({ q, minPrice, maxPrice, itemId, map, region, sort, limit, offset }) { + const { sql, params } = searchWhere({ q, minPrice, maxPrice, itemId, map, region }) + const order = SORTS[sort] || SORTS.price_asc + + const rows = await query( + `SELECT i.serial, i.item_id, i.hue, i.amount, i.price, i.name, i.cliloc, i.display_name, i.child, + v.serial AS vendor_serial, v.shop_name, v.owner_serial, v.owner_name, + v.map, v.x, v.y, v.z, v.region, v.house, v.updated_at + FROM shard_vendor_items i + JOIN shard_vendors v ON v.serial = i.vendor_serial + ${sql} + ORDER BY ${order} + LIMIT ? OFFSET ?`, + [...params, limit, offset], + ) + + const counted = await query( + `SELECT COUNT(*) AS n + FROM shard_vendor_items i + JOIN shard_vendors v ON v.serial = i.vendor_serial + ${sql}`, + params, + ) + + return { rows, total: Number(counted[0]?.n) || 0 } +} + +async function getVendor(serial) { + const rows = await query( + `SELECT serial, shop_name, owner_serial, owner_name, map, x, y, z, region, house, + item_count, item_total, truncated, t, updated_at + FROM shard_vendors WHERE serial = ?`, + [serial], + ) + return rows[0] || null +} + +async function listVendorItems(serial, { limit, offset }) { + return query( + `SELECT serial, item_id, hue, amount, price, name, cliloc, display_name, child + FROM shard_vendor_items + WHERE vendor_serial = ? + ORDER BY price ASC, id ASC + LIMIT ? OFFSET ?`, + [serial, limit, offset], + ) +} + +/** + * What the market page's header needs: how big the index is, and how stale it may + * be. `staleAt` is the OLDEST vendor row — the round-robin sweep means a shop can + * be a full cycle behind, and the page says so rather than implying live prices. + */ +async function meta() { + const rows = await query( + `SELECT COUNT(*) AS vendors, MIN(updated_at) AS stale_at, MAX(updated_at) AS fresh_at + FROM shard_vendors`, + ) + const items = await query('SELECT COUNT(*) AS n FROM shard_vendor_items') + return { + vendors: Number(rows[0]?.vendors) || 0, + items: Number(items[0]?.n) || 0, + staleAt: rows[0]?.stale_at || null, + freshAt: rows[0]?.fresh_at || null, + } +} + +/** The distinct facets and regions holding vendors — drives the page's filters. */ +async function listPlaces() { + const maps = await query( + 'SELECT DISTINCT map FROM shard_vendors WHERE map IS NOT NULL ORDER BY map', + ) + const regions = await query( + 'SELECT DISTINCT region FROM shard_vendors WHERE region IS NOT NULL ORDER BY region', + ) + return { maps: maps.map((r) => r.map), regions: regions.map((r) => r.region) } +} + +// ── Cliloc re-resolution ─────────────────────────────────────────────────── + +/** + * One page of listings whose name still needs resolving, for the bulk pass that + * runs after a cliloc import. + * + * Keyed on `id > after` rather than OFFSET: the pass updates the very rows it is + * scanning, and an OFFSET walk over a table being rewritten skips rows. Every + * row with a cliloc is re-read, not just the unresolved ones, because an import + * can also CHANGE a name — a shard overlay relabelling a stock item is the whole + * reason overlays exist. + */ +async function listResolvableItems(after, limit) { + return query( + `SELECT id, cliloc, name, display_name + FROM shard_vendor_items + WHERE cliloc IS NOT NULL AND cliloc > 0 AND id > ? + ORDER BY id + LIMIT ?`, + [after, limit], + ) +} + +/** Write back a batch of re-resolved display names. */ +async function updateDisplayNames(pairs) { + if (pairs.length === 0) return 0 + const conn = await core.pool.getConnection() + try { + await conn.batch('UPDATE shard_vendor_items SET display_name = ? WHERE id = ?', pairs) + return pairs.length + } finally { + conn.release() + } +} + +module.exports = { + replaceVendor, + removeVendor, + searchListings, + getVendor, + listVendorItems, + meta, + listPlaces, + listResolvableItems, + updateDisplayNames, + likeTerm, +} diff --git a/server/model/shardMarket/shardMarket.model.js b/server/model/shardMarket/shardMarket.model.js new file mode 100644 index 0000000..423fad4 --- /dev/null +++ b/server/model/shardMarket/shardMarket.model.js @@ -0,0 +1,329 @@ +// ── Player-vendor market index (Protocol 3.0 vendor.listing) ─────────────── +// +// The shard-wide shop index: what every player vendor is selling, for how much, +// and where it is standing. This is the website's half of the search the in-game +// Vendor Search gump offers — the same data, the same opt-out, reachable without +// logging in to the game. +// +// Ingest is per-vendor and authoritative: the shard's round-robin sweep emits one +// `vendor.listing` frame per shop whose contents, prices or location moved, and +// the frame is the whole shop (see docs/link/v3.md §8 and BridgeMarket.cs). This +// module normalizes it into shard_vendors + shard_vendor_items and, crucially, +// resolves each listing's cliloc to a DISPLAY NAME on the way in — a search for +// "kryss" is a search over names, and the shard only ever sends numbers. + +const db = require('./shardMarket.db') +const clilocs = require('../shardClilocs/shardClilocs.model') +const log = require('../../core').logger('shard-market') + +// Defense in depth on top of the shard's own MarketMaxListings cap. The shard is +// trusted, but it is a separately-versioned component: a frame from a plugin +// whose cap was raised (or a shard running modified scripts) must not be able to +// turn one ingest into an unbounded transaction. +const MAX_ITEMS_PER_VENDOR = 5000 + +// Column widths in schema.sql. Truncating here rather than letting MariaDB do it +// keeps the behavior the same in strict mode, where an over-length value is an +// ERROR and would fail the whole vendor rather than shortening one name. +const MAX_NAME = 160 +const MAX_SHOP = 160 +const MAX_OWNER = 64 +const MAX_MAP = 40 +const MAX_REGION = 80 +const MAX_SERIAL = 20 + +const clip = (value, max) => { + if (value == null) return null + const s = String(value) + return s.length > max ? s.slice(0, max) : s +} + +const int = (value, fallback = 0) => { + const n = Number(value) + return Number.isFinite(n) ? Math.trunc(n) : fallback +} + +// ── Ingest ───────────────────────────────────────────────────────────────── + +/** + * Flatten one `vendor.listing` frame into the row shapes the DB layer wants. + * + * `location` arrives as a nested object rather than flat map/x/y/region, and that + * shape is load-bearing rather than cosmetic: the visibility projection matches + * literal JSON keys, so ONE `market.location` rule can hide a vendor's + * whereabouts only if `location` is a single key on both the live frame and the + * stored read model. Flattening it here for storage and re-nesting it on read is + * what keeps that true on both paths. + * + * Exported for tests — it is the part with rules in it, and it is pure. + */ +function flattenFrame(ev) { + const loc = (ev && ev.location) || {} + return { + serial: clip(ev.serial, MAX_SERIAL), + shopName: clip(ev.shopName, MAX_SHOP), + ownerSerial: clip(ev.ownerSerial, MAX_SERIAL), + ownerName: clip(ev.ownerName, MAX_OWNER), + map: clip(loc.map, MAX_MAP), + x: Number.isFinite(loc.x) ? Math.trunc(loc.x) : null, + y: Number.isFinite(loc.y) ? Math.trunc(loc.y) : null, + z: Number.isFinite(loc.z) ? Math.trunc(loc.z) : null, + region: clip(loc.region, MAX_REGION), + house: clip(loc.house, MAX_SHOP), + // What the SHOP holds, which is not what the frame carries when it was + // truncated. Kept apart so the page can say "showing 250 of 3,104" rather + // than presenting a partial shop as a complete one. + itemTotal: int(ev.total, int(ev.count, 0)), + truncated: ev.truncated === true, + t: Number.isFinite(ev.t) ? ev.t : null, + } +} + +/** + * Resolve each listing's display name. + * + * Order of preference is the item's own literal `name` first, then the cliloc. + * That is the opposite of what "resolve the id" suggests and it is right: a + * literal name only exists because a player set one ("Bob's vanquishing kryss"), + * and it is strictly more specific than the generic cliloc the item still + * carries. + * + * One batched lookup per frame rather than per item; `resolveMany` is cached and + * never throws, so a cliloc table that is missing entirely just leaves + * `displayName` null and the page renders item ids, exactly as it did before the + * table existed. + */ +async function shapeItems(ev) { + const raw = Array.isArray(ev.items) ? ev.items.slice(0, MAX_ITEMS_PER_VENDOR) : [] + + const wanted = raw + .map((i) => int(i && i.cliloc, 0)) + .filter((n) => n > 0) + + const names = await clilocs.resolveMany(wanted) + + return raw + .filter((i) => i && i.serial) + .map((i) => { + const literal = clip(i.name, MAX_NAME) + const cliloc = int(i.cliloc, 0) || null + return { + serial: clip(i.serial, MAX_SERIAL), + itemId: int(i.itemId, 0), + hue: int(i.hue, 0), + amount: int(i.amount, 1), + price: int(i.price, 0), + name: literal, + cliloc, + displayName: literal || (cliloc ? clip(names.get(cliloc) ?? null, MAX_NAME) : null), + child: i.child === true, + } + }) + // Unpriced rows are inventory, not listings. The shard already drops them; + // this is the same rule enforced where the table is written, so a plugin that + // stops enforcing it cannot put un-buyable rows on the market page. + .filter((i) => i.price > 0) +} + +/** Ingest one `vendor.listing` frame. */ +async function upsertVendor(ev) { + if (!ev || !ev.serial) return + const vendor = flattenFrame(ev) + const items = await shapeItems(ev) + await db.replaceVendor(vendor, items) +} + +/** Ingest one `vendor.listing.remove` frame. */ +async function removeVendor(serial) { + if (!serial) return + await db.removeVendor(String(serial).slice(0, MAX_SERIAL)) +} + +// ── Read models ──────────────────────────────────────────────────────────── +// +// `location` is re-nested (see flattenFrame) so the stored read model and the +// live wire frame present the same keys to the visibility projection. + +const place = (r) => ({ + map: r.map, + x: r.x, + y: r.y, + z: r.z, + region: r.region, + house: r.house, +}) + +// A listing as the search returns it: the item, plus enough of its shop to be +// actionable without a second request. `displayName` falls back to nothing rather +// than to a fabricated "Item 3922" — the client decides how to render an +// unresolved id, and inventing a name here would make it indistinguishable from +// a real one. +const shapeListing = (r) => ({ + serial: r.serial, + itemId: r.item_id, + hue: r.hue, + amount: r.amount, + price: Number(r.price), + name: r.name, + cliloc: r.cliloc, + displayName: r.display_name, + child: !!r.child, + vendor: { + serial: r.vendor_serial, + shopName: r.shop_name, + ownerSerial: r.owner_serial, + ownerName: r.owner_name, + location: place(r), + updatedAt: r.updated_at, + }, +}) + +const shapeVendor = (r) => ({ + serial: r.serial, + shopName: r.shop_name, + ownerSerial: r.owner_serial, + ownerName: r.owner_name, + location: place(r), + count: r.item_count, + total: r.item_total, + truncated: !!r.truncated, + updatedAt: r.updated_at, +}) + +const shapeItem = (r) => ({ + serial: r.serial, + itemId: r.item_id, + hue: r.hue, + amount: r.amount, + price: Number(r.price), + name: r.name, + cliloc: r.cliloc, + displayName: r.display_name, + child: !!r.child, +}) + +/** + * Search the index. Returns a page of LISTINGS (not vendors) plus the + * unpaginated total and the staleness stamp the page's banner needs. + */ +async function search({ + q = '', + minPrice, + maxPrice, + itemId, + map = '', + region = '', + sort = 'price_asc', + limit = 50, + offset = 0, +} = {}) { + const { rows, total } = await db.searchListings({ + q: q.trim(), + minPrice: Number.isFinite(minPrice) ? minPrice : undefined, + maxPrice: Number.isFinite(maxPrice) ? maxPrice : undefined, + itemId: Number.isFinite(itemId) ? itemId : undefined, + map: map.trim(), + region: region.trim(), + sort, + limit, + offset, + }) + + const info = await db.meta() + + return { + listings: rows.map(shapeListing), + total, + limit, + offset, + // Repeated on every search response rather than left to a separate /meta + // call: the banner that says how old these prices are must age with the + // results it labels, and a client that fetched it once would keep showing a + // stamp from before the page it is looking at. + staleAt: info.staleAt, + vendors: info.vendors, + } +} + +/** One shop and its listings. `null` when the index has never seen that serial. */ +async function getVendor(serial, { limit = 250, offset = 0 } = {}) { + const row = await db.getVendor(serial) + if (!row) return null + const items = await db.listVendorItems(serial, { limit, offset }) + return { ...shapeVendor(row), items: items.map(shapeItem) } +} + +/** Index size, staleness, and the facet/region filter options. */ +async function meta() { + const [info, places] = await Promise.all([db.meta(), db.listPlaces()]) + return { ...info, ...places } +} + +// ── Cliloc re-resolution ─────────────────────────────────────────────────── + +// Batch size for the post-import pass. Big enough that a 40k-row table is ~40 +// round trips, small enough that a single batch is not a long-held connection. +const RESOLVE_BATCH = 1000 + +/** + * Re-resolve every listing's display name against the current cliloc table. + * + * Called after a cliloc import, and it has to be: the market's diff sweep will + * NOT re-send an unchanged shop just because the site learned what its items are + * called, so without this an operator who configures clilocs after the first + * market sweep sees item ids until every shop happens to change. That is the same + * class of staleness the spawn atlas avoids by re-parsing on boot — here the + * source of truth for names moved, not the data. + * + * Never throws. It is a cosmetic backfill on a table that is already serving; a + * failure means names stay as they were, which is exactly the pre-import state. + */ +async function refreshDisplayNames() { + let after = 0 + let scanned = 0 + let changed = 0 + + try { + for (;;) { + const rows = await db.listResolvableItems(after, RESOLVE_BATCH) + if (rows.length === 0) break + + after = rows[rows.length - 1].id + scanned += rows.length + + const names = await clilocs.resolveMany(rows.map((r) => Number(r.cliloc))) + + const pairs = [] + for (const row of rows) { + // The literal name still wins, so a re-resolution never overwrites a + // player-set name with the generic cliloc behind it. + const next = row.name + ? clip(row.name, MAX_NAME) + : clip(names.get(Number(row.cliloc)) ?? null, MAX_NAME) + if (next !== row.display_name) pairs.push([next, row.id]) + } + + changed += await db.updateDisplayNames(pairs) + } + + if (changed > 0) log.info('market display names refreshed', { scanned, changed }) + return { scanned, changed } + } catch (err) { + log.warn('market display-name refresh failed', { message: err.message, scanned, changed }) + return { scanned, changed, error: err.message } + } +} + +module.exports = { + upsertVendor, + removeVendor, + search, + getVendor, + meta, + refreshDisplayNames, + flattenFrame, + shapeItems, + shapeListing, + shapeVendor, + MAX_ITEMS_PER_VENDOR, +} diff --git a/server/model/shardState/shardState.db.js b/server/model/shardState/shardState.db.js new file mode 100644 index 0000000..5bdacfe --- /dev/null +++ b/server/model/shardState/shardState.db.js @@ -0,0 +1,369 @@ +const { query } = require('../../core') + +// Shared upsert builder for the shard-state tables. Each is keyed on a single +// primary column (`pkCol` = pk); `fields` carries only the columns the model +// wants to write, so a partial refresh touches nothing else. `coalesce` keeps +// the prior column value when the incoming one is NULL (used by shard_online so a +// vitals frame that omits acct/name doesn't blank what mob.login set); otherwise +// the incoming value wins (VALUES()). +function upsertRow(table, pkCol, pk, fields, { coalesce = false } = {}) { + const cols = Object.keys(fields) + const allCols = [pkCol, ...cols] + const insertCols = allCols.map((c) => `\`${c}\``).join(', ') + const placeholders = allCols.map(() => '?').join(', ') + const rhs = coalesce + ? (c) => `\`${c}\` = COALESCE(VALUES(\`${c}\`), \`${c}\`)` + : (c) => `\`${c}\` = VALUES(\`${c}\`)` + const updates = cols.map(rhs).join(', ') + return query( + `INSERT INTO ${table} (${insertCols}) VALUES (${placeholders}) + ON DUPLICATE KEY UPDATE ${updates}`, + [pk, ...cols.map((c) => fields[c])], + ) +} + +// ── Online players ───────────────────────────────────────────────────────── +const ONLINE_COLS = + 'serial, name, acct, web_id, map, x, y, z, hits, hits_max, mana, mana_max, stam, stam_max, str, dex, `int`, updated_at' + +// Upsert one online player. `fields` already prepared by the model (only the +// columns it wants to write); serial is required and is the primary key. +// COALESCE variant: a char.vitals frame that omits acct/name must not blank what +// mob.login set, so an incoming NULL keeps the prior column value. +const upsertOnline = (serial, fields) => + upsertRow('shard_online', 'serial', serial, fields, { coalesce: true }) + +const removeOnline = (serial) => query('DELETE FROM shard_online WHERE serial = ?', [serial]) +const clearOnline = () => query('DELETE FROM shard_online') + +async function countOnline() { + const rows = await query('SELECT COUNT(*) AS n FROM shard_online') + return rows[0] ? Number(rows[0].n) : 0 +} + +const listOnline = () => + query(`SELECT ${ONLINE_COLS} FROM shard_online ORDER BY name ASC`) + +// Online players on any of the given game accounts (admin: a user's linked +// accounts). Empty list short-circuits so we never emit `IN ()`. +const listOnlineByAccounts = (accounts) => + accounts.length === 0 + ? Promise.resolve([]) + : query( + `SELECT ${ONLINE_COLS} FROM shard_online + WHERE acct IN (${accounts.map(() => '?').join(', ')}) + ORDER BY name ASC`, + accounts, + ) + +// Staff roles whose online presence is shown on the public Shard page. Players +// who link an account are NOT surfaced publicly — only staff opt into visibility +// by virtue of being staff. +const PUBLIC_ONLINE_ROLES = ['admin', 'editor', 'moderator'] + +// Online players whose game account is linked to a STAFF website user. Joined +// against shard_account_links (not the sidecar-supplied web_id) so a link takes +// effect immediately, regardless of whether the player has re-logged since +// linking, then through to users so only staff roles are surfaced publicly. +const listOnlineLinked = () => { + const cols = ONLINE_COLS.split(', ') + .map((c) => `o.${c}`) + .join(', ') + return query( + `SELECT ${cols} + FROM shard_online o + JOIN shard_account_links l ON l.account = o.acct + JOIN users u ON u.id = l.user_id + WHERE u.role IN (${PUBLIC_ONLINE_ROLES.map(() => '?').join(', ')}) + ORDER BY o.name ASC`, + PUBLIC_ONLINE_ROLES, + ) +} + +// ── Economy supply series ──────────────────────────────────────────────── +const insertEconomy = ({ accounts, gold, t }) => + query('INSERT INTO shard_economy (accounts, gold, t) VALUES (?, ?, ?)', [ + accounts ?? null, + gold ?? null, + t, + ]) + +const listEconomy = (limit) => + query('SELECT accounts, gold, t FROM shard_economy ORDER BY t DESC LIMIT ?', [limit]) + +async function latestEconomy() { + const rows = await query('SELECT accounts, gold, t FROM shard_economy ORDER BY t DESC LIMIT 1') + return rows[0] || null +} + +// ── Houses / IDOC ──────────────────────────────────────────────────────── +const HOUSE_COLS = + 'serial, stage, map, x, y, z, region, name, owner_serial, owner_acct, built_on, last_refreshed, is_idoc, updated_at' + +const upsertHouse = (serial, fields) => upsertRow('shard_houses', 'serial', serial, fields) + +const listIdocHouses = () => + query(`SELECT ${HOUSE_COLS} FROM shard_houses WHERE is_idoc = 1 ORDER BY updated_at DESC`) + +// Houses owned by any of the given game accounts (admin: a user's linked +// accounts). IDOC houses first, then newest-refreshed. Empty list short-circuits. +const listHousesByAccounts = (accounts) => + accounts.length === 0 + ? Promise.resolve([]) + : query( + `SELECT ${HOUSE_REG_COLS} FROM shard_houses + WHERE owner_acct IN (${accounts.map(() => '?').join(', ')}) + ORDER BY is_idoc DESC, updated_at DESC`, + accounts, + ) + +// ── House registry (Protocol 2.0 house.update / house.remove) ────────────── +// The registry columns extend HOUSE_COLS; a registry row is one we've seen via +// house.update (in_registry = 1), as opposed to a decay-only transition row. +const HOUSE_REG_COLS = `${HOUSE_COLS}, owner_name, co_owners, friends, price, decay, in_registry` + +const removeHouse = (serial) => query('DELETE FROM shard_houses WHERE serial = ?', [serial]) + +// The full registered-house browser: every row we've seen via house.update. +const listRegistryHouses = () => + query(`SELECT ${HOUSE_REG_COLS} FROM shard_houses WHERE in_registry = 1 ORDER BY name ASC`) + +// ── Champion spawns ──────────────────────────────────────────────────────── +const CHAMP_COLS = + 'serial, category, type, name, status, active, map, x, y, z, boss_up, payload, t, updated_at' + +const upsertChamp = (serial, fields) => upsertRow('shard_champs', 'serial', serial, fields) + +const removeChamp = (serial) => query('DELETE FROM shard_champs WHERE serial = ?', [serial]) +const clearChamps = () => query('DELETE FROM shard_champs') +// Ordered by name (matches the sidecar's /champs ordering). +const listChamps = () => query(`SELECT ${CHAMP_COLS} FROM shard_champs ORDER BY name ASC`) + +// ── Help-page (support) queue ────────────────────────────────────────────── +const PAGE_COLS = + 'page_id, type, sender_name, sender_acct, web_id, message, map, x, y, z, sent_ms, handled, handler, payload, updated_at' + +async function upsertPage(pageId, fields) { + const cols = Object.keys(fields) + const allCols = ['page_id', ...cols] + const insertCols = allCols.map((c) => `\`${c}\``).join(', ') + const placeholders = allCols.map(() => '?').join(', ') + const updates = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(', ') + await query( + `INSERT INTO shard_pages (${insertCols}) VALUES (${placeholders}) + ON DUPLICATE KEY UPDATE ${updates}`, + [pageId, ...cols.map((c) => fields[c])], + ) +} + +const removePage = (pageId) => query('DELETE FROM shard_pages WHERE page_id = ?', [pageId]) +const clearPages = () => query('DELETE FROM shard_pages') +// Oldest-open first so the queue reads like a work list. +const listPages = () => query(`SELECT ${PAGE_COLS} FROM shard_pages ORDER BY sent_ms ASC`) + +// ── Guild board (Protocol 2.0) ───────────────────────────────────────────── +const GUILD_COLS = + 'id, name, abbr, members, online, alliance, leader_serial, leader_name, leader_acct, leader_web_id, payload, t, updated_at' + +const upsertGuild = (id, fields) => upsertRow('shard_guilds', 'id', id, fields) + +const removeGuild = (id) => query('DELETE FROM shard_guilds WHERE id = ?', [id]) +const clearGuilds = () => query('DELETE FROM shard_guilds') +const listGuilds = () => query(`SELECT ${GUILD_COLS} FROM shard_guilds ORDER BY name ASC`) + +// The guild an actor LEADS — matched on the current board (leader_serial or the +// linked leader_acct), so it reflects live state. Guild MEMBERSHIP for non-leaders +// is not modelled (the board carries only counts + leader), so we don't guess it. +const findGuildLedByActor = (serial, acct) => + query( + `SELECT id, name, abbr, alliance, leader_name FROM shard_guilds + WHERE leader_serial = ? OR (leader_acct IS NOT NULL AND leader_acct = ?) + LIMIT 1`, + [serial ?? null, acct ?? null], + ) + +// Guilds led by any of the given game accounts (admin: a user's linked accounts). +const listGuildsLedByAccounts = (accounts) => + accounts.length === 0 + ? Promise.resolve([]) + : query( + `SELECT id, name, abbr, alliance, leader_name FROM shard_guilds + WHERE leader_acct IN (${accounts.map(() => '?').join(', ')}) + ORDER BY name ASC`, + accounts, + ) + +// ── Governor board + term history (Protocol 2.0) ─────────────────────────── +const GOV_COLS = + 'city, governor_serial, governor_name, governor_acct, governor_web_id, elect_serial, elect_name, elect_acct, election_phase, candidates, auto_pick_at, payload, t, updated_at' + +const upsertGovernor = (city, fields) => upsertRow('shard_governors', 'city', city, fields) + +const listGovernors = () => query(`SELECT ${GOV_COLS} FROM shard_governors ORDER BY city ASC`) + +// Cities whose current governor is one of the given game accounts (cross-link: +// does this user hold a governorship?). Empty list short-circuits. +const listGovernorshipsByAccounts = (accounts) => + accounts.length === 0 + ? Promise.resolve([]) + : query( + `SELECT ${GOV_COLS} FROM shard_governors + WHERE governor_acct IN (${accounts.map(() => '?').join(', ')}) + ORDER BY city ASC`, + accounts, + ) + +// The single open term (ended_at IS NULL) for a city, if any. +async function currentGovernorTerm(city) { + const rows = await query( + 'SELECT id, city, governor_serial, governor_name, governor_acct, governor_web_id, started_at, ended_at, votes FROM shard_governor_terms WHERE city = ? AND ended_at IS NULL ORDER BY started_at DESC LIMIT 1', + [city], + ) + return rows[0] || null +} + +const closeGovernorTerm = (id, endedAt) => + query('UPDATE shard_governor_terms SET ended_at = ? WHERE id = ?', [endedAt, id]) + +const openGovernorTerm = ({ city, serial, name, acct, webId, startedAt }) => + query( + `INSERT INTO shard_governor_terms + (city, governor_serial, governor_name, governor_acct, governor_web_id, started_at) + VALUES (?, ?, ?, ?, ?, ?)`, + [city, serial ?? null, name ?? null, acct ?? null, webId ?? null, startedAt], + ) + +const listGovernorTerms = (city, limit) => + query( + 'SELECT id, city, governor_serial, governor_name, governor_acct, governor_web_id, started_at, ended_at, votes FROM shard_governor_terms WHERE city = ? ORDER BY started_at DESC LIMIT ?', + [city, limit], + ) + +// ── Online-population snapshot (Protocol 2.0 presence.online) ─────────────── +async function setPresence({ count, byFacet, byRegion, t }) { + await query( + `INSERT INTO shard_presence (id, count, by_facet, by_region, t) VALUES (1, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE count = VALUES(count), by_facet = VALUES(by_facet), + by_region = VALUES(by_region), t = VALUES(t)`, + [ + Number.isFinite(count) ? count : 0, + byFacet ? JSON.stringify(byFacet) : null, + byRegion ? JSON.stringify(byRegion) : null, + Number.isFinite(t) ? t : null, + ], + ) +} + +async function latestPresence() { + const rows = await query('SELECT count, by_facet, by_region, t FROM shard_presence WHERE id = 1') + return rows[0] || null +} + +// ── Shard ruleset (Protocol 3.0 world.ruleset) ───────────────────────────── +// Singleton, same shape as shard_presence: the shard re-emits the whole frame on +// every connect, so there is nothing to merge — the latest one wins outright. +async function setRuleset({ rev, expansion, payload, t }) { + await query( + `INSERT INTO shard_ruleset (id, rev, expansion, payload, t) VALUES (1, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE rev = VALUES(rev), expansion = VALUES(expansion), + payload = VALUES(payload), t = VALUES(t)`, + [rev ?? null, expansion ?? null, payload, Number.isFinite(t) ? t : null], + ) +} + +async function getRuleset() { + const rows = await query( + 'SELECT rev, expansion, payload, t, updated_at FROM shard_ruleset WHERE id = 1', + ) + return rows[0] || null +} + +// ── Points/loyalty boards (Protocol 3.0 points.board) ────────────────────── +// One row per point system. The shard only emits a system whose top N actually +// moved, so this is a sparse stream of overwrites; there is no delete, because +// the shard's set of systems is fixed at startup. +async function upsertPointsBoard({ system, name, nameCliloc, maxPoints, players, showOnGump, payload, t }) { + await query( + `INSERT INTO shard_points_boards + (system, name, name_cliloc, max_points, players, show_on_gump, payload, t) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE name = VALUES(name), name_cliloc = VALUES(name_cliloc), + max_points = VALUES(max_points), players = VALUES(players), + show_on_gump = VALUES(show_on_gump), payload = VALUES(payload), t = VALUES(t)`, + [ + system, + name ?? null, + Number.isFinite(nameCliloc) ? nameCliloc : null, + Number.isFinite(maxPoints) ? maxPoints : null, + Number.isFinite(players) ? players : null, + showOnGump ? 1 : 0, + payload, + Number.isFinite(t) ? t : null, + ], + ) +} + +// Ordered by display name, falling back to the system key for a board whose name +// arrived as a bare cliloc — otherwise every unresolved board would sort together +// under NULL. +async function listPointsBoards() { + return query( + `SELECT system, name, name_cliloc, max_points, players, show_on_gump, payload, t, updated_at + FROM shard_points_boards ORDER BY COALESCE(name, system), system`, + ) +} + +async function getPointsBoard(system) { + const rows = await query( + `SELECT system, name, name_cliloc, max_points, players, show_on_gump, payload, t, updated_at + FROM shard_points_boards WHERE system = ?`, + [system], + ) + return rows[0] || null +} + +module.exports = { + upsertOnline, + removeOnline, + clearOnline, + countOnline, + listOnline, + listOnlineLinked, + listOnlineByAccounts, + insertEconomy, + listEconomy, + latestEconomy, + upsertHouse, + listIdocHouses, + listHousesByAccounts, + removeHouse, + listRegistryHouses, + upsertGuild, + removeGuild, + clearGuilds, + listGuilds, + findGuildLedByActor, + listGuildsLedByAccounts, + upsertGovernor, + listGovernors, + listGovernorshipsByAccounts, + currentGovernorTerm, + closeGovernorTerm, + openGovernorTerm, + listGovernorTerms, + setPresence, + latestPresence, + setRuleset, + getRuleset, + upsertPointsBoard, + listPointsBoards, + getPointsBoard, + upsertChamp, + removeChamp, + clearChamps, + listChamps, + upsertPage, + removePage, + clearPages, + listPages, +} diff --git a/server/model/shardState/shardState.model.js b/server/model/shardState/shardState.model.js new file mode 100644 index 0000000..10a3ba2 --- /dev/null +++ b/server/model/shardState/shardState.model.js @@ -0,0 +1,638 @@ +// Live shard state derived from the WS feed: who is online, the gold-supply +// series, and per-house decay stage. The ingest dispatcher calls the write +// methods; the public read endpoints call the list/count methods. Writes take +// camelCase semantic objects and map to the snake_case columns; only the keys +// present are written (so a char.vitals refresh doesn't clobber login fields). + +const db = require('./shardState.db') + +const MAX_ECONOMY = 1000 + +// Small coercion helpers, kept out of the upsert builders below so those stay +// flat (each inline `?? null` / ternary otherwise adds to cognitive complexity). +const orNull = (v) => v ?? null +const toDate = (v) => (v ? new Date(v) : null) +// Owner is an actor object (or null for an abandoned house); flatten to columns. +const ownerFields = (owner) => ({ + owner_serial: orNull(owner?.serial), + owner_acct: orNull(owner?.acct), + owner_name: orNull(owner?.name), +}) + +// Map a camelCase online descriptor to DB columns, dropping undefined keys so a +// partial refresh only touches the fields it carries. +function onlineFields(data) { + const map = { + name: data.name, + acct: data.acct, + web_id: data.webId, + map: data.map, + x: data.x, + y: data.y, + z: data.z, + hits: data.hits, + hits_max: data.hitsMax, + mana: data.mana, + mana_max: data.manaMax, + stam: data.stam, + stam_max: data.stamMax, + str: data.str, + dex: data.dex, + int: data.int, + } + const fields = {} + for (const [k, v] of Object.entries(map)) if (v !== undefined) fields[k] = v + return fields +} + +// Upsert an online player (mob.login) or refresh their vitals (char.vitals). +async function upsertOnline(data) { + if (!data || !data.serial) return + await db.upsertOnline(data.serial, onlineFields(data)) +} + +const setOffline = (serial) => db.removeOnline(serial) +const clearOnline = () => db.clearOnline() +const onlineCount = () => db.countOnline() + +function shapeOnline(r) { + return { + serial: r.serial, + name: r.name, + acct: r.acct, + webId: r.web_id, + map: r.map, + x: r.x, + y: r.y, + z: r.z, + hits: r.hits, + hitsMax: r.hits_max, + mana: r.mana, + manaMax: r.mana_max, + stam: r.stam, + stamMax: r.stam_max, + str: r.str, + dex: r.dex, + int: r.int, + updatedAt: r.updated_at, + } +} + +// Only players whose account is linked to a website user (opt-in visibility). +async function listOnlineLinked() { + const rows = await db.listOnlineLinked() + return rows.map(shapeOnline) +} + +async function listOnline() { + const rows = await db.listOnline() + return rows.map(shapeOnline) +} + +// Append a gold-supply sample (economy.supply). +async function addEconomySample({ accounts, gold, t }) { + await db.insertEconomy({ accounts, gold, t }) +} + +async function listEconomy(limit = 100) { + const n = Math.min(Math.max(Number(limit) || 100, 1), MAX_ECONOMY) + const rows = await db.listEconomy(n) + // Return oldest → newest for charting. + return rows + .map((r) => ({ accounts: r.accounts, gold: r.gold == null ? null : Number(r.gold), t: r.t })) + .reverse() +} + +async function latestEconomy() { + const r = await db.latestEconomy() + return r ? { accounts: r.accounts, gold: r.gold == null ? null : Number(r.gold), t: r.t } : null +} + +// Upsert a house's decay stage (house.decay). is_idoc is derived from the stage. +async function upsertHouse(data) { + if (!data || !data.serial) return + const fields = { + stage: data.stage ?? null, + map: data.map ?? null, + x: data.x ?? null, + y: data.y ?? null, + z: data.z ?? null, + region: data.region ?? null, + name: data.name ?? null, + owner_serial: data.ownerSerial ?? null, + owner_acct: data.ownerAcct ?? null, + built_on: data.builtOn ? new Date(data.builtOn) : null, + last_refreshed: data.lastRefreshed ? new Date(data.lastRefreshed) : null, + is_idoc: String(data.stage).toUpperCase() === 'IDOC' ? 1 : 0, + } + await db.upsertHouse(data.serial, fields) +} + +function shapeHouse(r) { + return { + serial: r.serial, + stage: r.stage, + map: r.map, + x: r.x, + y: r.y, + z: r.z, + region: r.region, + name: r.name, + ownerSerial: r.owner_serial, + ownerAcct: r.owner_acct, + // Registry fields (Protocol 2.0 house.update); undefined on decay-only rows. + ownerName: r.owner_name, + coOwners: r.co_owners, + friends: r.friends, + price: r.price == null ? null : Number(r.price), + decay: r.decay, + inRegistry: r.in_registry == null ? undefined : Boolean(r.in_registry), + builtOn: r.built_on, + lastRefreshed: r.last_refreshed, + isIdoc: Boolean(r.is_idoc), + updatedAt: r.updated_at, + } +} + +async function listIdoc() { + const rows = await db.listIdocHouses() + return rows.map(shapeHouse) +} + +// Houses owned by the given game accounts (admin: a user's linked accounts). +async function listHousesForAccounts(accounts) { + const rows = await db.listHousesByAccounts(accounts) + return rows.map(shapeHouse) +} + +// ── House registry (Protocol 2.0 house.update / house.remove) ────────────── +// Richer per-house snapshot than the decay-transition feed. Writes only the +// registry columns (+ shared location/owner fields); is_idoc/stage stay owned by +// the house.decay path, so the two feeds never clobber each other. owner is an +// actor object (or null for an abandoned house). +async function upsertHouseRegistry(data) { + if (!data || !data.serial) return + const fields = { + name: orNull(data.name), + ...ownerFields(data.owner || null), + co_owners: orNull(data.coOwners), + friends: orNull(data.friends), + price: orNull(data.price), + decay: orNull(data.decay), + region: orNull(data.region), + map: orNull(data.map), + x: orNull(data.x), + y: orNull(data.y), + z: orNull(data.z), + built_on: toDate(data.builtOn), + last_refreshed: toDate(data.lastRefreshed), + in_registry: 1, + } + await db.upsertHouse(data.serial, fields) +} + +const removeHouse = (serial) => (serial ? db.removeHouse(serial) : Promise.resolve()) + +async function listHouses() { + const rows = await db.listRegistryHouses() + return rows.map(shapeHouse) +} + +// Online players on the given game accounts (admin: a user's linked accounts). +async function listOnlineForAccounts(accounts) { + const rows = await db.listOnlineByAccounts(accounts) + return rows.map(shapeOnline) +} + +// ── Champion spawns ──────────────────────────────────────────────────────── +// Upsert a champ spawn's state (champ.update). The full event is stored in +// `payload` for the category-specific fields; a few columns are hoisted out for +// querying/ordering. is-boss-up is derived from bossUp (sea bosses are always up). +async function upsertChamp(ev) { + if (!ev || !ev.serial) return + await db.upsertChamp(ev.serial, { + category: orNull(ev.category), + type: orNull(ev.type), + name: orNull(ev.name), + status: orNull(ev.status), + active: ev.active ? 1 : 0, + map: orNull(ev.map), + x: orNull(ev.x), + y: orNull(ev.y), + z: orNull(ev.z), + boss_up: ev.bossUp ? 1 : 0, + payload: JSON.stringify(ev), + t: Number.isFinite(ev.t) ? ev.t : null, + }) +} + +const removeChamp = (serial) => (serial ? db.removeChamp(serial) : Promise.resolve()) +const clearChamps = () => db.clearChamps() + +// Return the stored champ.update payload (the shape the sidecar/UI expect), +// falling back to the hoisted columns if an older row lacks a payload. +function shapeChamp(r) { + const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload + return payload || { + kind: 'champ.update', + serial: r.serial, + category: r.category, + type: r.type, + name: r.name, + status: r.status, + active: Boolean(r.active), + map: r.map, + x: r.x, + y: r.y, + z: r.z, + bossUp: Boolean(r.boss_up), + t: r.t, + } +} + +async function listChamps() { + const rows = await db.listChamps() + return rows.map(shapeChamp) +} + +// Replace the whole board with a fresh snapshot (sidecar GET /champs on connect). +async function replaceChamps(spawns) { + await db.clearChamps() + for (const ev of spawns || []) await upsertChamp(ev) +} + +// ── Help-page (support) queue ────────────────────────────────────────────── +// Upsert a page (page.new / page.updated). The `sender` actor object carries the +// name/acct/webId; the rest are top-level fields. +async function upsertPage(ev) { + const pageId = ev && (ev.pageId || (ev.sender && ev.sender.serial)) + if (!pageId) return + const sender = ev.sender || {} + await db.upsertPage(pageId, { + type: orNull(ev.type), + sender_name: orNull(sender.name), + sender_acct: orNull(sender.acct), + web_id: orNull(sender.webId), + message: orNull(ev.message), + map: orNull(ev.map), + x: orNull(ev.x), + y: orNull(ev.y), + z: orNull(ev.z), + sent_ms: Number.isFinite(ev.sentMs) ? ev.sentMs : null, + handled: ev.handled ? 1 : 0, + handler: orNull(ev.handler), + payload: JSON.stringify(ev), + }) +} + +const removePage = (pageId) => (pageId ? db.removePage(pageId) : Promise.resolve()) +const clearPages = () => db.clearPages() + +function shapePage(r) { + const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload + return { + pageId: r.page_id, + type: r.type, + sender: { serial: r.page_id, name: r.sender_name, acct: r.sender_acct, webId: r.web_id }, + message: r.message, + map: r.map, + x: r.x, + y: r.y, + z: r.z, + sentMs: r.sent_ms == null ? null : Number(r.sent_ms), + handled: Boolean(r.handled), + handler: r.handler, + updatedAt: r.updated_at, + // Keep the raw payload available for any field not hoisted above. + payload: payload || undefined, + } +} + +async function listPages() { + const rows = await db.listPages() + return rows.map(shapePage) +} + +// Replace the whole queue with a fresh snapshot (sidecar GET /pages on connect). +async function replacePages(pages) { + await db.clearPages() + for (const ev of pages || []) await upsertPage(ev) +} + +// ── Guild board (Protocol 2.0) ───────────────────────────────────────────── +// Upsert a guild's roster snapshot (guild.update). The leader is an actor object +// flattened into leader_* columns; the full event lives in `payload`. +async function upsertGuild(ev) { + if (!ev || ev.id == null) return + const leader = ev.leader || {} + await db.upsertGuild(ev.id, { + name: ev.name ?? null, + abbr: ev.abbr ?? null, + members: ev.members ?? null, + online: ev.online ?? null, + alliance: ev.alliance ?? null, + leader_serial: leader.serial ?? null, + leader_name: leader.name ?? null, + leader_acct: leader.acct ?? null, + leader_web_id: leader.webId ?? null, + payload: JSON.stringify(ev), + t: Number.isFinite(ev.t) ? ev.t : null, + }) +} + +const removeGuild = (id) => (id == null ? Promise.resolve() : db.removeGuild(id)) +const clearGuilds = () => db.clearGuilds() + +function shapeGuild(r) { + const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload + return payload || { + kind: 'guild.update', + id: r.id, + name: r.name, + abbr: r.abbr, + members: r.members, + online: r.online, + alliance: r.alliance, + leader: r.leader_serial + ? { serial: r.leader_serial, name: r.leader_name, acct: r.leader_acct, webId: r.leader_web_id } + : null, + t: r.t, + } +} + +async function listGuilds() { + const rows = await db.listGuilds() + return rows.map(shapeGuild) +} + +// Replace the board with a fresh snapshot (sidecar GET /guilds on connect). +async function replaceGuilds(guilds) { + await db.clearGuilds() + for (const ev of guilds || []) await upsertGuild(ev) +} + +// The guild an actor leads (cross-link on the character sheet). Leadership only — +// see the db note; membership for rank-and-file isn't in the feed, so we return +// null rather than show a possibly-stale guess. +async function findGuildForActor({ serial, acct }) { + const rows = await db.findGuildLedByActor(serial ?? null, acct ?? null) + const g = rows[0] + if (!g) return null + return { id: g.id, name: g.name, abbr: g.abbr, alliance: g.alliance, role: 'leader' } +} + +// Guilds led by any of a user's linked accounts (admin user-detail cross-link). +async function listGuildsLedForAccounts(accounts) { + const rows = await db.listGuildsLedByAccounts(accounts) + return rows.map((g) => ({ id: g.id, name: g.name, abbr: g.abbr, alliance: g.alliance, leaderName: g.leader_name })) +} + +// ── Town governors (Protocol 2.0) ────────────────────────────────────────── +// Upsert a city's governance snapshot (city.update) AND capture term history. +// Term capture runs first (it reads the CURRENT open term to decide whether the +// governor changed) and is idempotent: a repeat/backfill of the same governor is a +// no-op, so it's safe to call on the live feed and on reconnect snapshots alike. +async function upsertGovernor(ev) { + if (!ev || !ev.city) return + await recordGovernorTransition(ev) + const gov = ev.governor + const elect = ev.governorElect + await db.upsertGovernor(ev.city, { + governor_serial: orNull(gov?.serial), + governor_name: orNull(gov?.name), + governor_acct: orNull(gov?.acct), + governor_web_id: orNull(gov?.webId), + elect_serial: orNull(elect?.serial), + elect_name: orNull(elect?.name), + elect_acct: orNull(elect?.acct), + election_phase: orNull(ev.electionPhase), + candidates: orNull(ev.candidates), + auto_pick_at: toDate(ev.autoPickAt), + payload: JSON.stringify(ev), + t: Number.isFinite(ev.t) ? ev.t : null, + }) +} + +// Close the open term and open a new one when the governor CHANGES. Idempotent: +// same governor as the open term ⇒ nothing happens (so backfill/duplicate +// city.update events never spawn spurious terms). +async function recordGovernorTransition(ev) { + const gov = ev.governor || null + const newSerial = gov ? gov.serial ?? null : null + const t = Number.isFinite(ev.t) ? ev.t : Date.now() + const open = await db.currentGovernorTerm(ev.city) + const openSerial = open ? open.governor_serial : null + if (open && openSerial === newSerial) return // unchanged — nothing to record + if (open) await db.closeGovernorTerm(open.id, t) // governor changed or seat vacated + if (newSerial) { + await db.openGovernorTerm({ + city: ev.city, + serial: newSerial, + name: gov.name ?? null, + acct: gov.acct ?? null, + webId: gov.webId ?? null, + startedAt: t, + }) + } +} + +function shapeGovernor(r) { + const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload + return payload || { + kind: 'city.update', + city: r.city, + governor: r.governor_serial + ? { serial: r.governor_serial, name: r.governor_name, acct: r.governor_acct, webId: r.governor_web_id } + : null, + governorElect: r.elect_serial + ? { serial: r.elect_serial, name: r.elect_name, acct: r.elect_acct } + : null, + electionPhase: r.election_phase, + candidates: r.candidates, + t: r.t, + } +} + +async function listGovernors() { + const rows = await db.listGovernors() + return rows.map(shapeGovernor) +} + +// Cities the given game accounts currently govern (cross-link badge). +async function listGovernorshipsForAccounts(accounts) { + const rows = await db.listGovernorshipsByAccounts(accounts) + return rows.map(shapeGovernor) +} + +// Term history for a city (look-back), newest first. +async function listGovernorHistory(city, limit = 100) { + const n = Math.min(Math.max(Number(limit) || 100, 1), 500) + const rows = await db.listGovernorTerms(city, n) + return rows.map((r) => ({ + city: r.city, + governor: r.governor_serial + ? { serial: r.governor_serial, name: r.governor_name, acct: r.governor_acct, webId: r.governor_web_id } + : null, + startedAt: r.started_at == null ? null : Number(r.started_at), + endedAt: r.ended_at == null ? null : Number(r.ended_at), + votes: r.votes, + })) +} + +// Upsert governors without clearing (cities are fixed, no remove event); term +// capture inside upsertGovernor stays idempotent across reconnect snapshots. +async function replaceGovernors(cities) { + for (const ev of cities || []) await upsertGovernor(ev) +} + +// ── Online-population snapshot (Protocol 2.0 presence.online) ─────────────── +async function setPresence(ev) { + if (!ev) return + await db.setPresence({ + count: ev.count, + byFacet: ev.byFacet || null, + byRegion: ev.byRegion || null, + t: ev.t, + }) +} + +async function latestPresence() { + const r = await db.latestPresence() + if (!r) return { count: 0, byFacet: {}, byRegion: {}, t: null } + const parse = (v) => (typeof v === 'string' ? safeJson(v) || {} : v || {}) + return { + count: Number(r.count) || 0, + byFacet: parse(r.by_facet), + byRegion: parse(r.by_region), + t: r.t == null ? null : Number(r.t), + } +} + +// ── Shard ruleset (Protocol 3.0 world.ruleset) ───────────────────────────── +// +// The whole frame is stored in `payload` and served back whole. Nothing is +// normalized out of it: it is a flat description of config read as one page, and +// splitting it into columns would mean a schema change every time the shard grows +// a new block. `rev` and `expansion` are hoisted only because they are cheap to +// index/display, following shard_champs' payload-plus-hoisted-columns pattern. +async function setRuleset(ev) { + if (!ev) return + await db.setRuleset({ + rev: ev.rev ?? null, + expansion: ev.expansion ?? null, + payload: JSON.stringify(ev), + t: ev.t, + }) +} + +// The stored ruleset, or null when the shard has never published one (an old +// plugin, or Bridge.RulesetEnabled=false). Null is a real answer here — the page +// says "not published yet" rather than rendering an empty ruleset as if the shard +// had no rules — so it is deliberately not smoothed into {}. +async function getRuleset() { + const r = await db.getRuleset() + if (!r) return null + const payload = typeof r.payload === 'string' ? safeJson(r.payload) : r.payload + if (!payload) return null + return { ...payload, updatedAt: r.updated_at } +} + +// ── Points/loyalty boards (Protocol 3.0 points.board) ────────────────────── +// +// The whole frame is stored in `payload`; the columns beside it are hoisted for +// listing and ordering only. The top-N list deliberately stays inside the payload +// (see schema.sql) — it is a fixed-size list read whole, like the governor board's +// candidates. +async function upsertPointsBoard(ev) { + if (!ev || !ev.system) return + await db.upsertPointsBoard({ + system: String(ev.system).slice(0, 48), + name: ev.nameString ?? null, + nameCliloc: ev.nameNumber, + maxPoints: ev.maxPoints, + players: ev.players, + showOnGump: ev.showOnGump !== false, + payload: JSON.stringify(ev), + t: ev.t, + }) +} + +// A stored frame plus the freshness stamp. `top` is normalized to an array so a +// caller never has to guard it — a board with nobody on it is a real state (a +// system nobody has scored in yet), distinct from a system that was never +// published at all, which is absent from the table entirely. +function shapePointsBoard(r) { + const payload = (typeof r.payload === 'string' ? safeJson(r.payload) : r.payload) || {} + return { + ...payload, + system: r.system, + top: Array.isArray(payload.top) ? payload.top : [], + updatedAt: r.updated_at, + } +} + +async function listPointsBoards() { + const rows = await db.listPointsBoards() + return rows.map(shapePointsBoard) +} + +async function getPointsBoard(system) { + const r = await db.getPointsBoard(system) + return r ? shapePointsBoard(r) : null +} + +function safeJson(s) { + try { + return JSON.parse(s) + } catch { + return null + } +} + +module.exports = { + upsertOnline, + setOffline, + clearOnline, + onlineCount, + listOnline, + listOnlineLinked, + listOnlineForAccounts, + addEconomySample, + listEconomy, + latestEconomy, + upsertHouse, + listIdoc, + listHousesForAccounts, + upsertHouseRegistry, + removeHouse, + listHouses, + upsertChamp, + removeChamp, + clearChamps, + listChamps, + replaceChamps, + upsertPage, + removePage, + clearPages, + listPages, + replacePages, + upsertGuild, + removeGuild, + clearGuilds, + listGuilds, + replaceGuilds, + findGuildForActor, + listGuildsLedForAccounts, + upsertGovernor, + listGovernors, + listGovernorshipsForAccounts, + listGovernorHistory, + replaceGovernors, + setPresence, + latestPresence, + setRuleset, + getRuleset, + upsertPointsBoard, + listPointsBoards, + getPointsBoard, +} diff --git a/server/model/shardVisibility/shardVisibility.db.js b/server/model/shardVisibility/shardVisibility.db.js new file mode 100644 index 0000000..de8b7b8 --- /dev/null +++ b/server/model/shardVisibility/shardVisibility.db.js @@ -0,0 +1,37 @@ +const { query } = require('../../core') + +// One row per shard feature. Absent rows are fine — utils/shardVisibility.js +// compiles a default for every known feature and merges stored rows over it, so +// a fresh install with an empty table behaves exactly as the site did pre-v3. + +const COLS = 'feature, enabled, audience, stream, field_rules, updated_by, updated_at' + +const listAll = () => query(`SELECT ${COLS} FROM shard_feature_visibility`) + +const getOne = (feature) => + query(`SELECT ${COLS} FROM shard_feature_visibility WHERE feature = ?`, [feature]) + +// Upsert one feature's settings. `fieldRules` is stored as a JSON object of +// {field: rung}; the caller has already stripped locked fields and validated +// every rung against the ladder. +const upsert = ({ feature, enabled, audience, stream, fieldRules, updatedBy }) => + query( + `INSERT INTO shard_feature_visibility (feature, enabled, audience, stream, field_rules, updated_by) + VALUES (?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + enabled = VALUES(enabled), + audience = VALUES(audience), + stream = VALUES(stream), + field_rules = VALUES(field_rules), + updated_by = VALUES(updated_by)`, + [ + feature, + enabled ? 1 : 0, + audience, + stream ? 1 : 0, + fieldRules == null ? null : JSON.stringify(fieldRules), + updatedBy ?? null, + ], + ) + +module.exports = { listAll, getOne, upsert } diff --git a/server/model/shardVisibility/shardVisibility.model.js b/server/model/shardVisibility/shardVisibility.model.js new file mode 100644 index 0000000..094989a --- /dev/null +++ b/server/model/shardVisibility/shardVisibility.model.js @@ -0,0 +1,44 @@ +// ── Shard feature visibility (model) ─────────────────────────────────────── +// +// Thin row-shaping layer over shardVisibility.db. The policy — the ladder, the +// feature catalog, the locked fields, the kind→feature map — lives in +// utils/shardVisibility.js; this file only reads and writes rows. + +const db = require('./shardVisibility.db') + +// The `field_rules` JSON column comes back as a string on the mariadb driver. +function parseRules(raw) { + if (raw == null) return {} + if (typeof raw === 'object') return raw + try { + const parsed = JSON.parse(raw) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {} + } catch { + return {} + } +} + +const toSafe = (row) => + row && { + feature: row.feature, + enabled: !!row.enabled, + audience: row.audience, + stream: row.stream == null ? null : !!row.stream, + fieldRules: parseRules(row.field_rules), + updatedBy: row.updated_by, + updatedAt: row.updated_at, + } + +async function listAll() { + const rows = await db.listAll() + return rows.map(toSafe) +} + +async function getOne(feature) { + const rows = await db.getOne(feature) + return toSafe(rows[0]) +} + +const upsert = (entry) => db.upsert(entry) + +module.exports = { listAll, getOne, upsert } diff --git a/server/model/singletonConfigDb.js b/server/model/singletonConfigDb.js new file mode 100644 index 0000000..4797158 --- /dev/null +++ b/server/model/singletonConfigDb.js @@ -0,0 +1,35 @@ +const { query } = require('../core') + +// Factory for the singleton config tables (bot_config, email_config, +// uo_link_config). Each is a one-row table keyed on id = 1: `get()` returns the +// row (or null before the admin first saves it), and `upsert()` writes only the +// columns the model prepared, leaving the rest untouched. The three tables share +// this shape exactly, so the DB layer is generated rather than copy-pasted — +// only the table name and column list differ. +// +// `fields` are already prepared by the model (secrets pre-encrypted); ordering +// and encryption stay a model-layer concern. +function singletonConfigDb(table, cols) { + async function get() { + const rows = await query(`SELECT ${cols} FROM ${table} WHERE id = 1 LIMIT 1`) + return rows[0] || null + } + + async function upsert(fields) { + const columns = Object.keys(fields) + const vals = columns.map((c) => fields[c]) + const insertCols = ['id', ...columns].map((c) => `\`${c}\``).join(', ') + const placeholders = ['1', ...columns.map(() => '?')].join(', ') + const updates = columns.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(', ') + await query( + `INSERT INTO ${table} (${insertCols}) VALUES (${placeholders}) + ON DUPLICATE KEY UPDATE ${updates}`, + vals, + ) + return get() + } + + return { get, upsert } +} + +module.exports = singletonConfigDb diff --git a/server/model/uoLinkConfig/uoLinkConfig.db.js b/server/model/uoLinkConfig/uoLinkConfig.db.js new file mode 100644 index 0000000..a27d54d --- /dev/null +++ b/server/model/uoLinkConfig/uoLinkConfig.db.js @@ -0,0 +1,7 @@ +const singletonConfigDb = require('../singletonConfigDb') + +const COLS = + 'id, base_url, ws_url, auth_token_enc, protocol, enabled, status, status_detail, plugin_connected, last_event_at, boot_id, updated_by, created_at, updated_at' + +// Singleton row (id = 1). See ../singletonConfigDb for the get/upsert contract. +module.exports = singletonConfigDb('uo_link_config', COLS) diff --git a/server/model/uoLinkConfig/uoLinkConfig.model.js b/server/model/uoLinkConfig/uoLinkConfig.model.js new file mode 100644 index 0000000..0e04e23 --- /dev/null +++ b/server/model/uoLinkConfig/uoLinkConfig.model.js @@ -0,0 +1,88 @@ +// uo-link sidecar connection config store. Mirrors botConfig/emailConfig: the DB +// layer only ever sees ciphertext, and only getWithToken() (used server-side to +// call the sidecar over REST/WS) decrypts it. The admin-facing getSafe() never +// includes the token — it exposes only `hasToken`. A blank `token` on save means +// "leave the existing token unchanged" (same convention as the other configs). + +const db = require('./uoLinkConfig.db') +const { secretBox } = require('../../core') + +// The wire protocol this build speaks (link/sidecar/src/main.rs PROTOCOL_VERSION). +// Only used before an admin has saved anything — the stored row wins once it exists, +// and UOLINK_PROTOCOL still overrides for an operator running an older sidecar. +const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 3 + +function toSafe(row) { + if (!row) { + return { + baseUrl: process.env.UOLINK_BASE_URL || null, + wsUrl: process.env.UOLINK_WS_URL || null, + protocol: DEFAULT_PROTOCOL, + enabled: false, + hasToken: false, + status: 'disconnected', + statusDetail: null, + pluginConnected: false, + lastEventAt: null, + bootId: null, + } + } + return { + baseUrl: row.base_url || null, + wsUrl: row.ws_url || null, + protocol: row.protocol || DEFAULT_PROTOCOL, + enabled: Boolean(row.enabled), + hasToken: Boolean(row.auth_token_enc), + status: row.status || 'disconnected', + statusDetail: row.status_detail || null, + pluginConnected: Boolean(row.plugin_connected), + lastEventAt: row.last_event_at || null, + bootId: row.boot_id || null, + } +} + +async function getSafe() { + return toSafe(await db.get()) +} + +// Decrypted token included — server-side only (calling the sidecar's REST/WS +// API). Returns null when nothing has been saved yet. +async function getWithToken() { + const row = await db.get() + if (!row) return null + return { ...toSafe(row), token: row.auth_token_enc ? secretBox.decrypt(row.auth_token_enc) : null } +} + +// Save admin-supplied config. `token` undefined or '' means "leave the existing +// token unchanged" (same convention as botConfig.save). +async function save({ baseUrl, wsUrl, token, protocol, enabled, updatedBy }) { + const fields = {} + if (baseUrl !== undefined) fields.base_url = baseUrl + if (wsUrl !== undefined) fields.ws_url = wsUrl + if (token) fields.auth_token_enc = secretBox.encrypt(token) + if (protocol !== undefined) fields.protocol = protocol + if (enabled !== undefined) fields.enabled = enabled ? 1 : 0 + if (updatedBy !== undefined) fields.updated_by = updatedBy + const row = await db.upsert(fields) + return toSafe(row) +} + +// Mirror the sidecar's last-reported connection state into the DB so the admin +// panel has something to show between polls and the public status endpoint can +// read it without a live round-trip. +async function recordStatus({ status, statusDetail, pluginConnected, lastEventAt, bootId }) { + const fields = {} + if (status !== undefined) fields.status = status + if (statusDetail !== undefined) fields.status_detail = statusDetail + if (pluginConnected !== undefined) fields.plugin_connected = pluginConnected ? 1 : 0 + // lastEventAt may arrive as an ISO string (e.g. "2026-07-10T22:08:27Z"); the + // mariadb DATETIME parser rejects the "T"/"Z", so hand it a real Date (same + // fix as botConfig.recordStatus's last_connected_at). + if (lastEventAt !== undefined) fields.last_event_at = lastEventAt ? new Date(lastEventAt) : null + if (bootId !== undefined) fields.boot_id = bootId + if (Object.keys(fields).length === 0) return getSafe() + const row = await db.upsert(fields) + return toSafe(row) +} + +module.exports = { getSafe, getWithToken, save, recordStatus } diff --git a/server/package-lock.json b/server/package-lock.json index b396bcd..a8d1456 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -8,6 +8,9 @@ "name": "module-uo-server", "version": "0.1.0", "license": "GPL-3.0-or-later", + "dependencies": { + "ws": "^8.21.0" + }, "devDependencies": { "express": "^4.19.2", "express-validator": "^7.2.0" @@ -927,6 +930,27 @@ "engines": { "node": ">= 0.8" } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } } } } diff --git a/server/package.json b/server/package.json index 1e84d1e..0a3642c 100644 --- a/server/package.json +++ b/server/package.json @@ -6,15 +6,19 @@ "license": "GPL-3.0-or-later", "main": "index.js", "scripts": { - "test": "node --test", + "test": "node --test --require ./test/_setup.js", "check:imports": "node scripts/checkImports.js" }, "engines": { "node": ">=20" }, - "//dependencies": "Deliberately none. Everything the shipped server half needs arrives on ctx (MODULE_API.md §2.3) — a module lives outside core's server/ and cannot resolve core's node_modules. The two below are devDependencies because test/_fakes.js builds a REAL express router: a fake Router would test the fake.", + "//dependencies": "The ONE runtime dependency, and it ships inside the release tarball: CI runs npm ci --omit=dev and packs server/node_modules, because an operator never builds (MODULE_SYSTEM.md 1.14). Node resolves it by walking up from modules/uo/server/. Everything else the shipped half needs arrives on ctx (MODULE_API.md 2.3) - express, express-validator, the database, the logger, the middleware and the rate-limit factory are all core-owned and handed over.", "devDependencies": { "express": "^4.19.2", "express-validator": "^7.2.0" - } + }, + "dependencies": { + "ws": "^8.21.0" + }, + "//devDependencies": "Test-only. test/_fakes.js builds a REAL express router - a fake Router would test the fake." } diff --git a/server/scripts/importSpawnAtlas.js b/server/scripts/importSpawnAtlas.js new file mode 100644 index 0000000..b9ecbb8 --- /dev/null +++ b/server/scripts/importSpawnAtlas.js @@ -0,0 +1,127 @@ +#!/usr/bin/env node +// +// Refresh the spawn atlas from a ServUO tree, from the command line. +// +// npm run atlas:import # use the configured path +// npm run atlas:import -- --servuo # override it for this run +// npm run atlas:import -- --force # reimport even if unchanged +// npm run atlas:import -- --approve # apply a staged refresh +// npm run atlas:import -- --status # report without changing anything +// +// The server does this itself on every boot (see `shardAtlas.refreshOnBoot`), so +// this is for operators who want to apply a map change without a restart, and +// for approving a refresh that was staged because it would remove a facet. +// +// All the logic lives in `src/model/shardAtlas/shardAtlas.model.js`; this file +// is argument parsing and output formatting. + +const db = () => require('../src/utils/db') + +function parseArgs(argv) { + const args = {} + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i] + if (flag === '--servuo') args.servuo = argv[++i] + else if (flag === '--force') args.force = true + else if (flag === '--approve') args.approve = true + else if (flag === '--reject') args.reject = true + else if (flag === '--status') args.status = true + else if (flag === '--help' || flag === '-h') args.help = true + } + return args +} + +const USAGE = ` +Refresh the spawn atlas from a ServUO tree. + + node scripts/importSpawnAtlas.js [options] + + --servuo Use this tree for this run instead of the configured path. + --force Reimport even when the source files are unchanged. + --approve Apply a refresh that was staged for removing a facet. + --reject Keep the current atlas and dismiss the staged refresh. + --status Report atlas and source state; change nothing. + +With no options this imports only if the tree differs from what is loaded. +` + +function describe(result) { + switch (result.status) { + case 'skipped': + return ( + 'No ServUO path configured — nothing to import.\n' + + 'Set one with SERVUO_PATH, the admin panel, or --servuo .\n' + ) + case 'unavailable': + return `ServUO tree unavailable: ${result.reason}\n` + case 'unchanged': + return `Atlas is already up to date${result.reason ? ` (${result.reason})` : ''}.\n` + case 'needsReview': { + return ( + 'Refresh NOT applied — it would remove ' + + `${result.removedFacets.length} facet(s): ${result.removedFacets.join(', ')}.\n` + + 'This is what a half-copied or mid-update tree looks like, so it has been\n' + + 'staged for review. The current atlas is unchanged.\n' + + 'Apply it with --approve, or dismiss it with --reject.\n' + ) + } + case 'imported': { + const c = result.counts + const added = result.addedFacets?.length ? ` Added facets: ${result.addedFacets.join(', ')}.` : '' + const removed = result.removedFacets?.length + ? ` Removed facets: ${result.removedFacets.join(', ')}.` + : '' + return ( + `Atlas imported: ${c.points} points, ${c.creatures} creatures, ` + + `${c.pointTypes} point/type rows, ${c.regions} regions, ` + + `${c.landmarks} landmarks, ${c.champions} champion altars.${added}${removed}\n` + ) + } + case 'failed': + return `Atlas refresh failed: ${result.reason}\n` + default: + return `${JSON.stringify(result, null, 2)}\n` + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)) + if (args.help) { + process.stdout.write(USAGE) + return + } + + const shardAtlas = require('../src/model/shardAtlas/shardAtlas.model') + + // `--servuo` is a per-run override and deliberately does NOT persist to the + // configured path; changing where the atlas permanently reads from is an + // admin action, not a side effect of a one-off import. + const override = { path: args.servuo ?? '' } + + if (args.status) { + process.stdout.write(`${JSON.stringify(await shardAtlas.status(override), null, 2)}\n`) + return + } + if (args.reject) { + process.stdout.write(`${JSON.stringify(await shardAtlas.rejectPending(), null, 2)}\n`) + return + } + + const result = args.approve + ? await shardAtlas.approvePending(override) + : await shardAtlas.refresh({ ...override, force: Boolean(args.force) }) + + process.stdout.write(describe(result)) + if (result.status === 'failed') process.exitCode = 1 +} + +if (require.main === module) { + main() + .catch((err) => { + process.stderr.write(`atlas:import failed: ${err.message}\n`) + process.exitCode = 1 + }) + .finally(() => db().close()) +} + +module.exports = { describe, parseArgs } diff --git a/server/utils/announceLinks.js b/server/utils/announceLinks.js new file mode 100644 index 0000000..c29450f --- /dev/null +++ b/server/utils/announceLinks.js @@ -0,0 +1,52 @@ +// The three helpers the town-crier leg needs from core's announce pipeline. +// +// Core owns `announce_jobs`, the worker that drains it and the retry policy; +// this module owns one leg of it (MODULE_API.md §2.4). These three lived in +// core's `announceJobs.logic` and are reproduced here rather than added to +// `ctx`, because each is a few lines of pure string handling with no state and +// no policy — the kind of thing a contract member would only make harder to +// change on both sides. +// +// The one that could NOT be vendored is `baseUrl`. Core's version reads +// `process.env.APP_BASE_URL`, and §2.7 forbids a module reading core's +// environment — it is core's deployment fact, not the module's. So it comes off +// `ctx.site.baseUrl` (API 1.1.0), read per call rather than captured, which also +// means a module built before an env change keeps agreeing with core after it. + +// NOT destructured. `core.baseUrl` is a getter that resolves `ctx`, so pulling +// it out here would run at require time — before `register()` — and throw. Read +// it inside the function, where `ctx` exists. +const core = require('../core') + +/** Where this deployment is reachable, without a trailing slash. */ +function baseUrl() { + return core.baseUrl +} + +/** + * The public link that goes in an announcement. + * + * News has no per-post route — core's SPA has only the list — so this links the + * list, matching what the pre-pipeline Discord announce did. It names a CORE + * route on purpose: the news list is core's page and stays core's through the + * whole extraction, so this is a module linking to its host, not a leftover. + */ +function articleUrl(base) { + return `${String(base || '').replace(/\/+$/, '')}/site/news` +} + +/** + * Squeeze a leg client's `{ ok, status, data, error }` into the one line stored + * in `announce_job_legs.last_error` and shown in the admin panel. + */ +function legError(result) { + if (!result) return 'no response' + if (result.status) { + return result.data && result.data.message + ? `${result.status}: ${result.data.message}` + : result.error || `status ${result.status}` + } + return result.error || 'request failed' +} + +module.exports = { baseUrl, articleUrl, legError } diff --git a/server/utils/clilocParse.js b/server/utils/clilocParse.js new file mode 100644 index 0000000..cf1a113 --- /dev/null +++ b/server/utils/clilocParse.js @@ -0,0 +1,287 @@ +// Cliloc parsing — the pure half. +// +// A "cliloc" is UO's localization table: an integer id mapped to a display +// string. Items carry a `LabelNumber` rather than a name, so without this table +// the site can only render `id 1023721` where the game shows "quarter staff". +// The shard already sends the id on every equipment entry (`char.profile`'s +// `cliloc` field) and will send one per marketplace listing — the *number* was +// never the missing piece, the *table* was. +// +// This module is fs-free on purpose, exactly like `spawnAtlasParse.js`: the +// suite runs in CI where there is no UO client, so every parser here is driven +// from inline fixtures. `clilocSource.js` is the only thing that touches disk. +// +// ── Two input formats, and why ───────────────────────────────────────────── +// +// The client's own `Cliloc.enu` is COMPRESSED (Mythic format) on any modern +// client, and decompressing it is a bit-level port of an inverse-BWT coder that +// nothing in this stack needs at runtime. ServUO's own bundled `Ultima.StringList` +// cannot read it either — which is why `VendorSearch.GetItemName` is already inert +// on such a shard and the plugin could not supply names even if we asked it to. +// +// So the operator converts once, from their own client, and points the site at +// the result (see docs/website/CLILOCS.md). Two shapes are accepted because +// different tools produce different things: +// +// • PLAIN BINARY — the pre-compression cliloc layout: a 6-byte header, then +// records of {int32 number, byte flag, uint16 length, UTF-8 bytes}. +// • DELIMITED TEXT — `numbertext` per line, which is what the common +// GUI exports emit. Quoted CSV fields and a header row are tolerated. +// +// Nothing derived from the client is ever committed: the converted file lives at +// an operator-supplied path and is gitignored, the same rule the spawn atlas art +// map already follows. + +/** Raised for a file we can identify but deliberately refuse to guess at. */ +class ClilocFormatError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocFormatError' + this.code = code + } +} + +/** + * Bumped when this parser produces DIFFERENT data from an IDENTICAL source file. + * + * Stored beside the source hash so the boot path can tell "same file, but the + * parser moved on" from "same file, nothing to do". Without it a corrected parse + * would ship and never reach an install whose cliloc file never changes — the + * trap `spawnAtlasSource.PARSER_VERSION` documents. + */ +const PARSER_VERSION = 1 + +// The plain layout's header is `02 00 00 00 01 00` — a 4-byte version and a +// 2-byte language marker. Only the size matters for parsing; the values are +// checked to sniff the format, not to validate it. +const HEADER_BYTES = 6 +const RECORD_HEADER_BYTES = 7 // int32 number + byte flag + uint16 length + +// Every compressed cliloc file the client ships begins with a DWORD whose high +// byte is 0x8E (the XOR key UOFiddler calls `HeaderXorKey`, 0x8E2C9A3D). That is +// the single cheapest way to tell an operator they exported the wrong file — +// without it, the plain parser happily reads compressed bytes as ~19k records of +// negative ids and 60 KB "strings" before dying somewhere in the middle, and the +// resulting error names the wrong problem. +const MYTHIC_HIGH_BYTE = 0x8e + +/** True when `buffer` is a Mythic-compressed cliloc rather than the plain layout. */ +function isCompressedCliloc(buffer) { + return buffer.length >= 4 && buffer[3] === MYTHIC_HIGH_BYTE +} + +/** + * Parse the plain binary cliloc layout. + * + * Strict about truncation, and that strictness is load-bearing: a half-copied or + * partly-written file is the realistic failure here, and it must fail loudly + * rather than import a silently short table that then renders half the world as + * `id 1023721`. A record that runs past the end of the buffer throws. + */ +function parseClilocBinary(buffer) { + if (!Buffer.isBuffer(buffer)) throw new ClilocFormatError('Not a buffer', 'NOT_BUFFER') + if (isCompressedCliloc(buffer)) { + throw new ClilocFormatError( + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.', + 'COMPRESSED', + ) + } + if (buffer.length < HEADER_BYTES) { + throw new ClilocFormatError('File is shorter than a cliloc header', 'TRUNCATED') + } + + const entries = [] + let offset = HEADER_BYTES + + while (offset < buffer.length) { + if (offset + RECORD_HEADER_BYTES > buffer.length) { + throw new ClilocFormatError( + `Truncated record header at byte ${offset} (${entries.length} entries read)`, + 'TRUNCATED', + ) + } + const number = buffer.readInt32LE(offset) + const flag = buffer.readUInt8(offset + 4) + // The length is written by the client as an unsigned 16-bit value. Reading it + // signed (as ServUO's own SDK does) turns any string over 32 KB into a + // negative length; real tables top out around 12 KB, so this has no effect on + // current data and costs nothing to get right. + const length = buffer.readUInt16LE(offset + 5) + offset += RECORD_HEADER_BYTES + + if (offset + length > buffer.length) { + throw new ClilocFormatError( + `Truncated record body at byte ${offset} (${entries.length} entries read)`, + 'TRUNCATED', + ) + } + entries.push({ number, flag, text: buffer.toString('utf8', offset, offset + length) }) + offset += length + } + + return entries +} + +// A delimited line splits on the FIRST separator only: cliloc text is full of +// commas ("a scroll of magery, unfinished") and splitting on all of them would +// truncate every such entry at its first comma. +const TEXT_SEPARATORS = ['\t', ',', ';'] + +/** Unwrap one CSV field: strip surrounding quotes and unescape doubled quotes. */ +function unquote(value) { + const trimmed = value.trim() + if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) { + return trimmed.slice(1, -1).replace(/""/g, '"') + } + return trimmed +} + +/** + * Parse a delimited text export: `numbertext` per line. + * + * Tolerant by design — this is whatever an operator's GUI tool produced, not a + * format we control. A header row, blank lines, `#` comments and a trailing + * flags column are all ignored. A line whose first field is not an integer is + * skipped rather than fatal, because that is exactly what a header row is. + * + * The one thing it will NOT do is return an empty table quietly: a file that + * yields no entries at all is a wrong file, not an empty one. + */ +function parseClilocText(text) { + const entries = [] + for (const line of String(text).split(/\r?\n/)) { + // The line is deliberately NOT trimmed before the separator search. Roughly + // half of a real cliloc table is empty strings (unused ids), which export as + // `1005008` — and trimming eats that trailing separator, leaving a bare + // number that then looks like a header row and is skipped. That silently + // dropped 55,994 of 123,490 entries. Individual FIELDS are trimmed instead, + // by `unquote`. + if (line.trim() === '' || line.trimStart().startsWith('#')) continue + + // Pick the separator that actually appears first, so a tab-delimited line + // whose text contains a comma still splits on the tab. + let cut = -1 + for (const sep of TEXT_SEPARATORS) { + const at = line.indexOf(sep) + if (at !== -1 && (cut === -1 || at < cut)) cut = at + } + if (cut === -1) continue + + // An EMPTY first field must not become id 0: `Number('')` is 0, not NaN, so + // a line that merely starts with a separator would otherwise import as a + // bogus cliloc 0 instead of being skipped. + const head = unquote(line.slice(0, cut)) + if (head === '') continue + const number = Number(head) + if (!Number.isInteger(number)) continue // header row, or a wrapped line + + let rest = line.slice(cut + 1) + // Some exports carry `number,flag,text`. A bare integer in the second field + // is a flag; anything else is the text itself (and a text field that IS just + // a number is indistinguishable, so it stays as the text — the safer miss). + let flag = 0 + for (const sep of TEXT_SEPARATORS) { + const at = rest.indexOf(sep) + if (at === -1) continue + const head = unquote(rest.slice(0, at)) + if (/^\d{1,3}$/.test(head) && rest.slice(at + 1).trim() !== '') { + flag = Number(head) + rest = rest.slice(at + 1) + } + break + } + + entries.push({ number, flag, text: unquote(rest) }) + } + + if (entries.length === 0) { + throw new ClilocFormatError('No cliloc entries found in the text export', 'EMPTY') + } + return entries +} + +/** + * Parse either supported shape, sniffing which one this is. + * + * The sniff is on the binary header rather than the file extension: operators + * name these things whatever they like, and an `.enu` that is really a TSV (or a + * `.txt` that is really binary) should still import. + */ +function parseCliloc(buffer) { + const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer) + + if (isCompressedCliloc(buf)) { + throw new ClilocFormatError( + 'This is a compressed (Mythic-format) cliloc file, which the site cannot read. ' + + 'Convert it to the plain format first — see docs/website/CLILOCS.md.', + 'COMPRESSED', + ) + } + + // The plain layout always opens with version 2 / language 1. Anything else is + // treated as text, which is the recoverable guess: a mis-sniffed text file + // yields "no entries found", while a mis-sniffed binary yields nonsense. + if (buf.length >= HEADER_BYTES && buf.readInt32LE(0) === 2 && buf.readUInt16LE(4) === 1) { + return parseClilocBinary(buf) + } + return parseClilocText(buf.toString('utf8')) +} + +// ── Display ──────────────────────────────────────────────────────────────── + +// Cliloc strings interpolate arguments the client supplies out of an item's +// property list: `~1_val~`, `~2_NAME~`, `~1_ITEM~`. We never have those — the +// bridge sends the id, not the packet — so a name carrying them must be reduced +// to what is actually knowable rather than shown with the raw tokens in it. +const PLACEHOLDER_RE = /~\d+_[^~]*~/g + +/** + * Reduce a raw cliloc string to something displayable. + * + * Placeholders are dropped and the leftover punctuation tidied, so + * `"[~1_stuff~]"` becomes `""` (correctly nothing — the whole string was the + * argument) and `"cold damage ~1_val~%"` becomes `"cold damage"`. + * + * **Punctuation is only tidied when a placeholder was actually removed.** The + * trailing `%` above is the unit belonging to the number we never had, and the + * brackets in `[~1_stuff~]` only ever wrapped the argument — but a string with + * no placeholder has no such debris, and trimming it anyway corrupts real names. + * A shard's `"Runic Gateway Sigil (v2)"` came back as `"(v2"` while this was + * unconditional. + * + * Returns `''` when nothing survives, which callers treat as "no name" and fall + * back to the item id — better than showing a bracket. + */ +const DEBRIS = /^[\s\-–—,.;:%[\]()]+|[\s\-–—,.;:%[\]()]+$/g + +function displayText(raw) { + if (raw == null) return '' + const source = String(raw) + const hadPlaceholder = PLACEHOLDER_RE.test(source) + PLACEHOLDER_RE.lastIndex = 0 // the regex is global; `test` advances it + + if (!hadPlaceholder) return source.replace(/\s+/g, ' ').trim() + + return source + .replace(PLACEHOLDER_RE, ' ') + .replace(/\s+/g, ' ') + .replace(/\s+([,.;:!?])/g, '$1') + .replace(DEBRIS, '') + .trim() +} + +/** True when a raw cliloc string is nothing but interpolated arguments. */ +const isPlaceholderOnly = (raw) => raw != null && String(raw).trim() !== '' && displayText(raw) === '' + +module.exports = { + ClilocFormatError, + PARSER_VERSION, + HEADER_BYTES, + isCompressedCliloc, + parseCliloc, + parseClilocBinary, + parseClilocText, + displayText, + isPlaceholderOnly, +} diff --git a/server/utils/clilocSource.js b/server/utils/clilocSource.js new file mode 100644 index 0000000..93ae061 --- /dev/null +++ b/server/utils/clilocSource.js @@ -0,0 +1,316 @@ +// Cliloc table — the filesystem layer. +// +// `clilocParse.js` holds the pure parsers; this module is the only thing that +// touches cliloc files on disk, and it is shared by both callers: +// +// - the server, which refreshes the table on boot (`shardClilocs.model.js`) +// - the admin panel, which can force a reimport without a restart +// +// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived +// from them is committed: the repo holds no string table, exactly as it holds no +// map snapshot and no artwork. That rule is why this module reads a configured +// path instead of a path inside the repo. +// +// ── Why this reads a SET of files, not one ──────────────────────────────── +// +// Shards edit items and add new ones. Those carry cliloc ids that a stock client +// table does not have — and forcing a 5 MB client re-export every time an +// operator adds one item would be miserable enough that the table would simply +// go stale, which is the exact failure the spawn atlas was redesigned to avoid. +// +// So this mirrors `spawnAtlasSource.readSources()`: a BASE table (the converted +// client file) plus every operator-maintained OVERLAY beside it, all re-read on +// every boot and hash-gated as a SET. Adding, editing or removing any overlay +// counts as drift and re-imports. Later sources win, so an overlay both adds new +// ids and overrides stock ones. +// +// Measured on a real shard: the script tree references 16,434 cliloc ids and only +// 37 are absent from the stock client table. Tens of entries against a 67k base +// is what makes the overlay the right shape rather than a second full table. +// +// Reading and hashing ~5 MB costs a few milliseconds and a full parse ~50 ms, so +// the boot path hashes first and only parses when something actually changed. + +const crypto = require('crypto') +const fs = require('fs') +const path = require('path') + +const { ClilocFormatError, PARSER_VERSION, parseCliloc, isCompressedCliloc } = require('./clilocParse') + +/** + * Filenames looked for as the BASE table when the configured path is a directory. + * + * Ordered by how specific they are: an explicitly converted file wins over + * something that merely sits in a client folder, so an operator who dropped a + * `cliloc.plain.enu` next to the original compressed `cliloc.enu` gets the one + * they made rather than the one that will be rejected. + * + * Matching is case-insensitive against the real directory listing, because the + * client ships `Cliloc.enu` on Windows and the site usually runs on Linux, where + * a hardcoded lowercase open would simply miss. + */ +const CANDIDATE_NAMES = [ + 'clilocs.tsv', + 'clilocs.csv', + 'clilocs.plain', + 'cliloc.plain', + 'cliloc.plain.enu', + 'cliloc.enu.plain', + 'clilocs.txt', + 'cliloc.enu', +] + +/** + * Where shard-specific additions and overrides live: a `custom/` directory + * beside the base table. + * + * ServUO has **no server-side convention** for custom clilocs — they live in the + * patched client file a shard distributes to its players, and nothing in the + * tree declares them. There is therefore nothing to discover, and this is the + * one place in the cliloc pipeline that is a convention we chose rather than one + * the shard already has. It is a directory rather than a single file so an + * operator can keep additions grouped however they like (per system, per patch) + * without the site caring. + */ +const CUSTOM_DIR = 'custom' +const CUSTOM_EXTENSIONS = ['.tsv', '.csv', '.txt', '.enu', '.plain'] + +class ClilocSourceError extends Error { + constructor(message, code) { + super(message) + this.name = 'ClilocSourceError' + this.code = code + } +} + +function sha256(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex') +} + +/** + * Resolve the configured path to `{ root, base }`. + * + * Accepts either a direct file path or a directory to search, because operators + * reasonably supply both — "here is the file" and "here is the folder I put it + * in" are equally natural answers to the admin panel's prompt. When it is a + * file, `root` is the directory CONTAINING it, so overlays work either way: an + * operator who pointed at a file should not have to re-point at its folder just + * to add a `custom/` directory next to it. + */ +function resolveBase(configured) { + if (!configured || String(configured).trim() === '') { + throw new ClilocSourceError('No cliloc path configured', 'NO_PATH') + } + const target = String(configured).trim() + + let stat + try { + stat = fs.statSync(target) + } catch { + throw new ClilocSourceError(`Cliloc path does not exist: ${target}`, 'NOT_FOUND') + } + + if (stat.isFile()) return { root: path.dirname(target), base: target } + + if (!stat.isDirectory()) { + throw new ClilocSourceError(`Cliloc path is neither a file nor a directory: ${target}`, 'NOT_FOUND') + } + + let listing + try { + listing = fs.readdirSync(target) + } catch { + throw new ClilocSourceError(`Cliloc directory is not readable: ${target}`, 'NOT_FOUND') + } + + const byLower = new Map(listing.map((name) => [name.toLowerCase(), name])) + for (const candidate of CANDIDATE_NAMES) { + const actual = byLower.get(candidate) + if (actual) return { root: target, base: path.join(target, actual) } + } + + throw new ClilocSourceError( + `No cliloc file found in ${target} (looked for ${CANDIDATE_NAMES.join(', ')})`, + 'NO_FILE', + ) +} + +/** Overlay files under `/custom/`, sorted so precedence is deterministic. */ +function listCustom(root) { + const dir = path.join(root, CUSTOM_DIR) + let listing + try { + listing = fs.readdirSync(dir, { withFileTypes: true }) + } catch (err) { + // No overlay directory is the normal case, not an error. + if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return [] + throw new ClilocSourceError(`Cliloc overlay directory is not readable: ${dir}`, 'UNREADABLE') + } + return listing + .filter((e) => e.isFile() && CUSTOM_EXTENSIONS.includes(path.extname(e.name).toLowerCase())) + .map((e) => e.name) + .sort() + .map((name) => path.join(dir, name)) +} + +function readFileOrThrow(file) { + try { + return fs.readFileSync(file) + } catch { + throw new ClilocSourceError(`Cliloc file is not readable: ${file}`, 'UNREADABLE') + } +} + +/** + * Read every cliloc source under the configured path. + * + * Returns `{ root, files: [{ label, kind, file, buffer, sha256, bytes, compressed }] }` + * with the base first and overlays after, in the order they must be merged. + * + * Labels are root-relative and forward-slashed so a hash map compares equal + * across platforms — the same directory read on Windows and Linux must produce + * the same fingerprint, or every boot would look like a change. (The same + * reasoning, and the same bug, as `spawnAtlasSource.readSources`.) + */ +function readSources(configured) { + const { root, base } = resolveBase(configured) + + const describe = (file, kind) => { + const buffer = readFileOrThrow(file) + return { + label: path.relative(root, file).split(path.sep).join('/'), + kind, + file, + buffer, + sha256: sha256(buffer), + bytes: buffer.length, + compressed: isCompressedCliloc(buffer), + } + } + + const files = [describe(base, 'base')] + for (const overlay of listCustom(root)) files.push(describe(overlay, 'custom')) + + return { root, files } +} + +/** + * A fingerprint of every source: `{ "