Compare commits
6 Commits
v1.2.0
...
53b3aca0c1
| Author | SHA1 | Date | |
|---|---|---|---|
| 53b3aca0c1 | |||
| f335531538 | |||
| 6ece48f7d3 | |||
| a194ec68e0 | |||
| 55df03496d | |||
| 893a36618b |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$comment": "The core this module is proved against. MODULE_API.md §5.3: the frozen-manifest job clones RunicGateway/website at this exact ref, drops this module in as modules/uo and runs CORE's own routeManifest.js — nothing else can answer whether the URLs the module claims are the URLs it actually serves. Pinned rather than tracking a branch on purpose: core moves for reasons that have nothing to do with this module, and a bump is then a deliberate commit saying which core the module was last proved against, instead of an unexplained red X on someone else's PR. Bump it, regenerate routes.manifest.json, and commit both together. **It pointed at `edge` for the length of the Event System window** (org lead, 2026-09-04), and this commit ends that: `api.registerEventActions` exists only from MODULE_API 1.10.0, so under the previous `main` pin `register()` threw and the module did not load at all — the job would have been red by construction for eight phases and would have proved nothing while a real regression hid behind it. The Phase 16b cutover put 1.10.0 on `main`, so the pin comes home, and this is the same move that turns the Integration kit green again. **routes.manifest.json needed NO regeneration**: the job's own steps were run against this exact ref and answered `routes.manifest.json is current — 73 routes, all documented`, so the \"commit both together\" instruction above had nothing to pair with this time.",
|
||||
"$comment": "The core this module is proved against. MODULE_API.md §5.3: the frozen-manifest job clones RunicGateway/website at this exact ref, drops this module in as modules/uo and runs CORE's own routeManifest.js — nothing else can answer whether the URLs the module claims are the URLs it actually serves. Pinned rather than tracking `edge` on purpose: core moves for reasons that have nothing to do with this module, and a bump is then a deliberate commit saying which core the module was last proved against, instead of an unexplained red X on someone else's PR. Bump it, regenerate routes.manifest.json, and commit both together. **It points at `edge` for the length of the Event System window** (org lead, 2026-09-04), and that is the one line here a reader should not tidy back. This module registers event actions from EVENTS_PLAN.md Phase 9, and `api.registerEventActions` exists only from MODULE_API 1.10.0 -- under the previous `main` pin `register()` throws and the module does not load at all, so the job would be red by construction for eight phases and would prove nothing while a real regression hid behind it. Phase 16's cutover re-pins it to `main`, which is the same commit that turns the Integration kit green again.",
|
||||
"repo": "https://gitea.whitlocktech.com/RunicGateway/website.git",
|
||||
"ref": "655fbf3f69a6a1fd650ecbc81afd6cf9c2ad9f66",
|
||||
"refName": "main @ MODULE_API 1.10.0, the Event System cutover (website#199)"
|
||||
"ref": "d4516739b43de5cb83b8f0333f8f966280a5632f",
|
||||
"refName": "edge @ MODULE_API 1.10.0, the event module contract (website#189, #190)"
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
// player can reach is safe.
|
||||
|
||||
import ShardAccountActions from './ShardAccountActions.jsx'
|
||||
import ItemIcon from './ItemIcon'
|
||||
|
||||
const RESIST_LABELS = { phys: 'Physical', fire: 'Fire', cold: 'Cold', pois: 'Poison', energy: 'Energy' }
|
||||
|
||||
@@ -270,7 +271,16 @@ export default function CharacterSheet({ char, moderation = false }) {
|
||||
const detail = [label === layer ? null : layer, `id ${it.itemId}`, it.hue ? `hue ${it.hue}` : null]
|
||||
return (
|
||||
<div key={it.serial} style={{ display: 'flex', alignItems: 'center', gap: 12, padding: '10px 14px', border: '1px solid var(--line)', borderRadius: 8 }}>
|
||||
{/* The sheet has always drawn an empty swatch here to hold the
|
||||
row's alignment. As of phase 5 the shard can hand over the
|
||||
item's real picture, hued the way the client would draw it —
|
||||
so the swatch becomes the fallback rather than the only
|
||||
state, and a row with no picture looks exactly as it did. */}
|
||||
{it.art ? (
|
||||
<ItemIcon art={it.art} name={label} size={22} />
|
||||
) : (
|
||||
<span style={{ flex: 'none', width: 22, height: 22, borderRadius: 5, border: '1px solid var(--line)', background: 'rgba(255,255,255,0.05)' }} />
|
||||
)}
|
||||
<div style={{ flex: 1, minWidth: 0 }}>
|
||||
<div className="sans" style={{ color: 'var(--head)', fontSize: '0.88rem' }}>{label}</div>
|
||||
<div className="sans dim" style={{ fontSize: '0.74rem' }}>{detail.filter(Boolean).join(' · ')}</div>
|
||||
|
||||
47
client/src/components/ItemIcon.jsx
Normal file
47
client/src/components/ItemIcon.jsx
Normal file
@@ -0,0 +1,47 @@
|
||||
// One item's picture, when this site holds one (docs/link/v8.md §5, §11 — phase 5).
|
||||
//
|
||||
// `art` is a FILENAME under uploads/items/, never a path or a URL — the same
|
||||
// shape `CreaturePortrait` takes, so there is one place in this module that knows
|
||||
// where uploads are mounted rather than one per surface.
|
||||
//
|
||||
// **NULL is ordinary and permanent for some items, and this renders nothing for
|
||||
// it.** Three separate reasons an item has no picture, and none of them is a
|
||||
// fault: the site has no shard link and never fetched one; the warm pass has not
|
||||
// reached this key yet (pictures are fetched behind the page, never by it, so a
|
||||
// new listing shows text first and gains its icon a few minutes later); or the
|
||||
// operator's own client simply has no art at that id — 9,963 of a stock client's
|
||||
// static ids have an empty index entry. Every layout using this is written to sit
|
||||
// correctly with the icon absent, because that is the state all of them were
|
||||
// built in.
|
||||
//
|
||||
// A hued item is a DIFFERENT picture, not a tinted one: the shard applies the hue
|
||||
// out of `hues.mul` before it sends anything, because whether a hue repaints the
|
||||
// whole sprite or only its grey pixels is decided by a flag in `tiledata.mul`
|
||||
// that this browser has no way to read. So there is nothing to style here — the
|
||||
// bytes already are the right colour.
|
||||
//
|
||||
// `imageRendering: 'pixelated'` for the same reason the creature portraits use
|
||||
// it: UO art is pixel art, and a browser's default smoothing turns a 22×26
|
||||
// item into a smear at any size above its own.
|
||||
export default function ItemIcon({ art, name, size = 32 }) {
|
||||
if (!art) return null
|
||||
|
||||
return (
|
||||
<img
|
||||
src={`/uploads/items/${encodeURIComponent(art)}`}
|
||||
alt=""
|
||||
// Decorative: the item's name is already beside it as text, and an alt
|
||||
// repeating it would make a screen reader say it twice.
|
||||
aria-hidden="true"
|
||||
loading="lazy"
|
||||
style={{
|
||||
width: size,
|
||||
height: size,
|
||||
flex: 'none',
|
||||
objectFit: 'contain',
|
||||
imageRendering: 'pixelated',
|
||||
}}
|
||||
title={name}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -45,6 +45,46 @@ function Chip({ active, onClick, children }) {
|
||||
)
|
||||
}
|
||||
|
||||
// One creature's portrait, when there is one.
|
||||
//
|
||||
// `art` is a FILENAME under uploads/atlas/, never a path or a URL: it is either a
|
||||
// sprite the shard extracted from the operator's own UO client (docs/link/v8.md
|
||||
// §12) or a picture the operator drew and named in `spawnAtlas.art.json`, and the
|
||||
// two are indistinguishable here on purpose.
|
||||
//
|
||||
// **NULL is the ordinary case and always will be.** An install with no shard link
|
||||
// has never imported one; a shard whose host cannot render images has none; and
|
||||
// even on a complete import, two thirds of the playable ghost and gargoyle bodies
|
||||
// have no art in the client at all (§5.2). So this renders nothing rather than a
|
||||
// placeholder, and every layout around it is written to sit correctly with the
|
||||
// picture absent — which is the state the whole atlas was designed in.
|
||||
//
|
||||
// Sprites are small (a couple of dozen pixels square) and UO's art is pixel art,
|
||||
// so `imageRendering: 'pixelated'` matters: a browser's default smoothing turns a
|
||||
// 24×63 wolf into a smear at any size above its own.
|
||||
export function CreaturePortrait({ art, name, size = 40 }) {
|
||||
if (!art) return null
|
||||
|
||||
return (
|
||||
<img
|
||||
src={`/uploads/atlas/${encodeURIComponent(art)}`}
|
||||
alt=""
|
||||
// Decorative: the creature's name is already beside it as text, so an alt
|
||||
// repeating it would make a screen reader say it twice.
|
||||
aria-hidden="true"
|
||||
loading="lazy"
|
||||
style={{
|
||||
width: size,
|
||||
height: size,
|
||||
flex: 'none',
|
||||
objectFit: 'contain',
|
||||
imageRendering: 'pixelated',
|
||||
}}
|
||||
title={name}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function CreatureCard({ creature }) {
|
||||
const facets = Object.entries(creature.facets || {}).sort((a, b) => b[1] - a[1])
|
||||
return (
|
||||
@@ -60,6 +100,7 @@ function CreatureCard({ creature }) {
|
||||
color: 'inherit',
|
||||
}}
|
||||
>
|
||||
<CreaturePortrait art={creature.art} name={creature.name} />
|
||||
<div style={{ minWidth: 0, flex: 1 }}>
|
||||
<div
|
||||
className="display"
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useMemo, useState } from 'react'
|
||||
import { Link, useParams } from 'react-router-dom'
|
||||
import api from '../../api.js'
|
||||
import { EmptyState, ErrorState, Loading, PageHeader, PublicLayout, useAsync } from '../../core.js'
|
||||
import { CreaturePortrait } from './Atlas.jsx'
|
||||
|
||||
// One creature: where it spawns, and what spawns alongside it.
|
||||
//
|
||||
@@ -146,11 +147,22 @@ export default function AtlasCreature() {
|
||||
|
||||
{!loading && !error && data && (
|
||||
<>
|
||||
{/* The portrait sits BESIDE the header rather than inside it: `art`
|
||||
is NULL for most creatures on most installs — no shard link, a
|
||||
host that cannot render images, or simply a body this client has
|
||||
no art for — and a header component that had to lay out around an
|
||||
absent picture would be carrying that case forever. Here the row
|
||||
collapses to exactly the header, which is what it was before. */}
|
||||
<div style={{ display: 'flex', alignItems: 'flex-start', gap: 16 }}>
|
||||
<CreaturePortrait art={data.art} name={data.name} size={96} />
|
||||
<div style={{ minWidth: 0, flex: 1 }}>
|
||||
<PageHeader
|
||||
eyebrow="Bestiary"
|
||||
title={data.name}
|
||||
lead={`Up to ${num(data.total)} alive at once across ${num(data.points)} spawner${data.points === 1 ? '' : 's'}.`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 12 }}>
|
||||
<Panel
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from 'react'
|
||||
import { Link } from 'react-router-dom'
|
||||
import api from '../../api.js'
|
||||
import { EmptyState, ErrorState, Loading, PageHeader, PublicLayout, useAsync } from '../../core.js'
|
||||
import ItemIcon from '../../components/ItemIcon'
|
||||
|
||||
// ── The player-vendor marketplace ───────────────────────────────────────────
|
||||
//
|
||||
@@ -86,6 +87,7 @@ function ListingRow({ listing }) {
|
||||
|
||||
return (
|
||||
<div className="panel" style={{ padding: '13px 15px', display: 'flex', gap: 14, alignItems: 'center' }}>
|
||||
<ItemIcon art={listing.art} name={itemLabel(listing)} />
|
||||
<div style={{ minWidth: 0, flex: 1 }}>
|
||||
<div
|
||||
className="display"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Link, useParams } from 'react-router-dom'
|
||||
import api from '../../api.js'
|
||||
import { EmptyState, ErrorState, Loading, PageHeader, PublicLayout, useAsync } from '../../core.js'
|
||||
import ItemIcon from '../../components/ItemIcon'
|
||||
|
||||
// One player vendor: where to find it and everything it is selling.
|
||||
//
|
||||
@@ -75,8 +76,9 @@ export default function MarketVendor() {
|
||||
<div
|
||||
key={i.serial}
|
||||
className="panel"
|
||||
style={{ padding: '10px 14px', display: 'flex', gap: 12, alignItems: 'baseline' }}
|
||||
style={{ padding: '10px 14px', display: 'flex', gap: 12, alignItems: 'center' }}
|
||||
>
|
||||
<ItemIcon art={i.art} name={itemLabel(i)} size={28} />
|
||||
<span className="sans" style={{ flex: 1, minWidth: 0, color: 'var(--head)', fontSize: '0.88rem' }}>
|
||||
{i.amount > 1 ? `${num(i.amount)} × ` : ''}
|
||||
{itemLabel(i)}
|
||||
|
||||
@@ -16,6 +16,11 @@
|
||||
"path": "/api/v1/admin/shard/accounts",
|
||||
"tier": "public"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/v1/admin/shard/assets",
|
||||
"tier": "public"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/v1/admin/shard/atlas",
|
||||
@@ -271,6 +276,16 @@
|
||||
"path": "/api/v1/admin/shard/account",
|
||||
"tier": "public"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"path": "/api/v1/admin/shard/assets/import",
|
||||
"tier": "public"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"path": "/api/v1/admin/shard/assets/warm",
|
||||
"tier": "public"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"path": "/api/v1/admin/shard/atlas/approve",
|
||||
|
||||
@@ -33,6 +33,7 @@ const shardBroadcast = require('./utils/shardBroadcast')
|
||||
const shardAtlas = require('./model/shardAtlas/shardAtlas.model')
|
||||
const shardClilocs = require('./model/shardClilocs/shardClilocs.model')
|
||||
const shardMarket = require('./model/shardMarket/shardMarket.model')
|
||||
const shardItemArt = require('./model/shardAssets/shardItemArt.model')
|
||||
|
||||
/**
|
||||
* Best-effort startup probe of the uo-link sidecar.
|
||||
@@ -80,11 +81,18 @@ async function onBoot() {
|
||||
// REMOVE a facet is staged for admin approval instead of being applied.
|
||||
await shardAtlas.refreshOnBoot()
|
||||
|
||||
// Refresh the cliloc table (UO's id → display-string map) from the file the
|
||||
// operator converted out of their own client. Same contract as the atlas:
|
||||
// hash-gated so an unchanged file costs one read, and best-effort so a missing
|
||||
// or wrong-format file never stops the site coming up — it just means item
|
||||
// names render as ids, which is what they did before the table existed.
|
||||
// Refresh the cliloc table (UO's id → display-string map).
|
||||
//
|
||||
// **On an install with uo-link configured this imports nothing** — protocol 8
|
||||
// moved the base table to the shard, and asking for it would put a sidecar
|
||||
// round trip in the boot sequence to answer a question whose answer is "no"
|
||||
// except after a client patch. That is an operator action, so importing is an
|
||||
// operator action: Admin → Shard (docs/link/v8.md §9).
|
||||
//
|
||||
// Without a shard link it is the old file pipeline, unchanged: hash-gated so an
|
||||
// unchanged file costs one read, and best-effort so a missing or wrong-format
|
||||
// file never stops the site coming up — it just means item names render as ids,
|
||||
// which is what they did before the table existed.
|
||||
const clilocResult = await shardClilocs.refreshOnBoot()
|
||||
|
||||
// A cliloc import changes what item names RESOLVE to, and the marketplace
|
||||
@@ -107,6 +115,15 @@ async function onBoot() {
|
||||
// Deliberately not awaited — see the header. An unreachable sidecar would
|
||||
// otherwise hold the listener closed for the length of an HTTP timeout.
|
||||
checkUoLink().catch((err) => log.warn('uo-link startup probe failed', { error: err.message }))
|
||||
|
||||
// Item and land pictures for the keys this site's own rows name (§11, phase 5).
|
||||
//
|
||||
// A timer rather than a boot pass, and it is the same rule §9.2 set for clilocs:
|
||||
// **boot does not call the shard.** The first pass is one interval away, so an
|
||||
// unreachable sidecar costs a log line rather than a startup delay, and an
|
||||
// operator who has just configured the bridge does not have to restart to get
|
||||
// pictures. `unref`ed, so it never holds shutdown open.
|
||||
shardItemArt.startWarming()
|
||||
}
|
||||
|
||||
async function onShutdown() {
|
||||
@@ -114,6 +131,7 @@ async function onShutdown() {
|
||||
// still works — the pool is open, the push dispatcher is up, the SSE fan-out
|
||||
// is live. It is the only chance to close cleanly, and it is budgeted, so a
|
||||
// hook that will not let go costs five seconds rather than the whole shutdown.
|
||||
shardItemArt.stopWarming() // stop the item-art warm pass
|
||||
uoLinkSocket.stop() // close the uo-link WS ingest client
|
||||
shardBroadcast.closeAll() // end any open shard live-feed SSE streams
|
||||
}
|
||||
|
||||
@@ -27,6 +27,19 @@
|
||||
-- marker, and deleting it would re-arm a protocol bump against tables this
|
||||
-- file has just dropped.
|
||||
|
||||
-- The Asset Bridge's three (phase 3). No foreign keys of their own, so they lead:
|
||||
-- `shard_creature_bodies.slug` mirrors an atlas slug and `shard_assets.body` a body
|
||||
-- id, but neither is declared as a constraint — the atlas tables are rebuilt from
|
||||
-- scratch on every refresh, and an FK into a table that is emptied and refilled
|
||||
-- would make an ordinary re-parse fail on rows that are about to be re-inserted.
|
||||
--
|
||||
-- The uploaded PNGs are NOT removed here. They live under the uploads directory
|
||||
-- alongside the operator's own artwork, this file drops tables rather than files,
|
||||
-- and a purge that deleted an operator's hand-drawn creature portraits because
|
||||
-- they shared a directory with imported ones would be unrecoverable.
|
||||
DROP TABLE IF EXISTS `shard_asset_meta`;
|
||||
DROP TABLE IF EXISTS `shard_creature_bodies`;
|
||||
DROP TABLE IF EXISTS `shard_assets`;
|
||||
DROP TABLE IF EXISTS `shard_atlas_pending`;
|
||||
DROP TABLE IF EXISTS `shard_atlas_meta`;
|
||||
DROP TABLE IF EXISTS `shard_cliloc_meta`;
|
||||
|
||||
@@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS uo_link_config (
|
||||
base_url VARCHAR(255) NULL,
|
||||
ws_url VARCHAR(255) NULL,
|
||||
auth_token_enc TEXT NULL,
|
||||
protocol INT NOT NULL DEFAULT 7,
|
||||
protocol INT NOT NULL DEFAULT 8,
|
||||
enabled TINYINT(1) NOT NULL DEFAULT 0,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'disconnected',
|
||||
status_detail VARCHAR(500) NULL,
|
||||
@@ -660,6 +660,93 @@ CREATE TABLE IF NOT EXISTS shard_atlas_pending (
|
||||
CONSTRAINT chk_shard_atlas_pending_singleton CHECK (id = 1)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ── The Asset Bridge (docs/link/v8.md, protocol 8 phase 3) ─────────────────
|
||||
--
|
||||
-- One row per imported asset: the manifest side of §6, and what makes an Update
|
||||
-- a diff rather than a re-download. `sha256` is of the PNG the shard produced, so
|
||||
-- a re-import fetches only the keys whose hash moved.
|
||||
--
|
||||
-- **`file` is a filename under the uploads directory, never a path.** Images are
|
||||
-- written through `ctx.uploads`, the same door the operator's own atlas art comes
|
||||
-- in by, and storing a path here would let a row decide where the server reads
|
||||
-- from.
|
||||
--
|
||||
-- `bytes`/`width`/`height` are carried from the manifest rather than re-derived,
|
||||
-- because the manifest reports them before the pixels are fetched and a screen
|
||||
-- that lists what WOULD be imported needs them then.
|
||||
CREATE TABLE IF NOT EXISTS shard_assets (
|
||||
asset_key VARCHAR(191) NOT NULL PRIMARY KEY, -- §5's key: `body/34/a0`
|
||||
family VARCHAR(24) NOT NULL DEFAULT 'body',
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
bytes INT NOT NULL DEFAULT 0,
|
||||
width INT NOT NULL DEFAULT 0,
|
||||
height INT NOT NULL DEFAULT 0,
|
||||
body INT NULL, -- the body id, for the atlas join
|
||||
direction TINYINT NULL,
|
||||
file VARCHAR(191) NULL, -- filename under uploads/, NULL until fetched
|
||||
imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
-- The atlas art derivation joins creature → body → asset on every atlas refresh,
|
||||
-- so the body lookup is the read that has to be fast, not the key.
|
||||
INDEX idx_shard_assets_body (body)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- Which of the shard's catalogues a row was fetched under (§7, phase 5).
|
||||
--
|
||||
-- The body catalogue can answer "is this stale?" from `shard_asset_meta`, because
|
||||
-- it is imported as a SET: one manifest walk covers every key, so one stored
|
||||
-- fingerprint describes all of them. Item and land art has no manifest and never
|
||||
-- will — 49,152 static ids times three thousand hues is not a set anyone
|
||||
-- enumerates — so staleness has to be recorded per row, and this is it.
|
||||
--
|
||||
-- The shard derives the id from the files that decide the bytes (its art data
|
||||
-- file, hues.mul, tiledata.mul, verdata.mul, and its own extractor version), so a
|
||||
-- client patch changes it and a restart does not. A row whose `catalog` is not the
|
||||
-- shard's current one is stale: the warm pass re-fetches it the next time
|
||||
-- something asks for that key, and pictures nobody looks at any more are simply
|
||||
-- never re-fetched, which is the whole reason this is per-row and lazy rather than
|
||||
-- a sweep. NULL means "written before this column existed", which is stale by the
|
||||
-- same test and costs one re-fetch.
|
||||
ALTER TABLE shard_assets ADD COLUMN IF NOT EXISTS catalog VARCHAR(32) NULL;
|
||||
|
||||
-- Slug → body id, as the shard itself answered it (§8).
|
||||
--
|
||||
-- **Deliberately NOT a column on `shard_spawn_creatures`.** That table is
|
||||
-- IMPORT-OWNED: `replaceAtlas` empties and refills it inside one transaction on
|
||||
-- every atlas refresh. A body id living there would be destroyed by a routine
|
||||
-- re-parse of the ServUO tree — and the next asset Update would find the source
|
||||
-- hashes unchanged, report "nothing to do", and never put it back. The portrait
|
||||
-- would simply vanish from every creature page until somebody thought to force a
|
||||
-- re-import.
|
||||
--
|
||||
-- So the resolution lives here, outside the atlas's blast radius, and
|
||||
-- `replaceAtlas` READS it to derive `shard_spawn_creatures.art` on the way past.
|
||||
--
|
||||
-- `status` is the shard's own verdict and each value is a different thing an
|
||||
-- operator can act on: `ok`, `unknown` (the spawn file names a type this shard's
|
||||
-- scripts do not define — real drift), `notCreature` (a spawn file legitimately
|
||||
-- naming an item or decoration, a permanent answer), `failed` (its constructor
|
||||
-- threw). A row is kept for every one of them, because "asked and answered no" is
|
||||
-- what stops the next pass asking again.
|
||||
CREATE TABLE IF NOT EXISTS shard_creature_bodies (
|
||||
slug VARCHAR(120) NOT NULL PRIMARY KEY, -- → shard_spawn_creatures.slug (no FK)
|
||||
type_name VARCHAR(120) NOT NULL, -- the ServUO class name that was asked
|
||||
body INT NULL, -- NULL unless status = 'ok'
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ok',
|
||||
resolved_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_shard_creature_bodies_body (body)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- Singleton (id = 1) describing the asset import currently applied: the shard's
|
||||
-- catalogue id, its extractor version, the counts and when it ran. Same shape and
|
||||
-- same job as `shard_cliloc_meta` — it is what an Update compares against to
|
||||
-- decide there is nothing to do.
|
||||
CREATE TABLE IF NOT EXISTS shard_asset_meta (
|
||||
id TINYINT NOT NULL PRIMARY KEY DEFAULT 1,
|
||||
payload JSON NOT NULL,
|
||||
imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
CONSTRAINT chk_shard_asset_meta_singleton CHECK (id = 1)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- House registry (Protocol 2.0). The house.update full-state feed carries richer
|
||||
-- fields than the house.decay transition feed shard_houses was built for. Rather
|
||||
-- than a second table for one entity, extend shard_houses: house.update writes the
|
||||
@@ -837,3 +924,21 @@ UPDATE uo_link_config SET protocol = 7
|
||||
WHERE id = 1 AND protocol < 7
|
||||
AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_7_migrated');
|
||||
INSERT IGNORE INTO settings (`key`, value) VALUES ('uo_link_protocol_7_migrated', '1');
|
||||
|
||||
-- The protocol pin at 8 -- the Asset Bridge (docs/link/v8.md), and the first bump this
|
||||
-- module takes IN the phase that consumes it rather than a phase or two later.
|
||||
--
|
||||
-- Phase 1 of that work moved `link`'s PROTOCOL_VERSION and the overlay's `overlay.toml`
|
||||
-- together, because the installer refuses to pair a sidecar and an overlay that disagree.
|
||||
-- Nothing enforces the third declaration -- this one -- and the block above is the record
|
||||
-- of what that costs: two phases of every REST call answered `409 protocol version
|
||||
-- mismatch`, invisible because both live walks had set the column by hand.
|
||||
--
|
||||
-- Phase 2 is where this module first calls a protocol-8 route (`GET /cliloc`), so it is
|
||||
-- where the pin moves. Same one-shot shape and the same `protocol < 8`, so an install
|
||||
-- that missed an earlier bump is carried the whole way rather than one step.
|
||||
ALTER TABLE uo_link_config MODIFY COLUMN protocol INT NOT NULL DEFAULT 8;
|
||||
UPDATE uo_link_config SET protocol = 8
|
||||
WHERE id = 1 AND protocol < 8
|
||||
AND NOT EXISTS (SELECT 1 FROM settings WHERE `key` = 'uo_link_protocol_8_migrated');
|
||||
INSERT IGNORE INTO settings (`key`, value) VALUES ('uo_link_protocol_8_migrated', '1');
|
||||
|
||||
296
server/model/shardAssets/shardAssets.db.js
Normal file
296
server/model/shardAssets/shardAssets.db.js
Normal file
@@ -0,0 +1,296 @@
|
||||
const core = require('../../core')
|
||||
|
||||
const { query } = core
|
||||
|
||||
// Raw SQL for the Asset Bridge's three tables (docs/link/v8.md §6, §8, §12).
|
||||
//
|
||||
// Unlike `shard_clilocs` and the atlas tables, these are NOT import-owned in the
|
||||
// empty-and-refill sense, and the difference is the whole reason phase 3 put them
|
||||
// in their own tables rather than in columns on `shard_spawn_creatures`.
|
||||
//
|
||||
// An asset row is expensive to obtain — a decode on the shard, a PNG across the
|
||||
// wire, a file written under uploads/ — and it is valid until the operator
|
||||
// patches their client. An atlas refresh, by contrast, happens on every boot and
|
||||
// destroys everything it owns. Putting the two in one table would mean a routine
|
||||
// re-parse of the ServUO tree silently deleting every imported portrait, with the
|
||||
// next Update reporting "nothing changed" and never restoring them.
|
||||
//
|
||||
// So these are upserted per key, and the only thing that ever deletes from them
|
||||
// is an explicit removal of a key the shard no longer offers — which is staged
|
||||
// for review, never applied silently (§6).
|
||||
|
||||
const BATCH = 500
|
||||
|
||||
async function batched(conn, sql, rows) {
|
||||
for (let i = 0; i < rows.length; i += BATCH) {
|
||||
await conn.batch(sql, rows.slice(i, i + BATCH))
|
||||
}
|
||||
return rows.length
|
||||
}
|
||||
|
||||
// ── the manifest side ──────────────────────────────────────────────────────
|
||||
|
||||
/** Every asset row we hold, as a Map of key → row. */
|
||||
async function allAssets() {
|
||||
const rows = await query(
|
||||
'SELECT asset_key, family, sha256, bytes, width, height, body, direction, file, catalog FROM shard_assets',
|
||||
)
|
||||
|
||||
const map = new Map()
|
||||
|
||||
for (const row of rows) {
|
||||
map.set(row.asset_key, {
|
||||
key: row.asset_key,
|
||||
family: row.family,
|
||||
sha256: row.sha256,
|
||||
bytes: Number(row.bytes) || 0,
|
||||
width: Number(row.width) || 0,
|
||||
height: Number(row.height) || 0,
|
||||
body: row.body === null ? null : Number(row.body),
|
||||
direction: row.direction === null ? null : Number(row.direction),
|
||||
file: row.file || null,
|
||||
catalog: row.catalog || null,
|
||||
})
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the assets an import produced, and record what the import was.
|
||||
*
|
||||
* One transaction for the rows and the meta together: the meta row is what an
|
||||
* Update compares against to decide there is nothing to do, so a meta written
|
||||
* without its rows would make the site believe it holds a catalogue it does not.
|
||||
*
|
||||
* `ON DUPLICATE KEY UPDATE` rather than delete-and-insert, because an unchanged
|
||||
* key must keep the file it already points at — re-writing the file for every
|
||||
* asset on every Update is exactly the cost the manifest diff exists to avoid.
|
||||
*/
|
||||
async function saveAssets(rows, meta) {
|
||||
const conn = await core.pool.getConnection()
|
||||
|
||||
try {
|
||||
await conn.beginTransaction()
|
||||
|
||||
const values = rows.map((r) => [
|
||||
r.key,
|
||||
r.family || 'body',
|
||||
r.sha256,
|
||||
r.bytes ?? 0,
|
||||
r.width ?? 0,
|
||||
r.height ?? 0,
|
||||
r.body ?? null,
|
||||
r.direction ?? null,
|
||||
r.file ?? null,
|
||||
r.catalog ?? meta?.catalog ?? null,
|
||||
])
|
||||
|
||||
await batched(
|
||||
conn,
|
||||
'INSERT INTO shard_assets (asset_key, family, sha256, bytes, width, height, body, direction, file, catalog) ' +
|
||||
'VALUES (?,?,?,?,?,?,?,?,?,?) ' +
|
||||
'ON DUPLICATE KEY UPDATE family = VALUES(family), sha256 = VALUES(sha256), ' +
|
||||
'bytes = VALUES(bytes), width = VALUES(width), height = VALUES(height), ' +
|
||||
'body = VALUES(body), direction = VALUES(direction), file = VALUES(file), ' +
|
||||
'catalog = VALUES(catalog), imported_at = CURRENT_TIMESTAMP',
|
||||
values,
|
||||
)
|
||||
|
||||
if (meta) {
|
||||
await conn.query(
|
||||
'INSERT INTO shard_asset_meta (id, payload) VALUES (1, ?) ' +
|
||||
'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP',
|
||||
[JSON.stringify(meta)],
|
||||
)
|
||||
}
|
||||
|
||||
await conn.commit()
|
||||
|
||||
return values.length
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {})
|
||||
throw err
|
||||
} finally {
|
||||
conn.release()
|
||||
}
|
||||
}
|
||||
|
||||
// ── the on-demand side (§11, phase 5) ──────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The pictures we hold for an explicit list of keys, as a Map of key → filename.
|
||||
*
|
||||
* This is the read on the hot path — every marketplace page and every character
|
||||
* sheet runs it — so it is one statement over the primary key and it returns only
|
||||
* what it is asked for. It deliberately does NOT check staleness: a page renders
|
||||
* the picture it has, and deciding whether that picture is out of date is the warm
|
||||
* pass's job, off the request.
|
||||
*/
|
||||
async function filesForKeys(keys) {
|
||||
const list = [...new Set(keys.filter((k) => typeof k === 'string' && k !== ''))]
|
||||
|
||||
if (list.length === 0) return new Map()
|
||||
|
||||
const rows = await query(
|
||||
`SELECT asset_key, file FROM shard_assets WHERE file IS NOT NULL AND asset_key IN (${list
|
||||
.map(() => '?')
|
||||
.join(',')})`,
|
||||
list,
|
||||
)
|
||||
|
||||
const map = new Map()
|
||||
|
||||
for (const row of rows) map.set(row.asset_key, row.file)
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
/**
|
||||
* Which of these keys we already hold under the shard's CURRENT catalogue.
|
||||
*
|
||||
* The warm pass subtracts this from what it wants, so everything it does not
|
||||
* return gets fetched: a key we have never seen, and a key whose row was written
|
||||
* against a catalogue the shard has since moved past (§7 — an operator patched
|
||||
* their client). A row with no file is not held either, because the database and
|
||||
* the uploads volume can disagree and a broken image is worse than a re-fetch.
|
||||
*/
|
||||
async function freshKeys(keys, catalog) {
|
||||
const list = [...new Set(keys.filter((k) => typeof k === 'string' && k !== ''))]
|
||||
|
||||
if (list.length === 0) return new Set()
|
||||
|
||||
const rows = await query(
|
||||
`SELECT asset_key FROM shard_assets WHERE file IS NOT NULL AND catalog <=> ? ` +
|
||||
`AND asset_key IN (${list.map(() => '?').join(',')})`,
|
||||
[catalog ?? null, ...list],
|
||||
)
|
||||
|
||||
return new Set(rows.map((r) => r.asset_key))
|
||||
}
|
||||
|
||||
/** Counts for the admin surface, split by family. */
|
||||
async function countByFamily() {
|
||||
const rows = await query(
|
||||
'SELECT family, COUNT(*) AS total, SUM(file IS NOT NULL) AS stored FROM shard_assets GROUP BY family',
|
||||
)
|
||||
|
||||
const out = {}
|
||||
|
||||
for (const row of rows) {
|
||||
out[row.family] = { total: Number(row.total) || 0, stored: Number(row.stored) || 0 }
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
async function getMeta() {
|
||||
const rows = await query('SELECT payload, imported_at FROM shard_asset_meta WHERE id = 1')
|
||||
if (rows.length === 0) return null
|
||||
const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload
|
||||
return { ...payload, importedAt: rows[0].imported_at }
|
||||
}
|
||||
|
||||
async function countAssets() {
|
||||
const rows = await query(
|
||||
'SELECT COUNT(*) AS n, SUM(file IS NOT NULL) AS stored FROM shard_assets',
|
||||
)
|
||||
return { total: Number(rows[0]?.n) || 0, stored: Number(rows[0]?.stored) || 0 }
|
||||
}
|
||||
|
||||
// ── the body resolution side (§8) ──────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Replace the whole slug → body map.
|
||||
*
|
||||
* This one IS a replace, and for the opposite reason to the assets above: it is
|
||||
* derived from the atlas's creature list, so a slug that has left the atlas has
|
||||
* no meaning any more and keeping its row would leave the map growing forever
|
||||
* across map changes. The pass that produces it is cheap to redo — a shard round
|
||||
* trip, no files — which is what makes replacing safe here and not there.
|
||||
*/
|
||||
async function replaceBodies(rows) {
|
||||
const conn = await core.pool.getConnection()
|
||||
|
||||
try {
|
||||
await conn.beginTransaction()
|
||||
await conn.query('DELETE FROM shard_creature_bodies')
|
||||
|
||||
const values = rows.map((r) => [r.slug, r.typeName, r.body ?? null, r.status || 'ok'])
|
||||
|
||||
await batched(
|
||||
conn,
|
||||
'INSERT INTO shard_creature_bodies (slug, type_name, body, status) VALUES (?,?,?,?)',
|
||||
values,
|
||||
)
|
||||
|
||||
await conn.commit()
|
||||
|
||||
return values.length
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {})
|
||||
throw err
|
||||
} finally {
|
||||
conn.release()
|
||||
}
|
||||
}
|
||||
|
||||
async function allBodies() {
|
||||
return query(
|
||||
'SELECT slug, type_name, body, status, resolved_at FROM shard_creature_bodies ORDER BY slug',
|
||||
)
|
||||
}
|
||||
|
||||
async function countBodies() {
|
||||
const rows = await query(
|
||||
"SELECT COUNT(*) AS n, SUM(status = 'ok') AS resolved FROM shard_creature_bodies",
|
||||
)
|
||||
return { total: Number(rows[0]?.n) || 0, resolved: Number(rows[0]?.resolved) || 0 }
|
||||
}
|
||||
|
||||
/**
|
||||
* The derivation `replaceAtlas` applies on the way past: slug → uploaded filename.
|
||||
*
|
||||
* One join rather than two reads, because it runs inside the atlas transaction —
|
||||
* the atlas rows are being inserted at that moment and every extra round trip is
|
||||
* time the site's creature list does not exist.
|
||||
*
|
||||
* Rows with no body, no asset or an asset whose bytes were never fetched are
|
||||
* simply absent from the result, which is what leaves `art` NULL. That is a
|
||||
* first-class state everywhere it is consumed and the expected one for two thirds
|
||||
* of the player bodies (§5.2).
|
||||
*
|
||||
* **The join is pinned to the catalogue key, not merely to the body id.** Today
|
||||
* one body has exactly one asset, so `a.body = b.body` alone would be correct —
|
||||
* and it would stop being correct the moment phase 6 adds `body/400/a2/f0`, at
|
||||
* which point one slug would match dozens of rows and whichever the engine
|
||||
* returned last would become the portrait. Naming the key here means that phase
|
||||
* adds rows without changing what a creature page shows.
|
||||
*/
|
||||
async function artBySlug() {
|
||||
const rows = await query(
|
||||
'SELECT b.slug, a.file FROM shard_creature_bodies b ' +
|
||||
"JOIN shard_assets a ON a.asset_key = CONCAT('body/', b.body, '/a0') " +
|
||||
"WHERE b.status = 'ok' AND b.body IS NOT NULL AND a.file IS NOT NULL",
|
||||
)
|
||||
|
||||
const map = {}
|
||||
|
||||
for (const row of rows) map[row.slug] = row.file
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
allAssets,
|
||||
saveAssets,
|
||||
getMeta,
|
||||
countAssets,
|
||||
replaceBodies,
|
||||
allBodies,
|
||||
countBodies,
|
||||
artBySlug,
|
||||
filesForKeys,
|
||||
freshKeys,
|
||||
countByFamily,
|
||||
}
|
||||
518
server/model/shardAssets/shardAssets.model.js
Normal file
518
server/model/shardAssets/shardAssets.model.js
Normal file
@@ -0,0 +1,518 @@
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const db = require('./shardAssets.db')
|
||||
const atlasDb = require('../shardAtlas/shardAtlas.db')
|
||||
const core = require('../../core')
|
||||
const bridge = require('../../utils/assetBridge')
|
||||
const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model')
|
||||
const log = require('../../core').logger('shardAssets')
|
||||
|
||||
// Client artwork, over the bridge (docs/link/v8.md — protocol 8, phase 3).
|
||||
//
|
||||
// What this replaces: until now the only way a creature got a picture on this
|
||||
// site was for an operator to open UOFiddler on a desktop, export sprites by
|
||||
// hand, copy them to the web host and write a `spawnAtlas.art.json` naming each
|
||||
// one. Almost nobody did, so `shard_spawn_creatures.art` was NULL on every
|
||||
// install and the atlas rendered as text.
|
||||
//
|
||||
// The shard has had those files the whole time — a ServUO server cannot boot
|
||||
// without a UO client — so as of protocol 8 it decodes them itself and hands the
|
||||
// pictures over the same request/reply path as every other shard read.
|
||||
//
|
||||
// ── Two passes, and they answer different questions ───────────────────
|
||||
//
|
||||
// **The catalogue** (§4.8, §11) is one thumbnail per creature body: the shard
|
||||
// walks bodies 0–2047, validates each index entry, decodes the ones that are real
|
||||
// and hands back `{ key, sha256 }` first and the PNG second. On a stock client
|
||||
// that is **787 sprites**, not the 1,144 the decoder claims — see below.
|
||||
//
|
||||
// **Body resolution** (§8) is the join. The atlas knows a creature by the class
|
||||
// name in `Spawns/*.xml`; the client knows it by a body id; nothing in the ServUO
|
||||
// tree declares the mapping. Only code inside ServUO can answer it, by
|
||||
// constructing the creature and reading `Body.BodyID`, and that is the whole
|
||||
// reason this could not be done off the shard.
|
||||
//
|
||||
// ── The 357, and why nothing here trusts a success ────────────────────
|
||||
//
|
||||
// 357 of the bodies ServUO's decoder returns a bitmap for **have no art**. Their
|
||||
// index entry reads `length 0`, the library's stream buffer still holds the
|
||||
// previous creature, and what comes back is whichever body was decoded before —
|
||||
// a real, plausible, correctly-sized picture of the wrong animal. The shard now
|
||||
// validates every index entry before it decodes, which is what cut the catalogue
|
||||
// from 1,144 to 787, and the count going down is the point.
|
||||
//
|
||||
// The consequence for this file is a rule: **a missing asset is a normal
|
||||
// outcome, never an error.** Two thirds of the player bodies have no art on a
|
||||
// stock client (§5.2), so an import that reported eight failures every time would
|
||||
// teach an operator to ignore the panel.
|
||||
//
|
||||
// ── Where the pictures go, and what still wins ────────────────────────
|
||||
//
|
||||
// Into `<uploads>/atlas/`, through the same door the operator's own artwork uses,
|
||||
// and `shard_spawn_creatures.art` is DERIVED from them rather than written by
|
||||
// them. **The operator's `spawnAtlas.art.json` still wins outright**: someone who
|
||||
// has drawn their own creature portraits must not have them replaced by a sprite
|
||||
// rip on the next Update.
|
||||
//
|
||||
// ── Why the resolution does not live on the atlas row ─────────────────
|
||||
//
|
||||
// `shard_spawn_creatures` is emptied and refilled on every atlas refresh. A body
|
||||
// id or a filename stored there would be destroyed by an ordinary re-parse of the
|
||||
// ServUO tree, and the next asset Update would find the client files unchanged,
|
||||
// report "nothing to do" and never restore it. So both live in their own tables
|
||||
// and the atlas import reads them on the way past.
|
||||
|
||||
/** Where imported sprites land, under core's upload directory. */
|
||||
const ART_SUBDIR = 'atlas'
|
||||
|
||||
// ── configuration ──────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Is there a shard to ask?
|
||||
*
|
||||
* Both halves matter, exactly as in `shardClilocs.model`: a `baseUrl` on a
|
||||
* disabled config is an install that was set up and then switched off, and
|
||||
* calling it would spend a 12 s timeout to learn what the row already says.
|
||||
*/
|
||||
async function shardLinked() {
|
||||
try {
|
||||
const config = await uoLinkConfig.getSafe()
|
||||
return Boolean(config?.enabled && config?.baseUrl)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function artDir() {
|
||||
return path.join(core.uploads.UPLOAD_DIR, ART_SUBDIR)
|
||||
}
|
||||
|
||||
/**
|
||||
* The operator's own art map, which wins over anything imported.
|
||||
*
|
||||
* Read through the atlas model rather than re-implemented, so there is one
|
||||
* definition of where that file lives and what an absent one means.
|
||||
*/
|
||||
function operatorArt() {
|
||||
// eslint-disable-next-line global-require
|
||||
return require('../shardAtlas/shardAtlas.model').loadArtMap()
|
||||
}
|
||||
|
||||
// ── writing a sprite ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The filename one asset gets on disk.
|
||||
*
|
||||
* **Content-addressed on purpose.** A stable name per key (`uo-body-34.png`)
|
||||
* would be overwritten in place by an Update, and every browser and CDN that had
|
||||
* already cached it would keep serving last month's client's sprite — with
|
||||
* nothing anywhere to notice, because the database row would be correct. Putting
|
||||
* eight bytes of the hash in the name makes a changed sprite a changed URL.
|
||||
*
|
||||
* The old file is removed when a key's hash moves, so the directory tracks the
|
||||
* catalogue rather than accumulating one file per import forever.
|
||||
*/
|
||||
function fileNameFor(key, sha256) {
|
||||
const stem = key.replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')
|
||||
return `uo-${stem}-${String(sha256).slice(0, 8)}.png`
|
||||
}
|
||||
|
||||
/**
|
||||
* Write one sprite and return its filename, or null if it could not be written.
|
||||
*
|
||||
* Never throws. A full disk or a read-only volume must degrade to "this creature
|
||||
* has no picture" — which the whole site already renders correctly, because it is
|
||||
* the state every install was in until this phase — rather than failing an import
|
||||
* that has already fetched hundreds of others.
|
||||
*/
|
||||
function writeSprite(key, sha256, png) {
|
||||
const name = fileNameFor(key, sha256)
|
||||
|
||||
try {
|
||||
fs.mkdirSync(artDir(), { recursive: true })
|
||||
fs.writeFileSync(path.join(artDir(), name), png)
|
||||
return name
|
||||
} catch (err) {
|
||||
log.warn('could not write an imported sprite', { key, error: err.message })
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort removal of a sprite a key no longer points at. */
|
||||
function removeSprite(name) {
|
||||
if (!name) return
|
||||
|
||||
try {
|
||||
fs.unlinkSync(path.join(artDir(), name))
|
||||
} catch {
|
||||
// Already gone, or never written. Either way there is nothing to do, and an
|
||||
// import must not fail because a file it was tidying up was tidied already.
|
||||
}
|
||||
}
|
||||
|
||||
// ── the import ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Import (or update) the body catalogue and the slug → body map.
|
||||
*
|
||||
* Returns a result rather than throwing, so a controller can render it and an
|
||||
* operator can read it:
|
||||
*
|
||||
* `skipped` no shard configured — the file era had no equivalent here
|
||||
* `unavailable` the shard could not answer (down, plane off, no libgdiplus)
|
||||
* `unchanged` the client files match what was imported; nothing fetched
|
||||
* `imported` fetched and applied
|
||||
* `needsReview` a key we hold has vanished from the shard's manifest
|
||||
* `failed` something went wrong mid-import
|
||||
*
|
||||
* `force` re-imports even when the client files are unchanged (which is also how
|
||||
* an operator recovers from a deleted uploads directory — the database still
|
||||
* holds the hashes, but the files behind them are gone). `approve` accepts a
|
||||
* catalogue that no longer offers keys we hold.
|
||||
*/
|
||||
async function importAssets({ force = false, approve = false } = {}) {
|
||||
if (!(await shardLinked())) {
|
||||
return {
|
||||
status: 'skipped',
|
||||
reason: 'uo-link is not configured, so there is no shard to read client files from',
|
||||
}
|
||||
}
|
||||
|
||||
let sources
|
||||
|
||||
try {
|
||||
sources = await bridge.sourceFingerprint()
|
||||
} catch (err) {
|
||||
return failure(err, 'client file manifest')
|
||||
}
|
||||
|
||||
// §4.4: a Linux shard host without libgdiplus cannot render a sprite at all.
|
||||
// It is reported on the source gate precisely so an operator meets it while
|
||||
// setting the shard up rather than from an empty bestiary weeks later.
|
||||
if (sources.imaging && sources.imaging.ok === false) {
|
||||
return {
|
||||
status: 'unavailable',
|
||||
code: 'NO_IMAGING',
|
||||
reason: sources.imaging.reason || 'the shard host cannot render images',
|
||||
}
|
||||
}
|
||||
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
|
||||
if (!force && bridge.sameSources(sources, meta?.sources)) {
|
||||
const counts = await db.countAssets()
|
||||
const bodies = await db.countBodies()
|
||||
|
||||
return {
|
||||
status: 'unchanged',
|
||||
assets: counts.total,
|
||||
stored: counts.stored,
|
||||
bodies: bodies.resolved,
|
||||
hashing: sources.hashing,
|
||||
importedAt: meta?.importedAt ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
let manifest
|
||||
|
||||
try {
|
||||
manifest = await bridge.readManifest({ family: bridge.FAMILY })
|
||||
} catch (err) {
|
||||
return failure(err, 'asset manifest')
|
||||
}
|
||||
|
||||
const held = await db.allAssets()
|
||||
const offered = new Set(manifest.rows.map((r) => r.key))
|
||||
|
||||
// A key we hold that the shard no longer offers. An unmounted client volume and
|
||||
// a deliberate downgrade look identical from here, and the wrong guess deletes
|
||||
// artwork, so it is staged rather than applied — the same rule, and the same
|
||||
// reasoning, as a vanished cliloc overlay or a disappearing atlas facet.
|
||||
const vanished = [...held.keys()].filter((key) => !offered.has(key))
|
||||
|
||||
if (vanished.length > 0 && !approve) {
|
||||
return {
|
||||
status: 'needsReview',
|
||||
reason:
|
||||
`${vanished.length} asset(s) this site holds are no longer offered by the shard; ` +
|
||||
'nothing was changed',
|
||||
vanished: vanished.slice(0, 50),
|
||||
vanishedCount: vanished.length,
|
||||
}
|
||||
}
|
||||
|
||||
// The diff, and the whole reason stage 2 carries hashes and not pixels. An
|
||||
// unchanged key is skipped ONLY if its file is actually still on disk: the row
|
||||
// and the file can disagree (a wiped uploads volume, a restore from a database
|
||||
// dump), and re-fetching a sprite is far cheaper than a creature page with a
|
||||
// broken image on it.
|
||||
const wanted = manifest.rows.filter((row) => {
|
||||
const existing = held.get(row.key)
|
||||
if (!existing || existing.sha256 !== row.sha256) return true
|
||||
if (!existing.file) return true
|
||||
return !fs.existsSync(path.join(artDir(), existing.file))
|
||||
})
|
||||
|
||||
let fetched = { assets: new Map(), missing: { absent: 0, unsupported: 0 } }
|
||||
|
||||
if (wanted.length > 0) {
|
||||
try {
|
||||
fetched = await bridge.fetchAssets({
|
||||
keys: wanted.map((r) => r.key),
|
||||
catalog: manifest.catalog,
|
||||
})
|
||||
} catch (err) {
|
||||
return failure(err, 'asset content')
|
||||
}
|
||||
}
|
||||
|
||||
const rows = []
|
||||
let written = 0
|
||||
|
||||
for (const row of manifest.rows) {
|
||||
const existing = held.get(row.key)
|
||||
const got = fetched.assets.get(row.key)
|
||||
|
||||
if (!got) {
|
||||
// Either it was unchanged and skipped, or the shard could not serve it. The
|
||||
// row is kept either way, with whatever file it already had — a key the
|
||||
// shard suddenly cannot render must not lose the picture we already hold.
|
||||
rows.push({ ...row, file: existing?.file ?? null })
|
||||
continue
|
||||
}
|
||||
|
||||
const name = writeSprite(row.key, got.sha256, got.png)
|
||||
|
||||
if (name) {
|
||||
written++
|
||||
if (existing?.file && existing.file !== name) removeSprite(existing.file)
|
||||
}
|
||||
|
||||
rows.push({
|
||||
...row,
|
||||
sha256: got.sha256 || row.sha256,
|
||||
bytes: got.bytes || row.bytes,
|
||||
width: got.width || row.width,
|
||||
height: got.height || row.height,
|
||||
body: got.body ?? row.body,
|
||||
direction: got.direction ?? row.direction,
|
||||
file: name ?? existing?.file ?? null,
|
||||
})
|
||||
}
|
||||
|
||||
const removed = []
|
||||
|
||||
if (vanished.length > 0) {
|
||||
for (const key of vanished) {
|
||||
removeSprite(held.get(key)?.file)
|
||||
removed.push(key)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await db.saveAssets(rows, {
|
||||
catalog: manifest.catalog,
|
||||
extractorVersion: manifest.extractorVersion,
|
||||
family: bridge.FAMILY,
|
||||
playerBodies: manifest.playerBodies,
|
||||
sources: { files: sources.files, extractorVersion: sources.extractorVersion },
|
||||
count: rows.length,
|
||||
})
|
||||
} catch (err) {
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
|
||||
const bodies = await resolveAtlasBodies()
|
||||
const art = await applyArt()
|
||||
|
||||
log.info('asset import applied', {
|
||||
assets: rows.length,
|
||||
fetched: fetched.assets.size,
|
||||
written,
|
||||
absent: fetched.missing.absent,
|
||||
bodies: bodies.resolved,
|
||||
art: art.applied,
|
||||
})
|
||||
|
||||
return {
|
||||
status: 'imported',
|
||||
catalog: manifest.catalog,
|
||||
extractorVersion: manifest.extractorVersion,
|
||||
assets: rows.length,
|
||||
fetched: fetched.assets.size,
|
||||
written,
|
||||
absent: fetched.missing.absent,
|
||||
unsupported: fetched.missing.unsupported,
|
||||
removed: removed.length,
|
||||
scanned: manifest.scanned,
|
||||
pages: manifest.pages,
|
||||
playerBodies: manifest.playerBodies,
|
||||
bodies,
|
||||
art,
|
||||
}
|
||||
}
|
||||
|
||||
function failure(err, what) {
|
||||
if (err instanceof bridge.AssetBridgeError) {
|
||||
return { status: 'unavailable', code: err.code, reason: err.message }
|
||||
}
|
||||
|
||||
log.warn(`asset import failed reading the ${what}`, { error: err.message })
|
||||
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
|
||||
// ── the body pass (§8) ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Ask the shard for a body id for every creature the atlas knows.
|
||||
*
|
||||
* `shard_spawn_creatures.name` is the ServUO class name — the atlas build picks
|
||||
* the winning spelling of the spawn TYPE token rather than inventing a display
|
||||
* name — so this needs no new column to ask its question.
|
||||
*
|
||||
* Never throws: a shard that goes down between the asset fetch and this pass
|
||||
* leaves the assets imported and the map as it was, which is a strictly better
|
||||
* state than failing the whole import back to nothing.
|
||||
*/
|
||||
async function resolveAtlasBodies() {
|
||||
let creatures = []
|
||||
|
||||
try {
|
||||
creatures = await atlasDb.allCreatureTypes()
|
||||
} catch (err) {
|
||||
return { resolved: 0, asked: 0, reason: err.message }
|
||||
}
|
||||
|
||||
if (creatures.length === 0) {
|
||||
return { resolved: 0, asked: 0, reason: 'the spawn atlas has no creatures loaded' }
|
||||
}
|
||||
|
||||
let rows
|
||||
|
||||
try {
|
||||
rows = await bridge.resolveBodies({ creatures })
|
||||
} catch (err) {
|
||||
return { resolved: 0, asked: creatures.length, reason: err.message }
|
||||
}
|
||||
|
||||
try {
|
||||
await db.replaceBodies(rows)
|
||||
} catch (err) {
|
||||
return { resolved: 0, asked: creatures.length, reason: err.message }
|
||||
}
|
||||
|
||||
const tally = { ok: 0, unknown: 0, notCreature: 0, failed: 0 }
|
||||
|
||||
for (const row of rows) {
|
||||
if (tally[row.status] === undefined) tally.failed++
|
||||
else tally[row.status]++
|
||||
}
|
||||
|
||||
return { asked: creatures.length, answered: rows.length, resolved: tally.ok, tally }
|
||||
}
|
||||
|
||||
// ── the derivation (§12) ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Point every atlas creature at its imported portrait.
|
||||
*
|
||||
* Two rules, and the second is the one worth stating:
|
||||
*
|
||||
* 1. The operator's `spawnAtlas.art.json` wins. Someone who drew their own
|
||||
* creature portraits must not have them replaced by a sprite rip.
|
||||
* 2. A slug with neither is set back to NULL rather than left alone. A creature
|
||||
* whose body stopped resolving — the operator removed a script package, say
|
||||
* — would otherwise keep pointing at a file that is about to be deleted, and
|
||||
* a broken image is worse than no image.
|
||||
*/
|
||||
async function applyArt() {
|
||||
const derived = await db.artBySlug()
|
||||
const operator = operatorArt()
|
||||
|
||||
const map = { ...derived, ...operator }
|
||||
|
||||
try {
|
||||
const applied = await atlasDb.setCreatureArt(map)
|
||||
return { applied, derived: Object.keys(derived).length, operator: Object.keys(operator).length }
|
||||
} catch (err) {
|
||||
log.warn('could not apply imported creature art', { error: err.message })
|
||||
return { applied: 0, derived: Object.keys(derived).length, error: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
// ── status ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* What the admin panel renders: what is loaded, what the shard says, and whether
|
||||
* the two agree.
|
||||
*
|
||||
* Never throws and never fails a page: every branch that could — no shard, a
|
||||
* shard that is down, an asset plane the operator switched off — is a reported
|
||||
* state with a reason an operator can act on.
|
||||
*/
|
||||
async function getStatus() {
|
||||
const counts = await db.countAssets().catch(() => ({ total: 0, stored: 0 }))
|
||||
const bodies = await db.countBodies().catch(() => ({ total: 0, resolved: 0 }))
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
const families = await db.countByFamily().catch(() => ({}))
|
||||
|
||||
const status = {
|
||||
loaded: {
|
||||
assets: counts.total,
|
||||
stored: counts.stored,
|
||||
creatures: bodies.total,
|
||||
resolved: bodies.resolved,
|
||||
catalog: meta?.catalog ?? null,
|
||||
extractorVersion: meta?.extractorVersion ?? null,
|
||||
importedAt: meta?.importedAt ?? null,
|
||||
// Item and land pictures, counted separately because they are a different
|
||||
// KIND of thing (§11, phase 5): no manifest, no set, and no "how many are
|
||||
// there" to compare against. `items` is how many the site has been asked
|
||||
// for and holds, which is the only number that means anything here.
|
||||
items: families.static?.stored ?? 0,
|
||||
land: families.land?.stored ?? 0,
|
||||
},
|
||||
shard: null,
|
||||
drift: null,
|
||||
}
|
||||
|
||||
if (!(await shardLinked())) {
|
||||
status.reason = 'uo-link is not configured'
|
||||
return status
|
||||
}
|
||||
|
||||
try {
|
||||
const sources = await bridge.sourceFingerprint()
|
||||
|
||||
status.shard = {
|
||||
files: Object.keys(sources.files).length,
|
||||
extractorVersion: sources.extractorVersion,
|
||||
hashing: sources.hashing,
|
||||
complete: sources.complete,
|
||||
imaging: sources.imaging,
|
||||
// Which §5 families this overlay serves. A phase-3 or phase-4 overlay says
|
||||
// `['body']`, which is what an admin panel needs in order to say "update
|
||||
// your plugin" rather than showing an item-art pipeline that cannot work.
|
||||
families: sources.families,
|
||||
}
|
||||
|
||||
status.drift = meta ? !bridge.sameSources(sources, meta.sources) : true
|
||||
} catch (err) {
|
||||
status.reason = err.message
|
||||
status.code = err instanceof bridge.AssetBridgeError ? err.code : 'UNAVAILABLE'
|
||||
}
|
||||
|
||||
return status
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ART_SUBDIR,
|
||||
artDir,
|
||||
fileNameFor,
|
||||
importAssets,
|
||||
resolveAtlasBodies,
|
||||
applyArt,
|
||||
getStatus,
|
||||
}
|
||||
510
server/model/shardAssets/shardItemArt.model.js
Normal file
510
server/model/shardAssets/shardItemArt.model.js
Normal file
@@ -0,0 +1,510 @@
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const db = require('./shardAssets.db')
|
||||
const core = require('../../core')
|
||||
const bridge = require('../../utils/assetBridge')
|
||||
const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model')
|
||||
const log = require('../../core').logger('shardItemArt')
|
||||
|
||||
// Item and land pictures, fetched because something on this site names them
|
||||
// (docs/link/v8.md §5, §11 — protocol 8, phase 5).
|
||||
//
|
||||
// ── Why this is not the body catalogue with a different prefix ─────────
|
||||
//
|
||||
// The bestiary wants every creature, so phase 3 imports a SET: walk a manifest,
|
||||
// diff the hashes, fetch what moved. That works because the set is 1,022 rows
|
||||
// and one megabyte.
|
||||
//
|
||||
// This side has no set. The shard's client addresses 49,152 item graphics and
|
||||
// has art for 39,189 of them; multiply by three thousand hues and there is
|
||||
// nothing to enumerate, no manifest worth building and nothing worth importing
|
||||
// ahead of time. What there IS, at any moment, is a few hundred keys that the
|
||||
// site's own rows actually name: the items on a vendor, the things a character
|
||||
// is wearing. That is the working set, and it is what this fetches.
|
||||
//
|
||||
// ── Who is allowed to make the shard do work ──────────────────────────
|
||||
//
|
||||
// **Ingest warms; the route only serves** (org lead, 2026-09-11). A page never
|
||||
// waits on the shard and never causes a fetch: it renders the pictures already
|
||||
// on disk and leaves out the ones that are not, which is exactly the state every
|
||||
// install was in before this phase and which every surface already handles.
|
||||
// Fetching happens behind that, from the keys the site has stored.
|
||||
//
|
||||
// The alternative — fetch on the first request for a key — was rejected on one
|
||||
// number. The shard's asset plane serves **one request at a time** by design
|
||||
// (§3.2), so any anonymous visitor able to name a key could walk 49,152 ids
|
||||
// times 3,000 hues through that single slot and keep an operator's own import
|
||||
// waiting behind it, from a URL with nothing to authenticate. Warming from the
|
||||
// site's own data has no such surface: the ceiling is the number of distinct
|
||||
// (item, hue) pairs the shard itself has told us about.
|
||||
//
|
||||
// ── Why the wanted set is DERIVED and not a queue ─────────────────────
|
||||
//
|
||||
// A queue table would need writing on the ingest path, draining, retrying,
|
||||
// pruning and reconciling after a restart. The same answer falls out of a
|
||||
// `SELECT DISTINCT` over the rows that name the items — which is self-healing by
|
||||
// construction: a key lost to a restart comes back the next time the pass runs,
|
||||
// and a key for a vendor that has gone stops being wanted the moment its row is
|
||||
// deleted. The in-memory set below is an optimisation on top of that, never the
|
||||
// record: it exists so a picture seen on a LIVE character sheet — which is
|
||||
// fetched from the shard per request and stored nowhere — is not forgotten.
|
||||
//
|
||||
// ── Staleness, without a manifest (§7) ────────────────────────────────
|
||||
//
|
||||
// Every fetched row records the shard's `catalog` id, which is a hash of the
|
||||
// files that decide the bytes. A client patch changes it, a restart does not. So
|
||||
// "is this picture out of date?" is a per-row comparison rather than a manifest
|
||||
// diff, and the answer costs nothing for the pictures nobody is looking at any
|
||||
// more: they are simply never re-fetched.
|
||||
|
||||
/** Where item and land pictures land, under core's upload directory. */
|
||||
const ART_SUBDIR = 'items'
|
||||
|
||||
/**
|
||||
* How many keys one warm pass will fetch.
|
||||
*
|
||||
* A bound rather than a target. The shard serves one asset request at a time, so
|
||||
* a pass that asked for everything at once would hold that slot for as long as it
|
||||
* took — against an operator who might be trying to run an import. Passes are
|
||||
* cheap and repeat; a backlog drains over several of them and nothing waits.
|
||||
*/
|
||||
const WARM_BATCH = 400
|
||||
|
||||
/**
|
||||
* How many live-observed keys are remembered between passes.
|
||||
*
|
||||
* Bounded because this is a set fed by page views. It is an optimisation over the
|
||||
* derived set, so dropping from it costs a picture appearing one pass later, and
|
||||
* never a picture that is lost.
|
||||
*/
|
||||
const SEEN_CAP = 5000
|
||||
|
||||
const seen = new Set()
|
||||
|
||||
// ── keys (§5) ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The one place an item key is spelled.
|
||||
*
|
||||
* Hue 0 means "not hued" on the wire, so it produces the plain key rather than a
|
||||
* `/h0` one — the shard refuses `/h0` outright for the same reason, and the two
|
||||
* agreeing is what stops the same PNG being stored twice under two names.
|
||||
*/
|
||||
function staticKey(itemId, hue = 0) {
|
||||
const id = Number(itemId)
|
||||
|
||||
if (!Number.isInteger(id) || id < 0) return null
|
||||
|
||||
const h = Number(hue)
|
||||
|
||||
return Number.isInteger(h) && h > 0 ? `static/${id}/h${h}` : `static/${id}`
|
||||
}
|
||||
|
||||
function landKey(tileId) {
|
||||
const id = Number(tileId)
|
||||
|
||||
return Number.isInteger(id) && id >= 0 && id < 0x4000 ? `land/${id}` : null
|
||||
}
|
||||
|
||||
function artDir() {
|
||||
return path.join(core.uploads.UPLOAD_DIR, ART_SUBDIR)
|
||||
}
|
||||
|
||||
/**
|
||||
* Content-addressed, exactly as the body catalogue's names are and for the same
|
||||
* reason: a stable name overwritten in place leaves every browser and CDN serving
|
||||
* last month's client's sprite while the database row stays perfectly correct.
|
||||
*/
|
||||
function fileNameFor(key, sha256) {
|
||||
const stem = key.replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')
|
||||
return `uo-${stem}-${String(sha256).slice(0, 8)}.png`
|
||||
}
|
||||
|
||||
// ── noticing ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Remember that something on this site showed these (itemId, hue) pairs.
|
||||
*
|
||||
* Called from the market ingest and from the character sheet, and deliberately
|
||||
* synchronous and allocation-light: it is on a request path and a page must never
|
||||
* pay for a picture it is not going to get anyway.
|
||||
*/
|
||||
function notice(items) {
|
||||
if (!Array.isArray(items)) return 0
|
||||
|
||||
let added = 0
|
||||
|
||||
for (const item of items) {
|
||||
const key = staticKey(item?.itemId ?? item?.item_id, item?.hue)
|
||||
|
||||
if (!key || seen.has(key)) continue
|
||||
|
||||
// Oldest-first, and only when full. The derived set is the record; this is a
|
||||
// cache of hints, so forgetting one costs a pass, not a picture.
|
||||
if (seen.size >= SEEN_CAP) seen.delete(seen.values().next().value)
|
||||
|
||||
seen.add(key)
|
||||
added++
|
||||
}
|
||||
|
||||
return added
|
||||
}
|
||||
|
||||
/** For tests and the admin surface: how many hints are waiting. */
|
||||
function noticedCount() {
|
||||
return seen.size
|
||||
}
|
||||
|
||||
// ── serving ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Attach `art` to rows that name an item, in place, and notice what is missing.
|
||||
*
|
||||
* `art` is a FILENAME under `uploads/items/`, never a path or a URL — the same
|
||||
* shape `shard_spawn_creatures.art` uses, so the client builds one URL the same
|
||||
* way everywhere and the API never hard-codes a mount point.
|
||||
*
|
||||
* A row with no stored picture gets `art: null` rather than being changed in any
|
||||
* other way. That is a first-class state: it is what every row looked like before
|
||||
* this phase, every surface renders it, and it is what an item this client has no
|
||||
* art for looks like permanently.
|
||||
*/
|
||||
async function decorate(rows, { itemIdField = 'itemId', hueField = 'hue' } = {}) {
|
||||
const list = Array.isArray(rows) ? rows.filter((r) => r && typeof r === 'object') : []
|
||||
|
||||
if (list.length === 0) return list
|
||||
|
||||
const keys = list.map((row) => staticKey(row[itemIdField], row[hueField]))
|
||||
|
||||
let files = new Map()
|
||||
|
||||
try {
|
||||
files = await db.filesForKeys(keys.filter(Boolean))
|
||||
} catch (err) {
|
||||
// Decoration, not the page. A picture lookup that fails must not fail a
|
||||
// marketplace search.
|
||||
log.warn('could not read item art', { error: err.message })
|
||||
return list
|
||||
}
|
||||
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
list[i].art = (keys[i] && files.get(keys[i])) || null
|
||||
}
|
||||
|
||||
// Everything this page WANTED is worth warming, whether or not we had it: the
|
||||
// ones we had may be stale, and the ones we did not are the point.
|
||||
notice(list.map((row) => ({ itemId: row[itemIdField], hue: row[hueField] })))
|
||||
|
||||
return list
|
||||
}
|
||||
|
||||
// ── warming ────────────────────────────────────────────────────────────────
|
||||
|
||||
async function shardLinked() {
|
||||
try {
|
||||
const config = await uoLinkConfig.getSafe()
|
||||
return Boolean(config?.enabled && config?.baseUrl)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every item key the site's own rows name, newest-priced first.
|
||||
*
|
||||
* `shard_vendor_items` is the only stored table that carries (item_id, hue)
|
||||
* today. The character sheet's equipment is fetched live from the shard per
|
||||
* request and stored nowhere, which is precisely what the in-memory hint set is
|
||||
* for.
|
||||
*/
|
||||
async function wantedKeys() {
|
||||
const keys = []
|
||||
|
||||
try {
|
||||
const rows = await core.query(
|
||||
'SELECT DISTINCT item_id, hue FROM shard_vendor_items WHERE item_id > 0 LIMIT 20000',
|
||||
)
|
||||
|
||||
for (const row of rows) {
|
||||
const key = staticKey(row.item_id, row.hue)
|
||||
if (key) keys.push(key)
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn('could not read the marketplace for item art', { error: err.message })
|
||||
}
|
||||
|
||||
// Hints last, so a backlog of stored rows is never starved by page traffic.
|
||||
for (const key of seen) keys.push(key)
|
||||
|
||||
return [...new Set(keys)]
|
||||
}
|
||||
|
||||
function writePicture(key, sha256, png) {
|
||||
const name = fileNameFor(key, sha256)
|
||||
|
||||
try {
|
||||
fs.mkdirSync(artDir(), { recursive: true })
|
||||
fs.writeFileSync(path.join(artDir(), name), png)
|
||||
return name
|
||||
} catch (err) {
|
||||
log.warn('could not write an item picture', { key, error: err.message })
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function removePicture(name) {
|
||||
if (!name) return
|
||||
|
||||
try {
|
||||
fs.unlinkSync(path.join(artDir(), name))
|
||||
} catch {
|
||||
// Already gone, or never written. A warm pass must not fail because a file it
|
||||
// was tidying up was tidied already.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One warm pass: fetch the wanted keys we do not already hold, and store them.
|
||||
*
|
||||
* Returns a result rather than throwing, with the same vocabulary the body import
|
||||
* uses — `skipped`, `unavailable`, `unchanged`, `imported`, `failed` — so the
|
||||
* admin surface reports one set of words for both halves of this protocol.
|
||||
*
|
||||
* `limit` bounds one pass. `force` re-fetches keys we hold, which is how an
|
||||
* operator recovers from a wiped uploads volume without waiting for a client
|
||||
* patch to invalidate every row.
|
||||
*/
|
||||
async function warm({ limit = WARM_BATCH, force = false } = {}) {
|
||||
if (!(await shardLinked())) {
|
||||
return { status: 'skipped', reason: 'uo-link is not configured, so there is no shard to ask' }
|
||||
}
|
||||
|
||||
let sources
|
||||
|
||||
try {
|
||||
sources = await bridge.sourceFingerprint()
|
||||
} catch (err) {
|
||||
return failure(err, 'client file manifest')
|
||||
}
|
||||
|
||||
if (sources.imaging && sources.imaging.ok === false) {
|
||||
return {
|
||||
status: 'unavailable',
|
||||
code: 'NO_IMAGING',
|
||||
reason: sources.imaging.reason || 'the shard host cannot render images',
|
||||
}
|
||||
}
|
||||
|
||||
// A phase-3 or phase-4 overlay serves bodies and nothing else. Asking it for a
|
||||
// static is refused per request, which would be a warn on every pass forever —
|
||||
// so it is checked once, here, and reported as the ordinary state it is.
|
||||
// Defensive default rather than a trusted field: an older sidecar, an older
|
||||
// overlay or a stubbed fingerprint can all leave it off, and `['body']` is the
|
||||
// truthful reading of its absence (§6 — the families field arrived in phase 5).
|
||||
const families = Array.isArray(sources.families) ? sources.families : ['body']
|
||||
|
||||
if (!families.includes('static')) {
|
||||
return {
|
||||
status: 'unavailable',
|
||||
code: 'UNSUPPORTED',
|
||||
reason:
|
||||
"this shard's overlay does not serve item art; it offers " +
|
||||
`${families.join(', ')}. Update the plugin overlay to get it.`,
|
||||
}
|
||||
}
|
||||
|
||||
const wanted = await wantedKeys()
|
||||
|
||||
if (wanted.length === 0) {
|
||||
return { status: 'unchanged', wanted: 0, fetched: 0, written: 0 }
|
||||
}
|
||||
|
||||
// The catalogue is learned from the first reply rather than asked for, so this
|
||||
// pass cannot be the thing that decides what is stale. `catalog: null` on the
|
||||
// request means "whatever you have"; the mid-walk guard in `fetchAssets` is what
|
||||
// catches a client that moves underneath it.
|
||||
let held = new Set()
|
||||
|
||||
if (!force) {
|
||||
const current = await currentCatalog()
|
||||
|
||||
try {
|
||||
held = await db.freshKeys(wanted, current)
|
||||
} catch (err) {
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
const todo = wanted.filter((key) => !held.has(key)).slice(0, Math.max(1, limit))
|
||||
|
||||
if (todo.length === 0) {
|
||||
forget(wanted)
|
||||
return { status: 'unchanged', wanted: wanted.length, held: held.size, fetched: 0, written: 0 }
|
||||
}
|
||||
|
||||
let fetched
|
||||
|
||||
try {
|
||||
fetched = await bridge.fetchAssets({ keys: todo })
|
||||
} catch (err) {
|
||||
return failure(err, 'item art')
|
||||
}
|
||||
|
||||
// Only the filenames, and only for the keys in hand: the old file is removed
|
||||
// when a key's hash moves, so `uploads/items/` tracks the working set instead of
|
||||
// accumulating one file per client patch forever.
|
||||
const existing = await db.filesForKeys(todo).catch(() => new Map())
|
||||
const rows = []
|
||||
let written = 0
|
||||
|
||||
for (const key of todo) {
|
||||
const got = fetched.assets.get(key)
|
||||
|
||||
// A key the shard has no art for is not a failure and not a row: writing an
|
||||
// empty row would make it "held" and stop it ever being asked again, which is
|
||||
// wrong the moment an operator patches in the missing graphic.
|
||||
if (!got) continue
|
||||
|
||||
const name = writePicture(key, got.sha256, got.png)
|
||||
|
||||
if (!name) continue
|
||||
|
||||
written++
|
||||
|
||||
const before = existing.get(key)
|
||||
|
||||
if (before && before !== name) removePicture(before)
|
||||
|
||||
rows.push({
|
||||
key,
|
||||
family: key.startsWith('land/') ? 'land' : 'static',
|
||||
sha256: got.sha256,
|
||||
bytes: got.bytes,
|
||||
width: got.width,
|
||||
height: got.height,
|
||||
body: null,
|
||||
direction: null,
|
||||
file: name,
|
||||
catalog: fetched.catalog,
|
||||
})
|
||||
}
|
||||
|
||||
if (rows.length > 0) {
|
||||
try {
|
||||
// No meta: `shard_asset_meta` is the BODY catalogue's singleton — what an
|
||||
// Update compares a manifest against — and this family has no manifest. A
|
||||
// warm pass writing there would tell the body import that a client it never
|
||||
// looked at is unchanged.
|
||||
await db.saveAssets(rows, null)
|
||||
} catch (err) {
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
forget(todo)
|
||||
|
||||
const result = {
|
||||
status: 'imported',
|
||||
catalog: fetched.catalog,
|
||||
wanted: wanted.length,
|
||||
held: held.size,
|
||||
asked: todo.length,
|
||||
fetched: fetched.assets.size,
|
||||
written,
|
||||
absent: fetched.missing.absent,
|
||||
unsupported: fetched.missing.unsupported,
|
||||
remaining: Math.max(0, wanted.length - held.size - todo.length),
|
||||
}
|
||||
|
||||
log.info('item art warmed', result)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/** Drop hints a pass has dealt with, so the set does not grow without bound. */
|
||||
function forget(keys) {
|
||||
for (const key of keys) seen.delete(key)
|
||||
}
|
||||
|
||||
/**
|
||||
* The catalogue id the shard would answer under right now.
|
||||
*
|
||||
* Read from a one-key probe rather than from a dedicated command: the shard puts
|
||||
* `catalog` on every fetch reply, so the cheapest honest way to ask is to fetch
|
||||
* something. `static/0` is the smallest such question and its answer is thrown
|
||||
* away — what is wanted is the id beside it.
|
||||
*
|
||||
* A shard that cannot answer returns null, and null compares unequal to every
|
||||
* stored catalogue, so the pass falls back to "everything is stale" — which costs
|
||||
* a re-fetch and never serves a wrong picture. That is the right way round.
|
||||
*/
|
||||
async function currentCatalog() {
|
||||
try {
|
||||
const probe = await bridge.fetchAssets({ keys: ['static/0'] })
|
||||
return probe.catalog ?? null
|
||||
} catch (err) {
|
||||
log.warn('could not read the shard art catalogue', { error: err.message })
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function failure(err, what) {
|
||||
if (err instanceof bridge.AssetBridgeError) {
|
||||
return { status: 'unavailable', code: err.code, reason: err.message }
|
||||
}
|
||||
|
||||
log.warn(`item art failed reading the ${what}`, { error: err.message })
|
||||
|
||||
return { status: 'failed', reason: err.message }
|
||||
}
|
||||
|
||||
// ── the background pass ────────────────────────────────────────────────────
|
||||
|
||||
let timer = null
|
||||
|
||||
/**
|
||||
* Run a warm pass every few minutes, forever, while the process lives.
|
||||
*
|
||||
* Deliberately a plain interval and not a debounce on ingest. A market sweep
|
||||
* delivers dozens of `vendor.listing` frames in a burst and debouncing each of
|
||||
* them would either fire once per frame or need its own state machine; a pass is
|
||||
* cheap when there is nothing to do (one `SELECT DISTINCT` and one probe) and the
|
||||
* work it exists for is not urgent — a picture appearing a few minutes after the
|
||||
* listing that wants it is invisible to everyone.
|
||||
*
|
||||
* `unref()` so this never holds the process open at shutdown.
|
||||
*/
|
||||
function startWarming({ everyMs = 5 * 60 * 1000 } = {}) {
|
||||
if (timer) return
|
||||
|
||||
timer = setInterval(() => {
|
||||
warm().catch((err) => log.warn('item art warm pass failed', { error: err.message }))
|
||||
}, everyMs)
|
||||
|
||||
if (typeof timer.unref === 'function') timer.unref()
|
||||
}
|
||||
|
||||
function stopWarming() {
|
||||
if (!timer) return
|
||||
|
||||
clearInterval(timer)
|
||||
timer = null
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ART_SUBDIR,
|
||||
WARM_BATCH,
|
||||
artDir,
|
||||
fileNameFor,
|
||||
staticKey,
|
||||
landKey,
|
||||
notice,
|
||||
noticedCount,
|
||||
decorate,
|
||||
wantedKeys,
|
||||
warm,
|
||||
currentCatalog,
|
||||
startWarming,
|
||||
stopWarming,
|
||||
}
|
||||
@@ -452,8 +452,64 @@ function listChampions({ facet = '' } = {}) {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Every creature the atlas knows, as `{ slug, name }` (docs/link/v8.md §8).
|
||||
*
|
||||
* `name` is the ServUO CLASS NAME, not a display string invented here: the atlas
|
||||
* build picks the winning spelling of the spawn type token, so "GiantSpider" is
|
||||
* what the column holds and what `ScriptCompiler.FindTypeByName` will resolve.
|
||||
* That is the one property that lets the asset import ask its question without a
|
||||
* new column, and it is worth knowing before anyone "tidies" this into a
|
||||
* prettified label.
|
||||
*/
|
||||
async function allCreatureTypes() {
|
||||
return query('SELECT slug, name FROM shard_spawn_creatures ORDER BY slug')
|
||||
}
|
||||
|
||||
/**
|
||||
* Point creatures at their artwork, from a `{ slug: filename }` map.
|
||||
*
|
||||
* Everything NOT in the map is set back to NULL, which is deliberate: a creature
|
||||
* whose body stopped resolving must lose its portrait rather than keep pointing
|
||||
* at a file that is about to be deleted. A broken image is worse than no image,
|
||||
* and no image is the state the whole atlas UI was designed around.
|
||||
*
|
||||
* One transaction, and a single `CASE` update rather than a statement per slug —
|
||||
* at ~800 creatures the round trips are the cost, not the work.
|
||||
*/
|
||||
async function setCreatureArt(map) {
|
||||
const entries = Object.entries(map ?? {}).filter(
|
||||
([slug, file]) => typeof slug === 'string' && slug !== '' && typeof file === 'string' && file !== '',
|
||||
)
|
||||
|
||||
const conn = await core.pool.getConnection()
|
||||
|
||||
try {
|
||||
await conn.beginTransaction()
|
||||
await conn.query('UPDATE shard_spawn_creatures SET art = NULL WHERE art IS NOT NULL')
|
||||
|
||||
for (let i = 0; i < entries.length; i += BATCH) {
|
||||
await conn.batch(
|
||||
'UPDATE shard_spawn_creatures SET art = ? WHERE slug = ?',
|
||||
entries.slice(i, i + BATCH).map(([slug, file]) => [file, slug]),
|
||||
)
|
||||
}
|
||||
|
||||
await conn.commit()
|
||||
|
||||
return entries.length
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {})
|
||||
throw err
|
||||
} finally {
|
||||
conn.release()
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
replaceAtlas,
|
||||
allCreatureTypes,
|
||||
setCreatureArt,
|
||||
getMeta,
|
||||
getFacets,
|
||||
getPending,
|
||||
|
||||
@@ -107,8 +107,45 @@ function pointTypeRows(points) {
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* The art each creature gets when the atlas is rebuilt.
|
||||
*
|
||||
* **`replaceAtlas` empties `shard_spawn_creatures` and refills it**, so anything
|
||||
* on that row is destroyed on every refresh — and a refresh happens on every
|
||||
* boot. Before protocol 8 that cost nothing: `art` came from a file on disk and
|
||||
* was simply re-read. As of phase 3 it can also come from an IMPORT, which is
|
||||
* expensive to obtain and whose gate (the shard's client-file hashes) would say
|
||||
* "unchanged" for weeks afterwards. So the imported values are re-derived here,
|
||||
* on the way past, rather than being restored by an import that has no reason to
|
||||
* run again.
|
||||
*
|
||||
* **The operator's map is spread last and therefore wins.** Someone who drew
|
||||
* their own creature portraits must not have them replaced by a sprite rip on the
|
||||
* next Update — the one property §12 states outright.
|
||||
*
|
||||
* Never throws: the asset tables are the newer half of this pair, and an atlas
|
||||
* refresh must not start failing because an asset query did. Losing the imported
|
||||
* art for one boot is recoverable by pressing Import; a boot that cannot rebuild
|
||||
* the atlas is not.
|
||||
*/
|
||||
async function artForAtlas() {
|
||||
const operator = loadArtMap()
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line global-require
|
||||
const assetsDb = require('../shardAssets/shardAssets.db')
|
||||
const derived = await assetsDb.artBySlug()
|
||||
return { ...derived, ...operator }
|
||||
} catch (err) {
|
||||
log.warn('imported creature art could not be read; using the operator map alone', {
|
||||
error: err.message,
|
||||
})
|
||||
return operator
|
||||
}
|
||||
}
|
||||
|
||||
async function applyAtlas(atlas) {
|
||||
return db.replaceAtlas({ ...atlas, pointTypes: pointTypeRows(atlas.points) }, loadArtMap())
|
||||
return db.replaceAtlas({ ...atlas, pointTypes: pointTypeRows(atlas.points) }, await artForAtlas())
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
const db = require('./shardClilocs.db')
|
||||
const { settings } = require('../../core')
|
||||
const { displayText } = require('../../utils/clilocParse')
|
||||
const { displayText, parseCliloc } = require('../../utils/clilocParse')
|
||||
const {
|
||||
ClilocFormatError,
|
||||
ClilocSourceError,
|
||||
@@ -8,8 +8,12 @@ const {
|
||||
hashSources,
|
||||
sameSources,
|
||||
missingSources,
|
||||
missingOverlays,
|
||||
readOverlays,
|
||||
readCliloc,
|
||||
} = require('../../utils/clilocSource')
|
||||
const bridge = require('../../utils/clilocBridge')
|
||||
const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model')
|
||||
const log = require('../../core').logger('shardClilocs')
|
||||
|
||||
// The cliloc table — UO's id → display-string map, refreshed from a file the
|
||||
@@ -29,11 +33,35 @@ const log = require('../../core').logger('shardClilocs')
|
||||
// 2. **Nothing client-derived is committed.** The table is built from the
|
||||
// operator's own file at a configured path. The repo ships no strings.
|
||||
//
|
||||
// The table is built from a SET of sources — the converted client table plus
|
||||
// every operator-maintained overlay beside it — because shards edit items and
|
||||
// add new ones, and those carry cliloc ids no stock client table has. All of
|
||||
// them are re-read on every boot and hash-gated together, so adding one custom
|
||||
// item never means re-exporting a 5 MB client file. Later sources win.
|
||||
// The table is built from a SET of sources — a base table plus every
|
||||
// operator-maintained overlay beside it — because shards edit items and add new
|
||||
// ones, and those carry cliloc ids no stock client table has. Later sources win,
|
||||
// so an overlay both adds ids the client never had and overrides stock ones.
|
||||
//
|
||||
// ── Where the base comes from (protocol 8, docs/link/v8.md §9) ─────────
|
||||
//
|
||||
// **The shard**, on any install with uo-link configured. It has the operator's
|
||||
// client files already — a ServUO server cannot boot without them — and since
|
||||
// phase 2 it has the decompressor too, so `GET /cliloc` returns the table and
|
||||
// nobody installs UOFiddler or copies a 5 MB file anywhere.
|
||||
//
|
||||
// **A file on disk** otherwise. That is the pipeline this replaces, kept for
|
||||
// installs with no shard link and for development, and deprecated rather than
|
||||
// removed: an operator who has one keeps working, and an operator who has a shard
|
||||
// never builds one. Passing an explicit `path` to `refresh()` still selects it,
|
||||
// which is the escape hatch for "import from this file, this once".
|
||||
//
|
||||
// **Overlays are always the filesystem's**, either way. There is nothing on the
|
||||
// shard to ask for: ServUO has no server-side notion of a custom cliloc, so the
|
||||
// `custom/` directory is the only place those ids exist.
|
||||
//
|
||||
// ── What that changed about WHEN this runs ───────────────────────
|
||||
//
|
||||
// Boot no longer imports on the shard path. The file path could hash 5 MB locally
|
||||
// on every restart and skip; the shard path would mean a sidecar round trip in the
|
||||
// boot sequence, for a table that changes when an operator patches their client —
|
||||
// an event they know about and we do not. So on the bridge, importing is an admin
|
||||
// action (Admin → Shard), and boot leaves whatever is loaded serving.
|
||||
//
|
||||
// That set is also why this has the atlas's escalation, in a lighter form. A
|
||||
// single corrupt file fails the parse loudly, but a source that has simply
|
||||
@@ -96,8 +124,224 @@ const currentParser = (meta) => meta?.parserVersion === PARSER_VERSION
|
||||
*
|
||||
* `force` skips the hash check (an admin asking for a reimport). `approve`
|
||||
* additionally accepts a vanished source.
|
||||
*
|
||||
* Which SOURCE it reads is decided here and nowhere else: the shard when uo-link
|
||||
* is configured and enabled, a file otherwise, and always a file when the caller
|
||||
* named one.
|
||||
*/
|
||||
async function refresh({ force = false, approve = false, path: pathOverride = '' } = {}) {
|
||||
const override = String(pathOverride ?? '').trim()
|
||||
|
||||
if (override === '' && (await shardLinked())) {
|
||||
return refreshFromShard({ force, approve })
|
||||
}
|
||||
|
||||
return refreshFromFile({ force, approve, path: override })
|
||||
}
|
||||
|
||||
/**
|
||||
* Is there a shard to ask?
|
||||
*
|
||||
* Both halves matter. `baseUrl` alone is an install that has been configured and
|
||||
* then switched off, and calling it would spend a 12 s timeout to learn what the
|
||||
* row already says. Never throws: an unreadable config means "no shard", and the
|
||||
* file path is a working answer.
|
||||
*/
|
||||
async function shardLinked() {
|
||||
try {
|
||||
const config = await uoLinkConfig.getSafe()
|
||||
return Boolean(config?.enabled && config?.baseUrl)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge parsed sources in order, later winning.
|
||||
*
|
||||
* Shared by both paths, because the merge is the same question whichever end the
|
||||
* base arrived from: what did each source contribute, and what did it override.
|
||||
* The per-source breakdown is for the admin panel — an operator who adds an
|
||||
* overlay wants to see it took effect, and `overrode: 0` on a file meant to
|
||||
* re-label stock items says it did not.
|
||||
*/
|
||||
function mergeSources(groups) {
|
||||
const merged = new Map()
|
||||
const sources = []
|
||||
|
||||
for (const group of groups) {
|
||||
let added = 0
|
||||
let overrode = 0
|
||||
|
||||
for (const entry of group.entries) {
|
||||
if (!Number.isInteger(entry.number)) continue
|
||||
if (merged.has(entry.number)) overrode++
|
||||
else added++
|
||||
merged.set(entry.number, entry)
|
||||
}
|
||||
|
||||
sources.push({
|
||||
label: group.label,
|
||||
kind: group.kind,
|
||||
entries: group.entries.length,
|
||||
added,
|
||||
overrode,
|
||||
})
|
||||
}
|
||||
|
||||
return { entries: [...merged.values()], sources }
|
||||
}
|
||||
|
||||
/** Overlay hashes as a `{ label: sha256 }` map, in merge order. */
|
||||
function overlayHashes(files) {
|
||||
const hashes = {}
|
||||
for (const file of files) hashes[file.label] = file.sha256
|
||||
return hashes
|
||||
}
|
||||
|
||||
/** The overlay half of a stored fingerprint — everything under `custom/`. */
|
||||
function onlyOverlays(hashes) {
|
||||
if (!hashes) return null
|
||||
const out = {}
|
||||
for (const [label, sha] of Object.entries(hashes)) {
|
||||
if (label.startsWith('custom/')) out[label] = sha
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Import with the shard as the base source.
|
||||
*
|
||||
* The gate is two-part, and neither part is something the shard can answer for
|
||||
* us: has the client file changed (size/mtime/sha256, plus the shard's own
|
||||
* `EXTRACTOR_VERSION`), and has any overlay beside the configured path changed.
|
||||
* Either is drift; neither is the normal case.
|
||||
*/
|
||||
async function refreshFromShard({ force = false, approve = false } = {}) {
|
||||
let fingerprint
|
||||
|
||||
try {
|
||||
fingerprint = await bridge.fingerprint()
|
||||
} catch (err) {
|
||||
if (err instanceof bridge.ClilocBridgeError) {
|
||||
return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code }
|
||||
}
|
||||
return { status: 'failed', source: 'bridge', reason: err.message }
|
||||
}
|
||||
|
||||
const configured = await getClientPath()
|
||||
const overlays = readOverlays(configured)
|
||||
const hashes = overlayHashes(overlays.files)
|
||||
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
|
||||
if (
|
||||
!force &&
|
||||
bridge.sameSource(fingerprint, meta?.base) &&
|
||||
sameSources(hashes, onlyOverlays(meta?.hashes)) &&
|
||||
currentParser(meta)
|
||||
) {
|
||||
return {
|
||||
status: 'unchanged',
|
||||
source: 'bridge',
|
||||
file: fingerprint.file,
|
||||
count: meta.count ?? null,
|
||||
customCount: overlays.files.length,
|
||||
hashing: fingerprint.hashing,
|
||||
}
|
||||
}
|
||||
|
||||
// An overlay that was loaded last time and is not there now is refused rather
|
||||
// than applied — an unmounted volume and a deliberate deletion look identical
|
||||
// from here, and the wrong guess silently drops every name that file gave.
|
||||
// The BASE is deliberately not part of this question: an install upgraded from
|
||||
// the file pipeline is *supposed* to stop having one.
|
||||
const gone = missingOverlays(hashes, meta?.hashes)
|
||||
|
||||
if (gone.length > 0 && !approve) {
|
||||
return {
|
||||
status: 'needsReview',
|
||||
source: 'bridge',
|
||||
reason: `${gone.length} previously-loaded cliloc overlay(s) are missing; the existing table is unchanged`,
|
||||
missingSources: gone,
|
||||
file: fingerprint.file,
|
||||
}
|
||||
}
|
||||
|
||||
let base
|
||||
|
||||
try {
|
||||
base = await bridge.readCliloc({ lang: bridge.DEFAULT_LANGUAGE })
|
||||
} catch (err) {
|
||||
if (err instanceof bridge.ClilocBridgeError) {
|
||||
return { status: 'unavailable', source: 'bridge', reason: err.message, code: err.code }
|
||||
}
|
||||
return { status: 'failed', source: 'bridge', reason: err.message }
|
||||
}
|
||||
|
||||
const groups = [{ label: base.source.file, kind: 'shard', entries: base.entries }]
|
||||
|
||||
for (const file of overlays.files) {
|
||||
try {
|
||||
groups.push({ label: file.label, kind: 'custom', entries: parseCliloc(file.buffer) })
|
||||
} catch (err) {
|
||||
if (err instanceof ClilocFormatError) {
|
||||
// Named, because "which of my six overlay files is malformed" is
|
||||
// otherwise a guessing game.
|
||||
return {
|
||||
status: 'unavailable',
|
||||
source: 'bridge',
|
||||
reason: `${file.label}: ${err.message}`,
|
||||
code: err.code,
|
||||
}
|
||||
}
|
||||
return { status: 'failed', source: 'bridge', reason: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
const merged = mergeSources(groups)
|
||||
|
||||
try {
|
||||
const applied = await db.replaceAll(merged.entries, {
|
||||
source: 'bridge',
|
||||
base: fingerprint,
|
||||
hashes,
|
||||
parserVersion: PARSER_VERSION,
|
||||
sources: merged.sources,
|
||||
file: base.source.file,
|
||||
bytes: fingerprint.size,
|
||||
})
|
||||
|
||||
invalidate()
|
||||
|
||||
return {
|
||||
status: 'imported',
|
||||
source: 'bridge',
|
||||
file: base.source.file,
|
||||
count: applied.count,
|
||||
parsed: merged.entries.length,
|
||||
blank: applied.blank,
|
||||
pages: base.source.pages,
|
||||
sources: merged.sources,
|
||||
// The shard says how many rows it holds; this is how many arrived. They
|
||||
// agree, or the walk is wrong in a way no count on its own would show.
|
||||
reported: base.source.reported,
|
||||
received: base.source.received,
|
||||
overlayProblem: overlays.problem ?? undefined,
|
||||
acceptedMissing: gone.length > 0 ? gone : undefined,
|
||||
}
|
||||
} catch (err) {
|
||||
return { status: 'failed', source: 'bridge', reason: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Import from a converted file on disk — the pre-protocol-8 pipeline, unchanged.
|
||||
*
|
||||
* Deprecated but supported: an install with no shard link has no other way to get
|
||||
* a table, and development without a running ServUO is the same case.
|
||||
*/
|
||||
async function refreshFromFile({ force = false, approve = false, path: pathOverride = '' } = {}) {
|
||||
// An explicit override wins outright — a one-off "use this file", which must
|
||||
// not be silently overruled by the configured path the way an env default is.
|
||||
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
|
||||
@@ -153,7 +397,10 @@ async function refresh({ force = false, approve = false, path: pathOverride = ''
|
||||
}
|
||||
|
||||
try {
|
||||
const applied = await db.replaceAll(parsed.entries, parsed.source)
|
||||
// `source: 'file'` is what lets the NEXT refresh — and `status()` — tell a
|
||||
// table built from a converted file from one built over the bridge. Without
|
||||
// it an install that gains a shard link looks like it already imported.
|
||||
const applied = await db.replaceAll(parsed.entries, { ...parsed.source, source: 'file' })
|
||||
invalidate()
|
||||
return {
|
||||
status: 'imported',
|
||||
@@ -177,9 +424,22 @@ async function refresh({ force = false, approve = false, path: pathOverride = ''
|
||||
/**
|
||||
* Boot hook. Best-effort by contract: it logs and returns, never throws, so a
|
||||
* missing or malformed cliloc file can never stop the site coming up.
|
||||
*
|
||||
* **On the bridge it imports nothing**, deliberately. The file path can hash a
|
||||
* local 5 MB file on every restart and skip in 14 ms; asking the shard would put
|
||||
* a sidecar round trip in the boot sequence to answer a question whose answer is
|
||||
* "no" every time except after a client patch — which is an operator action, and
|
||||
* therefore something an operator can press a button for. Whatever table is
|
||||
* loaded keeps serving, which is exactly what happens today when a restart finds
|
||||
* nothing changed.
|
||||
*/
|
||||
async function refreshOnBoot() {
|
||||
try {
|
||||
if (await shardLinked()) {
|
||||
log.info('cliloc table comes from the shard; import is admin-triggered (Admin → Shard)')
|
||||
return { status: 'skipped', source: 'bridge', reason: 'the shard is the cliloc source' }
|
||||
}
|
||||
|
||||
const result = await refresh()
|
||||
switch (result.status) {
|
||||
case 'imported':
|
||||
@@ -220,8 +480,18 @@ async function refreshOnBoot() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Everything the admin panel needs to describe cliloc state. */
|
||||
/**
|
||||
* Everything the admin panel needs to describe cliloc state.
|
||||
*
|
||||
* Two shapes, one per source, sharing every field a panel actually renders
|
||||
* (`count`, `drift`, `problem`, `sources`, `missingSources`, `importedAt`). What
|
||||
* differs is what `file` means and what a problem with it looks like: on the
|
||||
* bridge it is the shard's own client file and the problems are transport ones,
|
||||
* on disk it is a path an operator typed.
|
||||
*/
|
||||
async function status({ path: pathOverride = '' } = {}) {
|
||||
if (pathOverride.trim() === '' && (await shardLinked())) return shardStatus()
|
||||
|
||||
const configured = pathOverride.trim() !== '' ? pathOverride.trim() : await getClientPath()
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
const loaded = await db.count().catch(() => 0)
|
||||
@@ -280,6 +550,72 @@ async function status({ path: pathOverride = '' } = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Status when the shard is the source.
|
||||
*
|
||||
* The one thing worth knowing here that the file path has no equivalent of:
|
||||
* `hashing`. The shard reports a null `sha256` for a client file it has not
|
||||
* hashed yet — hashing the 343 MB of art and animation it also serves cannot fit
|
||||
* in a 10 s reply, so it happens on its own thread — and a null hash means "ask
|
||||
* again", never "changed". Drift falls back to (size, mtime) meanwhile, which is
|
||||
* the same gate the shard itself applies, so an operator is never blocked from
|
||||
* importing by a hash that has not landed.
|
||||
*/
|
||||
async function shardStatus() {
|
||||
const configured = await getClientPath()
|
||||
const meta = await db.getMeta().catch(() => null)
|
||||
const loaded = await db.count().catch(() => 0)
|
||||
|
||||
const overlays = readOverlays(configured)
|
||||
const hashes = overlayHashes(overlays.files)
|
||||
|
||||
let fingerprint = null
|
||||
let problem = overlays.problem ?? null
|
||||
let code = null
|
||||
|
||||
try {
|
||||
fingerprint = await bridge.fingerprint()
|
||||
} catch (err) {
|
||||
problem = err.message
|
||||
code = err.code ?? null
|
||||
}
|
||||
|
||||
const drift = fingerprint
|
||||
? !bridge.sameSource(fingerprint, meta?.base) ||
|
||||
!sameSources(hashes, onlyOverlays(meta?.hashes)) ||
|
||||
!currentParser(meta)
|
||||
: null
|
||||
|
||||
return {
|
||||
source: 'bridge',
|
||||
configured: true,
|
||||
// The overlay directory, which is all the path setting still selects on this
|
||||
// source. Reported so a panel can say where `custom/` is being read from.
|
||||
path: configured,
|
||||
file: fingerprint?.file ?? bridge.SOURCE_FILE,
|
||||
fileReadable: Boolean(fingerprint),
|
||||
problem,
|
||||
code,
|
||||
drift,
|
||||
count: loaded,
|
||||
shard: fingerprint
|
||||
? {
|
||||
size: fingerprint.size,
|
||||
mtime: fingerprint.mtime,
|
||||
sha256: fingerprint.sha256,
|
||||
extractorVersion: fingerprint.extractorVersion,
|
||||
hashing: fingerprint.hashing,
|
||||
complete: fingerprint.complete,
|
||||
}
|
||||
: null,
|
||||
sources: Object.keys(hashes),
|
||||
loadedSources: meta?.sources ?? null,
|
||||
missingSources: missingOverlays(hashes, meta?.hashes),
|
||||
importedAt: meta?.importedAt ?? null,
|
||||
sourceBytes: meta?.base?.size ?? meta?.bytes ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Lookup ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Resolution happens SERVER-SIDE, not in the browser. Two reasons: the table is
|
||||
@@ -359,6 +695,7 @@ module.exports = {
|
||||
SETTING_KEY,
|
||||
getClientPath,
|
||||
setClientPath,
|
||||
shardLinked,
|
||||
refresh,
|
||||
refreshOnBoot,
|
||||
status,
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
const db = require('./shardMarket.db')
|
||||
const clilocs = require('../shardClilocs/shardClilocs.model')
|
||||
const itemArt = require('../shardAssets/shardItemArt.model')
|
||||
const log = require('../../core').logger('shard-market')
|
||||
|
||||
// Defense in depth on top of the shard's own MarketMaxListings cap. The shard is
|
||||
@@ -173,6 +174,13 @@ async function upsertVendor(ev) {
|
||||
const vendor = flattenFrame(ev)
|
||||
const items = await shapeItems(ev)
|
||||
await db.replaceVendor(vendor, items)
|
||||
|
||||
// The listings name (itemId, hue) pairs, which are §5 asset keys (phase 5).
|
||||
// Noticing them here is what makes the warm pass find a newly listed item's
|
||||
// picture before anyone looks at the shop, rather than one page view later.
|
||||
// A hint, never a queue — the pass derives its real set from this table, so a
|
||||
// hint lost to a restart costs nothing.
|
||||
itemArt.notice(items)
|
||||
}
|
||||
|
||||
/** Ingest one `vendor.listing.remove` frame. */
|
||||
@@ -273,8 +281,14 @@ async function search({
|
||||
|
||||
const info = await db.meta()
|
||||
|
||||
// Each listing gets `art`: the filename of the item's picture under
|
||||
// uploads/items/, or null where this site does not hold one (phase 5). One
|
||||
// query for the page, off the listing shape rather than the SQL, so the search
|
||||
// itself stays the search and a picture lookup that fails costs a picture.
|
||||
const listings = await itemArt.decorate(rows.map(shapeListing))
|
||||
|
||||
return {
|
||||
listings: rows.map(shapeListing),
|
||||
listings,
|
||||
total,
|
||||
limit,
|
||||
offset,
|
||||
@@ -292,7 +306,7 @@ async function getVendor(serial, { limit = 250, offset = 0 } = {}) {
|
||||
const row = await db.getVendor(serial)
|
||||
if (!row) return null
|
||||
const items = await db.listVendorItems(serial, { limit, offset })
|
||||
return { ...shapeVendor(row), items: items.map(shapeItem) }
|
||||
return { ...shapeVendor(row), items: await itemArt.decorate(items.map(shapeItem)) }
|
||||
}
|
||||
|
||||
/** Index size, staleness, and the facet/region filter options. */
|
||||
|
||||
@@ -11,8 +11,11 @@ const { secretBox } = require('../../core')
|
||||
// Only used before an admin has saved anything — the stored row wins once it exists,
|
||||
// and UOLINK_PROTOCOL still overrides for an operator running an older sidecar.
|
||||
//
|
||||
// This says 7 because this build speaks protocol 7: the idempotency key and the
|
||||
// participation ledger (6), and the world verbs plus the targeted lease planes (7).
|
||||
// This says 8 because this build speaks protocol 8: the idempotency key and the
|
||||
// participation ledger (6), the world verbs plus the targeted lease planes (7), and
|
||||
// the Asset Bridge (8) -- of which this module is the first consumer, importing the
|
||||
// cliloc table over `GET /cliloc` instead of reading a file an operator converted by
|
||||
// hand (docs/link/v8.md §9).
|
||||
//
|
||||
// It said 4 before 5, and 3 for a while after protocol 4 shipped — which is the bug this
|
||||
// constant was introduced to fix. A FRESH install pinned 3, the sidecar answered
|
||||
@@ -32,7 +35,7 @@ const { secretBox } = require('../../core')
|
||||
// (`PROTOCOL_VERSION`) and the overlay's in `servuo-plugins/overlay.toml`; the thing that
|
||||
// actually pairs them is the installer's bundle check, at deploy time. So bumping this in
|
||||
// the same change as the emitters is still the discipline, and no test here replaces it.
|
||||
const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 7
|
||||
const DEFAULT_PROTOCOL = Number(process.env.UOLINK_PROTOCOL) || 8
|
||||
|
||||
function toSafe(row) {
|
||||
if (!row) {
|
||||
|
||||
@@ -28,6 +28,7 @@ const shardOps = require('./shardOps.controller')
|
||||
const shardVisibility = require('./shardVisibility.controller')
|
||||
const shardAtlas = require('./shardAtlas.controller')
|
||||
const shardClilocs = require('./shardClilocs.controller')
|
||||
const shardAssets = require('./shardAssets.controller')
|
||||
const selfShard = require('../player/shard.controller')
|
||||
const { requireRole, validate } = core.middleware
|
||||
|
||||
@@ -307,10 +308,17 @@ shardRouter.put(
|
||||
)
|
||||
|
||||
// ── Cliloc table (admin only) ─────────────────────────────────────────────
|
||||
// UO's id → display-string map, converted once by the operator from their own
|
||||
// client (docs/website/CLILOCS.md). Sits beside the atlas for the same reason:
|
||||
// it is static content derived from operator-supplied files rather than anything
|
||||
// the sidecar sends, and operating it is shard administration.
|
||||
// UO's id → display-string map, read from the shard's own UO client over the
|
||||
// bridge (docs/link/v8.md §9, docs/website/CLILOCS.md). Sits beside the atlas for
|
||||
// the same reason: it is static content derived from the operator's own files
|
||||
// rather than anything the sidecar streams, and operating it is shard
|
||||
// administration.
|
||||
//
|
||||
// Protocol 8 changed where the base table comes from, not what these routes are:
|
||||
// the shard decompresses `Cliloc.enu` and serves it paged, so an operator no
|
||||
// longer converts anything by hand. Import stays an explicit admin action,
|
||||
// because the only thing that changes a client's table is an operator patching
|
||||
// their client.
|
||||
//
|
||||
// There is deliberately NO public counterpart. The table is never served as a
|
||||
// table — 123k rows would dwarf any page that used it, and the Android client
|
||||
@@ -320,7 +328,7 @@ shardRouter.get(
|
||||
'/clilocs',
|
||||
// #swagger.tags = ['Admin · Shard']
|
||||
// #swagger.summary = 'Cliloc table status: sources, drift, entry count (admin only)'
|
||||
// #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether the files on disk have drifted from them. The table is built from a SET of sources — the converted client table plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any source that was loaded before and is now gone; an import refuses that without `approve`. A shard with nothing configured is a supported state — item names simply render as ids.'
|
||||
// #swagger.description = 'Where the cliloc sources are, whether they can be read, how many entries are loaded, and whether they have drifted from what is loaded. `source` says which pipeline is in use: `bridge` (the shard reads its own client — the normal case once uo-link is configured) or `file` (a converted file on disk, deprecated, kept for installs with no shard link). On the bridge, `shard` carries the client file’s size, mtime, hash and the shard’s extractor version, and `shard.hashing: true` means a null hash is “not computed yet”, not “changed”. The table is always a SET: the base plus every operator-maintained overlay under `custom/`, which is how shard-added and shard-edited items get names. `missingSources` lists any overlay that was loaded before and is now gone; an import refuses that without `approve`. A shard with no source at all is a supported state — item names simply render as ids.'
|
||||
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
|
||||
/* #swagger.responses[200] = { description: 'Cliloc status', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocStatus" } } } } */
|
||||
/* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
|
||||
@@ -330,8 +338,8 @@ shardRouter.get(
|
||||
shardRouter.post(
|
||||
'/clilocs/import',
|
||||
// #swagger.tags = ['Admin · Shard']
|
||||
// #swagger.summary = 'Re-import the cliloc table from its source files (admin only)'
|
||||
// #swagger.description = 'Applies a client patch, or a change to the shard’s own overlay files, without a restart. `force` reimports even when the source hashes match what is loaded. `approve` accepts a refresh in which a previously-loaded source has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. A missing path — or the common mistake of pointing at the client’s own COMPRESSED Cliloc.enu — answers 200 with status "unavailable" and the reason, rather than 500: the refresh contract reports outcomes instead of throwing, and the admin needs to be told which file to convert.'
|
||||
// #swagger.summary = 'Re-import the cliloc table from its source (admin only)'
|
||||
// #swagger.description = 'Applies a client patch, or a change to the shard’s own overlay files, without a restart. On the bridge this is the ONLY thing that imports — boot deliberately does not call the shard — so it is what an operator presses after patching their client. `force` reimports even when the sources are unchanged. `approve` accepts a refresh in which a previously-loaded overlay has VANISHED — refused by default, because an unmounted volume and a deliberate deletion are indistinguishable from the server, and the wrong guess silently drops every name that file contributed. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane the operator has switched off, a client with no cliloc file, or a malformed overlay all answer 200 with status "unavailable" and a reason naming what to fix.'
|
||||
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
|
||||
/* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Reimport even if the sources are unchanged." }, approve: { type: "boolean", description: "Accept a refresh in which a previously-loaded source has vanished." } } } } } } */
|
||||
/* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocRefreshResult" } } } } */
|
||||
@@ -344,10 +352,10 @@ shardRouter.post(
|
||||
shardRouter.put(
|
||||
'/clilocs/path',
|
||||
// #swagger.tags = ['Admin · Shard']
|
||||
// #swagger.summary = 'Set the cliloc source the site reads from (admin only)'
|
||||
// #swagger.description = 'Accepts either the converted base file itself or a directory to search. Overlays are read from a `custom/` directory beside it either way — pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it and resolution is skipped on the next boot. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.'
|
||||
// #swagger.summary = 'Set the cliloc path the site reads overlays (and any file base) from (admin only)'
|
||||
// #swagger.description = 'On an install with uo-link configured this selects only where `custom/` overlays are read from — the base table comes from the shard. Without a shard link it is also where the converted base file is looked for, which is the deprecated pre-protocol-8 pipeline. Accepts either a file or a directory to search; overlays are read from a `custom/` directory beside it either way, so pointing at a file does not forfeit them. Persisted as a setting, which wins over the UO_CLIENT_PATH deploy default so the mount can move without a redeploy. Blank clears it. Deliberately does not import as a side effect — the response carries the refreshed status so the panel can offer that as the next step.'
|
||||
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
|
||||
/* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Path to the converted cliloc file, or a directory containing one. Blank disables resolution." } } } } } } */
|
||||
/* #swagger.requestBody = { required: true, content: { "application/json": { schema: { type: "object", required: ["path"], properties: { path: { type: "string", description: "Directory holding the custom/ overlays (and, with no shard link, a converted base file). Blank clears it." } } } } } } */
|
||||
/* #swagger.responses[200] = { description: 'Cliloc status after the change', content: { "application/json": { schema: { $ref: "#/components/schemas/UoClilocStatus" } } } } */
|
||||
adminOnly,
|
||||
body('path').isString().isLength({ max: 512 }),
|
||||
@@ -355,6 +363,58 @@ shardRouter.put(
|
||||
shardClilocs.setPath,
|
||||
)
|
||||
|
||||
// ── Client assets (admin only) ────────────────────────────────────────────
|
||||
// Creature artwork, read from the shard's own UO client over the bridge
|
||||
// (docs/link/v8.md §6, §8). Sits beside the cliloc routes for the same reason
|
||||
// they sit beside the atlas: static content derived from the operator's own
|
||||
// files, and operating it is shard administration.
|
||||
//
|
||||
// There is deliberately NO public counterpart. The pictures are served as
|
||||
// ordinary files under `/uploads`, and `shard_spawn_creatures.art` names them on
|
||||
// the atlas responses the site already returns — so nothing public needs to know
|
||||
// this pipeline exists.
|
||||
shardRouter.get(
|
||||
'/assets',
|
||||
// #swagger.tags = ['Admin · Shard']
|
||||
// #swagger.summary = 'Client asset import status: what is loaded, what the shard has, whether they differ (admin only)'
|
||||
// #swagger.description = 'What the site currently holds (the imported body catalogue, how many sprites are stored, how many atlas creatures resolved to a body id) beside what the shard reports for the client files those pictures come from. `drift: true` means the client files have changed since the last import — press Import. `shard.hashing: true` means a null hash is “not computed yet”, not “changed”: the shard hashes 195 MB anim files off the request path. `shard.imaging.ok: false` is the named NO_IMAGING state — a Linux shard host without libgdiplus cannot render a sprite at all, and the reason names the package to install. A shard with no link configured, or one that is down, is a reported state with a reason rather than an error.'
|
||||
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
|
||||
/* #swagger.responses[200] = { description: 'Asset import status', content: { "application/json": { schema: { $ref: "#/components/schemas/UoAssetStatus" } } } } */
|
||||
/* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
|
||||
adminOnly,
|
||||
shardAssets.getStatus,
|
||||
)
|
||||
shardRouter.post(
|
||||
'/assets/import',
|
||||
// #swagger.tags = ['Admin · Shard']
|
||||
// #swagger.summary = 'Import creature artwork from the shard’s UO client (admin only)'
|
||||
// #swagger.description = 'Walks the shard’s asset manifest, fetches only the sprites whose hash changed, stores them under uploads/atlas/, re-resolves every atlas creature to a body id and points each creature at its picture. This is the ONLY thing that imports — boot deliberately never calls the shard — so it is what an operator presses after patching their client. `force` re-imports even when the client files are unchanged. `approve` accepts a catalogue that no longer offers assets this site holds; refused by default, because an unmounted client volume and a deliberate downgrade are indistinguishable from the server and the wrong guess deletes artwork. An operator-supplied `spawnAtlas.art.json` always wins over an imported sprite. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane switched off, or a host that cannot render images all answer 200 with status "unavailable" and a reason naming what to fix. Assets a client simply does not have are NOT failures — two thirds of the playable ghost and gargoyle bodies have no art on a stock client.'
|
||||
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
|
||||
/* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Import even if the shard’s client files are unchanged." }, approve: { type: "boolean", description: "Accept a catalogue that no longer offers assets this site holds." } } } } } } */
|
||||
/* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/UoAssetImportResult" } } } } */
|
||||
adminOnly,
|
||||
body('force').optional().isBoolean(),
|
||||
body('approve').optional().isBoolean(),
|
||||
validate,
|
||||
shardAssets.importAssets,
|
||||
)
|
||||
|
||||
shardRouter.post(
|
||||
'/assets/warm',
|
||||
// #swagger.tags = ['Admin · Shard']
|
||||
// #swagger.summary = 'Fetch item and land artwork the site is missing, now (admin only)'
|
||||
// #swagger.description = 'Runs one pass of the item-art warm loop instead of waiting for its timer. The pass works out which item pictures this site's own rows name — every distinct (ItemID, hue) on a player vendor, plus anything a character sheet has shown since the last pass — and fetches the ones it does not already hold from the shard, hued and stored under uploads/items/. There is deliberately NO manifest and no bulk import here: the client addresses 49,152 item graphics times three thousand hues, so the working set is defined by what the site actually displays. `force` re-fetches pictures the site already holds, which is how an operator recovers a wiped uploads volume. `limit` bounds one pass; the default is 400, because the shard serves one asset request at a time and a pass must not hold that slot against an import. Nothing throws for an operator-visible problem: no shard configured, a shard that is down, an asset plane switched off, a host with no libgdiplus, or a plugin overlay too old to serve item art all answer 200 with status "unavailable"/"skipped" and a reason naming what to fix.'
|
||||
// #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }]
|
||||
/* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Re-fetch pictures this site already holds." }, limit: { type: "integer", description: "How many keys this pass may fetch (1-2000)." } } } } } } */
|
||||
/* #swagger.responses[200] = { description: 'What the pass did', content: { "application/json": { schema: { $ref: "#/components/schemas/UoItemArtWarmResult" } } } } */
|
||||
/* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */
|
||||
adminOnly,
|
||||
body('force').optional().isBoolean(),
|
||||
body('limit').optional().isInt({ min: 1, max: 2000 }),
|
||||
validate,
|
||||
shardAssets.warmItemArt,
|
||||
)
|
||||
|
||||
// ── Feature visibility (admin only) ───────────────────────────────────
|
||||
// Who can see which shard surface, and which sensitive fields within it. This
|
||||
// decides what ANONYMOUS visitors get, so it sits above the moderator tier.
|
||||
|
||||
136
server/router/admin/shardAssets.controller.js
Normal file
136
server/router/admin/shardAssets.controller.js
Normal file
@@ -0,0 +1,136 @@
|
||||
// ── Admin · Client assets ──────────────────────────────────────────────────
|
||||
//
|
||||
// Operating the asset import: what the site holds, what the shard's client files
|
||||
// currently are, and a re-import after a client patch (docs/link/v8.md §6, §8,
|
||||
// §14, protocol 8 phase 3).
|
||||
//
|
||||
// The policy lives in the model. This controller does three things and no more:
|
||||
// it validates input, it maps an import RESULT onto an HTTP status, and it
|
||||
// records the action in the admin activity log.
|
||||
//
|
||||
// **An import result is not an exception**, exactly as for clilocs. A shard that
|
||||
// is down, an asset plane the operator has switched off, a Linux host with no
|
||||
// libgdiplus, a client patched halfway through the walk — each is a 200 carrying
|
||||
// `status: 'unavailable'` and a reason naming what to fix, not a 500 that says
|
||||
// only "something broke". The one thing that DOES 500 is this file having a bug.
|
||||
//
|
||||
// **This is the only thing that imports.** Boot never calls the shard for assets,
|
||||
// for the same reason it stopped calling it for clilocs: the files change when an
|
||||
// operator patches their client, which is an event they know about and the site
|
||||
// does not. So this endpoint is what an operator presses afterwards.
|
||||
//
|
||||
// The full panel — per-key review, the activity view, approve/reject as buttons —
|
||||
// is phase 8. This pair is what makes phase 3 reachable at all.
|
||||
|
||||
const assets = require('../../model/shardAssets/shardAssets.model')
|
||||
const itemArt = require('../../model/shardAssets/shardItemArt.model')
|
||||
const { activity } = require('../../core')
|
||||
|
||||
const log = require('../../core').logger('admin-shard-assets')
|
||||
|
||||
// GET /admin/shard/assets — what is loaded, what the shard says, whether they
|
||||
// disagree. No public counterpart: the assets themselves are served as ordinary
|
||||
// files under /uploads, and this is the operating view of the import.
|
||||
async function getStatus(req, res) {
|
||||
try {
|
||||
return res.json(await assets.getStatus())
|
||||
} catch (err) {
|
||||
log.error('getStatus', err)
|
||||
return res.status(500).json({ message: 'Internal Server Error' })
|
||||
}
|
||||
}
|
||||
|
||||
// POST /admin/shard/assets/import — import or update the body catalogue, then
|
||||
// re-resolve the atlas's creatures and re-derive their artwork.
|
||||
//
|
||||
// `force` re-imports even when the client files are unchanged. It is also how an
|
||||
// operator recovers a wiped uploads volume: the database still holds every hash,
|
||||
// so the ordinary gate would report "unchanged" while every picture is missing.
|
||||
// (The import checks for the file on disk per key as well, so that case usually
|
||||
// heals itself — `force` is the answer when it does not.)
|
||||
//
|
||||
// `approve` accepts a catalogue that no longer offers keys this site holds.
|
||||
// Refused by default because an unmounted client volume and a deliberate
|
||||
// downgrade look identical from the server, and the wrong guess deletes artwork.
|
||||
async function importAssets(req, res) {
|
||||
try {
|
||||
const force = !!req.body?.force
|
||||
const approve = !!req.body?.approve
|
||||
const result = await assets.importAssets({ force, approve })
|
||||
|
||||
await activity.log({
|
||||
req,
|
||||
action: 'shard.assets.import',
|
||||
detail: {
|
||||
force,
|
||||
approve,
|
||||
status: result.status,
|
||||
code: result.code ?? null,
|
||||
assets: result.assets ?? null,
|
||||
fetched: result.fetched ?? null,
|
||||
written: result.written ?? null,
|
||||
removed: result.removed ?? null,
|
||||
// The body pass is logged as its own tally rather than as a single
|
||||
// number: `unknown` means the spawn files name a type this shard's
|
||||
// scripts do not define, which is real drift an operator should see, and
|
||||
// it reads identically to `failed` if both are summed into "not resolved".
|
||||
bodies: result.bodies?.tally ?? null,
|
||||
vanished: result.vanishedCount ?? null,
|
||||
},
|
||||
})
|
||||
|
||||
return res.json(result)
|
||||
} catch (err) {
|
||||
log.error('importAssets', err)
|
||||
return res.status(500).json({ message: 'Internal Server Error' })
|
||||
}
|
||||
}
|
||||
|
||||
// POST /admin/shard/assets/warm — run one item-art warm pass now.
|
||||
//
|
||||
// The pass runs on its own timer and needs no operator, so this exists for the
|
||||
// two moments where waiting for the interval is the wrong answer: an operator who
|
||||
// has just configured the bridge and wants to see it work, and one who has just
|
||||
// patched their client and would rather not wait for pictures to refresh.
|
||||
//
|
||||
// `force` re-fetches keys the site already holds. The body import's `force` means
|
||||
// the same thing for the same reason — a wiped uploads volume leaves every
|
||||
// database row correct and every picture missing, and only an explicit re-fetch
|
||||
// recovers it.
|
||||
//
|
||||
// It is bounded: one pass asks for at most `limit` keys, because the shard's
|
||||
// asset plane serves one request at a time and a pass must not hold that slot
|
||||
// against the operator's own import.
|
||||
async function warmItemArt(req, res) {
|
||||
try {
|
||||
const force = !!req.body?.force
|
||||
const limit = Number.isFinite(Number(req.body?.limit)) ? Number(req.body.limit) : undefined
|
||||
const result = await itemArt.warm({ force, ...(limit ? { limit } : {}) })
|
||||
|
||||
await activity.log({
|
||||
req,
|
||||
action: 'shard.assets.warm',
|
||||
detail: {
|
||||
force,
|
||||
limit: limit ?? null,
|
||||
status: result.status,
|
||||
code: result.code ?? null,
|
||||
wanted: result.wanted ?? null,
|
||||
asked: result.asked ?? null,
|
||||
written: result.written ?? null,
|
||||
remaining: result.remaining ?? null,
|
||||
},
|
||||
})
|
||||
|
||||
return res.json(result)
|
||||
} catch (err) {
|
||||
log.error('warmItemArt', err)
|
||||
return res.status(500).json({ message: 'Internal Server Error' })
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getStatus,
|
||||
importAssets,
|
||||
warmItemArt,
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
// ── Admin · Cliloc table ───────────────────────────────────────────────────
|
||||
//
|
||||
// Operating the cliloc import: where the converted cliloc file is, whether it
|
||||
// has drifted from what is loaded, and a forced reimport after a client patch
|
||||
// (docs/website/CLILOCS.md).
|
||||
// Operating the cliloc import: which source the table comes from, whether it has
|
||||
// drifted from what is loaded, and a reimport after a client patch
|
||||
// (docs/link/v8.md §9, docs/website/CLILOCS.md).
|
||||
//
|
||||
// The policy lives in the model. This controller does three things and no more:
|
||||
// it validates input, it maps a refresh RESULT onto an HTTP status, and it
|
||||
@@ -10,11 +10,16 @@
|
||||
//
|
||||
// **A refresh result is not an exception.** `shardClilocs.refresh()` reports
|
||||
// `unavailable` / `failed` rather than throwing, because the boot path must never
|
||||
// be stopped by a bad file. That contract is preserved here: a missing file, or
|
||||
// the single most likely operator mistake — pointing at the client's own
|
||||
// COMPRESSED `Cliloc.enu` — is a 200 carrying `status: 'unavailable'` and the
|
||||
// reason, not a 500. A 500 would say only "something broke"; the operator needs
|
||||
// to be told which file to convert.
|
||||
// be stopped by a bad source. That contract is preserved here, and protocol 8
|
||||
// widened the set of things it covers: a shard that is down, an asset plane the
|
||||
// operator has switched off, a client with no cliloc file, a client patched
|
||||
// halfway through the import — plus everything the file pipeline could already
|
||||
// report. Each is a 200 carrying `status: 'unavailable'` and a reason naming what
|
||||
// to fix, not a 500 that says only "something broke".
|
||||
//
|
||||
// **Import matters more than it used to.** On the bridge, boot deliberately does
|
||||
// not call the shard, so this endpoint is the only thing that refreshes the
|
||||
// table — the operator presses it after patching their client.
|
||||
|
||||
const clilocs = require('../../model/shardClilocs/shardClilocs.model')
|
||||
const market = require('../../model/shardMarket/shardMarket.model')
|
||||
@@ -69,6 +74,10 @@ async function importClilocs(req, res) {
|
||||
force,
|
||||
approve,
|
||||
status: result.status,
|
||||
// Which pipeline actually ran. Worth having in the audit log for the
|
||||
// same reason it is in the status: an operator debugging a stale table
|
||||
// needs to know whether the site asked the shard or read a file.
|
||||
source: result.source ?? null,
|
||||
count: result.count ?? null,
|
||||
missingSources: result.missingSources ?? result.acceptedMissing ?? null,
|
||||
},
|
||||
@@ -80,12 +89,16 @@ async function importClilocs(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
// PUT /admin/shard/clilocs/path — point the site at a different cliloc file.
|
||||
// PUT /admin/shard/clilocs/path — point the site at a different cliloc path.
|
||||
//
|
||||
// On an install with uo-link configured this selects where `custom/` OVERLAYS are
|
||||
// read from; the base table comes from the shard either way. Without a shard link
|
||||
// it is also where the converted base file is looked for.
|
||||
//
|
||||
// Persisted as a setting, which wins over the UO_CLIENT_PATH env default so an
|
||||
// operator can move the mount without a redeploy. Blank clears it, which turns
|
||||
// resolution off (boot skips, the loaded table keeps serving) — a legitimate
|
||||
// thing to want, so it is allowed rather than validated away.
|
||||
// overlay resolution off (the loaded table keeps serving) — a legitimate thing to
|
||||
// want, so it is allowed rather than validated away.
|
||||
//
|
||||
// Deliberately does NOT import as a side effect, for the same reason the atlas
|
||||
// path does not: changing where the table reads from and reloading it are
|
||||
|
||||
@@ -11,6 +11,7 @@ const uoLinkClient = require('../../utils/uoLinkClient')
|
||||
const shardLinks = require('../../model/shardLinks/shardLinks.model')
|
||||
const shardState = require('../../model/shardState/shardState.model')
|
||||
const shardClilocs = require('../../model/shardClilocs/shardClilocs.model')
|
||||
const itemArt = require('../../model/shardAssets/shardItemArt.model')
|
||||
const { activity } = require('../../core')
|
||||
const gameSignup = require('../../utils/gameSignup')
|
||||
const { salesForAccounts } = require('../../utils/shardSales')
|
||||
@@ -71,6 +72,28 @@ async function resolveProfileClilocs(profile) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach a picture to each equipped item (docs/link/v8.md §5, §11 — phase 5).
|
||||
*
|
||||
* The equipment list is the other place on this site that carries (itemId, hue),
|
||||
* and unlike the marketplace it is LIVE: the profile is fetched from the shard per
|
||||
* request and stored nowhere, so there is no table a warm pass could derive these
|
||||
* keys from. That is what `notice` is for, and `decorate` does both — it fills in
|
||||
* every picture we already hold and remembers the ones we do not, so a character
|
||||
* whose sheet renders without art once renders with it a few minutes later.
|
||||
*
|
||||
* It never asks the shard. §17.11: the page serves what is stored and the warm
|
||||
* pass does the fetching, because a route that fetched would let any visitor drive
|
||||
* the shard's single-slot asset plane from a URL.
|
||||
*/
|
||||
async function resolveProfileArt(profile) {
|
||||
const equipment = Array.isArray(profile?.equipment) ? profile.equipment : []
|
||||
|
||||
if (equipment.length === 0) return
|
||||
|
||||
await itemArt.decorate(equipment)
|
||||
}
|
||||
|
||||
// Decorate a char.profile with cross-links from our own board data: the guild the
|
||||
// character leads and any city governorship on its account, plus resolved cliloc
|
||||
// names. Best-effort — a failure here never fails the profile (it's a nicety,
|
||||
@@ -85,6 +108,7 @@ async function enrichCharProfile(profile) {
|
||||
if (govs.length) profile.governorOf = govs.map((g) => g.city)
|
||||
}
|
||||
await resolveProfileClilocs(profile)
|
||||
await resolveProfileArt(profile)
|
||||
} catch (err) {
|
||||
log.warn('enrichCharProfile failed', { serial: profile.serial, message: err.message })
|
||||
}
|
||||
|
||||
@@ -490,21 +490,40 @@ module.exports = {
|
||||
UoClilocStatus: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Admin view of cliloc state: where the converted file is, whether it is readable, how many entries are loaded, and whether the file has drifted from them. `configured: false` is a supported state — item names then render as ids.',
|
||||
'Admin view of cliloc state: which source the base table comes from, whether it can be read, how many entries are loaded, and whether anything has drifted from them. Nothing configured at all is a supported state — item names then render as ids.',
|
||||
properties: {
|
||||
source: {
|
||||
type: 'string',
|
||||
enum: ['bridge', 'file'],
|
||||
description: '`bridge`: the shard reads its own UO client (protocol 8, the normal case). `file`: a converted file on disk — the pre-protocol-8 pipeline, deprecated, kept for installs with no shard link.',
|
||||
example: 'bridge',
|
||||
},
|
||||
configured: { type: 'boolean', example: true },
|
||||
path: { type: 'string', example: '/srv/uo-client' },
|
||||
file: { type: 'string', nullable: true, description: 'The file actually resolved, when the path is a directory.', example: '/srv/uo-client/clilocs.tsv' },
|
||||
path: { type: 'string', description: 'On the bridge: where `custom/` overlays are read from. On a file source: the base path too.', example: '/srv/uo-client' },
|
||||
file: { type: 'string', nullable: true, description: 'The base file in use — the shard’s own `cliloc.enu` on the bridge, the resolved local file otherwise.', example: 'cliloc.enu' },
|
||||
fileReadable: { type: 'boolean', example: true },
|
||||
problem: { type: 'string', nullable: true, description: 'Why the file cannot be used, when it cannot. Set (with code COMPRESSED) for a readable-but-unconverted client file.', example: null },
|
||||
code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED'] },
|
||||
problem: { type: 'string', nullable: true, description: 'Why the base cannot be used, when it cannot: a shard that is down or has assets switched off, or (on a file source) a missing or still-compressed file.', example: null },
|
||||
code: { type: 'string', nullable: true, description: 'Machine-readable cause of `problem`.', enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'UNAVAILABLE'] },
|
||||
shard: {
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
description: 'Present on the bridge: the shard’s own cliloc file as it is right now. `hashing: true` with a null `sha256` means the hash has not been computed yet — “ask again”, not “changed”.',
|
||||
properties: {
|
||||
size: { type: 'integer', example: 4989921 },
|
||||
mtime: { type: 'integer', description: 'Unix milliseconds.', example: 1757462400000 },
|
||||
sha256: { type: 'string', nullable: true },
|
||||
extractorVersion: { type: 'integer', description: 'The version of the shard’s extraction code. A bump makes everything derived from it drift.', example: 1 },
|
||||
hashing: { type: 'boolean', example: false },
|
||||
complete: { type: 'boolean', description: 'Every client file has a hash.', example: true },
|
||||
},
|
||||
},
|
||||
drift: { type: 'boolean', nullable: true, description: 'True when any source hash differs from the loaded table. NULL when the sources could not be read or are not usable.', example: false },
|
||||
count: { type: 'integer', description: 'Entries currently loaded.', example: 67496 },
|
||||
sources: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
description: 'Every source found now, root-relative, base first then overlays in merge order.',
|
||||
example: ['clilocs.plain', 'custom/uomysticmoon.tsv'],
|
||||
example: ['custom/uomysticmoon.tsv'],
|
||||
},
|
||||
loadedSources: {
|
||||
type: 'array',
|
||||
@@ -514,7 +533,7 @@ module.exports = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
label: { type: 'string', example: 'custom/uomysticmoon.tsv' },
|
||||
kind: { type: 'string', enum: ['base', 'custom'], example: 'custom' },
|
||||
kind: { type: 'string', enum: ['shard', 'base', 'custom'], description: '`shard` is the table read over the bridge; `base` a converted file on disk.', example: 'custom' },
|
||||
entries: { type: 'integer', example: 37 },
|
||||
added: { type: 'integer', description: 'Ids this source introduced.', example: 25 },
|
||||
overrode: { type: 'integer', description: 'Ids it replaced from an earlier source.', example: 12 },
|
||||
@@ -543,17 +562,22 @@ module.exports = {
|
||||
example: 'imported',
|
||||
},
|
||||
reason: { type: 'string', nullable: true },
|
||||
source: { type: 'string', nullable: true, enum: ['bridge', 'file'], description: 'Which source this refresh read.', example: 'bridge' },
|
||||
code: {
|
||||
type: 'string',
|
||||
nullable: true,
|
||||
description: 'Machine-readable cause. `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.',
|
||||
enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER'],
|
||||
description: 'Machine-readable cause. Bridge codes describe the shard (`DISABLED`: the operator switched the asset plane off; `NO_SOURCE`: its client has no cliloc file; `SHARD_DOWN`; `SOURCE_CHANGED`: the client was patched mid-import, so nothing was applied). File codes describe the path — `COMPRESSED` means the client\'s own Cliloc.enu was supplied instead of a converted one.',
|
||||
enum: ['NO_PATH', 'NOT_FOUND', 'NO_FILE', 'UNREADABLE', 'COMPRESSED', 'TRUNCATED', 'EMPTY', 'NOT_BUFFER', 'DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'UNAVAILABLE', 'SOURCE_CHANGED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE'],
|
||||
},
|
||||
path: { type: 'string', nullable: true },
|
||||
file: { type: 'string', nullable: true },
|
||||
count: { type: 'integer', nullable: true, description: 'Entries stored (blank strings are dropped).', example: 67496 },
|
||||
parsed: { type: 'integer', nullable: true, description: 'Entries read across every source before blanks were dropped.', example: 123527 },
|
||||
blank: { type: 'integer', nullable: true, example: 55994 },
|
||||
blank: { type: 'integer', nullable: true, example: 0 },
|
||||
pages: { type: 'integer', nullable: true, description: 'Bridge only: how many pages the table arrived in (a stock English table is about eleven).', example: 11 },
|
||||
reported: { type: 'integer', nullable: true, description: 'Bridge only: how many rows the shard said it holds.', example: 67496 },
|
||||
received: { type: 'integer', nullable: true, description: 'Bridge only: how many arrived. Disagreeing with `reported` means the walk is wrong.', example: 67496 },
|
||||
overlayProblem: { type: 'string', nullable: true, description: 'The base imported, but the overlay directory could not be read. Reported rather than fatal.' },
|
||||
sources: {
|
||||
type: 'array',
|
||||
nullable: true,
|
||||
@@ -562,7 +586,7 @@ module.exports = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
label: { type: 'string' },
|
||||
kind: { type: 'string', enum: ['base', 'custom'] },
|
||||
kind: { type: 'string', enum: ['shard', 'base', 'custom'] },
|
||||
entries: { type: 'integer' },
|
||||
added: { type: 'integer' },
|
||||
overrode: { type: 'integer' },
|
||||
@@ -583,6 +607,187 @@ module.exports = {
|
||||
},
|
||||
},
|
||||
},
|
||||
UoAssetStatus: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Admin view of the client-asset import (docs/link/v8.md §6, §8). What the site holds beside what the shard’s UO client currently is. Holding nothing at all is a supported state — creature pages simply render without pictures, which is what every install did before this pipeline existed.',
|
||||
properties: {
|
||||
loaded: {
|
||||
type: 'object',
|
||||
description: 'What this site currently holds.',
|
||||
properties: {
|
||||
assets: { type: 'integer', description: 'Rows in the imported catalogue.', example: 787 },
|
||||
stored: { type: 'integer', description: 'How many of those have a picture on disk. Lower than `assets` when the shard listed a key it could not render.', example: 787 },
|
||||
creatures: { type: 'integer', description: 'Atlas creatures the shard has answered a body question about, resolved or not.', example: 812 },
|
||||
resolved: { type: 'integer', description: 'How many of those resolved to a body id. The rest are types this shard’s scripts do not define, or spawn entries naming an item rather than a creature.', example: 780 },
|
||||
catalog: { type: 'string', nullable: true, description: 'The shard’s catalogue id at the last import — derived from its client files, so it changes exactly when they do.', example: 'a3f9c21d4b8e0771' },
|
||||
extractorVersion: { type: 'integer', nullable: true, description: 'The version of the shard’s extraction code. A bump makes every derived byte drift even though the client files did not move.', example: 1 },
|
||||
importedAt: { type: 'string', format: 'date-time', nullable: true },
|
||||
items: { type: 'integer', description: 'Item pictures held. Unlike the catalogue this has no total to compare against: item art is fetched because something on the site names it, so this is the working set rather than a fraction of one.', example: 1840 },
|
||||
land: { type: 'integer', description: 'Land tile pictures held. Zero on every install until something asks for one.', example: 0 },
|
||||
},
|
||||
},
|
||||
shard: {
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
description: 'The shard’s own client files right now. NULL when there is no shard link or it could not be reached — see `reason`.',
|
||||
properties: {
|
||||
files: { type: 'integer', description: 'How many of the animation/definition files this catalogue reads the shard actually has. Few clients carry all five anim files.', example: 9 },
|
||||
extractorVersion: { type: 'integer', example: 1 },
|
||||
hashing: { type: 'boolean', description: 'A hash is being computed in the background. A null `sha256` while this is true means “not yet”, never “changed”.', example: false },
|
||||
complete: { type: 'boolean', description: 'Every client file has a content hash.', example: true },
|
||||
imaging: {
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
description: 'Whether the shard host can render an image at all. `ok: false` is the named NO_IMAGING state: ServUO under Mono needs libgdiplus, and without it a Linux shard cannot decode a sprite. Cliloc and atlas import are unaffected.',
|
||||
properties: {
|
||||
ok: { type: 'boolean', example: true },
|
||||
code: { type: 'string', nullable: true, example: null },
|
||||
reason: { type: 'string', nullable: true },
|
||||
},
|
||||
},
|
||||
families: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
description: 'Which asset key families this shard’s plugin overlay serves. An overlay older than phase 5 answers `["body"]` only — it has the creature catalogue and no item art.',
|
||||
example: ['body', 'land', 'static'],
|
||||
},
|
||||
},
|
||||
},
|
||||
drift: {
|
||||
type: 'boolean',
|
||||
nullable: true,
|
||||
description: 'True when the shard’s client files no longer match what was imported — press Import. NULL when they could not be read.',
|
||||
example: false,
|
||||
},
|
||||
reason: { type: 'string', nullable: true, description: 'Why the shard could not be asked, when it could not.' },
|
||||
code: {
|
||||
type: 'string',
|
||||
nullable: true,
|
||||
description: 'Machine-readable cause of `reason`.',
|
||||
enum: ['DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'NO_IMAGING', 'SOURCE_CHANGED', 'UNAVAILABLE'],
|
||||
},
|
||||
},
|
||||
},
|
||||
UoAssetImportResult: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Outcome of an asset import. Reported rather than thrown, so a shard that is down or a host that cannot render images is an answer and not a 500.',
|
||||
properties: {
|
||||
status: {
|
||||
type: 'string',
|
||||
enum: ['skipped', 'unavailable', 'unchanged', 'imported', 'needsReview', 'failed'],
|
||||
description: '`skipped`: no shard is configured. `unchanged`: the client files match what was imported and nothing was fetched. `needsReview`: assets this site holds are no longer offered by the shard, and nothing was changed — re-run with `approve` to accept it.',
|
||||
example: 'imported',
|
||||
},
|
||||
reason: { type: 'string', nullable: true },
|
||||
code: {
|
||||
type: 'string',
|
||||
nullable: true,
|
||||
description: 'Machine-readable cause. `NO_IMAGING` is a shard host with no libgdiplus; `SOURCE_CHANGED` is a client patched partway through the walk, in which case nothing was applied.',
|
||||
enum: ['DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'NO_IMAGING', 'SOURCE_CHANGED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE', 'UNAVAILABLE'],
|
||||
},
|
||||
catalog: { type: 'string', nullable: true, example: 'a3f9c21d4b8e0771' },
|
||||
extractorVersion: { type: 'integer', nullable: true, example: 1 },
|
||||
assets: { type: 'integer', nullable: true, description: 'Catalogue rows after the import.', example: 787 },
|
||||
fetched: { type: 'integer', nullable: true, description: 'How many sprites actually crossed the wire. On an Update after a client patch this is far smaller than `assets`, which is the point of the manifest.', example: 12 },
|
||||
written: { type: 'integer', nullable: true, description: 'How many were written to disk.', example: 12 },
|
||||
absent: {
|
||||
type: 'integer',
|
||||
nullable: true,
|
||||
description: 'Keys the shard listed but could not render. NOT a failure: this client has no art at that key, which is the expected answer for two thirds of the playable ghost and gargoyle bodies.',
|
||||
example: 0,
|
||||
},
|
||||
unsupported: { type: 'integer', nullable: true, description: 'Keys the shard does not serve at all. Unlike `absent` this indicates a bug on the site’s side, not a gap in the client.', example: 0 },
|
||||
removed: { type: 'integer', nullable: true, description: 'Assets deleted because the shard no longer offers them (only with `approve`).', example: 0 },
|
||||
scanned: { type: 'integer', nullable: true, description: 'Body ids the shard walked. Far larger than `assets` — most of the addressable range has no art.', example: 2047 },
|
||||
pages: { type: 'integer', nullable: true, description: 'Manifest pages. This family pages on the shard’s scan budget rather than on bytes, so several is normal.', example: 4 },
|
||||
playerBodies: {
|
||||
type: 'array',
|
||||
nullable: true,
|
||||
items: { type: 'integer' },
|
||||
description: 'The body ids the shard reports as player-character bodies — every registered race’s male, female and ghost bodies, asked of the shard rather than hardcoded. These render head-on; everything else renders three-quarter.',
|
||||
example: [400, 401, 402, 403, 605, 606, 607, 608, 666, 667, 694, 695],
|
||||
},
|
||||
vanished: { type: 'array', nullable: true, items: { type: 'string' }, description: 'On `needsReview`: up to fifty of the keys that disappeared.' },
|
||||
vanishedCount: { type: 'integer', nullable: true },
|
||||
bodies: {
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
description: 'The slug → body id pass (§8). The shard constructs each creature and reads its body id, which is the only thing correct for a shard’s own custom creatures.',
|
||||
properties: {
|
||||
asked: { type: 'integer', example: 812 },
|
||||
answered: { type: 'integer', example: 812 },
|
||||
resolved: { type: 'integer', example: 780 },
|
||||
tally: {
|
||||
type: 'object',
|
||||
description: 'Per-outcome counts. `unknown` is real drift worth acting on — a spawn file naming a type this shard’s scripts do not define. `notCreature` is a spawn entry for an item or decoration and is permanent.',
|
||||
properties: {
|
||||
ok: { type: 'integer', example: 780 },
|
||||
unknown: { type: 'integer', example: 20 },
|
||||
notCreature: { type: 'integer', example: 12 },
|
||||
failed: { type: 'integer', example: 0 },
|
||||
},
|
||||
},
|
||||
reason: { type: 'string', nullable: true },
|
||||
},
|
||||
},
|
||||
art: {
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
description: 'The derivation onto `shard_spawn_creatures.art`. An operator-supplied `spawnAtlas.art.json` always wins over an imported sprite.',
|
||||
properties: {
|
||||
applied: { type: 'integer', description: 'Creatures now pointing at a picture.', example: 763 },
|
||||
derived: { type: 'integer', description: 'From the import.', example: 763 },
|
||||
operator: { type: 'integer', description: 'From the operator’s own map.', example: 0 },
|
||||
error: { type: 'string', nullable: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
UoItemArtWarmResult: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Outcome of one item-art warm pass (docs/link/v8.md §11, phase 5). Unlike the body catalogue there is no manifest and no set: the client addresses 49,152 item graphics times three thousand hues, so what gets fetched is defined by what this site’s own rows name — every distinct (ItemID, hue) on a player vendor, plus anything a character sheet has shown since the last pass. Reported rather than thrown, so a shard that is down is an answer and not a 500.',
|
||||
properties: {
|
||||
status: {
|
||||
type: 'string',
|
||||
enum: ['skipped', 'unavailable', 'unchanged', 'imported', 'failed'],
|
||||
description:
|
||||
'`skipped`: no shard is configured. `unchanged`: every wanted picture is already held and current. `unavailable`: the shard could not be asked, or its plugin overlay is too old to serve item art.',
|
||||
example: 'imported',
|
||||
},
|
||||
reason: { type: 'string', nullable: true },
|
||||
code: {
|
||||
type: 'string',
|
||||
nullable: true,
|
||||
description:
|
||||
'Machine-readable cause. `NO_IMAGING` is a shard host with no libgdiplus. `UNSUPPORTED` is a plugin overlay that serves the creature catalogue but not item art — update the overlay.',
|
||||
enum: ['DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'NO_IMAGING', 'UNSUPPORTED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE', 'UNAVAILABLE'],
|
||||
},
|
||||
catalog: {
|
||||
type: 'string',
|
||||
nullable: true,
|
||||
description:
|
||||
'The shard’s art catalogue id these pictures were fetched under — a hash of the files that decide their bytes. Stored per row, which is how staleness is answered without a manifest.',
|
||||
example: '7c1e04b9aa2f3d58',
|
||||
},
|
||||
wanted: { type: 'integer', nullable: true, description: 'Distinct keys this site’s rows name right now.', example: 1840 },
|
||||
held: { type: 'integer', nullable: true, description: 'How many of those are already stored and current.', example: 1440 },
|
||||
asked: { type: 'integer', nullable: true, description: 'How many this pass actually requested. Bounded by `limit`.', example: 400 },
|
||||
fetched: { type: 'integer', nullable: true, description: 'How many the shard returned a picture for.', example: 396 },
|
||||
written: { type: 'integer', nullable: true, description: 'How many were written to disk.', example: 396 },
|
||||
absent: {
|
||||
type: 'integer',
|
||||
nullable: true,
|
||||
description:
|
||||
'Keys the shard has no art for. NOT a failure — 9,963 of this client’s static ids have an empty index entry, and an item using one simply has no picture.',
|
||||
example: 4,
|
||||
},
|
||||
unsupported: { type: 'integer', nullable: true, description: 'Keys the shard does not serve at all. A bug on the site’s side rather than a gap in the client.', example: 0 },
|
||||
remaining: { type: 'integer', nullable: true, description: 'Wanted keys left for the next pass. Passes repeat on a timer, so a backlog drains without an operator.', example: 0 },
|
||||
},
|
||||
},
|
||||
UoShardLinkRequest: {
|
||||
type: 'object',
|
||||
required: ['code'],
|
||||
|
||||
368
server/test/assetBridge.test.js
Normal file
368
server/test/assetBridge.test.js
Normal file
@@ -0,0 +1,368 @@
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
|
||||
const uoLinkClient = require('../utils/uoLinkClient')
|
||||
const bridge = require('../utils/assetBridge')
|
||||
|
||||
// The three walks over the asset plane, driven against a stubbed sidecar client
|
||||
// (docs/link/v8.md §5, §6, §8 — protocol 8, phase 3).
|
||||
//
|
||||
// Two families of failure are asserted here and they are not the same shape.
|
||||
//
|
||||
// **The envelope failures** are ways the shard can be wrong that leave this side
|
||||
// holding a catalogue it believes is complete. They are invisible downstream: a
|
||||
// catalogue missing its last three hundred bodies renders as a site where some
|
||||
// creatures have pictures and some do not, which is exactly what NO catalogue
|
||||
// looks like. Each corresponds to a field §3.4 puts on the wire specifically so
|
||||
// this side can tell the difference.
|
||||
//
|
||||
// **The absence failures** are the opposite mistake, and phase 3's more likely
|
||||
// one: treating a body this client has no art for as an error. Two thirds of the
|
||||
// playable ghost and gargoyle bodies are in that state on a stock client, and an
|
||||
// import that failed — or even warned loudly — on them would teach an operator to
|
||||
// ignore the panel.
|
||||
|
||||
const saved = {}
|
||||
|
||||
function stub({ sources, manifest = [], fetch = [], bodies = [] } = {}) {
|
||||
saved.getAssetSources = uoLinkClient.getAssetSources
|
||||
saved.getAssetManifest = uoLinkClient.getAssetManifest
|
||||
saved.fetchAssets = uoLinkClient.fetchAssets
|
||||
saved.resolveBodies = uoLinkClient.resolveBodies
|
||||
|
||||
const calls = { manifest: [], fetch: [], bodies: [] }
|
||||
|
||||
uoLinkClient.getAssetSources = async () => sources
|
||||
uoLinkClient.getAssetManifest = async ({ family, cursor } = {}) => {
|
||||
calls.manifest.push({ family: family ?? null, cursor: cursor ?? null })
|
||||
const next = manifest.shift()
|
||||
if (!next) throw new Error('the walk asked for more manifest pages than the test supplied')
|
||||
return next
|
||||
}
|
||||
uoLinkClient.fetchAssets = async ({ keys, catalog, cursor } = {}) => {
|
||||
calls.fetch.push({ keys, catalog: catalog ?? null, cursor: cursor ?? null })
|
||||
const next = fetch.shift()
|
||||
if (!next) throw new Error('the walk asked for more fetch pages than the test supplied')
|
||||
return next
|
||||
}
|
||||
uoLinkClient.resolveBodies = async (types) => {
|
||||
calls.bodies.push(types)
|
||||
const next = bodies.shift()
|
||||
if (!next) throw new Error('the walk asked for more body chunks than the test supplied')
|
||||
return next
|
||||
}
|
||||
|
||||
return calls
|
||||
}
|
||||
|
||||
function restore() {
|
||||
for (const [name, fn] of Object.entries(saved)) {
|
||||
if (fn) uoLinkClient[name] = fn
|
||||
}
|
||||
}
|
||||
|
||||
const ok = (data) => ({ ok: true, status: 200, data })
|
||||
const fail = (status, data) => ({ ok: false, status, data })
|
||||
|
||||
const CATALOG = 'a3f9c21d4b8e0771'
|
||||
|
||||
const manifestPage = (rows, extra = {}) =>
|
||||
ok({
|
||||
kind: 'assets.manifest.ok',
|
||||
family: 'body',
|
||||
catalog: CATALOG,
|
||||
extractorVersion: 1,
|
||||
playerBodies: [400, 401, 402, 403],
|
||||
scanned: rows.length,
|
||||
rows,
|
||||
more: false,
|
||||
cut: 'end',
|
||||
...extra,
|
||||
})
|
||||
|
||||
const fetchPage = (rows, extra = {}) =>
|
||||
ok({
|
||||
kind: 'assets.fetch.ok',
|
||||
family: 'body',
|
||||
catalog: CATALOG,
|
||||
rows,
|
||||
more: false,
|
||||
cut: 'end',
|
||||
...extra,
|
||||
})
|
||||
|
||||
const row = (body, sha = 'aa') => ({
|
||||
key: `body/${body}/a0`,
|
||||
sha256: sha,
|
||||
bytes: 900,
|
||||
width: 24,
|
||||
height: 63,
|
||||
body,
|
||||
direction: 1,
|
||||
})
|
||||
|
||||
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47]).toString('base64')
|
||||
|
||||
const sourcesReply = (extra = {}) =>
|
||||
ok({
|
||||
kind: 'assets.sources.ok',
|
||||
extractorVersion: 1,
|
||||
imaging: { ok: true },
|
||||
hashing: false,
|
||||
complete: true,
|
||||
files: [
|
||||
{ name: 'anim.idx', size: 10, mtime: 1, sha256: 'a' },
|
||||
{ name: 'anim.mul', size: 20, mtime: 2, sha256: 'b' },
|
||||
{ name: 'body.def', size: 30, mtime: 3, sha256: 'c' },
|
||||
// Not a source this family reads: `art.mul` decides item pictures, not
|
||||
// creature ones, and folding it in would make every item-art change look
|
||||
// like a reason to re-import the whole body catalogue.
|
||||
{ name: 'art.mul', size: 148000000, mtime: 4, sha256: 'd' },
|
||||
],
|
||||
...extra,
|
||||
})
|
||||
|
||||
// ── the source gate (§6 stage 1) ──────────────────────────────────────────
|
||||
|
||||
test('the source fingerprint keeps only the files the body catalogue reads', async (t) => {
|
||||
stub({ sources: sourcesReply() })
|
||||
t.after(restore)
|
||||
|
||||
const fingerprint = await bridge.sourceFingerprint()
|
||||
|
||||
assert.deepEqual(Object.keys(fingerprint.files).sort(), ['anim.idx', 'anim.mul', 'body.def'])
|
||||
assert.equal(fingerprint.extractorVersion, 1)
|
||||
})
|
||||
|
||||
test('a bumped extractor version is drift even when every client file is identical', () => {
|
||||
const files = { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } }
|
||||
|
||||
assert.equal(
|
||||
bridge.sameSources({ files, extractorVersion: 1 }, { files, extractorVersion: 1 }),
|
||||
true,
|
||||
)
|
||||
// §7: a corrected frame offset changes every derived byte while every source
|
||||
// file stays byte-identical. If this returned true the fix would never reach
|
||||
// an install whose client never moves.
|
||||
assert.equal(
|
||||
bridge.sameSources({ files, extractorVersion: 2 }, { files, extractorVersion: 1 }),
|
||||
false,
|
||||
)
|
||||
})
|
||||
|
||||
test('a client that GAINED an anim file is drift, not a match', () => {
|
||||
const before = { files: { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } }, extractorVersion: 1 }
|
||||
const after = {
|
||||
files: {
|
||||
'anim.mul': { size: 1, mtime: 2, sha256: 'x' },
|
||||
// A client that grows an anim5.mul is a client whose gargoyles suddenly
|
||||
// resolve. Comparing only the files present in both would call that
|
||||
// unchanged and never import them.
|
||||
'anim5.mul': { size: 9, mtime: 9, sha256: 'y' },
|
||||
},
|
||||
extractorVersion: 1,
|
||||
}
|
||||
|
||||
assert.equal(bridge.sameSources(before, after), false)
|
||||
})
|
||||
|
||||
test('a null hash falls back to size and mtime rather than reading as changed', () => {
|
||||
// The shard hashes 195 MB anim files off the request path, so a null sha256 is
|
||||
// "not computed yet". Treating it as a difference would re-import the whole
|
||||
// catalogue on every restart until the background pass finished.
|
||||
const a = { files: { 'anim.mul': { size: 5, mtime: 7, sha256: null } }, extractorVersion: 1 }
|
||||
const b = { files: { 'anim.mul': { size: 5, mtime: 7, sha256: 'later' } }, extractorVersion: 1 }
|
||||
|
||||
assert.equal(bridge.sameSources(a, b), true)
|
||||
})
|
||||
|
||||
// ── the manifest walk (§6 stage 2) ────────────────────────────────────────
|
||||
|
||||
test('the manifest walks every page and stops only on cut: end', async (t) => {
|
||||
const calls = stub({
|
||||
manifest: [
|
||||
manifestPage([row(12), row(34)], { more: true, cursor: 'b:34', cut: 'limit' }),
|
||||
manifestPage([row(400)]),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await bridge.readManifest()
|
||||
|
||||
assert.equal(result.rows.length, 3)
|
||||
assert.equal(result.catalog, CATALOG)
|
||||
assert.deepEqual(result.playerBodies, [400, 401, 402, 403])
|
||||
assert.deepEqual(
|
||||
calls.manifest.map((c) => c.cursor),
|
||||
[null, 'b:34'],
|
||||
)
|
||||
})
|
||||
|
||||
test('a short page that did not end the catalogue is refused', async (t) => {
|
||||
// `cut: 'limit'` with `more: false` is the shard saying it stopped for its own
|
||||
// reason. Importing what arrived would silently drop every body after it, and
|
||||
// the result is indistinguishable from a client with fewer creatures.
|
||||
stub({ manifest: [manifestPage([row(12)], { more: false, cut: 'limit' })] })
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(() => bridge.readManifest(), /stopped sending assets/)
|
||||
})
|
||||
|
||||
test('a cursor that does not advance is refused rather than looped on', async (t) => {
|
||||
stub({
|
||||
manifest: [
|
||||
manifestPage([row(12)], { more: true, cursor: 'b:12', cut: 'budget' }),
|
||||
manifestPage([row(13)], { more: true, cursor: 'b:12', cut: 'budget' }),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(() => bridge.readManifest(), /without advancing its cursor/)
|
||||
})
|
||||
|
||||
test('the client files changing mid-walk aborts the whole import', async (t) => {
|
||||
// The catalogue id is derived from the client files themselves, so a change
|
||||
// between two pages means half of what we hold describes files that no longer
|
||||
// exist — and nothing later can tell which half.
|
||||
stub({
|
||||
manifest: [
|
||||
manifestPage([row(12)], { more: true, cursor: 'b:12', cut: 'limit' }),
|
||||
manifestPage([row(34)], { catalog: 'something-else' }),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(() => bridge.readManifest(), /changed while the manifest was being read/)
|
||||
})
|
||||
|
||||
// ── the fetch (§5) ────────────────────────────────────────────────────────
|
||||
|
||||
test('a fetch passes the catalogue id and decodes the PNG', async (t) => {
|
||||
const calls = stub({
|
||||
fetch: [fetchPage([{ key: 'body/12/a0', status: 'ok', sha256: 'aa', bytes: 4, width: 24, height: 63, body: 12, direction: 1, png }])],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const { assets } = await bridge.fetchAssets({ keys: ['body/12/a0'], catalog: CATALOG })
|
||||
|
||||
assert.equal(calls.fetch[0].catalog, CATALOG)
|
||||
assert.equal(assets.get('body/12/a0').png.length, 4)
|
||||
assert.equal(assets.get('body/12/a0').width, 24)
|
||||
})
|
||||
|
||||
test('an absent asset is a counted row, not a failed fetch', async (t) => {
|
||||
// The whole reason this is not an error: two thirds of the playable ghost and
|
||||
// gargoyle bodies have no art on a stock client (§5.2), and an import that
|
||||
// failed on them could never succeed.
|
||||
stub({
|
||||
fetch: [
|
||||
fetchPage([
|
||||
{ key: 'body/12/a0', status: 'ok', sha256: 'aa', bytes: 4, png },
|
||||
{ key: 'body/666/a0', status: 'absent' },
|
||||
{ key: 'body/400/a2/f3', status: 'unsupported' },
|
||||
]),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const { assets, missing } = await bridge.fetchAssets({
|
||||
keys: ['body/12/a0', 'body/666/a0', 'body/400/a2/f3'],
|
||||
catalog: CATALOG,
|
||||
})
|
||||
|
||||
assert.equal(assets.size, 1)
|
||||
// Counted apart, because they mean different things: `absent` is a gap in the
|
||||
// operator's client and `unsupported` is a bug on this side.
|
||||
assert.equal(missing.absent, 1)
|
||||
assert.equal(missing.unsupported, 1)
|
||||
})
|
||||
|
||||
test('a busy shard is retried rather than failing the walk', async (t) => {
|
||||
saved.fetchAssets = uoLinkClient.fetchAssets
|
||||
t.after(restore)
|
||||
|
||||
let attempts = 0
|
||||
|
||||
uoLinkClient.fetchAssets = async () => {
|
||||
attempts++
|
||||
if (attempts < 3) return fail(425, { reason: 'busy' })
|
||||
return fetchPage([{ key: 'body/12/a0', status: 'ok', sha256: 'aa', bytes: 4, png }])
|
||||
}
|
||||
|
||||
const { assets } = await bridge.fetchAssets({ keys: ['body/12/a0'], catalog: CATALOG })
|
||||
|
||||
assert.equal(attempts, 3)
|
||||
assert.equal(assets.size, 1)
|
||||
})
|
||||
|
||||
test('a shard host with no libgdiplus is named, not reported as a dead shard', async (t) => {
|
||||
saved.getAssetManifest = uoLinkClient.getAssetManifest
|
||||
t.after(restore)
|
||||
|
||||
uoLinkClient.getAssetManifest = async () =>
|
||||
fail(503, { reason: "this shard host cannot render images - Mono's System.Drawing needs libgdiplus" })
|
||||
|
||||
await assert.rejects(
|
||||
() => bridge.readManifest(),
|
||||
(err) => err.code === 'NO_IMAGING',
|
||||
)
|
||||
})
|
||||
|
||||
// ── the body pass (§8) ────────────────────────────────────────────────────
|
||||
|
||||
test('body resolution chunks to the shard cap and records every outcome', async (t) => {
|
||||
const creatures = []
|
||||
|
||||
for (let i = 0; i < bridge.BODY_CHUNK + 5; i++) {
|
||||
creatures.push({ slug: `c-${i}`, name: `Creature${i}` })
|
||||
}
|
||||
|
||||
const reply = (types) =>
|
||||
ok({
|
||||
kind: 'assets.bodies.ok',
|
||||
rows: types.map((type, i) => (i === 0 ? { type, status: 'unknown' } : { type, status: 'ok', body: 100 + i })),
|
||||
more: false,
|
||||
cut: 'end',
|
||||
})
|
||||
|
||||
const calls = stub({ bodies: [] })
|
||||
t.after(restore)
|
||||
|
||||
uoLinkClient.resolveBodies = async (types) => {
|
||||
calls.bodies.push(types)
|
||||
return reply(types)
|
||||
}
|
||||
|
||||
const rows = await bridge.resolveBodies({ creatures })
|
||||
|
||||
// Two chunks, and neither over the cap: the shard REFUSES an over-long list
|
||||
// rather than truncating it, so a chunk size above its cap does not degrade —
|
||||
// every request fails.
|
||||
assert.equal(calls.bodies.length, 2)
|
||||
assert.ok(calls.bodies.every((chunk) => chunk.length <= bridge.BODY_CHUNK))
|
||||
|
||||
assert.equal(rows.length, creatures.length)
|
||||
// The negative answers are kept. Without them the next pass asks again, and
|
||||
// the pass costs a real constructor per name on the shard's Core thread.
|
||||
assert.equal(rows.filter((r) => r.status === 'unknown').length, 2)
|
||||
})
|
||||
|
||||
test('two slugs sharing a class name are asked once and both get the answer', async (t) => {
|
||||
const calls = stub({
|
||||
bodies: [
|
||||
ok({ kind: 'assets.bodies.ok', rows: [{ type: 'GiantSpider', status: 'ok', body: 28 }], more: false, cut: 'end' }),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const rows = await bridge.resolveBodies({
|
||||
creatures: [
|
||||
{ slug: 'giant-spider', name: 'GiantSpider' },
|
||||
{ slug: 'giantspider', name: 'GiantSpider' },
|
||||
],
|
||||
})
|
||||
|
||||
assert.deepEqual(calls.bodies[0], ['GiantSpider'])
|
||||
assert.equal(rows.length, 2)
|
||||
assert.ok(rows.every((r) => r.body === 28))
|
||||
})
|
||||
303
server/test/clilocBridge.test.js
Normal file
303
server/test/clilocBridge.test.js
Normal file
@@ -0,0 +1,303 @@
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
|
||||
const uoLinkClient = require('../utils/uoLinkClient')
|
||||
const bridge = require('../utils/clilocBridge')
|
||||
|
||||
// The walk over `GET /cliloc`, driven against a stubbed sidecar client.
|
||||
//
|
||||
// Everything asserted here is a way the shard can be wrong that leaves the
|
||||
// website holding a table it believes is complete. That is the failure worth
|
||||
// testing, because it is invisible downstream: a truncated cliloc table renders
|
||||
// some items with names and some with ids, which is exactly what NO table looks
|
||||
// like. None of these are hypothetical shapes — each corresponds to a field the
|
||||
// paging envelope carries specifically so this side can tell the difference
|
||||
// (docs/link/v8.md §3.4).
|
||||
|
||||
const saved = {}
|
||||
|
||||
function stub({ sources, pages }) {
|
||||
saved.getAssetSources = uoLinkClient.getAssetSources
|
||||
saved.getClilocTable = uoLinkClient.getClilocTable
|
||||
|
||||
const calls = []
|
||||
|
||||
uoLinkClient.getAssetSources = async () => sources
|
||||
uoLinkClient.getClilocTable = async ({ lang, cursor } = {}) => {
|
||||
calls.push({ lang, cursor: cursor ?? null })
|
||||
const next = pages.shift()
|
||||
if (!next) throw new Error('the walk asked for more pages than the test supplied')
|
||||
return next
|
||||
}
|
||||
|
||||
return calls
|
||||
}
|
||||
|
||||
function restore() {
|
||||
if (saved.getAssetSources) uoLinkClient.getAssetSources = saved.getAssetSources
|
||||
if (saved.getClilocTable) uoLinkClient.getClilocTable = saved.getClilocTable
|
||||
}
|
||||
|
||||
const ok = (data) => ({ ok: true, status: 200, data })
|
||||
|
||||
/** One page of rows, with the source fingerprint every page echoes. */
|
||||
const page = (rows, extra = {}) =>
|
||||
ok({
|
||||
kind: 'cliloc.table.ok',
|
||||
lang: 'enu',
|
||||
file: 'cliloc.enu',
|
||||
size: 4989921,
|
||||
mtime: 1757462400000,
|
||||
total: 3,
|
||||
rows,
|
||||
more: false,
|
||||
cut: 'end',
|
||||
...extra,
|
||||
})
|
||||
|
||||
const sourcesReply = (file = {}) =>
|
||||
ok({
|
||||
kind: 'assets.sources.ok',
|
||||
extractorVersion: 1,
|
||||
imaging: { ok: true },
|
||||
hashing: false,
|
||||
complete: true,
|
||||
files: [
|
||||
{ name: 'cliloc.enu', path: '/uo/cliloc.enu', size: 4989921, mtime: 1757462400000, sha256: 'abc', ...file },
|
||||
{ name: 'art.mul', path: '/uo/art.mul', size: 148000000, mtime: 1, sha256: null },
|
||||
],
|
||||
})
|
||||
|
||||
// ── Stage 1: the fingerprint ───────────────────────────────────────────────
|
||||
|
||||
test('fingerprint picks the cliloc file out of the client manifest', async (t) => {
|
||||
stub({ sources: sourcesReply(), pages: [] })
|
||||
t.after(restore)
|
||||
|
||||
const fp = await bridge.fingerprint()
|
||||
|
||||
assert.equal(fp.file, 'cliloc.enu')
|
||||
assert.equal(fp.size, 4989921)
|
||||
assert.equal(fp.sha256, 'abc')
|
||||
assert.equal(fp.extractorVersion, 1)
|
||||
})
|
||||
|
||||
test('a client with no cliloc file is NO_SOURCE, not a crash', async (t) => {
|
||||
stub({
|
||||
sources: ok({ extractorVersion: 1, files: [{ name: 'art.mul', size: 1, mtime: 1 }] }),
|
||||
pages: [],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.fingerprint(), (err) => {
|
||||
assert.equal(err.code, 'NO_SOURCE')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
test('the asset plane being switched off reads as a refusal, not a bug', async (t) => {
|
||||
stub({
|
||||
sources: { ok: false, status: 403, data: { reason: 'asset extraction is disabled on this shard' } },
|
||||
pages: [],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.fingerprint(), (err) => {
|
||||
assert.equal(err.code, 'DISABLED')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
// A hash that has not been computed yet is the shard's ordinary first answer:
|
||||
// hashing the 343 MB of art and animation it also serves cannot fit in a 10 s
|
||||
// reply, so it happens off the request path. Treating a null hash as a CHANGE
|
||||
// would make the panel show drift forever on a shard nobody has imported from.
|
||||
test('a missing hash falls back to (size, mtime) rather than reading as drift', () => {
|
||||
const before = { size: 10, mtime: 20, sha256: null, extractorVersion: 1 }
|
||||
const after = { size: 10, mtime: 20, sha256: null, extractorVersion: 1 }
|
||||
|
||||
assert.equal(bridge.sameSource(before, after), true)
|
||||
assert.equal(bridge.sameSource(before, { ...after, mtime: 21 }), false)
|
||||
})
|
||||
|
||||
test('a hash on both sides beats size and mtime, which a patched-in-place file can preserve', () => {
|
||||
const a = { size: 10, mtime: 20, sha256: 'aaa', extractorVersion: 1 }
|
||||
|
||||
assert.equal(bridge.sameSource(a, { ...a, sha256: 'bbb' }), false)
|
||||
assert.equal(bridge.sameSource(a, { ...a, size: 11, mtime: 99 }), true)
|
||||
})
|
||||
|
||||
test('the extractor version is part of the fingerprint, so a corrected reader drifts', () => {
|
||||
const a = { size: 10, mtime: 20, sha256: 'aaa', extractorVersion: 1 }
|
||||
|
||||
assert.equal(bridge.sameSource(a, { ...a, extractorVersion: 2 }), false)
|
||||
})
|
||||
|
||||
// ── Stage 2: the walk ──────────────────────────────────────────────────────
|
||||
|
||||
test('a one-page table comes back whole', async (t) => {
|
||||
const calls = stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [page([{ n: 3, f: 0, t: 'c' }, { n: 1, f: 2, t: 'a' }])],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const { entries, source } = await bridge.readCliloc()
|
||||
|
||||
assert.deepEqual(entries, [
|
||||
{ number: 3, flag: 0, text: 'c' },
|
||||
{ number: 1, flag: 2, text: 'a' },
|
||||
])
|
||||
assert.equal(source.pages, 1)
|
||||
assert.equal(source.received, 2)
|
||||
assert.equal(source.reported, 3)
|
||||
assert.deepEqual(calls, [{ lang: 'enu', cursor: null }])
|
||||
})
|
||||
|
||||
test('pages are walked by echoing the cursor back until more is false', async (t) => {
|
||||
const calls = stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [
|
||||
page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }),
|
||||
page([{ n: 2, f: 0, t: 'b' }], { more: true, cursor: 'n:2', cut: 'budget' }),
|
||||
page([{ n: 3, f: 0, t: 'c' }]),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const { entries, source } = await bridge.readCliloc()
|
||||
|
||||
assert.equal(entries.length, 3)
|
||||
assert.equal(source.pages, 3)
|
||||
assert.deepEqual(
|
||||
calls.map((c) => c.cursor),
|
||||
[null, 'n:1', 'n:2'],
|
||||
)
|
||||
})
|
||||
|
||||
// `cut` is the field that is easy to omit and expensive not to have. A short
|
||||
// page means the source ended, the byte budget was spent, or the family hit its
|
||||
// own limit — and only the first means finished.
|
||||
test('a last page that did not end the table is refused, not imported', async (t) => {
|
||||
stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [page([{ n: 1, f: 0, t: 'a' }], { more: false, cut: 'limit' })],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.readCliloc(), (err) => {
|
||||
assert.equal(err.code, 'INCOMPLETE')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
test('a shard that does not advance its cursor is stopped rather than spun on', async (t) => {
|
||||
stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [
|
||||
page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }),
|
||||
page([{ n: 2, f: 0, t: 'b' }], { more: true, cursor: 'n:1', cut: 'budget' }),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.readCliloc(), (err) => {
|
||||
assert.equal(err.code, 'STUCK')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
test('more:true with no cursor at all is the same refusal', async (t) => {
|
||||
stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [page([{ n: 1, f: 0, t: 'a' }], { more: true, cut: 'budget' })],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.readCliloc(), (err) => {
|
||||
assert.equal(err.code, 'STUCK')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
// The one failure a count cannot catch: an operator patches their client while
|
||||
// the import is walking it. Half of what arrived is from a file that no longer
|
||||
// exists, and nothing later can tell which half.
|
||||
test('a client patched mid-walk aborts the whole import', async (t) => {
|
||||
stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [
|
||||
page([{ n: 1, f: 0, t: 'a' }], { more: true, cursor: 'n:1', cut: 'budget' }),
|
||||
page([{ n: 2, f: 0, t: 'b' }], { size: 5000000, mtime: 1757470000000 }),
|
||||
],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.readCliloc(), (err) => {
|
||||
assert.equal(err.code, 'SOURCE_CHANGED')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
// 425 is flow control and the ORDINARY answer during an import — the shard's
|
||||
// asset plane serves one request at a time on purpose — so it is retried rather
|
||||
// than failed. (The backoff is real time, so this exercises one retry only.)
|
||||
test('a busy shard is retried, because the work is happening', async (t) => {
|
||||
saved.getAssetSources = uoLinkClient.getAssetSources
|
||||
saved.getClilocTable = uoLinkClient.getClilocTable
|
||||
t.after(restore)
|
||||
|
||||
let attempts = 0
|
||||
uoLinkClient.getAssetSources = async () => sourcesReply()
|
||||
uoLinkClient.getClilocTable = async () => {
|
||||
attempts++
|
||||
if (attempts === 1) return { ok: false, status: 425, data: { kind: 'bridge.busy' } }
|
||||
return page([{ n: 1, f: 0, t: 'a' }])
|
||||
}
|
||||
|
||||
const { entries } = await bridge.readCliloc()
|
||||
|
||||
assert.equal(attempts, 2)
|
||||
assert.equal(entries.length, 1)
|
||||
})
|
||||
|
||||
test('a page with no rows array is malformed, not an empty table', async (t) => {
|
||||
stub({ sources: sourcesReply(), pages: [ok({ kind: 'cliloc.table.ok', more: false, cut: 'end' })] })
|
||||
t.after(restore)
|
||||
|
||||
await assert.rejects(bridge.readCliloc(), (err) => {
|
||||
assert.equal(err.code, 'MALFORMED')
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
test('a shard that never ends the table is bounded by the page cap', async (t) => {
|
||||
saved.getAssetSources = uoLinkClient.getAssetSources
|
||||
saved.getClilocTable = uoLinkClient.getClilocTable
|
||||
t.after(restore)
|
||||
|
||||
let n = 0
|
||||
uoLinkClient.getAssetSources = async () => sourcesReply()
|
||||
uoLinkClient.getClilocTable = async () => {
|
||||
n++
|
||||
return page([{ n, f: 0, t: 'x' }], { more: true, cursor: `n:${n}`, cut: 'budget' })
|
||||
}
|
||||
|
||||
await assert.rejects(bridge.readCliloc(), (err) => {
|
||||
assert.equal(err.code, 'TOO_LARGE')
|
||||
return true
|
||||
})
|
||||
assert.equal(n, bridge.MAX_PAGES)
|
||||
})
|
||||
|
||||
test('rows with an unusable id are dropped rather than stored as NaN', async (t) => {
|
||||
stub({
|
||||
sources: sourcesReply(),
|
||||
pages: [page([{ n: 'nonsense', f: 0, t: 'a' }, { n: 7, f: 0, t: 'b' }])],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const { entries } = await bridge.readCliloc()
|
||||
|
||||
assert.deepEqual(entries, [{ number: 7, flag: 0, text: 'b' }])
|
||||
})
|
||||
330
server/test/clilocSourceSelection.test.js
Normal file
330
server/test/clilocSourceSelection.test.js
Normal file
@@ -0,0 +1,330 @@
|
||||
// Which cliloc source runs, and what the shard path does with the answer
|
||||
// (docs/link/v8.md §9, docs/website/CLILOCS.md).
|
||||
//
|
||||
// The model is the only place that decides between the two pipelines, so these
|
||||
// drive it with the shard, the database and the filesystem all stubbed. Nothing
|
||||
// here reaches the real sidecar or a real table.
|
||||
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
|
||||
const clilocs = require('../model/shardClilocs/shardClilocs.model')
|
||||
const db = require('../model/shardClilocs/shardClilocs.db')
|
||||
const bridge = require('../utils/clilocBridge')
|
||||
const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
|
||||
const { ctx } = require('./_setup')
|
||||
|
||||
const saved = {
|
||||
getMeta: db.getMeta,
|
||||
replaceAll: db.replaceAll,
|
||||
count: db.count,
|
||||
fingerprint: bridge.fingerprint,
|
||||
readCliloc: bridge.readCliloc,
|
||||
getSafe: uoLinkConfig.getSafe,
|
||||
settingsGet: ctx.settings.get,
|
||||
}
|
||||
|
||||
function restore() {
|
||||
db.getMeta = saved.getMeta
|
||||
db.replaceAll = saved.replaceAll
|
||||
db.count = saved.count
|
||||
bridge.fingerprint = saved.fingerprint
|
||||
bridge.readCliloc = saved.readCliloc
|
||||
uoLinkConfig.getSafe = saved.getSafe
|
||||
ctx.settings.get = saved.settingsGet
|
||||
}
|
||||
|
||||
const FINGERPRINT = {
|
||||
kind: 'bridge',
|
||||
file: 'cliloc.enu',
|
||||
size: 4989921,
|
||||
mtime: 1757462400000,
|
||||
sha256: 'abc',
|
||||
extractorVersion: 1,
|
||||
hashing: false,
|
||||
complete: true,
|
||||
}
|
||||
|
||||
/**
|
||||
* A rig with the shard reachable (or not), the configured overlay path pointed
|
||||
* at a temp directory, and every write captured rather than made.
|
||||
*/
|
||||
function rig({ linked = true, meta = null, clientPath = '', rows = [] } = {}) {
|
||||
const applied = []
|
||||
|
||||
uoLinkConfig.getSafe = async () => ({ enabled: linked, baseUrl: linked ? 'http://127.0.0.1:8099' : null })
|
||||
ctx.settings.get = async (key) => (key === clilocs.SETTING_KEY ? clientPath : null)
|
||||
|
||||
db.getMeta = async () => meta
|
||||
db.count = async () => meta?.count ?? 0
|
||||
db.replaceAll = async (entries, writtenMeta) => {
|
||||
applied.push({ entries, meta: writtenMeta })
|
||||
return { count: entries.length, blank: 0, duplicates: 0 }
|
||||
}
|
||||
|
||||
bridge.fingerprint = async () => FINGERPRINT
|
||||
bridge.readCliloc = async () => ({
|
||||
entries: rows,
|
||||
source: {
|
||||
kind: 'bridge',
|
||||
lang: 'enu',
|
||||
file: 'cliloc.enu',
|
||||
size: FINGERPRINT.size,
|
||||
mtime: FINGERPRINT.mtime,
|
||||
pages: 1,
|
||||
reported: rows.length,
|
||||
received: rows.length,
|
||||
},
|
||||
})
|
||||
|
||||
return applied
|
||||
}
|
||||
|
||||
function tmpWithOverlay(contents) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cliloc-sel-'))
|
||||
fs.mkdirSync(path.join(dir, 'custom'), { recursive: true })
|
||||
if (contents !== undefined) fs.writeFileSync(path.join(dir, 'custom', 'shard.tsv'), contents)
|
||||
return dir
|
||||
}
|
||||
|
||||
// ── Which source runs ──────────────────────────────────────────────────────
|
||||
|
||||
test('a configured shard is the base source, and the file path is not consulted', async (t) => {
|
||||
const applied = rig({ rows: [{ number: 1, flag: 0, text: 'a' }] })
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'imported')
|
||||
assert.equal(result.source, 'bridge')
|
||||
assert.equal(applied.length, 1)
|
||||
assert.equal(applied[0].meta.source, 'bridge')
|
||||
})
|
||||
|
||||
test('no shard link falls back to the file pipeline, unchanged', async (t) => {
|
||||
rig({ linked: false })
|
||||
t.after(restore)
|
||||
|
||||
// No path configured either, so the file path reports exactly what it always
|
||||
// did — which is the assertion: the fallback is the OLD code, not a new one.
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'skipped')
|
||||
assert.equal(result.reason, 'no cliloc path configured')
|
||||
})
|
||||
|
||||
test('an explicit path is still an escape hatch, even with a shard linked', async (t) => {
|
||||
let asked = false
|
||||
rig({ linked: true })
|
||||
bridge.fingerprint = async () => {
|
||||
asked = true
|
||||
return FINGERPRINT
|
||||
}
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refresh({ path: path.join(os.tmpdir(), 'nope-does-not-exist') })
|
||||
|
||||
assert.equal(asked, false, 'the shard must not be asked when a file was named')
|
||||
assert.equal(result.status, 'unavailable')
|
||||
})
|
||||
|
||||
// Boot deliberately does not call the shard: it would put a sidecar round trip
|
||||
// in the startup sequence to answer a question whose answer is "no" except after
|
||||
// a client patch, which is an operator action.
|
||||
test('boot imports nothing over the bridge and leaves the loaded table serving', async (t) => {
|
||||
let asked = false
|
||||
rig({ linked: true })
|
||||
bridge.fingerprint = async () => {
|
||||
asked = true
|
||||
return FINGERPRINT
|
||||
}
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refreshOnBoot()
|
||||
|
||||
assert.equal(result.status, 'skipped')
|
||||
assert.equal(result.source, 'bridge')
|
||||
assert.equal(asked, false)
|
||||
})
|
||||
|
||||
// ── The gate ───────────────────────────────────────────────────────────────
|
||||
|
||||
test('an unchanged client file and no overlays is a no-op', async (t) => {
|
||||
const applied = rig({
|
||||
meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 },
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'unchanged')
|
||||
assert.equal(result.count, 67496)
|
||||
assert.equal(applied.length, 0)
|
||||
})
|
||||
|
||||
test('a patched client re-imports', async (t) => {
|
||||
const applied = rig({
|
||||
meta: {
|
||||
source: 'bridge',
|
||||
base: { ...FINGERPRINT, sha256: 'older' },
|
||||
hashes: {},
|
||||
parserVersion: 1,
|
||||
count: 10,
|
||||
},
|
||||
rows: [{ number: 1, flag: 0, text: 'a' }],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
assert.equal((await clilocs.refresh()).status, 'imported')
|
||||
assert.equal(applied.length, 1)
|
||||
})
|
||||
|
||||
// The upgrade path. An install that used the converted-file pipeline carries its
|
||||
// base label in the stored fingerprint; on the bridge that label is SUPPOSED to
|
||||
// disappear. Counting it as a vanished source would make the first import after
|
||||
// the upgrade demand an approval for a change the upgrade itself made.
|
||||
test('the retired file base is not reported as a vanished source', async (t) => {
|
||||
const applied = rig({
|
||||
meta: {
|
||||
source: 'file',
|
||||
hashes: { 'clilocs.plain': 'aaa' },
|
||||
parserVersion: 1,
|
||||
count: 67496,
|
||||
},
|
||||
rows: [{ number: 1, flag: 0, text: 'a' }],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'imported', result.reason)
|
||||
assert.equal(applied.length, 1)
|
||||
})
|
||||
|
||||
// An overlay is a different matter: it vanished, and an unmounted volume looks
|
||||
// exactly like a deliberate deletion from here.
|
||||
test('a vanished OVERLAY still stages for review', async (t) => {
|
||||
const applied = rig({
|
||||
meta: {
|
||||
source: 'bridge',
|
||||
base: FINGERPRINT,
|
||||
hashes: { 'custom/shard.tsv': 'aaa' },
|
||||
parserVersion: 1,
|
||||
count: 5,
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'needsReview')
|
||||
assert.deepEqual(result.missingSources, ['custom/shard.tsv'])
|
||||
assert.equal(applied.length, 0)
|
||||
|
||||
const accepted = await clilocs.refresh({ approve: true })
|
||||
assert.equal(accepted.status, 'imported')
|
||||
assert.deepEqual(accepted.acceptedMissing, ['custom/shard.tsv'])
|
||||
})
|
||||
|
||||
// ── The merge ──────────────────────────────────────────────────────────────
|
||||
|
||||
test('an overlay overrides the shard table, and says so', async (t) => {
|
||||
const dir = tmpWithOverlay('1023721\ta better staff\n900001\ta shard-only item\n')
|
||||
const applied = rig({
|
||||
clientPath: dir,
|
||||
rows: [
|
||||
{ number: 1023721, flag: 0, text: 'quarter staff' },
|
||||
{ number: 3000001, flag: 0, text: 'Entering Britannia...' },
|
||||
],
|
||||
})
|
||||
t.after(() => {
|
||||
restore()
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'imported', result.reason)
|
||||
|
||||
const stored = new Map(applied[0].entries.map((e) => [e.number, e.text]))
|
||||
assert.equal(stored.get(1023721), 'a better staff', 'the overlay must win')
|
||||
assert.equal(stored.get(3000001), 'Entering Britannia...')
|
||||
assert.equal(stored.get(900001), 'a shard-only item')
|
||||
|
||||
const overlay = result.sources.find((s) => s.kind === 'custom')
|
||||
assert.equal(overlay.label, 'custom/shard.tsv')
|
||||
assert.equal(overlay.added, 1)
|
||||
assert.equal(overlay.overrode, 1)
|
||||
|
||||
// Only overlay hashes are stored now — the base is fingerprinted separately,
|
||||
// and mixing them is what made the upgrade case above ambiguous.
|
||||
assert.deepEqual(Object.keys(applied[0].meta.hashes), ['custom/shard.tsv'])
|
||||
assert.equal(applied[0].meta.base.sha256, 'abc')
|
||||
})
|
||||
|
||||
test('a malformed overlay names the file rather than failing the import namelessly', async (t) => {
|
||||
const dir = tmpWithOverlay('not a cliloc file at all\n')
|
||||
rig({ clientPath: dir, rows: [{ number: 1, flag: 0, text: 'a' }] })
|
||||
t.after(() => {
|
||||
restore()
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'unavailable')
|
||||
assert.match(result.reason, /custom\/shard\.tsv/)
|
||||
})
|
||||
|
||||
// An overlay path an operator has mistyped must not stop a base table that
|
||||
// arrived perfectly well — but it must be visible, or the site silently serves a
|
||||
// table missing every shard-added name.
|
||||
test('an unreadable overlay path is reported beside a successful import', async (t) => {
|
||||
const applied = rig({
|
||||
clientPath: path.join(os.tmpdir(), 'cliloc-does-not-exist-at-all'),
|
||||
rows: [{ number: 1, flag: 0, text: 'a' }],
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await clilocs.refresh()
|
||||
|
||||
assert.equal(result.status, 'imported')
|
||||
assert.match(result.overlayProblem, /does not exist/)
|
||||
assert.equal(applied.length, 1)
|
||||
})
|
||||
|
||||
// ── Status ─────────────────────────────────────────────────────────────────
|
||||
|
||||
test('status describes the shard source, hash state and drift', async (t) => {
|
||||
rig({ meta: { source: 'bridge', base: FINGERPRINT, hashes: {}, parserVersion: 1, count: 67496 } })
|
||||
t.after(restore)
|
||||
|
||||
const status = await clilocs.status()
|
||||
|
||||
assert.equal(status.source, 'bridge')
|
||||
assert.equal(status.file, 'cliloc.enu')
|
||||
assert.equal(status.fileReadable, true)
|
||||
assert.equal(status.drift, false)
|
||||
assert.equal(status.shard.extractorVersion, 1)
|
||||
assert.equal(status.shard.hashing, false)
|
||||
})
|
||||
|
||||
test('a shard that cannot be reached is a problem on the status, not a throw', async (t) => {
|
||||
rig({})
|
||||
bridge.fingerprint = async () => {
|
||||
throw new bridge.ClilocBridgeError('The shard did not answer: timeout', 'SHARD_DOWN')
|
||||
}
|
||||
t.after(restore)
|
||||
|
||||
const status = await clilocs.status()
|
||||
|
||||
assert.equal(status.source, 'bridge')
|
||||
assert.equal(status.fileReadable, false)
|
||||
assert.equal(status.code, 'SHARD_DOWN')
|
||||
// Null, not false: with no fingerprint there is nothing to compare, and
|
||||
// reporting "no drift" would read as "up to date".
|
||||
assert.equal(status.drift, null)
|
||||
})
|
||||
334
server/test/shardAssets.model.test.js
Normal file
334
server/test/shardAssets.model.test.js
Normal file
@@ -0,0 +1,334 @@
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
|
||||
const core = require('../core')
|
||||
const model = require('../model/shardAssets/shardAssets.model')
|
||||
const db = require('../model/shardAssets/shardAssets.db')
|
||||
const atlasDb = require('../model/shardAtlas/shardAtlas.db')
|
||||
const atlasModel = require('../model/shardAtlas/shardAtlas.model')
|
||||
const bridge = require('../utils/assetBridge')
|
||||
const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
|
||||
|
||||
// The import as a decision, with the shard and the database both stubbed
|
||||
// (docs/link/v8.md §6, §12 — protocol 8, phase 3).
|
||||
//
|
||||
// Each of these is a way the import can be wrong that an operator would either
|
||||
// never notice or notice only weeks later, on a page:
|
||||
//
|
||||
// - Re-fetching every sprite on every Update. Correct output, and it makes the
|
||||
// manifest — the entire reason stage 2 carries hashes instead of pixels —
|
||||
// dead weight.
|
||||
// - Silently dropping an asset the shard stopped offering. An unmounted client
|
||||
// volume and a deliberate downgrade are the same thing from here, and the
|
||||
// wrong guess deletes artwork nobody asked to delete.
|
||||
// - Overwriting artwork the operator drew themselves. §12 states outright that
|
||||
// theirs wins, and a sprite rip replacing hand-drawn portraits is not
|
||||
// recoverable by pressing anything.
|
||||
// - Treating a body with no art as a failure. Two thirds of the playable ghost
|
||||
// and gargoyle bodies are in that state on a stock client.
|
||||
|
||||
const saved = {}
|
||||
let uploadDir
|
||||
|
||||
function stubEverything({ manifest, fetched, held = new Map(), meta = null, sources } = {}) {
|
||||
saved.sourceFingerprint = bridge.sourceFingerprint
|
||||
saved.readManifest = bridge.readManifest
|
||||
saved.fetchAssets = bridge.fetchAssets
|
||||
saved.resolveBodies = bridge.resolveBodies
|
||||
saved.allAssets = db.allAssets
|
||||
saved.saveAssets = db.saveAssets
|
||||
saved.getMeta = db.getMeta
|
||||
saved.countAssets = db.countAssets
|
||||
saved.countBodies = db.countBodies
|
||||
saved.replaceBodies = db.replaceBodies
|
||||
saved.artBySlug = db.artBySlug
|
||||
saved.allCreatureTypes = atlasDb.allCreatureTypes
|
||||
saved.setCreatureArt = atlasDb.setCreatureArt
|
||||
saved.loadArtMap = atlasModel.loadArtMap
|
||||
saved.getSafe = uoLinkConfig.getSafe
|
||||
|
||||
const seen = { saved: null, fetchedKeys: null, art: null }
|
||||
|
||||
uoLinkConfig.getSafe = async () => ({ enabled: true, baseUrl: 'http://127.0.0.1:8080' })
|
||||
|
||||
bridge.sourceFingerprint = async () =>
|
||||
sources ?? {
|
||||
files: { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } },
|
||||
extractorVersion: 1,
|
||||
hashing: false,
|
||||
complete: true,
|
||||
imaging: { ok: true },
|
||||
}
|
||||
|
||||
bridge.readManifest = async () => manifest
|
||||
bridge.fetchAssets = async ({ keys }) => {
|
||||
seen.fetchedKeys = keys
|
||||
return fetched ?? { assets: new Map(), missing: { absent: 0, unsupported: 0 } }
|
||||
}
|
||||
bridge.resolveBodies = async () => []
|
||||
|
||||
db.allAssets = async () => held
|
||||
db.getMeta = async () => meta
|
||||
db.countAssets = async () => ({ total: held.size, stored: held.size })
|
||||
db.countBodies = async () => ({ total: 0, resolved: 0 })
|
||||
db.saveAssets = async (rows) => {
|
||||
seen.saved = rows
|
||||
return rows.length
|
||||
}
|
||||
db.replaceBodies = async () => 0
|
||||
db.artBySlug = async () => ({})
|
||||
|
||||
atlasDb.allCreatureTypes = async () => []
|
||||
atlasDb.setCreatureArt = async (map) => {
|
||||
seen.art = map
|
||||
return Object.keys(map).length
|
||||
}
|
||||
atlasModel.loadArtMap = () => ({})
|
||||
|
||||
return seen
|
||||
}
|
||||
|
||||
function restore() {
|
||||
for (const [name, fn] of Object.entries(saved)) {
|
||||
if (!fn) continue
|
||||
if (name in db) db[name] = fn
|
||||
if (name in bridge) bridge[name] = fn
|
||||
if (name in atlasDb) atlasDb[name] = fn
|
||||
if (name === 'loadArtMap') atlasModel.loadArtMap = fn
|
||||
if (name === 'getSafe') uoLinkConfig.getSafe = fn
|
||||
}
|
||||
}
|
||||
|
||||
/** A real uploads directory, because the import checks the disk as well as the row. */
|
||||
function useTempUploads(t) {
|
||||
uploadDir = fs.mkdtempSync(path.join(os.tmpdir(), 'uo-assets-'))
|
||||
const previous = core.uploads
|
||||
|
||||
Object.defineProperty(core, 'uploads', {
|
||||
configurable: true,
|
||||
get: () => ({ ...previous, UPLOAD_DIR: uploadDir }),
|
||||
})
|
||||
|
||||
t.after(() => {
|
||||
Object.defineProperty(core, 'uploads', { configurable: true, get: () => previous })
|
||||
fs.rmSync(uploadDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
return uploadDir
|
||||
}
|
||||
|
||||
const row = (body, sha) => ({
|
||||
key: `body/${body}/a0`,
|
||||
family: 'body',
|
||||
sha256: sha,
|
||||
bytes: 900,
|
||||
width: 24,
|
||||
height: 63,
|
||||
body,
|
||||
direction: 1,
|
||||
})
|
||||
|
||||
const manifestOf = (rows) => ({
|
||||
rows,
|
||||
catalog: 'cat1',
|
||||
extractorVersion: 1,
|
||||
playerBodies: [400],
|
||||
pages: 1,
|
||||
scanned: 2047,
|
||||
})
|
||||
|
||||
const sprite = (sha) => ({
|
||||
sha256: sha,
|
||||
bytes: 4,
|
||||
width: 24,
|
||||
height: 63,
|
||||
body: 12,
|
||||
direction: 1,
|
||||
png: Buffer.from([0x89, 0x50, 0x4e, 0x47]),
|
||||
})
|
||||
|
||||
// ── the gate ──────────────────────────────────────────────────────────────
|
||||
|
||||
test('unchanged client files import nothing at all', async (t) => {
|
||||
const sources = {
|
||||
files: { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } },
|
||||
extractorVersion: 1,
|
||||
hashing: false,
|
||||
complete: true,
|
||||
imaging: { ok: true },
|
||||
}
|
||||
|
||||
stubEverything({ manifest: manifestOf([]), meta: { sources }, sources })
|
||||
t.after(restore)
|
||||
|
||||
bridge.readManifest = async () => {
|
||||
throw new Error('the gate should have stopped before reading a manifest')
|
||||
}
|
||||
|
||||
const result = await model.importAssets()
|
||||
|
||||
assert.equal(result.status, 'unchanged')
|
||||
})
|
||||
|
||||
test('a host that cannot render images is named rather than walked', async (t) => {
|
||||
// §4.4: reported on the SOURCE gate, so an operator meets it while setting the
|
||||
// shard up rather than from an empty bestiary weeks later.
|
||||
stubEverything({
|
||||
manifest: manifestOf([]),
|
||||
sources: {
|
||||
files: {},
|
||||
extractorVersion: 1,
|
||||
hashing: false,
|
||||
complete: true,
|
||||
imaging: { ok: false, code: 'NO_IMAGING', reason: 'needs libgdiplus' },
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.importAssets()
|
||||
|
||||
assert.equal(result.status, 'unavailable')
|
||||
assert.equal(result.code, 'NO_IMAGING')
|
||||
})
|
||||
|
||||
// ── the diff (§6) ─────────────────────────────────────────────────────────
|
||||
|
||||
test('only the keys whose hash moved are fetched', async (t) => {
|
||||
const dir = useTempUploads(t)
|
||||
fs.mkdirSync(path.join(dir, model.ART_SUBDIR), { recursive: true })
|
||||
fs.writeFileSync(path.join(dir, model.ART_SUBDIR, 'kept.png'), 'x')
|
||||
|
||||
const held = new Map([
|
||||
['body/12/a0', { key: 'body/12/a0', sha256: 'same', file: 'kept.png' }],
|
||||
['body/34/a0', { key: 'body/34/a0', sha256: 'old', file: 'kept.png' }],
|
||||
])
|
||||
|
||||
const seen = stubEverything({
|
||||
held,
|
||||
manifest: manifestOf([row(12, 'same'), row(34, 'new')]),
|
||||
fetched: { assets: new Map([['body/34/a0', sprite('new')]]), missing: { absent: 0, unsupported: 0 } },
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.importAssets({ force: true })
|
||||
|
||||
assert.equal(result.status, 'imported')
|
||||
// The whole point of a manifest that carries hashes and not pixels.
|
||||
assert.deepEqual(seen.fetchedKeys, ['body/34/a0'])
|
||||
assert.equal(result.written, 1)
|
||||
})
|
||||
|
||||
test('an unchanged key whose file is missing from disk is fetched again', async (t) => {
|
||||
// The row and the file can disagree — a wiped uploads volume, a restore from a
|
||||
// database dump. Trusting the row alone leaves a broken image on a creature
|
||||
// page with nothing anywhere reporting a problem, and re-fetching a sprite is
|
||||
// far cheaper than that.
|
||||
useTempUploads(t)
|
||||
|
||||
const held = new Map([['body/12/a0', { key: 'body/12/a0', sha256: 'same', file: 'gone.png' }]])
|
||||
|
||||
const seen = stubEverything({
|
||||
held,
|
||||
manifest: manifestOf([row(12, 'same')]),
|
||||
fetched: { assets: new Map([['body/12/a0', sprite('same')]]), missing: { absent: 0, unsupported: 0 } },
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await model.importAssets({ force: true })
|
||||
|
||||
assert.deepEqual(seen.fetchedKeys, ['body/12/a0'])
|
||||
})
|
||||
|
||||
test('a key that vanished from the manifest needs review before anything changes', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const held = new Map([['body/99/a0', { key: 'body/99/a0', sha256: 'a', file: 'x.png' }]])
|
||||
|
||||
const seen = stubEverything({ held, manifest: manifestOf([row(12, 'a')]) })
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.importAssets({ force: true })
|
||||
|
||||
assert.equal(result.status, 'needsReview')
|
||||
assert.equal(result.vanishedCount, 1)
|
||||
// Nothing was applied. An unmounted client volume and a deliberate downgrade
|
||||
// look identical from here.
|
||||
assert.equal(seen.saved, null)
|
||||
})
|
||||
|
||||
test('approve accepts the vanished key and removes its file', async (t) => {
|
||||
const dir = useTempUploads(t)
|
||||
fs.mkdirSync(path.join(dir, model.ART_SUBDIR), { recursive: true })
|
||||
fs.writeFileSync(path.join(dir, model.ART_SUBDIR, 'gone.png'), 'x')
|
||||
|
||||
const held = new Map([['body/99/a0', { key: 'body/99/a0', sha256: 'a', file: 'gone.png' }]])
|
||||
|
||||
stubEverything({
|
||||
held,
|
||||
manifest: manifestOf([row(12, 'a')]),
|
||||
fetched: { assets: new Map([['body/12/a0', sprite('a')]]), missing: { absent: 0, unsupported: 0 } },
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.importAssets({ force: true, approve: true })
|
||||
|
||||
assert.equal(result.status, 'imported')
|
||||
assert.equal(result.removed, 1)
|
||||
assert.equal(fs.existsSync(path.join(dir, model.ART_SUBDIR, 'gone.png')), false)
|
||||
})
|
||||
|
||||
// ── absence is not failure (§5.2) ─────────────────────────────────────────
|
||||
|
||||
test('a key the shard could not render keeps the picture already held', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const held = new Map([['body/12/a0', { key: 'body/12/a0', sha256: 'old', file: 'existing.png' }]])
|
||||
|
||||
const seen = stubEverything({
|
||||
held,
|
||||
manifest: manifestOf([row(12, 'new')]),
|
||||
// Listed, asked for, and not served. A shard that suddenly cannot render one
|
||||
// sprite must not cost the picture we already have.
|
||||
fetched: { assets: new Map(), missing: { absent: 1, unsupported: 0 } },
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.importAssets({ force: true })
|
||||
|
||||
assert.equal(result.status, 'imported')
|
||||
assert.equal(result.absent, 1)
|
||||
assert.equal(seen.saved[0].file, 'existing.png')
|
||||
})
|
||||
|
||||
// ── the derivation (§12) ──────────────────────────────────────────────────
|
||||
|
||||
test("the operator's own artwork wins over an imported sprite", async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const seen = stubEverything({ manifest: manifestOf([]) })
|
||||
t.after(restore)
|
||||
|
||||
db.artBySlug = async () => ({ 'giant-spider': 'uo-body-28-aaaabbbb.png', wolf: 'uo-body-34-ccccdddd.png' })
|
||||
// Someone who drew their own giant spider must not have it replaced by a
|
||||
// sprite rip on the next Update. §12 states this outright.
|
||||
atlasModel.loadArtMap = () => ({ 'giant-spider': 'my-own-spider.png' })
|
||||
|
||||
await model.importAssets({ force: true })
|
||||
|
||||
assert.equal(seen.art['giant-spider'], 'my-own-spider.png')
|
||||
assert.equal(seen.art.wolf, 'uo-body-34-ccccdddd.png')
|
||||
})
|
||||
|
||||
test('a sprite filename carries its hash so a changed picture is a changed URL', () => {
|
||||
const before = model.fileNameFor('body/34/a0', 'aaaaaaaabbbb')
|
||||
const after = model.fileNameFor('body/34/a0', 'ccccccccdddd')
|
||||
|
||||
// A stable name would be overwritten in place, and every browser and CDN that
|
||||
// had cached it would keep serving last month's client's sprite — with the
|
||||
// database row correct and nothing to notice.
|
||||
assert.notEqual(before, after)
|
||||
assert.match(before, /^uo-body-34-a0-[0-9a-f]{8}\.png$/)
|
||||
})
|
||||
414
server/test/shardItemArt.model.test.js
Normal file
414
server/test/shardItemArt.model.test.js
Normal file
@@ -0,0 +1,414 @@
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
|
||||
const core = require('../core')
|
||||
const model = require('../model/shardAssets/shardItemArt.model')
|
||||
const db = require('../model/shardAssets/shardAssets.db')
|
||||
const bridge = require('../utils/assetBridge')
|
||||
const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model')
|
||||
|
||||
// The warm pass as a decision, with the shard and the database stubbed
|
||||
// (docs/link/v8.md §5, §11 — protocol 8, phase 5).
|
||||
//
|
||||
// Item art has no manifest, so almost everything the body import gets from a
|
||||
// hash diff this side has to get right by construction instead. Each test below
|
||||
// is a way that goes wrong quietly:
|
||||
//
|
||||
// - Asking an overlay that cannot answer. A phase-4 plugin serves the creature
|
||||
// catalogue and nothing else, and every static key it is sent is refused —
|
||||
// once per pass, forever, in the log, with no picture ever appearing.
|
||||
// - Re-fetching pictures the site already holds. There is no manifest to make
|
||||
// that obvious, so the only thing standing between a working install and a
|
||||
// pass that re-downloads its whole working set every five minutes is the
|
||||
// per-row catalogue id.
|
||||
// - NOT re-fetching after a client patch. The same field, read the other way.
|
||||
// - Writing a row for a key the shard has no art for. It would make the key
|
||||
// "held", and it would never be asked again — including after the operator
|
||||
// patches in the graphic that was missing.
|
||||
// - Spelling `static/3922/h0`. The shard refuses it outright (hue 0 means "not
|
||||
// hued"), so a disagreement here is a picture that never arrives.
|
||||
|
||||
const saved = {}
|
||||
|
||||
function stub({
|
||||
families = ['body', 'land', 'static'],
|
||||
wanted = [],
|
||||
fresh = new Set(),
|
||||
files = new Map(),
|
||||
fetched,
|
||||
catalog = 'cat-current',
|
||||
linked = true,
|
||||
} = {}) {
|
||||
saved.sourceFingerprint = bridge.sourceFingerprint
|
||||
saved.fetchAssets = bridge.fetchAssets
|
||||
saved.freshKeys = db.freshKeys
|
||||
saved.filesForKeys = db.filesForKeys
|
||||
saved.saveAssets = db.saveAssets
|
||||
saved.getSafe = uoLinkConfig.getSafe
|
||||
saved.query = core.query
|
||||
|
||||
const seen = { asked: [], saved: null, freshAsked: null, calls: 0 }
|
||||
|
||||
uoLinkConfig.getSafe = async () =>
|
||||
linked ? { enabled: true, baseUrl: 'http://127.0.0.1:8080' } : { enabled: false }
|
||||
|
||||
bridge.sourceFingerprint = async () => ({
|
||||
files: { 'art.mul': { size: 1, mtime: 2, sha256: 'x' } },
|
||||
extractorVersion: 2,
|
||||
hashing: false,
|
||||
complete: true,
|
||||
imaging: { ok: true },
|
||||
families,
|
||||
})
|
||||
|
||||
bridge.fetchAssets = async ({ keys }) => {
|
||||
seen.calls++
|
||||
seen.asked.push(keys)
|
||||
|
||||
// The catalogue probe asks for exactly one key and throws the answer away.
|
||||
if (keys.length === 1 && keys[0] === 'static/0' && !fetched?.assets?.has('static/0')) {
|
||||
return { assets: new Map(), missing: { absent: 1, unsupported: 0 }, pages: 1, catalog }
|
||||
}
|
||||
|
||||
return (
|
||||
fetched ?? { assets: new Map(), missing: { absent: 0, unsupported: 0 }, pages: 1, catalog }
|
||||
)
|
||||
}
|
||||
|
||||
// The derived set: what `SELECT DISTINCT item_id, hue FROM shard_vendor_items`
|
||||
// would return.
|
||||
core.query = async () => wanted
|
||||
|
||||
db.freshKeys = async (keys, askedCatalog) => {
|
||||
seen.freshAsked = { keys, catalog: askedCatalog }
|
||||
return fresh
|
||||
}
|
||||
db.filesForKeys = async () => files
|
||||
db.saveAssets = async (rows, meta) => {
|
||||
seen.saved = { rows, meta }
|
||||
return rows.length
|
||||
}
|
||||
|
||||
return seen
|
||||
}
|
||||
|
||||
function restore() {
|
||||
if (saved.sourceFingerprint) bridge.sourceFingerprint = saved.sourceFingerprint
|
||||
if (saved.fetchAssets) bridge.fetchAssets = saved.fetchAssets
|
||||
if (saved.freshKeys) db.freshKeys = saved.freshKeys
|
||||
if (saved.filesForKeys) db.filesForKeys = saved.filesForKeys
|
||||
if (saved.saveAssets) db.saveAssets = saved.saveAssets
|
||||
if (saved.getSafe) uoLinkConfig.getSafe = saved.getSafe
|
||||
if (saved.query) core.query = saved.query
|
||||
}
|
||||
|
||||
function useTempUploads(t) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'uo-items-'))
|
||||
const previous = core.uploads
|
||||
|
||||
Object.defineProperty(core, 'uploads', {
|
||||
configurable: true,
|
||||
get: () => ({ ...previous, UPLOAD_DIR: dir }),
|
||||
})
|
||||
|
||||
t.after(() => {
|
||||
Object.defineProperty(core, 'uploads', { configurable: true, get: () => previous })
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
const picture = (sha) => ({
|
||||
sha256: sha,
|
||||
bytes: 294,
|
||||
width: 22,
|
||||
height: 26,
|
||||
hue: null,
|
||||
partialHue: null,
|
||||
source: 'uop',
|
||||
png: Buffer.from('not really a png'),
|
||||
})
|
||||
|
||||
// ── keys ───────────────────────────────────────────────────────────────────
|
||||
|
||||
test('hue 0 is the plain key, because the shard refuses /h0 for the same reason', () => {
|
||||
// The wire's hue 0 means "this item is not hued". If this spelled `/h0` the
|
||||
// shard would answer `unsupported` and the picture would never arrive; if the
|
||||
// shard accepted it, the identical PNG would be stored twice under two names
|
||||
// and diffed separately forever. The two sides agreeing is the whole point.
|
||||
assert.equal(model.staticKey(3922, 0), 'static/3922')
|
||||
assert.equal(model.staticKey(3922), 'static/3922')
|
||||
assert.equal(model.staticKey(3922, null), 'static/3922')
|
||||
assert.equal(model.staticKey(3922, 33), 'static/3922/h33')
|
||||
})
|
||||
|
||||
test('a key is refused rather than fabricated for input that is not an item id', () => {
|
||||
assert.equal(model.staticKey(-5), null)
|
||||
assert.equal(model.staticKey('frog'), null)
|
||||
assert.equal(model.staticKey(undefined), null)
|
||||
assert.equal(model.landKey(0x4000), null)
|
||||
assert.equal(model.landKey(3), 'land/3')
|
||||
})
|
||||
|
||||
// ── the overlay gate ───────────────────────────────────────────────────────
|
||||
|
||||
test('an overlay that serves only the creature catalogue is reported, not asked', async (t) => {
|
||||
// A phase-3 or phase-4 plugin. Every static key sent to it comes back refused,
|
||||
// so discovering this per request would mean a warn per pass forever and no
|
||||
// picture ever. It is one check, once, with a sentence naming the fix.
|
||||
const seen = stub({ families: ['body'], wanted: [{ item_id: 3922, hue: 0 }] })
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.warm()
|
||||
|
||||
assert.equal(result.status, 'unavailable')
|
||||
assert.equal(result.code, 'UNSUPPORTED')
|
||||
assert.match(result.reason, /does not serve item art/)
|
||||
assert.equal(seen.calls, 0, 'nothing should have been asked of the shard')
|
||||
})
|
||||
|
||||
test('no shard link is skipped, not failed', async (t) => {
|
||||
stub({ linked: false })
|
||||
t.after(restore)
|
||||
|
||||
assert.equal((await model.warm()).status, 'skipped')
|
||||
})
|
||||
|
||||
test('a host that cannot render images is the named NO_IMAGING state', async (t) => {
|
||||
stub()
|
||||
t.after(restore)
|
||||
|
||||
bridge.sourceFingerprint = async () => ({
|
||||
files: {},
|
||||
extractorVersion: 2,
|
||||
hashing: false,
|
||||
complete: true,
|
||||
imaging: { ok: false, reason: 'libgdiplus is not installed' },
|
||||
families: ['body', 'static'],
|
||||
})
|
||||
|
||||
const result = await model.warm()
|
||||
|
||||
assert.equal(result.status, 'unavailable')
|
||||
assert.equal(result.code, 'NO_IMAGING')
|
||||
})
|
||||
|
||||
// ── what gets asked for ────────────────────────────────────────────────────
|
||||
|
||||
test('only the keys we do not already hold under the shard’s current catalogue are fetched', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const seen = stub({
|
||||
wanted: [
|
||||
{ item_id: 3922, hue: 0 },
|
||||
{ item_id: 597, hue: 33 },
|
||||
{ item_id: 1, hue: 0 },
|
||||
],
|
||||
// 3922 is held and current; the other two are not.
|
||||
fresh: new Set(['static/3922']),
|
||||
fetched: {
|
||||
assets: new Map([
|
||||
['static/597/h33', picture('aaa')],
|
||||
['static/1', picture('bbb')],
|
||||
]),
|
||||
missing: { absent: 0, unsupported: 0 },
|
||||
pages: 1,
|
||||
catalog: 'cat-current',
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.warm()
|
||||
|
||||
assert.equal(result.status, 'imported')
|
||||
|
||||
// The first call is the catalogue probe; the second is the real fetch.
|
||||
const asked = seen.asked[seen.asked.length - 1]
|
||||
|
||||
assert.deepEqual(asked.sort(), ['static/1', 'static/597/h33'])
|
||||
assert.equal(
|
||||
seen.freshAsked.catalog,
|
||||
'cat-current',
|
||||
'staleness must be asked against the catalogue the shard answers under right now, ' +
|
||||
'or a client patch never invalidates anything',
|
||||
)
|
||||
})
|
||||
|
||||
test('every stored row records the catalogue it was fetched under', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const seen = stub({
|
||||
wanted: [{ item_id: 1, hue: 0 }],
|
||||
fetched: {
|
||||
assets: new Map([['static/1', picture('bbb')]]),
|
||||
missing: { absent: 0, unsupported: 0 },
|
||||
pages: 1,
|
||||
catalog: 'cat-after-patch',
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await model.warm()
|
||||
|
||||
// Without this field there is no way to answer "is this picture out of date?"
|
||||
// for a family that has no manifest — which is the entire §7 story on this side.
|
||||
assert.equal(seen.saved.rows.length, 1)
|
||||
assert.equal(seen.saved.rows[0].catalog, 'cat-after-patch')
|
||||
assert.equal(seen.saved.rows[0].family, 'static')
|
||||
})
|
||||
|
||||
test('the body catalogue’s meta singleton is never written by a warm pass', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const seen = stub({
|
||||
wanted: [{ item_id: 1, hue: 0 }],
|
||||
fetched: {
|
||||
assets: new Map([['static/1', picture('bbb')]]),
|
||||
missing: { absent: 0, unsupported: 0 },
|
||||
pages: 1,
|
||||
catalog: 'cat-current',
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await model.warm()
|
||||
|
||||
// `shard_asset_meta` is what an Update compares a BODY manifest against. A
|
||||
// warm pass writing there would tell the body import that a client it never
|
||||
// looked at is unchanged, and the creature catalogue would stop updating.
|
||||
assert.equal(seen.saved.meta, null)
|
||||
})
|
||||
|
||||
test('a key the shard has no art for produces no row, so it can be asked again', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const seen = stub({
|
||||
wanted: [
|
||||
{ item_id: 1, hue: 0 },
|
||||
{ item_id: 60000, hue: 0 },
|
||||
],
|
||||
fetched: {
|
||||
assets: new Map([['static/1', picture('bbb')]]),
|
||||
missing: { absent: 1, unsupported: 0 },
|
||||
pages: 1,
|
||||
catalog: 'cat-current',
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.warm()
|
||||
|
||||
assert.equal(result.absent, 1)
|
||||
assert.deepEqual(
|
||||
seen.saved.rows.map((r) => r.key),
|
||||
['static/1'],
|
||||
'an empty row would make the key held, and it would never be asked again — ' +
|
||||
'including after the operator patches in the graphic that was missing',
|
||||
)
|
||||
})
|
||||
|
||||
test('a pass is bounded, and says how much it left behind', async (t) => {
|
||||
useTempUploads(t)
|
||||
|
||||
const wanted = []
|
||||
for (let i = 1; i <= 10; i++) wanted.push({ item_id: i, hue: 0 })
|
||||
|
||||
const seen = stub({
|
||||
wanted,
|
||||
fetched: {
|
||||
assets: new Map([['static/1', picture('bbb')]]),
|
||||
missing: { absent: 0, unsupported: 0 },
|
||||
pages: 1,
|
||||
catalog: 'cat-current',
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
const result = await model.warm({ limit: 4 })
|
||||
|
||||
assert.equal(seen.asked[seen.asked.length - 1].length, 4)
|
||||
assert.equal(result.asked, 4)
|
||||
assert.equal(result.remaining, 6)
|
||||
})
|
||||
|
||||
test('a picture whose bytes changed replaces its file instead of shadowing it', async (t) => {
|
||||
const dir = useTempUploads(t)
|
||||
|
||||
const old = model.fileNameFor('static/1', 'old00000')
|
||||
fs.mkdirSync(model.artDir(), { recursive: true })
|
||||
fs.writeFileSync(path.join(model.artDir(), old), 'stale')
|
||||
|
||||
stub({
|
||||
wanted: [{ item_id: 1, hue: 0 }],
|
||||
files: new Map([['static/1', old]]),
|
||||
fetched: {
|
||||
assets: new Map([['static/1', picture('new00000')]]),
|
||||
missing: { absent: 0, unsupported: 0 },
|
||||
pages: 1,
|
||||
catalog: 'cat-after-patch',
|
||||
},
|
||||
})
|
||||
t.after(restore)
|
||||
|
||||
await model.warm()
|
||||
|
||||
const names = fs.readdirSync(path.join(dir, model.ART_SUBDIR))
|
||||
|
||||
// Content-addressed names mean a changed picture is a changed URL, so nothing
|
||||
// keeps serving last client's sprite from a cache — and the superseded file is
|
||||
// removed rather than left to accumulate one per client patch forever.
|
||||
assert.deepEqual(names, [model.fileNameFor('static/1', 'new00000')])
|
||||
})
|
||||
|
||||
// ── serving ────────────────────────────────────────────────────────────────
|
||||
|
||||
test('decorate attaches a filename, never a URL, and null where there is none', async (t) => {
|
||||
stub({ files: new Map([['static/3922', 'uo-static-3922-abcd1234.png']]) })
|
||||
t.after(restore)
|
||||
|
||||
const rows = [
|
||||
{ itemId: 3922, hue: 0 },
|
||||
{ itemId: 597, hue: 33 },
|
||||
]
|
||||
|
||||
await model.decorate(rows)
|
||||
|
||||
// A filename, because the client is what knows where uploads are mounted —
|
||||
// the same contract `shard_spawn_creatures.art` already uses.
|
||||
assert.equal(rows[0].art, 'uo-static-3922-abcd1234.png')
|
||||
assert.equal(rows[1].art, null)
|
||||
})
|
||||
|
||||
test('decorate never throws a page away over a picture', async (t) => {
|
||||
stub()
|
||||
t.after(restore)
|
||||
|
||||
db.filesForKeys = async () => {
|
||||
throw new Error('the database is on fire')
|
||||
}
|
||||
|
||||
const rows = [{ itemId: 3922, hue: 0 }]
|
||||
|
||||
await model.decorate(rows)
|
||||
|
||||
assert.deepEqual(rows, [{ itemId: 3922, hue: 0 }], 'the row is returned unchanged, not lost')
|
||||
})
|
||||
|
||||
test('what a page asked for is remembered, including what it could not show', async (t) => {
|
||||
stub({ files: new Map() })
|
||||
t.after(restore)
|
||||
|
||||
const before = model.noticedCount()
|
||||
|
||||
await model.decorate([{ itemId: 12345, hue: 7 }])
|
||||
|
||||
// The character sheet is fetched live from the shard and stored nowhere, so
|
||||
// nothing on disk would ever name this key. Noticing it here is the only reason
|
||||
// a warm pass can find it.
|
||||
assert.ok(model.noticedCount() > before)
|
||||
assert.ok((await model.wantedKeys()).includes('static/12345/h7'))
|
||||
})
|
||||
572
server/utils/assetBridge.js
Normal file
572
server/utils/assetBridge.js
Normal file
@@ -0,0 +1,572 @@
|
||||
// The Asset Bridge client (docs/link/v8.md §5, §6, §8 — protocol 8, phase 3).
|
||||
//
|
||||
// Three walks over the same request/reply path `clilocBridge.js` already uses,
|
||||
// and everything that file says about the envelope holds here unchanged: only
|
||||
// `cut: 'end'` means finished, the cursor must advance, and 425 is the ordinary
|
||||
// answer during an import rather than an error.
|
||||
//
|
||||
// What is different is what each walk is FOR.
|
||||
//
|
||||
// ── `readManifest` — what the shard could serve, without the pixels ────────
|
||||
//
|
||||
// §6's stage 2. Every row is `{ key, sha256, bytes, width, height }`, so the
|
||||
// site can diff against what it already holds and ask for only the keys whose
|
||||
// hash moved. On the ordinary case — a shard restart that changed nothing —
|
||||
// that diff is empty and no pixels cross at all.
|
||||
//
|
||||
// This family pages on the shard's WALL CLOCK, not on bytes. Its rows are about
|
||||
// ninety bytes and the whole catalogue is one page by the byte budget, but
|
||||
// producing that page means decoding hundreds of sprites and the sidecar waits
|
||||
// ten seconds for a reply. So `cut: 'limit'` is the normal page ending here,
|
||||
// where for clilocs it would have signalled something wrong.
|
||||
//
|
||||
// ── `fetchAssets` — the pixels, for keys we chose ─────────────────────────
|
||||
//
|
||||
// Each row carries a base64 PNG. The shard encodes it: `System.Drawing` is
|
||||
// already in its decode path, so PNG costs it no new dependency, and having the
|
||||
// hash cover exactly the bytes we store is what makes the next Update a diff.
|
||||
//
|
||||
// **`catalog` is passed on every fetch and it is not optional in practice.** It
|
||||
// is an id the shard derives from the client files themselves, so handing it back
|
||||
// makes the shard refuse if those files moved since the manifest was read.
|
||||
// Without it an operator patching their client mid-import produces one asset set
|
||||
// stitched out of two, with no error anywhere — the same failure `clilocBridge`
|
||||
// guards against by comparing (size, mtime) across pages.
|
||||
//
|
||||
// ── `resolveBodies` — the atlas's creatures, by class name ────────────────
|
||||
//
|
||||
// §8. The shard constructs each type and reads `Body.BodyID`, which is the only
|
||||
// thing that is correct for a shard's own custom creatures. That runs on its Core
|
||||
// thread, so the batch is small and the shard REFUSES an over-long list rather
|
||||
// than truncating it — hence the chunking here, and hence a chunk size that is a
|
||||
// constant rather than "as many as fit".
|
||||
|
||||
// Required as a namespace, not destructured: a test that stubs the sidecar
|
||||
// replaces these on the module object, and a destructured copy taken at load
|
||||
// time would keep calling the real one.
|
||||
const uoLinkClient = require('./uoLinkClient')
|
||||
const log = require('../core').logger('asset-bridge')
|
||||
|
||||
/** The only family phase 3 serves. §5's key scheme covers statics and land later. */
|
||||
const FAMILY = 'body'
|
||||
|
||||
// Chunk size for the body pass. The shard's own cap defaults to 100 and it
|
||||
// refuses rather than truncates, so this must stay at or under it — a mismatch
|
||||
// here does not degrade, it fails every chunk.
|
||||
const BODY_CHUNK = 100
|
||||
|
||||
// Chunk size for a fetch request. The shard cuts the PAGE by byte budget within
|
||||
// whatever it is handed, so this only bounds how large a single request is; a
|
||||
// chunk of 400 one-kilobyte sprites is a couple of pages.
|
||||
const FETCH_CHUNK = 400
|
||||
|
||||
// Bounds on each walk. None is expected to be reached — the catalogue is under a
|
||||
// thousand rows — and each exists so that a shard answering nonsense costs a
|
||||
// bounded amount of time rather than an unbounded amount of memory.
|
||||
const MAX_PAGES = 200
|
||||
const MAX_ROWS = 100000
|
||||
|
||||
// 425 is flow control, not failure: the shard's asset plane serves one request at
|
||||
// a time because its outbound queue is bounded in lines rather than bytes. During
|
||||
// an import a page coming back busy is expected, so it is retried with a backoff
|
||||
// rather than failing the walk.
|
||||
const BUSY_RETRIES = 6
|
||||
const BUSY_BACKOFF_MS = [200, 400, 800, 1600, 3200, 5000]
|
||||
|
||||
class AssetBridgeError extends Error {
|
||||
constructor(message, code) {
|
||||
super(message)
|
||||
this.name = 'AssetBridgeError'
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
/**
|
||||
* Map a sidecar response onto one of this module's codes.
|
||||
*
|
||||
* Deliberately the same vocabulary `clilocBridge.describeFailure` uses, because
|
||||
* the admin panel reports them side by side and an operator should not have to
|
||||
* learn two names for "you have not switched this on".
|
||||
*
|
||||
* 422 is the one that means something different here: on the cliloc path it is a
|
||||
* file the shard cannot decode, and on this one it is *also* the mid-import guard
|
||||
* firing — the client files moved between the manifest and the fetch.
|
||||
*/
|
||||
function describeFailure(res, what) {
|
||||
const reason = res?.data?.reason || res?.error || `sidecar responded ${res?.status}`
|
||||
|
||||
switch (res?.status) {
|
||||
case 403:
|
||||
return new AssetBridgeError(
|
||||
`The shard is refusing to serve client assets (Bridge.AssetsEnabled is off): ${reason}`,
|
||||
'DISABLED',
|
||||
)
|
||||
case 404:
|
||||
return new AssetBridgeError(`The shard has no ${what}: ${reason}`, 'NO_SOURCE')
|
||||
case 409:
|
||||
return new AssetBridgeError(
|
||||
`The sidecar refused the protocol version this build declares: ${reason}`,
|
||||
'PROTOCOL',
|
||||
)
|
||||
case 422:
|
||||
return new AssetBridgeError(reason, 'SOURCE_CHANGED')
|
||||
case 425:
|
||||
return new AssetBridgeError(
|
||||
'The shard stayed busy serving another asset request',
|
||||
'BUSY',
|
||||
)
|
||||
case 503:
|
||||
// The named `NO_IMAGING` outcome arrives this way: a Linux shard host with
|
||||
// no libgdiplus cannot render a sprite at all, and §4.4 requires that be an
|
||||
// actionable sentence rather than a stack trace. The shard's own wording
|
||||
// already names the package and the command, so it is passed through.
|
||||
return new AssetBridgeError(reason, /libgdiplus/i.test(reason) ? 'NO_IMAGING' : 'SHARD_DOWN')
|
||||
case 504:
|
||||
return new AssetBridgeError(`The shard did not answer: ${reason}`, 'SHARD_DOWN')
|
||||
default:
|
||||
return new AssetBridgeError(reason, 'UNAVAILABLE')
|
||||
}
|
||||
}
|
||||
|
||||
/** One call, with the 425 backoff. `send` returns the client's `{ ok, ... }`. */
|
||||
async function withBusyRetry(send, what) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const res = await send()
|
||||
if (res.ok) return res.data
|
||||
|
||||
if (res.status === 425 && attempt < BUSY_RETRIES) {
|
||||
await sleep(BUSY_BACKOFF_MS[Math.min(attempt, BUSY_BACKOFF_MS.length - 1)])
|
||||
continue
|
||||
}
|
||||
|
||||
throw describeFailure(res, what)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared page-envelope checks (§3.4).
|
||||
*
|
||||
* Every one of these is a way a walk can end in something that LOOKS like a
|
||||
* complete import and is not, which is why they are assertions rather than
|
||||
* warnings: a truncated catalogue is indistinguishable downstream from a client
|
||||
* that simply has fewer creatures.
|
||||
*/
|
||||
function checkPage(page, { arrayName, cursor, pages }) {
|
||||
if (!page || !Array.isArray(page[arrayName])) {
|
||||
throw new AssetBridgeError(
|
||||
`The shard sent an asset page with no ${arrayName} array`,
|
||||
'MALFORMED',
|
||||
)
|
||||
}
|
||||
|
||||
if (!page.more) {
|
||||
if (page.cut !== 'end') {
|
||||
throw new AssetBridgeError(
|
||||
`The shard stopped sending assets after ${pages} page(s) (cut: ${page.cut || 'unknown'})`,
|
||||
'INCOMPLETE',
|
||||
)
|
||||
}
|
||||
return { done: true }
|
||||
}
|
||||
|
||||
if (!page.cursor || page.cursor === cursor) {
|
||||
throw new AssetBridgeError(
|
||||
`The shard asked for another asset page without advancing its cursor (${page.cursor || 'none'})`,
|
||||
'STUCK',
|
||||
)
|
||||
}
|
||||
|
||||
return { done: false, cursor: page.cursor }
|
||||
}
|
||||
|
||||
// The client files the body catalogue is derived from. `assets.sources` reports
|
||||
// every file the shard can see; these are the ones that decide a sprite.
|
||||
//
|
||||
// `body.def` and `bodyconv.def` are in the list and it would be easy to leave
|
||||
// them out — they hold no pixels. They decide WHICH record a body id resolves to,
|
||||
// so an operator editing one changes what every affected creature looks like
|
||||
// while every anim file stays byte-identical. That is precisely the drift a
|
||||
// content hash of the art files cannot see.
|
||||
const SOURCE_FILES = [
|
||||
'anim.idx', 'anim.mul',
|
||||
'anim2.idx', 'anim2.mul',
|
||||
'anim3.idx', 'anim3.mul',
|
||||
'anim4.idx', 'anim4.mul',
|
||||
'anim5.idx', 'anim5.mul',
|
||||
'body.def', 'bodyconv.def',
|
||||
'verdata.mul',
|
||||
]
|
||||
|
||||
/**
|
||||
* Stage 1 of the import gate (§6): have the client files this family reads
|
||||
* changed at all?
|
||||
*
|
||||
* Returns `{ files, extractorVersion, hashing, complete, imaging }` where `files`
|
||||
* is a `{ name: { size, mtime, sha256 } }` map over `SOURCE_FILES` — a file the
|
||||
* shard does not have is simply absent, which is normal (few clients carry all
|
||||
* five anim files).
|
||||
*
|
||||
* **A null `sha256` means "not computed yet", never "changed".** The shard hashes
|
||||
* off the request path because `anim.mul` alone is 195 MB and hashing it cannot
|
||||
* fit inside a reply, and it reports `hashing: true` while that runs.
|
||||
* `sameSources` below falls back to (size, mtime) in that case, which is the same
|
||||
* gate the shard itself applies.
|
||||
*/
|
||||
async function sourceFingerprint() {
|
||||
const res = await uoLinkClient.getAssetSources()
|
||||
if (!res.ok) throw describeFailure(res, 'client file manifest')
|
||||
|
||||
const wanted = new Set(SOURCE_FILES)
|
||||
const files = {}
|
||||
|
||||
for (const entry of res.data?.files ?? []) {
|
||||
const name = String(entry?.name || '').toLowerCase()
|
||||
if (!wanted.has(name)) continue
|
||||
|
||||
files[name] = {
|
||||
size: Number(entry.size) || 0,
|
||||
mtime: Number(entry.mtime) || 0,
|
||||
sha256: entry.sha256 ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
files,
|
||||
extractorVersion: Number(res.data?.extractorVersion) || 0,
|
||||
hashing: Boolean(res.data?.hashing),
|
||||
complete: Boolean(res.data?.complete),
|
||||
imaging: res.data?.imaging ?? null,
|
||||
// Which §5 key families this overlay can be asked for (phase 5). Absent on a
|
||||
// phase-3 or phase-4 overlay, which served bodies and nothing else — so the
|
||||
// fallback is `['body']` rather than `[]`: an older shard is not a shard with
|
||||
// no assets, and treating it as one would turn a working bestiary off.
|
||||
families: Array.isArray(res.data?.families) && res.data.families.length > 0
|
||||
? res.data.families.map(String)
|
||||
: [FAMILY],
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when two source fingerprints describe the same client files.
|
||||
*
|
||||
* The file SET has to match as well as each file's contents: a client that gained
|
||||
* an `anim5.mul` it did not have before is a client whose gargoyles suddenly
|
||||
* resolve, and comparing only the files present in both would call that
|
||||
* unchanged.
|
||||
*/
|
||||
function sameSources(a, b) {
|
||||
if (!a || !b) return false
|
||||
if (a.extractorVersion !== b.extractorVersion) return false
|
||||
|
||||
const names = new Set([...Object.keys(a.files ?? {}), ...Object.keys(b.files ?? {})])
|
||||
|
||||
for (const name of names) {
|
||||
const left = a.files?.[name]
|
||||
const right = b.files?.[name]
|
||||
|
||||
if (!left || !right) return false
|
||||
|
||||
if (left.sha256 && right.sha256) {
|
||||
if (left.sha256 !== right.sha256) return false
|
||||
continue
|
||||
}
|
||||
|
||||
if (left.size !== right.size || left.mtime !== right.mtime || left.size <= 0) return false
|
||||
}
|
||||
|
||||
return names.size > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2: the whole manifest for the body family.
|
||||
*
|
||||
* Returns `{ rows, catalog, extractorVersion, playerBodies, pages, scanned }`.
|
||||
* No pixels — `rows` is `[{ key, sha256, bytes, width, height, body, direction }]`.
|
||||
*/
|
||||
async function readManifest({ family = FAMILY } = {}) {
|
||||
const started = Date.now()
|
||||
const rows = []
|
||||
|
||||
let cursor = null
|
||||
let pages = 0
|
||||
let catalog = null
|
||||
let extractorVersion = 0
|
||||
let playerBodies = []
|
||||
let scanned = 0
|
||||
let finished = false
|
||||
|
||||
while (pages < MAX_PAGES) {
|
||||
const page = await withBusyRetry(
|
||||
() => uoLinkClient.getAssetManifest({ family, cursor }),
|
||||
`${family} asset manifest`,
|
||||
)
|
||||
pages++
|
||||
|
||||
if (catalog === null) {
|
||||
catalog = page.catalog ?? null
|
||||
extractorVersion = Number(page.extractorVersion) || 0
|
||||
playerBodies = Array.isArray(page.playerBodies) ? page.playerBodies.map(Number) : []
|
||||
} else if (page.catalog !== catalog) {
|
||||
// The client files moved between two pages of one walk. Refusing is the
|
||||
// only honest answer: half of what we hold describes files that no longer
|
||||
// exist, and nothing later can tell which half.
|
||||
throw new AssetBridgeError(
|
||||
"The shard's client files changed while the manifest was being read; nothing was imported",
|
||||
'SOURCE_CHANGED',
|
||||
)
|
||||
}
|
||||
|
||||
scanned += Number(page.scanned) || 0
|
||||
|
||||
for (const row of page.rows) {
|
||||
const key = String(row?.key ?? '')
|
||||
if (key === '') continue
|
||||
|
||||
rows.push({
|
||||
key,
|
||||
family,
|
||||
sha256: String(row?.sha256 ?? ''),
|
||||
bytes: Number(row?.bytes) || 0,
|
||||
width: Number(row?.width) || 0,
|
||||
height: Number(row?.height) || 0,
|
||||
body: Number.isFinite(Number(row?.body)) ? Number(row.body) : null,
|
||||
direction: Number.isFinite(Number(row?.direction)) ? Number(row.direction) : null,
|
||||
})
|
||||
}
|
||||
|
||||
if (rows.length > MAX_ROWS) {
|
||||
throw new AssetBridgeError(
|
||||
`The shard listed more than ${MAX_ROWS} assets; refusing to keep reading`,
|
||||
'TOO_LARGE',
|
||||
)
|
||||
}
|
||||
|
||||
const state = checkPage(page, { arrayName: 'rows', cursor, pages })
|
||||
|
||||
if (state.done) {
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
|
||||
cursor = state.cursor
|
||||
}
|
||||
|
||||
if (!finished) {
|
||||
throw new AssetBridgeError(
|
||||
`The asset manifest did not end within ${MAX_PAGES} pages; nothing was imported`,
|
||||
'TOO_LARGE',
|
||||
)
|
||||
}
|
||||
|
||||
log.info('asset manifest read from the shard', {
|
||||
family,
|
||||
rows: rows.length,
|
||||
scanned,
|
||||
pages,
|
||||
ms: Date.now() - started,
|
||||
})
|
||||
|
||||
return { rows, catalog, extractorVersion, playerBodies, pages, scanned }
|
||||
}
|
||||
|
||||
/**
|
||||
* The bytes for an explicit list of keys.
|
||||
*
|
||||
* Returns a Map of key → `{ sha256, bytes, width, height, body, direction, png }`
|
||||
* where `png` is a Buffer. A key the shard could not serve is **absent from the
|
||||
* map** rather than present with a null — the caller then decides what that means
|
||||
* for its own row, and the two ways it happens (`absent`, `unsupported`) are
|
||||
* counted separately in the returned tallies so an operator can tell "this client
|
||||
* has no art for that body" from "the site asked for a key shape this shard does
|
||||
* not serve", which is a bug rather than a gap.
|
||||
*/
|
||||
async function fetchAssets({ keys, catalog } = {}) {
|
||||
const started = Date.now()
|
||||
const out = new Map()
|
||||
const missing = { absent: 0, unsupported: 0 }
|
||||
|
||||
const list = Array.isArray(keys) ? keys.filter((k) => typeof k === 'string' && k !== '') : []
|
||||
|
||||
if (list.length === 0) return { assets: out, missing, pages: 0, catalog: catalog ?? null }
|
||||
|
||||
let pages = 0
|
||||
|
||||
// The catalogue the shard actually answered under. The body import already knows
|
||||
// it from the manifest, but the on-demand families have no manifest to learn it
|
||||
// from (§11) — so it is read back off the reply and stored with the rows, which
|
||||
// is what makes a later "is this stale?" answerable per key.
|
||||
let answered = catalog ?? null
|
||||
|
||||
for (let i = 0; i < list.length; i += FETCH_CHUNK) {
|
||||
const chunk = list.slice(i, i + FETCH_CHUNK)
|
||||
|
||||
let cursor = null
|
||||
let finished = false
|
||||
let walked = 0
|
||||
|
||||
while (walked < MAX_PAGES) {
|
||||
const page = await withBusyRetry(
|
||||
() => uoLinkClient.fetchAssets({ keys: chunk, catalog, cursor }),
|
||||
'asset content',
|
||||
)
|
||||
pages++
|
||||
walked++
|
||||
|
||||
if (typeof page.catalog === 'string' && page.catalog !== '') {
|
||||
if (answered !== null && page.catalog !== answered) {
|
||||
// Two pages of one walk describing two different clients. The shard
|
||||
// refuses this when it is told what to expect; when it was not told —
|
||||
// the first fetch of a warm pass — this is where it is caught.
|
||||
throw new AssetBridgeError(
|
||||
`The shard's client files changed mid-fetch (catalog ${answered} became ${page.catalog})`,
|
||||
'UNAVAILABLE',
|
||||
)
|
||||
}
|
||||
|
||||
answered = page.catalog
|
||||
}
|
||||
|
||||
for (const row of page.rows ?? []) {
|
||||
const key = String(row?.key ?? '')
|
||||
if (key === '') continue
|
||||
|
||||
if (row?.status !== 'ok') {
|
||||
if (row?.status === 'unsupported') missing.unsupported++
|
||||
else missing.absent++
|
||||
continue
|
||||
}
|
||||
|
||||
if (typeof row.png !== 'string' || row.png === '') {
|
||||
missing.absent++
|
||||
continue
|
||||
}
|
||||
|
||||
out.set(key, {
|
||||
sha256: String(row.sha256 ?? ''),
|
||||
bytes: Number(row.bytes) || 0,
|
||||
width: Number(row.width) || 0,
|
||||
height: Number(row.height) || 0,
|
||||
body: Number.isFinite(Number(row.body)) ? Number(row.body) : null,
|
||||
direction: Number.isFinite(Number(row.direction)) ? Number(row.direction) : null,
|
||||
// Phase 5's art families carry these; the body catalogue does not, and a
|
||||
// consumer that wants neither is unaffected by either.
|
||||
hue: Number.isFinite(Number(row.hue)) ? Number(row.hue) : null,
|
||||
partialHue: typeof row.partialHue === 'boolean' ? row.partialHue : null,
|
||||
source: typeof row.source === 'string' ? row.source : null,
|
||||
png: Buffer.from(row.png, 'base64'),
|
||||
})
|
||||
}
|
||||
|
||||
const state = checkPage(page, { arrayName: 'rows', cursor, pages: walked })
|
||||
|
||||
if (state.done) {
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
|
||||
cursor = state.cursor
|
||||
}
|
||||
|
||||
if (!finished) {
|
||||
throw new AssetBridgeError(
|
||||
`An asset fetch did not end within ${MAX_PAGES} pages; nothing was imported`,
|
||||
'TOO_LARGE',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
log.info('asset content fetched from the shard', {
|
||||
catalog: answered,
|
||||
asked: list.length,
|
||||
got: out.size,
|
||||
absent: missing.absent,
|
||||
unsupported: missing.unsupported,
|
||||
pages,
|
||||
ms: Date.now() - started,
|
||||
})
|
||||
|
||||
return { assets: out, missing, pages, catalog: answered }
|
||||
}
|
||||
|
||||
/**
|
||||
* Slug → body id, for the atlas's own creature list (§8).
|
||||
*
|
||||
* `creatures` is `[{ slug, name }]` where `name` is the ServUO class name — which
|
||||
* `shard_spawn_creatures.name` already holds, because the atlas build picks the
|
||||
* winning spelling of the spawn TYPE token rather than inventing a display name.
|
||||
* That is why this needs no new column to ask its question.
|
||||
*
|
||||
* Returns `[{ slug, typeName, body, status }]`, one row per creature asked, with
|
||||
* every outcome recorded — including the negative ones. A creature the shard says
|
||||
* it does not have is a fact worth keeping: without it, the next pass asks again,
|
||||
* and the pass costs a real constructor per name on the shard's Core thread.
|
||||
*/
|
||||
async function resolveBodies({ creatures } = {}) {
|
||||
const started = Date.now()
|
||||
const list = Array.isArray(creatures) ? creatures : []
|
||||
const out = []
|
||||
|
||||
for (let i = 0; i < list.length; i += BODY_CHUNK) {
|
||||
const chunk = list.slice(i, i + BODY_CHUNK)
|
||||
const bySlug = new Map()
|
||||
|
||||
for (const creature of chunk) {
|
||||
const typeName = String(creature?.name ?? '').trim()
|
||||
if (typeName === '') continue
|
||||
// Several slugs can share a type name only if the atlas slugified two
|
||||
// spellings to one slug, in which case they ARE one creature; asking once
|
||||
// per distinct name is what keeps the batch inside the shard's cap.
|
||||
if (!bySlug.has(typeName)) bySlug.set(typeName, [])
|
||||
bySlug.get(typeName).push(String(creature.slug))
|
||||
}
|
||||
|
||||
const types = [...bySlug.keys()]
|
||||
if (types.length === 0) continue
|
||||
|
||||
const page = await withBusyRetry(() => uoLinkClient.resolveBodies(types), 'body resolution')
|
||||
|
||||
if (!page || !Array.isArray(page.rows)) {
|
||||
throw new AssetBridgeError('The shard sent a body resolution with no rows array', 'MALFORMED')
|
||||
}
|
||||
|
||||
for (const row of page.rows) {
|
||||
const typeName = String(row?.type ?? '')
|
||||
const slugs = bySlug.get(typeName)
|
||||
|
||||
if (!slugs) continue
|
||||
|
||||
const status = String(row?.status ?? 'failed')
|
||||
const body = status === 'ok' && Number.isFinite(Number(row?.body)) ? Number(row.body) : null
|
||||
|
||||
for (const slug of slugs) out.push({ slug, typeName, body, status })
|
||||
}
|
||||
}
|
||||
|
||||
const resolved = out.filter((r) => r.status === 'ok').length
|
||||
|
||||
log.info('creature bodies resolved by the shard', {
|
||||
asked: list.length,
|
||||
answered: out.length,
|
||||
resolved,
|
||||
ms: Date.now() - started,
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
AssetBridgeError,
|
||||
FAMILY,
|
||||
BODY_CHUNK,
|
||||
FETCH_CHUNK,
|
||||
MAX_PAGES,
|
||||
MAX_ROWS,
|
||||
SOURCE_FILES,
|
||||
sourceFingerprint,
|
||||
sameSources,
|
||||
readManifest,
|
||||
fetchAssets,
|
||||
resolveBodies,
|
||||
}
|
||||
306
server/utils/clilocBridge.js
Normal file
306
server/utils/clilocBridge.js
Normal file
@@ -0,0 +1,306 @@
|
||||
// Cliloc table — the SHARD source (docs/link/v8.md §9, protocol 8 phase 2).
|
||||
//
|
||||
// `clilocSource.js` is the filesystem half of this story and predates it. This is
|
||||
// the half that replaces the part of it nobody enjoyed: until protocol 8 the base
|
||||
// table reached the site because an operator installed UOFiddler, built a
|
||||
// converter against its `Ultima.dll`, ran it over their client's compressed
|
||||
// `Cliloc.enu` and copied a five-megabyte file to the web host — every time they
|
||||
// patched their client.
|
||||
//
|
||||
// The shard has always had those files (a ServUO server cannot boot without a UO
|
||||
// client) and, as of phase 2, has the decompressor too. So the base table now
|
||||
// arrives over the same request/reply path as every other shard read, and the
|
||||
// operator installs nothing.
|
||||
//
|
||||
// **What is NOT here.** Overlays. Shard-added items carry cliloc ids no client
|
||||
// table has, ServUO has no server-side notion of a custom cliloc, and there is
|
||||
// therefore nothing on the shard to ask for. `custom/` stays a directory the site
|
||||
// reads (`clilocSource.readOverlays`), and the model merges it OVER whatever
|
||||
// arrives here. That division is the whole of CLILOCS.md §Shard-added items and
|
||||
// it is unchanged by this file.
|
||||
//
|
||||
// ── Why this walks pages instead of asking for a table ────────────────────
|
||||
//
|
||||
// The sidecar's reply timeout is 10 s and its inbound line cap is 1 MiB, so a
|
||||
// five-megabyte table cannot be one answer. The shard cuts pages at a 512 KiB
|
||||
// byte budget and hands back a cursor; this walks them. A stock English table is
|
||||
// about eleven pages.
|
||||
//
|
||||
// Three properties of that envelope are load-bearing and each has a check below:
|
||||
//
|
||||
// - **Only `cut: 'end'` means finished.** A short page can equally mean the
|
||||
// budget was spent (`budget`) or the family stopped at its own limit
|
||||
// (`limit`). Treating a short page as the end would import a truncated table,
|
||||
// which is indistinguishable downstream from a complete one — some items
|
||||
// named, some not, exactly what "no table at all" looks like.
|
||||
// - **The cursor must advance.** A shard that answered the same cursor forever
|
||||
// would spin this loop until the request timeout with nothing to show.
|
||||
// - **The file must not change underneath the walk.** Every page echoes the
|
||||
// source's size and mtime; an operator patching their client mid-import would
|
||||
// otherwise produce one table stitched from two, with no error anywhere.
|
||||
|
||||
// Required as a namespace, not destructured: a test that stubs the sidecar
|
||||
// replaces these on the module object, and a destructured copy taken at load
|
||||
// time would keep calling the real one.
|
||||
const uoLinkClient = require('./uoLinkClient')
|
||||
const log = require('../core').logger('cliloc-bridge')
|
||||
|
||||
/** The client file the base table comes from, as `assets.sources` names it. */
|
||||
const SOURCE_FILE = 'cliloc.enu'
|
||||
|
||||
const DEFAULT_LANGUAGE = 'enu'
|
||||
|
||||
// Bounds on the walk. Neither is expected to be reached — a stock table is ~11
|
||||
// pages and ~67k rows — and both exist so that a shard answering nonsense costs a
|
||||
// bounded amount of time rather than an unbounded amount of memory.
|
||||
const MAX_PAGES = 200
|
||||
const MAX_ROWS = 500000
|
||||
|
||||
// 425 is the ordinary answer during an import, not an error: the shard's asset
|
||||
// plane serves one request at a time on purpose, because its outbound queue is
|
||||
// bounded in lines rather than bytes. So a page that comes back busy is retried
|
||||
// with a short backoff rather than failing the import.
|
||||
const BUSY_RETRIES = 5
|
||||
const BUSY_BACKOFF_MS = [200, 400, 800, 1600, 3200]
|
||||
|
||||
class ClilocBridgeError extends Error {
|
||||
constructor(message, code) {
|
||||
super(message)
|
||||
this.name = 'ClilocBridgeError'
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
/**
|
||||
* Map a sidecar response onto one of this module's codes.
|
||||
*
|
||||
* The statuses are the ones `respond_assets` produces, and the distinction that
|
||||
* matters most to an operator is 403 vs 404: "you have not switched this on" and
|
||||
* "your client does not have that file" are different jobs, and both are things
|
||||
* they can fix.
|
||||
*/
|
||||
function describeFailure(res, what) {
|
||||
const reason = res?.data?.reason || res?.error || `sidecar responded ${res?.status}`
|
||||
|
||||
switch (res?.status) {
|
||||
case 403:
|
||||
return new ClilocBridgeError(
|
||||
`The shard is refusing to serve client assets (Bridge.AssetsEnabled is off): ${reason}`,
|
||||
'DISABLED',
|
||||
)
|
||||
case 404:
|
||||
return new ClilocBridgeError(`The shard has no ${what}: ${reason}`, 'NO_SOURCE')
|
||||
case 409:
|
||||
return new ClilocBridgeError(
|
||||
`The sidecar refused the protocol version this build declares: ${reason}`,
|
||||
'PROTOCOL',
|
||||
)
|
||||
case 422:
|
||||
return new ClilocBridgeError(`The shard could not read its own ${what}: ${reason}`, 'UNREADABLE')
|
||||
case 425:
|
||||
return new ClilocBridgeError(
|
||||
'The shard is busy serving another asset request and stayed busy',
|
||||
'BUSY',
|
||||
)
|
||||
case 503:
|
||||
case 504:
|
||||
return new ClilocBridgeError(`The shard did not answer: ${reason}`, 'SHARD_DOWN')
|
||||
default:
|
||||
return new ClilocBridgeError(reason, 'UNAVAILABLE')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 1: the fingerprint of the shard's own cliloc file.
|
||||
*
|
||||
* Returns `{ file, size, mtime, sha256, extractorVersion, hashing, complete }`.
|
||||
*
|
||||
* `sha256` may be **null** — the shard reports hashes only once it has computed
|
||||
* them off the request path, because hashing the client files it also serves
|
||||
* (343 MB of art and animation) cannot fit inside a 10 s reply. A null hash means
|
||||
* "not yet", never "changed", and `sameSource` below compares (size, mtime) in
|
||||
* that case, which is the same gate the shard itself uses.
|
||||
*/
|
||||
async function fingerprint() {
|
||||
const res = await uoLinkClient.getAssetSources()
|
||||
if (!res.ok) throw describeFailure(res, 'client file manifest')
|
||||
|
||||
const files = Array.isArray(res.data?.files) ? res.data.files : []
|
||||
const entry = files.find((f) => String(f?.name || '').toLowerCase() === SOURCE_FILE)
|
||||
|
||||
if (!entry) {
|
||||
throw new ClilocBridgeError(
|
||||
`The shard's UO client has no ${SOURCE_FILE} (it reported ${files.length} client file(s))`,
|
||||
'NO_SOURCE',
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'bridge',
|
||||
file: entry.name,
|
||||
path: entry.path ?? null,
|
||||
size: Number(entry.size) || 0,
|
||||
mtime: Number(entry.mtime) || 0,
|
||||
sha256: entry.sha256 ?? null,
|
||||
extractorVersion: Number(res.data?.extractorVersion) || 0,
|
||||
hashing: Boolean(res.data?.hashing),
|
||||
complete: Boolean(res.data?.complete),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when two fingerprints describe the same client file.
|
||||
*
|
||||
* Hash first when both sides have one, because a hash is the only thing that
|
||||
* catches a file rewritten with the same length and timestamp. Falls back to
|
||||
* (size, mtime) when either side's hash is missing, which is the case on the
|
||||
* first poll after a shard restart and the reason `hashing` exists at all.
|
||||
*/
|
||||
function sameSource(a, b) {
|
||||
if (!a || !b) return false
|
||||
if (a.extractorVersion !== b.extractorVersion) return false
|
||||
if (a.sha256 && b.sha256) return a.sha256 === b.sha256
|
||||
return a.size === b.size && a.mtime === b.mtime && a.size > 0
|
||||
}
|
||||
|
||||
/** One page, with the 425 backoff. */
|
||||
async function fetchPage({ lang, cursor }) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const res = await uoLinkClient.getClilocTable({ lang, cursor })
|
||||
if (res.ok) return res.data
|
||||
|
||||
if (res.status === 425 && attempt < BUSY_RETRIES) {
|
||||
await sleep(BUSY_BACKOFF_MS[Math.min(attempt, BUSY_BACKOFF_MS.length - 1)])
|
||||
continue
|
||||
}
|
||||
|
||||
throw describeFailure(res, `cliloc.${lang}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the whole table.
|
||||
*
|
||||
* Returns `{ entries, source }` where `entries` is `[{ number, flag, text }]` in
|
||||
* the shape `clilocParse` produces, so the merge in `shardClilocs.model` does not
|
||||
* care which source an entry came from.
|
||||
*
|
||||
* Blanks are already gone: the shard drops the ~56,000 empty strings a stock
|
||||
* table carries before they reach the wire, since the site would drop them at
|
||||
* import anyway. Nothing downstream changes — `db.replaceAll` still filters, and
|
||||
* still would if a source ever sent one.
|
||||
*/
|
||||
async function readCliloc({ lang = DEFAULT_LANGUAGE } = {}) {
|
||||
const started = Date.now()
|
||||
const entries = []
|
||||
|
||||
let cursor = null
|
||||
let pages = 0
|
||||
let first = null
|
||||
let finished = false
|
||||
let total = null
|
||||
|
||||
while (pages < MAX_PAGES) {
|
||||
const page = await fetchPage({ lang, cursor })
|
||||
pages++
|
||||
|
||||
if (!page || !Array.isArray(page.rows)) {
|
||||
throw new ClilocBridgeError('The shard sent a cliloc page with no rows array', 'MALFORMED')
|
||||
}
|
||||
|
||||
if (first === null) {
|
||||
first = { size: Number(page.size) || 0, mtime: Number(page.mtime) || 0 }
|
||||
total = Number.isFinite(Number(page.total)) ? Number(page.total) : null
|
||||
} else if (Number(page.size) !== first.size || Number(page.mtime) !== first.mtime) {
|
||||
// The client was patched (or a different one mounted) between two pages.
|
||||
// Refusing is the only honest answer: half of what we hold is from a file
|
||||
// that no longer exists, and nothing later can tell which half.
|
||||
throw new ClilocBridgeError(
|
||||
'The shard\'s cliloc file changed while it was being read; nothing was imported',
|
||||
'SOURCE_CHANGED',
|
||||
)
|
||||
}
|
||||
|
||||
for (const row of page.rows) {
|
||||
const number = Number(row?.n)
|
||||
if (!Number.isInteger(number)) continue
|
||||
entries.push({ number, flag: Number(row?.f) || 0, text: String(row?.t ?? '') })
|
||||
}
|
||||
|
||||
if (entries.length > MAX_ROWS) {
|
||||
throw new ClilocBridgeError(
|
||||
`The shard sent more than ${MAX_ROWS} cliloc rows; refusing to keep reading`,
|
||||
'TOO_LARGE',
|
||||
)
|
||||
}
|
||||
|
||||
if (!page.more) {
|
||||
// `cut` is the field that says WHY a page was the last one, and only one of
|
||||
// its values means the table ended. A shard that stopped for its own limit
|
||||
// has not finished, and importing what arrived would silently drop the tail.
|
||||
if (page.cut !== 'end') {
|
||||
throw new ClilocBridgeError(
|
||||
`The shard stopped sending cliloc rows after ${entries.length} (cut: ${page.cut || 'unknown'})`,
|
||||
'INCOMPLETE',
|
||||
)
|
||||
}
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
|
||||
if (!page.cursor || page.cursor === cursor) {
|
||||
// Either would loop forever: no cursor to advance with, or the same one
|
||||
// back again.
|
||||
throw new ClilocBridgeError(
|
||||
`The shard asked for another cliloc page without advancing its cursor (${page.cursor || 'none'})`,
|
||||
'STUCK',
|
||||
)
|
||||
}
|
||||
|
||||
cursor = page.cursor
|
||||
}
|
||||
|
||||
if (!finished) {
|
||||
throw new ClilocBridgeError(
|
||||
`The cliloc table did not end within ${MAX_PAGES} pages; nothing was imported`,
|
||||
'TOO_LARGE',
|
||||
)
|
||||
}
|
||||
|
||||
log.info('cliloc table read from the shard', {
|
||||
lang,
|
||||
entries: entries.length,
|
||||
pages,
|
||||
ms: Date.now() - started,
|
||||
})
|
||||
|
||||
return {
|
||||
entries,
|
||||
source: {
|
||||
kind: 'bridge',
|
||||
lang,
|
||||
file: `cliloc.${lang}`,
|
||||
size: first?.size ?? 0,
|
||||
mtime: first?.mtime ?? 0,
|
||||
pages,
|
||||
// What the shard said it holds, kept beside what actually arrived. They
|
||||
// agree or the walk is wrong, and an operator seeing them disagree in the
|
||||
// panel learns more than a single number would tell them.
|
||||
reported: total,
|
||||
received: entries.length,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ClilocBridgeError,
|
||||
SOURCE_FILE,
|
||||
DEFAULT_LANGUAGE,
|
||||
MAX_PAGES,
|
||||
MAX_ROWS,
|
||||
fingerprint,
|
||||
sameSource,
|
||||
readCliloc,
|
||||
}
|
||||
@@ -6,6 +6,24 @@
|
||||
// - the server, which refreshes the table on boot (`shardClilocs.model.js`)
|
||||
// - the admin panel, which can force a reimport without a restart
|
||||
//
|
||||
// ── What protocol 8 took away, and what it left ───────────────────────────
|
||||
//
|
||||
// The BASE table no longer comes from here on a shard that has uo-link
|
||||
// configured: `clilocBridge.js` asks the shard for it, because the shard has the
|
||||
// operator's client files already and, since phase 2, the decompressor to read
|
||||
// them (docs/link/v8.md §9). Nobody converts a file by hand any more.
|
||||
//
|
||||
// Two things keep this module alive rather than deleting it:
|
||||
//
|
||||
// - **Overlays.** Shard-added items carry cliloc ids no client table has, and
|
||||
// ServUO has no server-side notion of a custom cliloc — there is nothing on
|
||||
// the shard to ask for. `custom/` is still a directory the site reads, and
|
||||
// `readOverlays` below is the entry point the bridge path uses.
|
||||
// - **Installs with no shard link**, and development. A site that has never
|
||||
// configured uo-link can still be pointed at a converted file; that path is
|
||||
// deprecated, not removed, and it stays the whole of this module's base-table
|
||||
// behaviour.
|
||||
//
|
||||
// The files are the OPERATOR'S (see docs/website/CLILOCS.md). Nothing derived
|
||||
// from them is committed: the repo holds no string table, exactly as it holds no
|
||||
// map snapshot and no artwork. That rule is why this module reads a configured
|
||||
@@ -194,6 +212,63 @@ function readSources(configured) {
|
||||
return { root, files }
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the OVERLAY files only, with no base table.
|
||||
*
|
||||
* The bridge path needs exactly this: the base arrives from the shard and the
|
||||
* `custom/` directory beside the configured path still has to be merged over it.
|
||||
* `readSources` cannot answer it, because resolving a base is the first thing it
|
||||
* does and there may not be one — an operator on the bridge is entitled to point
|
||||
* this setting at a directory that holds nothing but `custom/`.
|
||||
*
|
||||
* **Never throws.** A path that is blank, missing or unreadable is reported as a
|
||||
* `problem` string and an empty file list, because none of those may stop a base
|
||||
* table that arrived perfectly well from being imported. The model decides what
|
||||
* to do about it — and it has a real decision to make, since an overlay that was
|
||||
* loaded last time and is missing now is the vanished-source hazard, not a
|
||||
* config typo.
|
||||
*/
|
||||
function readOverlays(configured) {
|
||||
const target = String(configured ?? '').trim()
|
||||
if (target === '') return { root: null, files: [], problem: null }
|
||||
|
||||
let root
|
||||
try {
|
||||
const stat = fs.statSync(target)
|
||||
root = stat.isFile() ? path.dirname(target) : target
|
||||
} catch {
|
||||
return { root: null, files: [], problem: `Cliloc path does not exist: ${target}` }
|
||||
}
|
||||
|
||||
let overlays
|
||||
try {
|
||||
overlays = listCustom(root)
|
||||
} catch (err) {
|
||||
return { root, files: [], problem: err.message }
|
||||
}
|
||||
|
||||
const files = []
|
||||
for (const file of overlays) {
|
||||
let buffer
|
||||
try {
|
||||
buffer = fs.readFileSync(file)
|
||||
} catch {
|
||||
return { root, files: [], problem: `Cliloc overlay is not readable: ${file}` }
|
||||
}
|
||||
files.push({
|
||||
label: path.relative(root, file).split(path.sep).join('/'),
|
||||
kind: 'custom',
|
||||
file,
|
||||
buffer,
|
||||
sha256: sha256(buffer),
|
||||
bytes: buffer.length,
|
||||
compressed: isCompressedCliloc(buffer),
|
||||
})
|
||||
}
|
||||
|
||||
return { root, files, problem: null }
|
||||
}
|
||||
|
||||
/**
|
||||
* A fingerprint of every source: `{ "<label>": "<sha256>" }`, plus the base's
|
||||
* details for the admin panel.
|
||||
@@ -244,6 +319,25 @@ function missingSources(current, loaded) {
|
||||
return Object.keys(loaded).filter((label) => !Object.hasOwn(current, label))
|
||||
}
|
||||
|
||||
/**
|
||||
* The same question asked of OVERLAYS only.
|
||||
*
|
||||
* Needed because the base table moved to the bridge. An install upgraded from the
|
||||
* file pipeline carries a base label (`clilocs.plain`, say) in its loaded
|
||||
* fingerprint, and that label is *supposed* to disappear when the base starts
|
||||
* arriving from the shard — reporting it as a vanished source would make every
|
||||
* first import after the upgrade demand an approval for a change the upgrade
|
||||
* itself made. Overlay labels are the ones whose absence is genuinely ambiguous,
|
||||
* and they are exactly the labels under `custom/`.
|
||||
*/
|
||||
function missingOverlays(current, loaded) {
|
||||
if (!loaded) return []
|
||||
const prefix = `${CUSTOM_DIR}/`
|
||||
return Object.keys(loaded).filter(
|
||||
(label) => label.startsWith(prefix) && !Object.hasOwn(current, label),
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read and parse every source, merged into one entry list.
|
||||
*
|
||||
@@ -309,8 +403,10 @@ module.exports = {
|
||||
resolveBase,
|
||||
listCustom,
|
||||
readSources,
|
||||
readOverlays,
|
||||
hashSources,
|
||||
sameSources,
|
||||
missingSources,
|
||||
missingOverlays,
|
||||
readCliloc,
|
||||
}
|
||||
|
||||
@@ -178,6 +178,74 @@ const getPointsBoard = (system) => call(`/points/${encodeURIComponent(system)}`)
|
||||
const getMarket = ({ limit = 200, offset = 0 } = {}) =>
|
||||
call(`/market?limit=${encodeURIComponent(limit)}&offset=${encodeURIComponent(offset)}`)
|
||||
|
||||
// ── Protocol 8: the Asset Bridge (docs/link/v8.md) ────────────────────────
|
||||
//
|
||||
// The shard reads the operator's own UO client files and hands the results over
|
||||
// this link, which is why nobody has to install UOFiddler any more.
|
||||
|
||||
// Stage 1 of the import gate: what those client files currently ARE — size, mtime
|
||||
// and content hash of each, plus the version of the shard's extractor that would
|
||||
// read them. No pixels and no strings cross on this call; its whole job is to let
|
||||
// the site decide that nothing has changed and stop, which is the normal case on
|
||||
// every restart.
|
||||
//
|
||||
// `sha256` comes back NULL for a file the shard has not hashed yet (anim.mul is
|
||||
// 195 MB and hashing it cannot fit in a reply), with `hashing: true` alongside.
|
||||
// That is "ask again in a moment", not "the file changed".
|
||||
const getAssetSources = () => call('/assets/sources')
|
||||
|
||||
// The cliloc table out of the shard's own client, PAGED: each reply carries `rows`
|
||||
// plus `more` / `cursor` / `cut`, and the caller echoes the cursor back until a
|
||||
// reply says `more: false`. Only `cut: 'end'` means the table is finished — a short
|
||||
// page can equally mean the byte budget was spent.
|
||||
//
|
||||
// `clilocBridge.js` is the thing that walks it; nothing else should call this
|
||||
// directly, because a half-walked table is worse than none.
|
||||
const getClilocTable = ({ lang, cursor } = {}) => {
|
||||
const params = new URLSearchParams()
|
||||
if (lang) params.set('lang', lang)
|
||||
if (cursor) params.set('cursor', cursor)
|
||||
const qs = params.toString()
|
||||
return call(`/cliloc${qs ? `?${qs}` : ''}`)
|
||||
}
|
||||
|
||||
// Stage 2 of the import gate, PAGED: every asset the shard could serve, with a
|
||||
// hash and a size and no pixels. The website diffs it against what it holds and
|
||||
// fetches only the keys whose hash moved — which on an ordinary restart is none
|
||||
// of them, and is the whole difference between an Update and a re-download.
|
||||
//
|
||||
// This family pages on the shard's WALL CLOCK rather than on bytes: its rows are
|
||||
// ~90 bytes, but building one means decoding a sprite, so a page ends when the
|
||||
// shard's scan budget is spent (`cut: 'limit'`) far more often than when the byte
|
||||
// budget is (`cut: 'budget'`). Neither means finished; only `cut: 'end'` does.
|
||||
const getAssetManifest = ({ family, cursor } = {}) => {
|
||||
const params = new URLSearchParams()
|
||||
if (family) params.set('family', family)
|
||||
if (cursor) params.set('cursor', cursor)
|
||||
const qs = params.toString()
|
||||
return call(`/assets/manifest${qs ? `?${qs}` : ''}`)
|
||||
}
|
||||
|
||||
// The pixels, for keys the caller names. POST because the key list IS the request.
|
||||
//
|
||||
// `catalog` is the mid-import guard and should always be passed: it is an id the
|
||||
// manifest derived from the client files themselves, and handing it back makes the
|
||||
// shard refuse (422) if those files moved in between. Without it, an operator who
|
||||
// patched their client halfway through an import gets one asset set stitched out
|
||||
// of two, with nothing anywhere reporting a problem.
|
||||
const fetchAssets = ({ keys, catalog, cursor } = {}) =>
|
||||
call('/assets/fetch', { method: 'POST', body: { keys, catalog, cursor } })
|
||||
|
||||
// Slug → body id (docs/link/v8.md §8). The atlas knows a creature by its ServUO
|
||||
// class name; the client knows it by a body id; nothing in the ServUO tree
|
||||
// declares the mapping, so the shard answers it by constructing the creature and
|
||||
// reading `Body.BodyID`.
|
||||
//
|
||||
// That runs on the shard's CORE THREAD, so the batch is small and the shard
|
||||
// refuses an over-long list rather than truncating it. `assetBridge.js` chunks;
|
||||
// nothing else should call this directly.
|
||||
const resolveBodies = (types) => call('/assets/bodies', { method: 'POST', body: { types } })
|
||||
|
||||
// ── Commands ──────────────────────────────────────────────────────────────
|
||||
const confirmLink = (code, websiteUserId) =>
|
||||
call('/link/confirm', { method: 'POST', body: { code, websiteUserId: String(websiteUserId) } })
|
||||
@@ -398,6 +466,11 @@ module.exports = {
|
||||
getPoints,
|
||||
getPointsBoard,
|
||||
getMarket,
|
||||
getAssetSources,
|
||||
getClilocTable,
|
||||
getAssetManifest,
|
||||
fetchAssets,
|
||||
resolveBodies,
|
||||
confirmLink,
|
||||
linkLookup,
|
||||
createAccount,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user