diff --git a/client/src/components/CharacterSheet.jsx b/client/src/components/CharacterSheet.jsx index 58c8fb4..c192f2d 100644 --- a/client/src/components/CharacterSheet.jsx +++ b/client/src/components/CharacterSheet.jsx @@ -7,6 +7,7 @@ // player can reach is safe. import ShardAccountActions from './ShardAccountActions.jsx' +import ItemIcon from './ItemIcon' const RESIST_LABELS = { phys: 'Physical', fire: 'Fire', cold: 'Cold', pois: 'Poison', energy: 'Energy' } @@ -270,7 +271,16 @@ export default function CharacterSheet({ char, moderation = false }) { const detail = [label === layer ? null : layer, `id ${it.itemId}`, it.hue ? `hue ${it.hue}` : null] return (
- + {/* The sheet has always drawn an empty swatch here to hold the + row's alignment. As of phase 5 the shard can hand over the + item's real picture, hued the way the client would draw it — + so the swatch becomes the fallback rather than the only + state, and a row with no picture looks exactly as it did. */} + {it.art ? ( + + ) : ( + + )}
{label}
{detail.filter(Boolean).join(' · ')}
diff --git a/client/src/components/ItemIcon.jsx b/client/src/components/ItemIcon.jsx new file mode 100644 index 0000000..b30ab3f --- /dev/null +++ b/client/src/components/ItemIcon.jsx @@ -0,0 +1,47 @@ +// One item's picture, when this site holds one (docs/link/v8.md §5, §11 — phase 5). +// +// `art` is a FILENAME under uploads/items/, never a path or a URL — the same +// shape `CreaturePortrait` takes, so there is one place in this module that knows +// where uploads are mounted rather than one per surface. +// +// **NULL is ordinary and permanent for some items, and this renders nothing for +// it.** Three separate reasons an item has no picture, and none of them is a +// fault: the site has no shard link and never fetched one; the warm pass has not +// reached this key yet (pictures are fetched behind the page, never by it, so a +// new listing shows text first and gains its icon a few minutes later); or the +// operator's own client simply has no art at that id — 9,963 of a stock client's +// static ids have an empty index entry. Every layout using this is written to sit +// correctly with the icon absent, because that is the state all of them were +// built in. +// +// A hued item is a DIFFERENT picture, not a tinted one: the shard applies the hue +// out of `hues.mul` before it sends anything, because whether a hue repaints the +// whole sprite or only its grey pixels is decided by a flag in `tiledata.mul` +// that this browser has no way to read. So there is nothing to style here — the +// bytes already are the right colour. +// +// `imageRendering: 'pixelated'` for the same reason the creature portraits use +// it: UO art is pixel art, and a browser's default smoothing turns a 22×26 +// item into a smear at any size above its own. +export default function ItemIcon({ art, name, size = 32 }) { + if (!art) return null + + return ( + + ) +} diff --git a/client/src/routes/public/Market.jsx b/client/src/routes/public/Market.jsx index 8a36e3c..584adb9 100644 --- a/client/src/routes/public/Market.jsx +++ b/client/src/routes/public/Market.jsx @@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from 'react' import { Link } from 'react-router-dom' import api from '../../api.js' import { EmptyState, ErrorState, Loading, PageHeader, PublicLayout, useAsync } from '../../core.js' +import ItemIcon from '../../components/ItemIcon' // ── The player-vendor marketplace ─────────────────────────────────────────── // @@ -86,6 +87,7 @@ function ListingRow({ listing }) { return (
+
+ {i.amount > 1 ? `${num(i.amount)} × ` : ''} {itemLabel(i)} diff --git a/routes.manifest.json b/routes.manifest.json index 79a81be..d2e715d 100644 --- a/routes.manifest.json +++ b/routes.manifest.json @@ -281,6 +281,11 @@ "path": "/api/v1/admin/shard/assets/import", "tier": "public" }, + { + "method": "POST", + "path": "/api/v1/admin/shard/assets/warm", + "tier": "public" + }, { "method": "POST", "path": "/api/v1/admin/shard/atlas/approve", diff --git a/server/boot.js b/server/boot.js index 468a633..30d9dca 100644 --- a/server/boot.js +++ b/server/boot.js @@ -33,6 +33,7 @@ const shardBroadcast = require('./utils/shardBroadcast') const shardAtlas = require('./model/shardAtlas/shardAtlas.model') const shardClilocs = require('./model/shardClilocs/shardClilocs.model') const shardMarket = require('./model/shardMarket/shardMarket.model') +const shardItemArt = require('./model/shardAssets/shardItemArt.model') /** * Best-effort startup probe of the uo-link sidecar. @@ -114,6 +115,15 @@ async function onBoot() { // Deliberately not awaited — see the header. An unreachable sidecar would // otherwise hold the listener closed for the length of an HTTP timeout. checkUoLink().catch((err) => log.warn('uo-link startup probe failed', { error: err.message })) + + // Item and land pictures for the keys this site's own rows name (§11, phase 5). + // + // A timer rather than a boot pass, and it is the same rule §9.2 set for clilocs: + // **boot does not call the shard.** The first pass is one interval away, so an + // unreachable sidecar costs a log line rather than a startup delay, and an + // operator who has just configured the bridge does not have to restart to get + // pictures. `unref`ed, so it never holds shutdown open. + shardItemArt.startWarming() } async function onShutdown() { @@ -121,6 +131,7 @@ async function onShutdown() { // still works — the pool is open, the push dispatcher is up, the SSE fan-out // is live. It is the only chance to close cleanly, and it is budgeted, so a // hook that will not let go costs five seconds rather than the whole shutdown. + shardItemArt.stopWarming() // stop the item-art warm pass uoLinkSocket.stop() // close the uo-link WS ingest client shardBroadcast.closeAll() // end any open shard live-feed SSE streams } diff --git a/server/db/schema.sql b/server/db/schema.sql index 250ba60..62f64db 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -690,6 +690,24 @@ CREATE TABLE IF NOT EXISTS shard_assets ( INDEX idx_shard_assets_body (body) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- Which of the shard's catalogues a row was fetched under (§7, phase 5). +-- +-- The body catalogue can answer "is this stale?" from `shard_asset_meta`, because +-- it is imported as a SET: one manifest walk covers every key, so one stored +-- fingerprint describes all of them. Item and land art has no manifest and never +-- will — 49,152 static ids times three thousand hues is not a set anyone +-- enumerates — so staleness has to be recorded per row, and this is it. +-- +-- The shard derives the id from the files that decide the bytes (its art data +-- file, hues.mul, tiledata.mul, verdata.mul, and its own extractor version), so a +-- client patch changes it and a restart does not. A row whose `catalog` is not the +-- shard's current one is stale: the warm pass re-fetches it the next time +-- something asks for that key, and pictures nobody looks at any more are simply +-- never re-fetched, which is the whole reason this is per-row and lazy rather than +-- a sweep. NULL means "written before this column existed", which is stale by the +-- same test and costs one re-fetch. +ALTER TABLE shard_assets ADD COLUMN IF NOT EXISTS catalog VARCHAR(32) NULL; + -- Slug → body id, as the shard itself answered it (§8). -- -- **Deliberately NOT a column on `shard_spawn_creatures`.** That table is diff --git a/server/model/shardAssets/shardAssets.db.js b/server/model/shardAssets/shardAssets.db.js index 97ec9dc..9a611b6 100644 --- a/server/model/shardAssets/shardAssets.db.js +++ b/server/model/shardAssets/shardAssets.db.js @@ -33,7 +33,7 @@ async function batched(conn, sql, rows) { /** Every asset row we hold, as a Map of key → row. */ async function allAssets() { const rows = await query( - 'SELECT asset_key, family, sha256, bytes, width, height, body, direction, file FROM shard_assets', + 'SELECT asset_key, family, sha256, bytes, width, height, body, direction, file, catalog FROM shard_assets', ) const map = new Map() @@ -49,6 +49,7 @@ async function allAssets() { body: row.body === null ? null : Number(row.body), direction: row.direction === null ? null : Number(row.direction), file: row.file || null, + catalog: row.catalog || null, }) } @@ -82,16 +83,17 @@ async function saveAssets(rows, meta) { r.body ?? null, r.direction ?? null, r.file ?? null, + r.catalog ?? meta?.catalog ?? null, ]) await batched( conn, - 'INSERT INTO shard_assets (asset_key, family, sha256, bytes, width, height, body, direction, file) ' + - 'VALUES (?,?,?,?,?,?,?,?,?) ' + + 'INSERT INTO shard_assets (asset_key, family, sha256, bytes, width, height, body, direction, file, catalog) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?) ' + 'ON DUPLICATE KEY UPDATE family = VALUES(family), sha256 = VALUES(sha256), ' + 'bytes = VALUES(bytes), width = VALUES(width), height = VALUES(height), ' + 'body = VALUES(body), direction = VALUES(direction), file = VALUES(file), ' + - 'imported_at = CURRENT_TIMESTAMP', + 'catalog = VALUES(catalog), imported_at = CURRENT_TIMESTAMP', values, ) @@ -114,6 +116,74 @@ async function saveAssets(rows, meta) { } } +// ── the on-demand side (§11, phase 5) ────────────────────────────────────── + +/** + * The pictures we hold for an explicit list of keys, as a Map of key → filename. + * + * This is the read on the hot path — every marketplace page and every character + * sheet runs it — so it is one statement over the primary key and it returns only + * what it is asked for. It deliberately does NOT check staleness: a page renders + * the picture it has, and deciding whether that picture is out of date is the warm + * pass's job, off the request. + */ +async function filesForKeys(keys) { + const list = [...new Set(keys.filter((k) => typeof k === 'string' && k !== ''))] + + if (list.length === 0) return new Map() + + const rows = await query( + `SELECT asset_key, file FROM shard_assets WHERE file IS NOT NULL AND asset_key IN (${list + .map(() => '?') + .join(',')})`, + list, + ) + + const map = new Map() + + for (const row of rows) map.set(row.asset_key, row.file) + + return map +} + +/** + * Which of these keys we already hold under the shard's CURRENT catalogue. + * + * The warm pass subtracts this from what it wants, so everything it does not + * return gets fetched: a key we have never seen, and a key whose row was written + * against a catalogue the shard has since moved past (§7 — an operator patched + * their client). A row with no file is not held either, because the database and + * the uploads volume can disagree and a broken image is worse than a re-fetch. + */ +async function freshKeys(keys, catalog) { + const list = [...new Set(keys.filter((k) => typeof k === 'string' && k !== ''))] + + if (list.length === 0) return new Set() + + const rows = await query( + `SELECT asset_key FROM shard_assets WHERE file IS NOT NULL AND catalog <=> ? ` + + `AND asset_key IN (${list.map(() => '?').join(',')})`, + [catalog ?? null, ...list], + ) + + return new Set(rows.map((r) => r.asset_key)) +} + +/** Counts for the admin surface, split by family. */ +async function countByFamily() { + const rows = await query( + 'SELECT family, COUNT(*) AS total, SUM(file IS NOT NULL) AS stored FROM shard_assets GROUP BY family', + ) + + const out = {} + + for (const row of rows) { + out[row.family] = { total: Number(row.total) || 0, stored: Number(row.stored) || 0 } + } + + return out +} + async function getMeta() { const rows = await query('SELECT payload, imported_at FROM shard_asset_meta WHERE id = 1') if (rows.length === 0) return null @@ -220,4 +290,7 @@ module.exports = { allBodies, countBodies, artBySlug, + filesForKeys, + freshKeys, + countByFamily, } diff --git a/server/model/shardAssets/shardAssets.model.js b/server/model/shardAssets/shardAssets.model.js index d1eb6a0..3d8301c 100644 --- a/server/model/shardAssets/shardAssets.model.js +++ b/server/model/shardAssets/shardAssets.model.js @@ -456,6 +456,7 @@ async function getStatus() { const counts = await db.countAssets().catch(() => ({ total: 0, stored: 0 })) const bodies = await db.countBodies().catch(() => ({ total: 0, resolved: 0 })) const meta = await db.getMeta().catch(() => null) + const families = await db.countByFamily().catch(() => ({})) const status = { loaded: { @@ -466,6 +467,12 @@ async function getStatus() { catalog: meta?.catalog ?? null, extractorVersion: meta?.extractorVersion ?? null, importedAt: meta?.importedAt ?? null, + // Item and land pictures, counted separately because they are a different + // KIND of thing (§11, phase 5): no manifest, no set, and no "how many are + // there" to compare against. `items` is how many the site has been asked + // for and holds, which is the only number that means anything here. + items: families.static?.stored ?? 0, + land: families.land?.stored ?? 0, }, shard: null, drift: null, @@ -485,6 +492,10 @@ async function getStatus() { hashing: sources.hashing, complete: sources.complete, imaging: sources.imaging, + // Which §5 families this overlay serves. A phase-3 or phase-4 overlay says + // `['body']`, which is what an admin panel needs in order to say "update + // your plugin" rather than showing an item-art pipeline that cannot work. + families: sources.families, } status.drift = meta ? !bridge.sameSources(sources, meta.sources) : true diff --git a/server/model/shardAssets/shardItemArt.model.js b/server/model/shardAssets/shardItemArt.model.js new file mode 100644 index 0000000..3a87775 --- /dev/null +++ b/server/model/shardAssets/shardItemArt.model.js @@ -0,0 +1,510 @@ +const fs = require('fs') +const path = require('path') + +const db = require('./shardAssets.db') +const core = require('../../core') +const bridge = require('../../utils/assetBridge') +const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model') +const log = require('../../core').logger('shardItemArt') + +// Item and land pictures, fetched because something on this site names them +// (docs/link/v8.md §5, §11 — protocol 8, phase 5). +// +// ── Why this is not the body catalogue with a different prefix ───────── +// +// The bestiary wants every creature, so phase 3 imports a SET: walk a manifest, +// diff the hashes, fetch what moved. That works because the set is 1,022 rows +// and one megabyte. +// +// This side has no set. The shard's client addresses 49,152 item graphics and +// has art for 39,189 of them; multiply by three thousand hues and there is +// nothing to enumerate, no manifest worth building and nothing worth importing +// ahead of time. What there IS, at any moment, is a few hundred keys that the +// site's own rows actually name: the items on a vendor, the things a character +// is wearing. That is the working set, and it is what this fetches. +// +// ── Who is allowed to make the shard do work ────────────────────────── +// +// **Ingest warms; the route only serves** (org lead, 2026-09-11). A page never +// waits on the shard and never causes a fetch: it renders the pictures already +// on disk and leaves out the ones that are not, which is exactly the state every +// install was in before this phase and which every surface already handles. +// Fetching happens behind that, from the keys the site has stored. +// +// The alternative — fetch on the first request for a key — was rejected on one +// number. The shard's asset plane serves **one request at a time** by design +// (§3.2), so any anonymous visitor able to name a key could walk 49,152 ids +// times 3,000 hues through that single slot and keep an operator's own import +// waiting behind it, from a URL with nothing to authenticate. Warming from the +// site's own data has no such surface: the ceiling is the number of distinct +// (item, hue) pairs the shard itself has told us about. +// +// ── Why the wanted set is DERIVED and not a queue ───────────────────── +// +// A queue table would need writing on the ingest path, draining, retrying, +// pruning and reconciling after a restart. The same answer falls out of a +// `SELECT DISTINCT` over the rows that name the items — which is self-healing by +// construction: a key lost to a restart comes back the next time the pass runs, +// and a key for a vendor that has gone stops being wanted the moment its row is +// deleted. The in-memory set below is an optimisation on top of that, never the +// record: it exists so a picture seen on a LIVE character sheet — which is +// fetched from the shard per request and stored nowhere — is not forgotten. +// +// ── Staleness, without a manifest (§7) ──────────────────────────────── +// +// Every fetched row records the shard's `catalog` id, which is a hash of the +// files that decide the bytes. A client patch changes it, a restart does not. So +// "is this picture out of date?" is a per-row comparison rather than a manifest +// diff, and the answer costs nothing for the pictures nobody is looking at any +// more: they are simply never re-fetched. + +/** Where item and land pictures land, under core's upload directory. */ +const ART_SUBDIR = 'items' + +/** + * How many keys one warm pass will fetch. + * + * A bound rather than a target. The shard serves one asset request at a time, so + * a pass that asked for everything at once would hold that slot for as long as it + * took — against an operator who might be trying to run an import. Passes are + * cheap and repeat; a backlog drains over several of them and nothing waits. + */ +const WARM_BATCH = 400 + +/** + * How many live-observed keys are remembered between passes. + * + * Bounded because this is a set fed by page views. It is an optimisation over the + * derived set, so dropping from it costs a picture appearing one pass later, and + * never a picture that is lost. + */ +const SEEN_CAP = 5000 + +const seen = new Set() + +// ── keys (§5) ────────────────────────────────────────────────────────────── + +/** + * The one place an item key is spelled. + * + * Hue 0 means "not hued" on the wire, so it produces the plain key rather than a + * `/h0` one — the shard refuses `/h0` outright for the same reason, and the two + * agreeing is what stops the same PNG being stored twice under two names. + */ +function staticKey(itemId, hue = 0) { + const id = Number(itemId) + + if (!Number.isInteger(id) || id < 0) return null + + const h = Number(hue) + + return Number.isInteger(h) && h > 0 ? `static/${id}/h${h}` : `static/${id}` +} + +function landKey(tileId) { + const id = Number(tileId) + + return Number.isInteger(id) && id >= 0 && id < 0x4000 ? `land/${id}` : null +} + +function artDir() { + return path.join(core.uploads.UPLOAD_DIR, ART_SUBDIR) +} + +/** + * Content-addressed, exactly as the body catalogue's names are and for the same + * reason: a stable name overwritten in place leaves every browser and CDN serving + * last month's client's sprite while the database row stays perfectly correct. + */ +function fileNameFor(key, sha256) { + const stem = key.replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '') + return `uo-${stem}-${String(sha256).slice(0, 8)}.png` +} + +// ── noticing ─────────────────────────────────────────────────────────────── + +/** + * Remember that something on this site showed these (itemId, hue) pairs. + * + * Called from the market ingest and from the character sheet, and deliberately + * synchronous and allocation-light: it is on a request path and a page must never + * pay for a picture it is not going to get anyway. + */ +function notice(items) { + if (!Array.isArray(items)) return 0 + + let added = 0 + + for (const item of items) { + const key = staticKey(item?.itemId ?? item?.item_id, item?.hue) + + if (!key || seen.has(key)) continue + + // Oldest-first, and only when full. The derived set is the record; this is a + // cache of hints, so forgetting one costs a pass, not a picture. + if (seen.size >= SEEN_CAP) seen.delete(seen.values().next().value) + + seen.add(key) + added++ + } + + return added +} + +/** For tests and the admin surface: how many hints are waiting. */ +function noticedCount() { + return seen.size +} + +// ── serving ──────────────────────────────────────────────────────────────── + +/** + * Attach `art` to rows that name an item, in place, and notice what is missing. + * + * `art` is a FILENAME under `uploads/items/`, never a path or a URL — the same + * shape `shard_spawn_creatures.art` uses, so the client builds one URL the same + * way everywhere and the API never hard-codes a mount point. + * + * A row with no stored picture gets `art: null` rather than being changed in any + * other way. That is a first-class state: it is what every row looked like before + * this phase, every surface renders it, and it is what an item this client has no + * art for looks like permanently. + */ +async function decorate(rows, { itemIdField = 'itemId', hueField = 'hue' } = {}) { + const list = Array.isArray(rows) ? rows.filter((r) => r && typeof r === 'object') : [] + + if (list.length === 0) return list + + const keys = list.map((row) => staticKey(row[itemIdField], row[hueField])) + + let files = new Map() + + try { + files = await db.filesForKeys(keys.filter(Boolean)) + } catch (err) { + // Decoration, not the page. A picture lookup that fails must not fail a + // marketplace search. + log.warn('could not read item art', { error: err.message }) + return list + } + + for (let i = 0; i < list.length; i++) { + list[i].art = (keys[i] && files.get(keys[i])) || null + } + + // Everything this page WANTED is worth warming, whether or not we had it: the + // ones we had may be stale, and the ones we did not are the point. + notice(list.map((row) => ({ itemId: row[itemIdField], hue: row[hueField] }))) + + return list +} + +// ── warming ──────────────────────────────────────────────────────────────── + +async function shardLinked() { + try { + const config = await uoLinkConfig.getSafe() + return Boolean(config?.enabled && config?.baseUrl) + } catch { + return false + } +} + +/** + * Every item key the site's own rows name, newest-priced first. + * + * `shard_vendor_items` is the only stored table that carries (item_id, hue) + * today. The character sheet's equipment is fetched live from the shard per + * request and stored nowhere, which is precisely what the in-memory hint set is + * for. + */ +async function wantedKeys() { + const keys = [] + + try { + const rows = await core.query( + 'SELECT DISTINCT item_id, hue FROM shard_vendor_items WHERE item_id > 0 LIMIT 20000', + ) + + for (const row of rows) { + const key = staticKey(row.item_id, row.hue) + if (key) keys.push(key) + } + } catch (err) { + log.warn('could not read the marketplace for item art', { error: err.message }) + } + + // Hints last, so a backlog of stored rows is never starved by page traffic. + for (const key of seen) keys.push(key) + + return [...new Set(keys)] +} + +function writePicture(key, sha256, png) { + const name = fileNameFor(key, sha256) + + try { + fs.mkdirSync(artDir(), { recursive: true }) + fs.writeFileSync(path.join(artDir(), name), png) + return name + } catch (err) { + log.warn('could not write an item picture', { key, error: err.message }) + return null + } +} + +function removePicture(name) { + if (!name) return + + try { + fs.unlinkSync(path.join(artDir(), name)) + } catch { + // Already gone, or never written. A warm pass must not fail because a file it + // was tidying up was tidied already. + } +} + +/** + * One warm pass: fetch the wanted keys we do not already hold, and store them. + * + * Returns a result rather than throwing, with the same vocabulary the body import + * uses — `skipped`, `unavailable`, `unchanged`, `imported`, `failed` — so the + * admin surface reports one set of words for both halves of this protocol. + * + * `limit` bounds one pass. `force` re-fetches keys we hold, which is how an + * operator recovers from a wiped uploads volume without waiting for a client + * patch to invalidate every row. + */ +async function warm({ limit = WARM_BATCH, force = false } = {}) { + if (!(await shardLinked())) { + return { status: 'skipped', reason: 'uo-link is not configured, so there is no shard to ask' } + } + + let sources + + try { + sources = await bridge.sourceFingerprint() + } catch (err) { + return failure(err, 'client file manifest') + } + + if (sources.imaging && sources.imaging.ok === false) { + return { + status: 'unavailable', + code: 'NO_IMAGING', + reason: sources.imaging.reason || 'the shard host cannot render images', + } + } + + // A phase-3 or phase-4 overlay serves bodies and nothing else. Asking it for a + // static is refused per request, which would be a warn on every pass forever — + // so it is checked once, here, and reported as the ordinary state it is. + // Defensive default rather than a trusted field: an older sidecar, an older + // overlay or a stubbed fingerprint can all leave it off, and `['body']` is the + // truthful reading of its absence (§6 — the families field arrived in phase 5). + const families = Array.isArray(sources.families) ? sources.families : ['body'] + + if (!families.includes('static')) { + return { + status: 'unavailable', + code: 'UNSUPPORTED', + reason: + "this shard's overlay does not serve item art; it offers " + + `${families.join(', ')}. Update the plugin overlay to get it.`, + } + } + + const wanted = await wantedKeys() + + if (wanted.length === 0) { + return { status: 'unchanged', wanted: 0, fetched: 0, written: 0 } + } + + // The catalogue is learned from the first reply rather than asked for, so this + // pass cannot be the thing that decides what is stale. `catalog: null` on the + // request means "whatever you have"; the mid-walk guard in `fetchAssets` is what + // catches a client that moves underneath it. + let held = new Set() + + if (!force) { + const current = await currentCatalog() + + try { + held = await db.freshKeys(wanted, current) + } catch (err) { + return { status: 'failed', reason: err.message } + } + } + + const todo = wanted.filter((key) => !held.has(key)).slice(0, Math.max(1, limit)) + + if (todo.length === 0) { + forget(wanted) + return { status: 'unchanged', wanted: wanted.length, held: held.size, fetched: 0, written: 0 } + } + + let fetched + + try { + fetched = await bridge.fetchAssets({ keys: todo }) + } catch (err) { + return failure(err, 'item art') + } + + // Only the filenames, and only for the keys in hand: the old file is removed + // when a key's hash moves, so `uploads/items/` tracks the working set instead of + // accumulating one file per client patch forever. + const existing = await db.filesForKeys(todo).catch(() => new Map()) + const rows = [] + let written = 0 + + for (const key of todo) { + const got = fetched.assets.get(key) + + // A key the shard has no art for is not a failure and not a row: writing an + // empty row would make it "held" and stop it ever being asked again, which is + // wrong the moment an operator patches in the missing graphic. + if (!got) continue + + const name = writePicture(key, got.sha256, got.png) + + if (!name) continue + + written++ + + const before = existing.get(key) + + if (before && before !== name) removePicture(before) + + rows.push({ + key, + family: key.startsWith('land/') ? 'land' : 'static', + sha256: got.sha256, + bytes: got.bytes, + width: got.width, + height: got.height, + body: null, + direction: null, + file: name, + catalog: fetched.catalog, + }) + } + + if (rows.length > 0) { + try { + // No meta: `shard_asset_meta` is the BODY catalogue's singleton — what an + // Update compares a manifest against — and this family has no manifest. A + // warm pass writing there would tell the body import that a client it never + // looked at is unchanged. + await db.saveAssets(rows, null) + } catch (err) { + return { status: 'failed', reason: err.message } + } + } + + forget(todo) + + const result = { + status: 'imported', + catalog: fetched.catalog, + wanted: wanted.length, + held: held.size, + asked: todo.length, + fetched: fetched.assets.size, + written, + absent: fetched.missing.absent, + unsupported: fetched.missing.unsupported, + remaining: Math.max(0, wanted.length - held.size - todo.length), + } + + log.info('item art warmed', result) + + return result +} + +/** Drop hints a pass has dealt with, so the set does not grow without bound. */ +function forget(keys) { + for (const key of keys) seen.delete(key) +} + +/** + * The catalogue id the shard would answer under right now. + * + * Read from a one-key probe rather than from a dedicated command: the shard puts + * `catalog` on every fetch reply, so the cheapest honest way to ask is to fetch + * something. `static/0` is the smallest such question and its answer is thrown + * away — what is wanted is the id beside it. + * + * A shard that cannot answer returns null, and null compares unequal to every + * stored catalogue, so the pass falls back to "everything is stale" — which costs + * a re-fetch and never serves a wrong picture. That is the right way round. + */ +async function currentCatalog() { + try { + const probe = await bridge.fetchAssets({ keys: ['static/0'] }) + return probe.catalog ?? null + } catch (err) { + log.warn('could not read the shard art catalogue', { error: err.message }) + return null + } +} + +function failure(err, what) { + if (err instanceof bridge.AssetBridgeError) { + return { status: 'unavailable', code: err.code, reason: err.message } + } + + log.warn(`item art failed reading the ${what}`, { error: err.message }) + + return { status: 'failed', reason: err.message } +} + +// ── the background pass ──────────────────────────────────────────────────── + +let timer = null + +/** + * Run a warm pass every few minutes, forever, while the process lives. + * + * Deliberately a plain interval and not a debounce on ingest. A market sweep + * delivers dozens of `vendor.listing` frames in a burst and debouncing each of + * them would either fire once per frame or need its own state machine; a pass is + * cheap when there is nothing to do (one `SELECT DISTINCT` and one probe) and the + * work it exists for is not urgent — a picture appearing a few minutes after the + * listing that wants it is invisible to everyone. + * + * `unref()` so this never holds the process open at shutdown. + */ +function startWarming({ everyMs = 5 * 60 * 1000 } = {}) { + if (timer) return + + timer = setInterval(() => { + warm().catch((err) => log.warn('item art warm pass failed', { error: err.message })) + }, everyMs) + + if (typeof timer.unref === 'function') timer.unref() +} + +function stopWarming() { + if (!timer) return + + clearInterval(timer) + timer = null +} + +module.exports = { + ART_SUBDIR, + WARM_BATCH, + artDir, + fileNameFor, + staticKey, + landKey, + notice, + noticedCount, + decorate, + wantedKeys, + warm, + currentCatalog, + startWarming, + stopWarming, +} diff --git a/server/model/shardMarket/shardMarket.model.js b/server/model/shardMarket/shardMarket.model.js index 9e2d818..b913715 100644 --- a/server/model/shardMarket/shardMarket.model.js +++ b/server/model/shardMarket/shardMarket.model.js @@ -14,6 +14,7 @@ const db = require('./shardMarket.db') const clilocs = require('../shardClilocs/shardClilocs.model') +const itemArt = require('../shardAssets/shardItemArt.model') const log = require('../../core').logger('shard-market') // Defense in depth on top of the shard's own MarketMaxListings cap. The shard is @@ -173,6 +174,13 @@ async function upsertVendor(ev) { const vendor = flattenFrame(ev) const items = await shapeItems(ev) await db.replaceVendor(vendor, items) + + // The listings name (itemId, hue) pairs, which are §5 asset keys (phase 5). + // Noticing them here is what makes the warm pass find a newly listed item's + // picture before anyone looks at the shop, rather than one page view later. + // A hint, never a queue — the pass derives its real set from this table, so a + // hint lost to a restart costs nothing. + itemArt.notice(items) } /** Ingest one `vendor.listing.remove` frame. */ @@ -273,8 +281,14 @@ async function search({ const info = await db.meta() + // Each listing gets `art`: the filename of the item's picture under + // uploads/items/, or null where this site does not hold one (phase 5). One + // query for the page, off the listing shape rather than the SQL, so the search + // itself stays the search and a picture lookup that fails costs a picture. + const listings = await itemArt.decorate(rows.map(shapeListing)) + return { - listings: rows.map(shapeListing), + listings, total, limit, offset, @@ -292,7 +306,7 @@ async function getVendor(serial, { limit = 250, offset = 0 } = {}) { const row = await db.getVendor(serial) if (!row) return null const items = await db.listVendorItems(serial, { limit, offset }) - return { ...shapeVendor(row), items: items.map(shapeItem) } + return { ...shapeVendor(row), items: await itemArt.decorate(items.map(shapeItem)) } } /** Index size, staleness, and the facet/region filter options. */ diff --git a/server/router/admin/shard.router.js b/server/router/admin/shard.router.js index f419e88..8b35ab8 100644 --- a/server/router/admin/shard.router.js +++ b/server/router/admin/shard.router.js @@ -399,6 +399,22 @@ shardRouter.post( shardAssets.importAssets, ) +shardRouter.post( + '/assets/warm', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Fetch item and land artwork the site is missing, now (admin only)' + // #swagger.description = 'Runs one pass of the item-art warm loop instead of waiting for its timer. The pass works out which item pictures this site's own rows name — every distinct (ItemID, hue) on a player vendor, plus anything a character sheet has shown since the last pass — and fetches the ones it does not already hold from the shard, hued and stored under uploads/items/. There is deliberately NO manifest and no bulk import here: the client addresses 49,152 item graphics times three thousand hues, so the working set is defined by what the site actually displays. `force` re-fetches pictures the site already holds, which is how an operator recovers a wiped uploads volume. `limit` bounds one pass; the default is 400, because the shard serves one asset request at a time and a pass must not hold that slot against an import. Nothing throws for an operator-visible problem: no shard configured, a shard that is down, an asset plane switched off, a host with no libgdiplus, or a plugin overlay too old to serve item art all answer 200 with status "unavailable"/"skipped" and a reason naming what to fix.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Re-fetch pictures this site already holds." }, limit: { type: "integer", description: "How many keys this pass may fetch (1-2000)." } } } } } } */ + /* #swagger.responses[200] = { description: 'What the pass did', content: { "application/json": { schema: { $ref: "#/components/schemas/UoItemArtWarmResult" } } } } */ + /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + adminOnly, + body('force').optional().isBoolean(), + body('limit').optional().isInt({ min: 1, max: 2000 }), + validate, + shardAssets.warmItemArt, +) + // ── Feature visibility (admin only) ─────────────────────────────────── // Who can see which shard surface, and which sensitive fields within it. This // decides what ANONYMOUS visitors get, so it sits above the moderator tier. diff --git a/server/router/admin/shardAssets.controller.js b/server/router/admin/shardAssets.controller.js index 1d9db48..f4aa17e 100644 --- a/server/router/admin/shardAssets.controller.js +++ b/server/router/admin/shardAssets.controller.js @@ -23,6 +23,7 @@ // is phase 8. This pair is what makes phase 3 reachable at all. const assets = require('../../model/shardAssets/shardAssets.model') +const itemArt = require('../../model/shardAssets/shardItemArt.model') const { activity } = require('../../core') const log = require('../../core').logger('admin-shard-assets') @@ -85,7 +86,51 @@ async function importAssets(req, res) { } } +// POST /admin/shard/assets/warm — run one item-art warm pass now. +// +// The pass runs on its own timer and needs no operator, so this exists for the +// two moments where waiting for the interval is the wrong answer: an operator who +// has just configured the bridge and wants to see it work, and one who has just +// patched their client and would rather not wait for pictures to refresh. +// +// `force` re-fetches keys the site already holds. The body import's `force` means +// the same thing for the same reason — a wiped uploads volume leaves every +// database row correct and every picture missing, and only an explicit re-fetch +// recovers it. +// +// It is bounded: one pass asks for at most `limit` keys, because the shard's +// asset plane serves one request at a time and a pass must not hold that slot +// against the operator's own import. +async function warmItemArt(req, res) { + try { + const force = !!req.body?.force + const limit = Number.isFinite(Number(req.body?.limit)) ? Number(req.body.limit) : undefined + const result = await itemArt.warm({ force, ...(limit ? { limit } : {}) }) + + await activity.log({ + req, + action: 'shard.assets.warm', + detail: { + force, + limit: limit ?? null, + status: result.status, + code: result.code ?? null, + wanted: result.wanted ?? null, + asked: result.asked ?? null, + written: result.written ?? null, + remaining: result.remaining ?? null, + }, + }) + + return res.json(result) + } catch (err) { + log.error('warmItemArt', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + module.exports = { getStatus, importAssets, + warmItemArt, } diff --git a/server/router/player/shard.controller.js b/server/router/player/shard.controller.js index fd36705..4d4e893 100644 --- a/server/router/player/shard.controller.js +++ b/server/router/player/shard.controller.js @@ -11,6 +11,7 @@ const uoLinkClient = require('../../utils/uoLinkClient') const shardLinks = require('../../model/shardLinks/shardLinks.model') const shardState = require('../../model/shardState/shardState.model') const shardClilocs = require('../../model/shardClilocs/shardClilocs.model') +const itemArt = require('../../model/shardAssets/shardItemArt.model') const { activity } = require('../../core') const gameSignup = require('../../utils/gameSignup') const { salesForAccounts } = require('../../utils/shardSales') @@ -71,6 +72,28 @@ async function resolveProfileClilocs(profile) { } } +/** + * Attach a picture to each equipped item (docs/link/v8.md §5, §11 — phase 5). + * + * The equipment list is the other place on this site that carries (itemId, hue), + * and unlike the marketplace it is LIVE: the profile is fetched from the shard per + * request and stored nowhere, so there is no table a warm pass could derive these + * keys from. That is what `notice` is for, and `decorate` does both — it fills in + * every picture we already hold and remembers the ones we do not, so a character + * whose sheet renders without art once renders with it a few minutes later. + * + * It never asks the shard. §17.11: the page serves what is stored and the warm + * pass does the fetching, because a route that fetched would let any visitor drive + * the shard's single-slot asset plane from a URL. + */ +async function resolveProfileArt(profile) { + const equipment = Array.isArray(profile?.equipment) ? profile.equipment : [] + + if (equipment.length === 0) return + + await itemArt.decorate(equipment) +} + // Decorate a char.profile with cross-links from our own board data: the guild the // character leads and any city governorship on its account, plus resolved cliloc // names. Best-effort — a failure here never fails the profile (it's a nicety, @@ -85,6 +108,7 @@ async function enrichCharProfile(profile) { if (govs.length) profile.governorOf = govs.map((g) => g.city) } await resolveProfileClilocs(profile) + await resolveProfileArt(profile) } catch (err) { log.warn('enrichCharProfile failed', { serial: profile.serial, message: err.message }) } diff --git a/server/swagger/doc.js b/server/swagger/doc.js index b5d9be0..9c4e24a 100644 --- a/server/swagger/doc.js +++ b/server/swagger/doc.js @@ -623,6 +623,8 @@ module.exports = { catalog: { type: 'string', nullable: true, description: 'The shard’s catalogue id at the last import — derived from its client files, so it changes exactly when they do.', example: 'a3f9c21d4b8e0771' }, extractorVersion: { type: 'integer', nullable: true, description: 'The version of the shard’s extraction code. A bump makes every derived byte drift even though the client files did not move.', example: 1 }, importedAt: { type: 'string', format: 'date-time', nullable: true }, + items: { type: 'integer', description: 'Item pictures held. Unlike the catalogue this has no total to compare against: item art is fetched because something on the site names it, so this is the working set rather than a fraction of one.', example: 1840 }, + land: { type: 'integer', description: 'Land tile pictures held. Zero on every install until something asks for one.', example: 0 }, }, }, shard: { @@ -644,6 +646,12 @@ module.exports = { reason: { type: 'string', nullable: true }, }, }, + families: { + type: 'array', + items: { type: 'string' }, + description: 'Which asset key families this shard’s plugin overlay serves. An overlay older than phase 5 answers `["body"]` only — it has the creature catalogue and no item art.', + example: ['body', 'land', 'static'], + }, }, }, drift: { @@ -737,6 +745,49 @@ module.exports = { }, }, }, + UoItemArtWarmResult: { + type: 'object', + description: + 'Outcome of one item-art warm pass (docs/link/v8.md §11, phase 5). Unlike the body catalogue there is no manifest and no set: the client addresses 49,152 item graphics times three thousand hues, so what gets fetched is defined by what this site’s own rows name — every distinct (ItemID, hue) on a player vendor, plus anything a character sheet has shown since the last pass. Reported rather than thrown, so a shard that is down is an answer and not a 500.', + properties: { + status: { + type: 'string', + enum: ['skipped', 'unavailable', 'unchanged', 'imported', 'failed'], + description: + '`skipped`: no shard is configured. `unchanged`: every wanted picture is already held and current. `unavailable`: the shard could not be asked, or its plugin overlay is too old to serve item art.', + example: 'imported', + }, + reason: { type: 'string', nullable: true }, + code: { + type: 'string', + nullable: true, + description: + 'Machine-readable cause. `NO_IMAGING` is a shard host with no libgdiplus. `UNSUPPORTED` is a plugin overlay that serves the creature catalogue but not item art — update the overlay.', + enum: ['DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'NO_IMAGING', 'UNSUPPORTED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE', 'UNAVAILABLE'], + }, + catalog: { + type: 'string', + nullable: true, + description: + 'The shard’s art catalogue id these pictures were fetched under — a hash of the files that decide their bytes. Stored per row, which is how staleness is answered without a manifest.', + example: '7c1e04b9aa2f3d58', + }, + wanted: { type: 'integer', nullable: true, description: 'Distinct keys this site’s rows name right now.', example: 1840 }, + held: { type: 'integer', nullable: true, description: 'How many of those are already stored and current.', example: 1440 }, + asked: { type: 'integer', nullable: true, description: 'How many this pass actually requested. Bounded by `limit`.', example: 400 }, + fetched: { type: 'integer', nullable: true, description: 'How many the shard returned a picture for.', example: 396 }, + written: { type: 'integer', nullable: true, description: 'How many were written to disk.', example: 396 }, + absent: { + type: 'integer', + nullable: true, + description: + 'Keys the shard has no art for. NOT a failure — 9,963 of this client’s static ids have an empty index entry, and an item using one simply has no picture.', + example: 4, + }, + unsupported: { type: 'integer', nullable: true, description: 'Keys the shard does not serve at all. A bug on the site’s side rather than a gap in the client.', example: 0 }, + remaining: { type: 'integer', nullable: true, description: 'Wanted keys left for the next pass. Passes repeat on a timer, so a backlog drains without an operator.', example: 0 }, + }, + }, UoShardLinkRequest: { type: 'object', required: ['code'], diff --git a/server/test/shardItemArt.model.test.js b/server/test/shardItemArt.model.test.js new file mode 100644 index 0000000..ab94f7a --- /dev/null +++ b/server/test/shardItemArt.model.test.js @@ -0,0 +1,414 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const core = require('../core') +const model = require('../model/shardAssets/shardItemArt.model') +const db = require('../model/shardAssets/shardAssets.db') +const bridge = require('../utils/assetBridge') +const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model') + +// The warm pass as a decision, with the shard and the database stubbed +// (docs/link/v8.md §5, §11 — protocol 8, phase 5). +// +// Item art has no manifest, so almost everything the body import gets from a +// hash diff this side has to get right by construction instead. Each test below +// is a way that goes wrong quietly: +// +// - Asking an overlay that cannot answer. A phase-4 plugin serves the creature +// catalogue and nothing else, and every static key it is sent is refused — +// once per pass, forever, in the log, with no picture ever appearing. +// - Re-fetching pictures the site already holds. There is no manifest to make +// that obvious, so the only thing standing between a working install and a +// pass that re-downloads its whole working set every five minutes is the +// per-row catalogue id. +// - NOT re-fetching after a client patch. The same field, read the other way. +// - Writing a row for a key the shard has no art for. It would make the key +// "held", and it would never be asked again — including after the operator +// patches in the graphic that was missing. +// - Spelling `static/3922/h0`. The shard refuses it outright (hue 0 means "not +// hued"), so a disagreement here is a picture that never arrives. + +const saved = {} + +function stub({ + families = ['body', 'land', 'static'], + wanted = [], + fresh = new Set(), + files = new Map(), + fetched, + catalog = 'cat-current', + linked = true, +} = {}) { + saved.sourceFingerprint = bridge.sourceFingerprint + saved.fetchAssets = bridge.fetchAssets + saved.freshKeys = db.freshKeys + saved.filesForKeys = db.filesForKeys + saved.saveAssets = db.saveAssets + saved.getSafe = uoLinkConfig.getSafe + saved.query = core.query + + const seen = { asked: [], saved: null, freshAsked: null, calls: 0 } + + uoLinkConfig.getSafe = async () => + linked ? { enabled: true, baseUrl: 'http://127.0.0.1:8080' } : { enabled: false } + + bridge.sourceFingerprint = async () => ({ + files: { 'art.mul': { size: 1, mtime: 2, sha256: 'x' } }, + extractorVersion: 2, + hashing: false, + complete: true, + imaging: { ok: true }, + families, + }) + + bridge.fetchAssets = async ({ keys }) => { + seen.calls++ + seen.asked.push(keys) + + // The catalogue probe asks for exactly one key and throws the answer away. + if (keys.length === 1 && keys[0] === 'static/0' && !fetched?.assets?.has('static/0')) { + return { assets: new Map(), missing: { absent: 1, unsupported: 0 }, pages: 1, catalog } + } + + return ( + fetched ?? { assets: new Map(), missing: { absent: 0, unsupported: 0 }, pages: 1, catalog } + ) + } + + // The derived set: what `SELECT DISTINCT item_id, hue FROM shard_vendor_items` + // would return. + core.query = async () => wanted + + db.freshKeys = async (keys, askedCatalog) => { + seen.freshAsked = { keys, catalog: askedCatalog } + return fresh + } + db.filesForKeys = async () => files + db.saveAssets = async (rows, meta) => { + seen.saved = { rows, meta } + return rows.length + } + + return seen +} + +function restore() { + if (saved.sourceFingerprint) bridge.sourceFingerprint = saved.sourceFingerprint + if (saved.fetchAssets) bridge.fetchAssets = saved.fetchAssets + if (saved.freshKeys) db.freshKeys = saved.freshKeys + if (saved.filesForKeys) db.filesForKeys = saved.filesForKeys + if (saved.saveAssets) db.saveAssets = saved.saveAssets + if (saved.getSafe) uoLinkConfig.getSafe = saved.getSafe + if (saved.query) core.query = saved.query +} + +function useTempUploads(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'uo-items-')) + const previous = core.uploads + + Object.defineProperty(core, 'uploads', { + configurable: true, + get: () => ({ ...previous, UPLOAD_DIR: dir }), + }) + + t.after(() => { + Object.defineProperty(core, 'uploads', { configurable: true, get: () => previous }) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + return dir +} + +const picture = (sha) => ({ + sha256: sha, + bytes: 294, + width: 22, + height: 26, + hue: null, + partialHue: null, + source: 'uop', + png: Buffer.from('not really a png'), +}) + +// ── keys ─────────────────────────────────────────────────────────────────── + +test('hue 0 is the plain key, because the shard refuses /h0 for the same reason', () => { + // The wire's hue 0 means "this item is not hued". If this spelled `/h0` the + // shard would answer `unsupported` and the picture would never arrive; if the + // shard accepted it, the identical PNG would be stored twice under two names + // and diffed separately forever. The two sides agreeing is the whole point. + assert.equal(model.staticKey(3922, 0), 'static/3922') + assert.equal(model.staticKey(3922), 'static/3922') + assert.equal(model.staticKey(3922, null), 'static/3922') + assert.equal(model.staticKey(3922, 33), 'static/3922/h33') +}) + +test('a key is refused rather than fabricated for input that is not an item id', () => { + assert.equal(model.staticKey(-5), null) + assert.equal(model.staticKey('frog'), null) + assert.equal(model.staticKey(undefined), null) + assert.equal(model.landKey(0x4000), null) + assert.equal(model.landKey(3), 'land/3') +}) + +// ── the overlay gate ─────────────────────────────────────────────────────── + +test('an overlay that serves only the creature catalogue is reported, not asked', async (t) => { + // A phase-3 or phase-4 plugin. Every static key sent to it comes back refused, + // so discovering this per request would mean a warn per pass forever and no + // picture ever. It is one check, once, with a sentence naming the fix. + const seen = stub({ families: ['body'], wanted: [{ item_id: 3922, hue: 0 }] }) + t.after(restore) + + const result = await model.warm() + + assert.equal(result.status, 'unavailable') + assert.equal(result.code, 'UNSUPPORTED') + assert.match(result.reason, /does not serve item art/) + assert.equal(seen.calls, 0, 'nothing should have been asked of the shard') +}) + +test('no shard link is skipped, not failed', async (t) => { + stub({ linked: false }) + t.after(restore) + + assert.equal((await model.warm()).status, 'skipped') +}) + +test('a host that cannot render images is the named NO_IMAGING state', async (t) => { + stub() + t.after(restore) + + bridge.sourceFingerprint = async () => ({ + files: {}, + extractorVersion: 2, + hashing: false, + complete: true, + imaging: { ok: false, reason: 'libgdiplus is not installed' }, + families: ['body', 'static'], + }) + + const result = await model.warm() + + assert.equal(result.status, 'unavailable') + assert.equal(result.code, 'NO_IMAGING') +}) + +// ── what gets asked for ──────────────────────────────────────────────────── + +test('only the keys we do not already hold under the shard’s current catalogue are fetched', async (t) => { + useTempUploads(t) + + const seen = stub({ + wanted: [ + { item_id: 3922, hue: 0 }, + { item_id: 597, hue: 33 }, + { item_id: 1, hue: 0 }, + ], + // 3922 is held and current; the other two are not. + fresh: new Set(['static/3922']), + fetched: { + assets: new Map([ + ['static/597/h33', picture('aaa')], + ['static/1', picture('bbb')], + ]), + missing: { absent: 0, unsupported: 0 }, + pages: 1, + catalog: 'cat-current', + }, + }) + t.after(restore) + + const result = await model.warm() + + assert.equal(result.status, 'imported') + + // The first call is the catalogue probe; the second is the real fetch. + const asked = seen.asked[seen.asked.length - 1] + + assert.deepEqual(asked.sort(), ['static/1', 'static/597/h33']) + assert.equal( + seen.freshAsked.catalog, + 'cat-current', + 'staleness must be asked against the catalogue the shard answers under right now, ' + + 'or a client patch never invalidates anything', + ) +}) + +test('every stored row records the catalogue it was fetched under', async (t) => { + useTempUploads(t) + + const seen = stub({ + wanted: [{ item_id: 1, hue: 0 }], + fetched: { + assets: new Map([['static/1', picture('bbb')]]), + missing: { absent: 0, unsupported: 0 }, + pages: 1, + catalog: 'cat-after-patch', + }, + }) + t.after(restore) + + await model.warm() + + // Without this field there is no way to answer "is this picture out of date?" + // for a family that has no manifest — which is the entire §7 story on this side. + assert.equal(seen.saved.rows.length, 1) + assert.equal(seen.saved.rows[0].catalog, 'cat-after-patch') + assert.equal(seen.saved.rows[0].family, 'static') +}) + +test('the body catalogue’s meta singleton is never written by a warm pass', async (t) => { + useTempUploads(t) + + const seen = stub({ + wanted: [{ item_id: 1, hue: 0 }], + fetched: { + assets: new Map([['static/1', picture('bbb')]]), + missing: { absent: 0, unsupported: 0 }, + pages: 1, + catalog: 'cat-current', + }, + }) + t.after(restore) + + await model.warm() + + // `shard_asset_meta` is what an Update compares a BODY manifest against. A + // warm pass writing there would tell the body import that a client it never + // looked at is unchanged, and the creature catalogue would stop updating. + assert.equal(seen.saved.meta, null) +}) + +test('a key the shard has no art for produces no row, so it can be asked again', async (t) => { + useTempUploads(t) + + const seen = stub({ + wanted: [ + { item_id: 1, hue: 0 }, + { item_id: 60000, hue: 0 }, + ], + fetched: { + assets: new Map([['static/1', picture('bbb')]]), + missing: { absent: 1, unsupported: 0 }, + pages: 1, + catalog: 'cat-current', + }, + }) + t.after(restore) + + const result = await model.warm() + + assert.equal(result.absent, 1) + assert.deepEqual( + seen.saved.rows.map((r) => r.key), + ['static/1'], + 'an empty row would make the key held, and it would never be asked again — ' + + 'including after the operator patches in the graphic that was missing', + ) +}) + +test('a pass is bounded, and says how much it left behind', async (t) => { + useTempUploads(t) + + const wanted = [] + for (let i = 1; i <= 10; i++) wanted.push({ item_id: i, hue: 0 }) + + const seen = stub({ + wanted, + fetched: { + assets: new Map([['static/1', picture('bbb')]]), + missing: { absent: 0, unsupported: 0 }, + pages: 1, + catalog: 'cat-current', + }, + }) + t.after(restore) + + const result = await model.warm({ limit: 4 }) + + assert.equal(seen.asked[seen.asked.length - 1].length, 4) + assert.equal(result.asked, 4) + assert.equal(result.remaining, 6) +}) + +test('a picture whose bytes changed replaces its file instead of shadowing it', async (t) => { + const dir = useTempUploads(t) + + const old = model.fileNameFor('static/1', 'old00000') + fs.mkdirSync(model.artDir(), { recursive: true }) + fs.writeFileSync(path.join(model.artDir(), old), 'stale') + + stub({ + wanted: [{ item_id: 1, hue: 0 }], + files: new Map([['static/1', old]]), + fetched: { + assets: new Map([['static/1', picture('new00000')]]), + missing: { absent: 0, unsupported: 0 }, + pages: 1, + catalog: 'cat-after-patch', + }, + }) + t.after(restore) + + await model.warm() + + const names = fs.readdirSync(path.join(dir, model.ART_SUBDIR)) + + // Content-addressed names mean a changed picture is a changed URL, so nothing + // keeps serving last client's sprite from a cache — and the superseded file is + // removed rather than left to accumulate one per client patch forever. + assert.deepEqual(names, [model.fileNameFor('static/1', 'new00000')]) +}) + +// ── serving ──────────────────────────────────────────────────────────────── + +test('decorate attaches a filename, never a URL, and null where there is none', async (t) => { + stub({ files: new Map([['static/3922', 'uo-static-3922-abcd1234.png']]) }) + t.after(restore) + + const rows = [ + { itemId: 3922, hue: 0 }, + { itemId: 597, hue: 33 }, + ] + + await model.decorate(rows) + + // A filename, because the client is what knows where uploads are mounted — + // the same contract `shard_spawn_creatures.art` already uses. + assert.equal(rows[0].art, 'uo-static-3922-abcd1234.png') + assert.equal(rows[1].art, null) +}) + +test('decorate never throws a page away over a picture', async (t) => { + stub() + t.after(restore) + + db.filesForKeys = async () => { + throw new Error('the database is on fire') + } + + const rows = [{ itemId: 3922, hue: 0 }] + + await model.decorate(rows) + + assert.deepEqual(rows, [{ itemId: 3922, hue: 0 }], 'the row is returned unchanged, not lost') +}) + +test('what a page asked for is remembered, including what it could not show', async (t) => { + stub({ files: new Map() }) + t.after(restore) + + const before = model.noticedCount() + + await model.decorate([{ itemId: 12345, hue: 7 }]) + + // The character sheet is fetched live from the shard and stored nowhere, so + // nothing on disk would ever name this key. Noticing it here is the only reason + // a warm pass can find it. + assert.ok(model.noticedCount() > before) + assert.ok((await model.wantedKeys()).includes('static/12345/h7')) +}) diff --git a/server/utils/assetBridge.js b/server/utils/assetBridge.js index 432bf25..d295154 100644 --- a/server/utils/assetBridge.js +++ b/server/utils/assetBridge.js @@ -238,6 +238,13 @@ async function sourceFingerprint() { hashing: Boolean(res.data?.hashing), complete: Boolean(res.data?.complete), imaging: res.data?.imaging ?? null, + // Which §5 key families this overlay can be asked for (phase 5). Absent on a + // phase-3 or phase-4 overlay, which served bodies and nothing else — so the + // fallback is `['body']` rather than `[]`: an older shard is not a shard with + // no assets, and treating it as one would turn a working bestiary off. + families: Array.isArray(res.data?.families) && res.data.families.length > 0 + ? res.data.families.map(String) + : [FAMILY], } } @@ -382,10 +389,16 @@ async function fetchAssets({ keys, catalog } = {}) { const list = Array.isArray(keys) ? keys.filter((k) => typeof k === 'string' && k !== '') : [] - if (list.length === 0) return { assets: out, missing, pages: 0 } + if (list.length === 0) return { assets: out, missing, pages: 0, catalog: catalog ?? null } let pages = 0 + // The catalogue the shard actually answered under. The body import already knows + // it from the manifest, but the on-demand families have no manifest to learn it + // from (§11) — so it is read back off the reply and stored with the rows, which + // is what makes a later "is this stale?" answerable per key. + let answered = catalog ?? null + for (let i = 0; i < list.length; i += FETCH_CHUNK) { const chunk = list.slice(i, i + FETCH_CHUNK) @@ -401,6 +414,20 @@ async function fetchAssets({ keys, catalog } = {}) { pages++ walked++ + if (typeof page.catalog === 'string' && page.catalog !== '') { + if (answered !== null && page.catalog !== answered) { + // Two pages of one walk describing two different clients. The shard + // refuses this when it is told what to expect; when it was not told — + // the first fetch of a warm pass — this is where it is caught. + throw new AssetBridgeError( + `The shard's client files changed mid-fetch (catalog ${answered} became ${page.catalog})`, + 'UNAVAILABLE', + ) + } + + answered = page.catalog + } + for (const row of page.rows ?? []) { const key = String(row?.key ?? '') if (key === '') continue @@ -423,6 +450,11 @@ async function fetchAssets({ keys, catalog } = {}) { height: Number(row.height) || 0, body: Number.isFinite(Number(row.body)) ? Number(row.body) : null, direction: Number.isFinite(Number(row.direction)) ? Number(row.direction) : null, + // Phase 5's art families carry these; the body catalogue does not, and a + // consumer that wants neither is unaffected by either. + hue: Number.isFinite(Number(row.hue)) ? Number(row.hue) : null, + partialHue: typeof row.partialHue === 'boolean' ? row.partialHue : null, + source: typeof row.source === 'string' ? row.source : null, png: Buffer.from(row.png, 'base64'), }) } @@ -446,6 +478,7 @@ async function fetchAssets({ keys, catalog } = {}) { } log.info('asset content fetched from the shard', { + catalog: answered, asked: list.length, got: out.size, absent: missing.absent, @@ -454,7 +487,7 @@ async function fetchAssets({ keys, catalog } = {}) { ms: Date.now() - started, }) - return { assets: out, missing, pages } + return { assets: out, missing, pages, catalog: answered } } /** diff --git a/swagger-fragment.json b/swagger-fragment.json index 4049d38..318faa7 100644 --- a/swagger-fragment.json +++ b/swagger-fragment.json @@ -204,6 +204,68 @@ } } }, + "/api/v1/admin/shard/assets/warm": { + "post": { + "tags": [ + "Admin · Shard" + ], + "summary": "Fetch item and land artwork the site is missing, now (admin only)", + "description": "Runs one pass of the item-art warm loop instead of waiting for its timer. The pass works out which item pictures this site", + "responses": { + "200": { + "description": "What the pass did", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UoItemArtWarmResult" + } + } + } + }, + "403": { + "description": "Admin role required", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "500": { + "description": "Internal Server Error" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "force": { + "type": "boolean", + "description": "Re-fetch pictures this site already holds." + }, + "limit": { + "type": "integer", + "description": "How many keys this pass may fetch (1-2000)." + } + } + } + } + } + } + } + }, "/api/v1/admin/shard/atlas": { "get": { "tags": [ @@ -8477,6 +8539,40 @@ "example": true } } + }, + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "Item pictures held. Unlike the catalogue this has no total to compare against: item art is fetched because something on the site names it, so this is the working set rather than a fraction of one." + }, + "example": { + "type": "number", + "example": 1840 + } + } + }, + "land": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "Land tile pictures held. Zero on every install until something asks for one." + }, + "example": { + "type": "number", + "example": 0 + } + } } } } @@ -8625,6 +8721,39 @@ } } } + }, + "families": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "array" + }, + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + } + } + }, + "description": { + "type": "string", + "example": "Which asset key families this shard’s plugin overlay serves. An overlay older than phase 5 answers `[\"body\"]` only — it has the creature catalogue and no item art." + }, + "example": { + "type": "array", + "example": [ + "body", + "land", + "static" + ], + "items": { + "type": "string" + } + } + } } } } @@ -9321,6 +9450,293 @@ } } }, + "UoItemArtWarmResult": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "Outcome of one item-art warm pass (docs/link/v8.md §11, phase 5). Unlike the body catalogue there is no manifest and no set: the client addresses 49,152 item graphics times three thousand hues, so what gets fetched is defined by what this site’s own rows name — every distinct (ItemID, hue) on a player vendor, plus anything a character sheet has shown since the last pass. Reported rather than thrown, so a shard that is down is an answer and not a 500." + }, + "properties": { + "type": "object", + "properties": { + "status": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "enum": { + "type": "array", + "example": [ + "skipped", + "unavailable", + "unchanged", + "imported", + "failed" + ], + "items": { + "type": "string" + } + }, + "description": { + "type": "string", + "example": "`skipped`: no shard is configured. `unchanged`: every wanted picture is already held and current. `unavailable`: the shard could not be asked, or its plugin overlay is too old to serve item art." + }, + "example": { + "type": "string", + "example": "imported" + } + } + }, + "reason": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "code": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Machine-readable cause. `NO_IMAGING` is a shard host with no libgdiplus. `UNSUPPORTED` is a plugin overlay that serves the creature catalogue but not item art — update the overlay." + }, + "enum": { + "type": "array", + "example": [ + "DISABLED", + "NO_SOURCE", + "SHARD_DOWN", + "PROTOCOL", + "BUSY", + "NO_IMAGING", + "UNSUPPORTED", + "INCOMPLETE", + "STUCK", + "MALFORMED", + "TOO_LARGE", + "UNAVAILABLE" + ], + "items": { + "type": "string" + } + } + } + }, + "catalog": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The shard’s art catalogue id these pictures were fetched under — a hash of the files that decide their bytes. Stored per row, which is how staleness is answered without a manifest." + }, + "example": { + "type": "string", + "example": "7c1e04b9aa2f3d58" + } + } + }, + "wanted": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Distinct keys this site’s rows name right now." + }, + "example": { + "type": "number", + "example": 1840 + } + } + }, + "held": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "How many of those are already stored and current." + }, + "example": { + "type": "number", + "example": 1440 + } + } + }, + "asked": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "How many this pass actually requested. Bounded by `limit`." + }, + "example": { + "type": "number", + "example": 400 + } + } + }, + "fetched": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "How many the shard returned a picture for." + }, + "example": { + "type": "number", + "example": 396 + } + } + }, + "written": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "How many were written to disk." + }, + "example": { + "type": "number", + "example": 396 + } + } + }, + "absent": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Keys the shard has no art for. NOT a failure — 9,963 of this client’s static ids have an empty index entry, and an item using one simply has no picture." + }, + "example": { + "type": "number", + "example": 4 + } + } + }, + "unsupported": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Keys the shard does not serve at all. A bug on the site’s side rather than a gap in the client." + }, + "example": { + "type": "number", + "example": 0 + } + } + }, + "remaining": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Wanted keys left for the next pass. Passes repeat on a timer, so a backlog drains without an operator." + }, + "example": { + "type": "number", + "example": 0 + } + } + } + } + } + } + }, "UoShardLinkRequest": { "type": "object", "properties": {