From a194ec68e0259071f2d804d88c409e1ce8153798 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Thu, 10 Sep 2026 18:40:46 -0500 Subject: [PATCH] feat(assets): creature artwork from the shard's own client (Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now the only way a creature got a picture on this site was for an operator to open UOFiddler on a desktop, export sprites by hand, copy them to the web host and write a spawnAtlas.art.json naming each one. Almost nobody did, so shard_spawn_creatures.art was NULL on every install. The shard has had those files the whole time. Admin -> Shard -> Import now walks its asset manifest, fetches only the sprites whose hash changed, writes them under uploads/atlas/, asks the shard for a body id per atlas creature (§8: it CONSTRUCTS the creature and reads Body.BodyID, which is the only thing that is right for a shard's own custom creatures) and points each creature at its picture. On a stock client that is 787 portraits, about a megabyte. **The one thing v8.md §12 got wrong, and it is not cosmetic.** It says `shard_spawn_creatures.art` "starts being filled by the import". That table is emptied and refilled by replaceAtlas on EVERY atlas refresh, and a refresh runs on every boot -- so a filename stored there would be destroyed by an ordinary re-parse of the ServUO tree, with the next Update finding the client files unchanged, reporting "nothing to do", and never restoring it. Nothing would report a fault; the pictures would just be gone. So the assets and the body map live in their own tables outside that blast radius, and applyAtlas re-derives `art` on the way past as `{ ...derived, ...operatorMap }` -- which is also the one place "the operator's own artwork wins" is enforced, on every rebuild rather than only at import. Smaller decisions worth not rediscovering: - The derivation joins on the catalogue KEY, not on the body id. The simpler join is correct today and stops being correct the moment phase 6 adds body/400/a2/f0, at which point one slug matches dozens of rows. - Filenames are content-addressed. A stable name overwritten in place leaves every browser and CDN serving the previous client's sprite, with the database row perfectly correct. - An unchanged key whose FILE is missing is fetched again. The row and the disk can disagree (a wiped uploads volume, a restore from a dump), and a broken image on a creature page is worse than one re-fetched sprite. - A key the shard cannot render is not a failure. Two thirds of the playable ghost and gargoyle bodies have no art on a stock client, and an import that reported eight failures every time would teach an operator to ignore the panel. - A key that VANISHED from the manifest needs review before anything changes: an unmounted client volume and a deliberate downgrade look identical here. 23 new tests; 674 server and 42 client tests pass. The SQL was also run against a real MariaDB, which is what proved the CONCAT join and the singleton CHECK. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016wDDVXWMDz82WqE1i969r4 --- client/src/routes/public/Atlas.jsx | 41 + client/src/routes/public/AtlasCreature.jsx | 22 +- routes.manifest.json | 10 + server/db/purge.sql | 13 + server/db/schema.sql | 69 ++ server/model/shardAssets/shardAssets.db.js | 223 ++++ server/model/shardAssets/shardAssets.model.js | 507 ++++++++ server/model/shardAtlas/shardAtlas.db.js | 56 + server/model/shardAtlas/shardAtlas.model.js | 39 +- server/router/admin/shard.router.js | 37 + server/router/admin/shardAssets.controller.js | 91 ++ server/swagger/doc.js | 130 ++ server/test/assetBridge.test.js | 368 ++++++ server/test/shardAssets.model.test.js | 334 +++++ server/utils/assetBridge.js | 539 ++++++++ server/utils/uoLinkClient.js | 40 + swagger-fragment.json | 1092 +++++++++++++++++ 17 files changed, 3605 insertions(+), 6 deletions(-) create mode 100644 server/model/shardAssets/shardAssets.db.js create mode 100644 server/model/shardAssets/shardAssets.model.js create mode 100644 server/router/admin/shardAssets.controller.js create mode 100644 server/test/assetBridge.test.js create mode 100644 server/test/shardAssets.model.test.js create mode 100644 server/utils/assetBridge.js diff --git a/client/src/routes/public/Atlas.jsx b/client/src/routes/public/Atlas.jsx index dd4f1e0..4c13f9e 100644 --- a/client/src/routes/public/Atlas.jsx +++ b/client/src/routes/public/Atlas.jsx @@ -45,6 +45,46 @@ function Chip({ active, onClick, children }) { ) } +// One creature's portrait, when there is one. +// +// `art` is a FILENAME under uploads/atlas/, never a path or a URL: it is either a +// sprite the shard extracted from the operator's own UO client (docs/link/v8.md +// §12) or a picture the operator drew and named in `spawnAtlas.art.json`, and the +// two are indistinguishable here on purpose. +// +// **NULL is the ordinary case and always will be.** An install with no shard link +// has never imported one; a shard whose host cannot render images has none; and +// even on a complete import, two thirds of the playable ghost and gargoyle bodies +// have no art in the client at all (§5.2). So this renders nothing rather than a +// placeholder, and every layout around it is written to sit correctly with the +// picture absent — which is the state the whole atlas was designed in. +// +// Sprites are small (a couple of dozen pixels square) and UO's art is pixel art, +// so `imageRendering: 'pixelated'` matters: a browser's default smoothing turns a +// 24×63 wolf into a smear at any size above its own. +export function CreaturePortrait({ art, name, size = 40 }) { + if (!art) return null + + return ( + + ) +} + function CreatureCard({ creature }) { const facets = Object.entries(creature.facets || {}).sort((a, b) => b[1] - a[1]) return ( @@ -60,6 +100,7 @@ function CreatureCard({ creature }) { color: 'inherit', }} > +
- + {/* The portrait sits BESIDE the header rather than inside it: `art` + is NULL for most creatures on most installs — no shard link, a + host that cannot render images, or simply a body this client has + no art for — and a header component that had to lay out around an + absent picture would be carrying that case forever. Here the row + collapses to exactly the header, which is what it was before. */} +
+ +
+ +
+
[ + r.key, + r.family || 'body', + r.sha256, + r.bytes ?? 0, + r.width ?? 0, + r.height ?? 0, + r.body ?? null, + r.direction ?? null, + r.file ?? null, + ]) + + await batched( + conn, + 'INSERT INTO shard_assets (asset_key, family, sha256, bytes, width, height, body, direction, file) ' + + 'VALUES (?,?,?,?,?,?,?,?,?) ' + + 'ON DUPLICATE KEY UPDATE family = VALUES(family), sha256 = VALUES(sha256), ' + + 'bytes = VALUES(bytes), width = VALUES(width), height = VALUES(height), ' + + 'body = VALUES(body), direction = VALUES(direction), file = VALUES(file), ' + + 'imported_at = CURRENT_TIMESTAMP', + values, + ) + + if (meta) { + await conn.query( + 'INSERT INTO shard_asset_meta (id, payload) VALUES (1, ?) ' + + 'ON DUPLICATE KEY UPDATE payload = VALUES(payload), imported_at = CURRENT_TIMESTAMP', + [JSON.stringify(meta)], + ) + } + + await conn.commit() + + return values.length + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +async function getMeta() { + const rows = await query('SELECT payload, imported_at FROM shard_asset_meta WHERE id = 1') + if (rows.length === 0) return null + const payload = typeof rows[0].payload === 'string' ? JSON.parse(rows[0].payload) : rows[0].payload + return { ...payload, importedAt: rows[0].imported_at } +} + +async function countAssets() { + const rows = await query( + 'SELECT COUNT(*) AS n, SUM(file IS NOT NULL) AS stored FROM shard_assets', + ) + return { total: Number(rows[0]?.n) || 0, stored: Number(rows[0]?.stored) || 0 } +} + +// ── the body resolution side (§8) ────────────────────────────────────────── + +/** + * Replace the whole slug → body map. + * + * This one IS a replace, and for the opposite reason to the assets above: it is + * derived from the atlas's creature list, so a slug that has left the atlas has + * no meaning any more and keeping its row would leave the map growing forever + * across map changes. The pass that produces it is cheap to redo — a shard round + * trip, no files — which is what makes replacing safe here and not there. + */ +async function replaceBodies(rows) { + const conn = await core.pool.getConnection() + + try { + await conn.beginTransaction() + await conn.query('DELETE FROM shard_creature_bodies') + + const values = rows.map((r) => [r.slug, r.typeName, r.body ?? null, r.status || 'ok']) + + await batched( + conn, + 'INSERT INTO shard_creature_bodies (slug, type_name, body, status) VALUES (?,?,?,?)', + values, + ) + + await conn.commit() + + return values.length + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + +async function allBodies() { + return query( + 'SELECT slug, type_name, body, status, resolved_at FROM shard_creature_bodies ORDER BY slug', + ) +} + +async function countBodies() { + const rows = await query( + "SELECT COUNT(*) AS n, SUM(status = 'ok') AS resolved FROM shard_creature_bodies", + ) + return { total: Number(rows[0]?.n) || 0, resolved: Number(rows[0]?.resolved) || 0 } +} + +/** + * The derivation `replaceAtlas` applies on the way past: slug → uploaded filename. + * + * One join rather than two reads, because it runs inside the atlas transaction — + * the atlas rows are being inserted at that moment and every extra round trip is + * time the site's creature list does not exist. + * + * Rows with no body, no asset or an asset whose bytes were never fetched are + * simply absent from the result, which is what leaves `art` NULL. That is a + * first-class state everywhere it is consumed and the expected one for two thirds + * of the player bodies (§5.2). + * + * **The join is pinned to the catalogue key, not merely to the body id.** Today + * one body has exactly one asset, so `a.body = b.body` alone would be correct — + * and it would stop being correct the moment phase 6 adds `body/400/a2/f0`, at + * which point one slug would match dozens of rows and whichever the engine + * returned last would become the portrait. Naming the key here means that phase + * adds rows without changing what a creature page shows. + */ +async function artBySlug() { + const rows = await query( + 'SELECT b.slug, a.file FROM shard_creature_bodies b ' + + "JOIN shard_assets a ON a.asset_key = CONCAT('body/', b.body, '/a0') " + + "WHERE b.status = 'ok' AND b.body IS NOT NULL AND a.file IS NOT NULL", + ) + + const map = {} + + for (const row of rows) map[row.slug] = row.file + + return map +} + +module.exports = { + allAssets, + saveAssets, + getMeta, + countAssets, + replaceBodies, + allBodies, + countBodies, + artBySlug, +} diff --git a/server/model/shardAssets/shardAssets.model.js b/server/model/shardAssets/shardAssets.model.js new file mode 100644 index 0000000..d1eb6a0 --- /dev/null +++ b/server/model/shardAssets/shardAssets.model.js @@ -0,0 +1,507 @@ +const fs = require('fs') +const path = require('path') + +const db = require('./shardAssets.db') +const atlasDb = require('../shardAtlas/shardAtlas.db') +const core = require('../../core') +const bridge = require('../../utils/assetBridge') +const uoLinkConfig = require('../uoLinkConfig/uoLinkConfig.model') +const log = require('../../core').logger('shardAssets') + +// Client artwork, over the bridge (docs/link/v8.md — protocol 8, phase 3). +// +// What this replaces: until now the only way a creature got a picture on this +// site was for an operator to open UOFiddler on a desktop, export sprites by +// hand, copy them to the web host and write a `spawnAtlas.art.json` naming each +// one. Almost nobody did, so `shard_spawn_creatures.art` was NULL on every +// install and the atlas rendered as text. +// +// The shard has had those files the whole time — a ServUO server cannot boot +// without a UO client — so as of protocol 8 it decodes them itself and hands the +// pictures over the same request/reply path as every other shard read. +// +// ── Two passes, and they answer different questions ─────────────────── +// +// **The catalogue** (§4.8, §11) is one thumbnail per creature body: the shard +// walks bodies 0–2047, validates each index entry, decodes the ones that are real +// and hands back `{ key, sha256 }` first and the PNG second. On a stock client +// that is **787 sprites**, not the 1,144 the decoder claims — see below. +// +// **Body resolution** (§8) is the join. The atlas knows a creature by the class +// name in `Spawns/*.xml`; the client knows it by a body id; nothing in the ServUO +// tree declares the mapping. Only code inside ServUO can answer it, by +// constructing the creature and reading `Body.BodyID`, and that is the whole +// reason this could not be done off the shard. +// +// ── The 357, and why nothing here trusts a success ──────────────────── +// +// 357 of the bodies ServUO's decoder returns a bitmap for **have no art**. Their +// index entry reads `length 0`, the library's stream buffer still holds the +// previous creature, and what comes back is whichever body was decoded before — +// a real, plausible, correctly-sized picture of the wrong animal. The shard now +// validates every index entry before it decodes, which is what cut the catalogue +// from 1,144 to 787, and the count going down is the point. +// +// The consequence for this file is a rule: **a missing asset is a normal +// outcome, never an error.** Two thirds of the player bodies have no art on a +// stock client (§5.2), so an import that reported eight failures every time would +// teach an operator to ignore the panel. +// +// ── Where the pictures go, and what still wins ──────────────────────── +// +// Into `/atlas/`, through the same door the operator's own artwork uses, +// and `shard_spawn_creatures.art` is DERIVED from them rather than written by +// them. **The operator's `spawnAtlas.art.json` still wins outright**: someone who +// has drawn their own creature portraits must not have them replaced by a sprite +// rip on the next Update. +// +// ── Why the resolution does not live on the atlas row ───────────────── +// +// `shard_spawn_creatures` is emptied and refilled on every atlas refresh. A body +// id or a filename stored there would be destroyed by an ordinary re-parse of the +// ServUO tree, and the next asset Update would find the client files unchanged, +// report "nothing to do" and never restore it. So both live in their own tables +// and the atlas import reads them on the way past. + +/** Where imported sprites land, under core's upload directory. */ +const ART_SUBDIR = 'atlas' + +// ── configuration ────────────────────────────────────────────────────────── + +/** + * Is there a shard to ask? + * + * Both halves matter, exactly as in `shardClilocs.model`: a `baseUrl` on a + * disabled config is an install that was set up and then switched off, and + * calling it would spend a 12 s timeout to learn what the row already says. + */ +async function shardLinked() { + try { + const config = await uoLinkConfig.getSafe() + return Boolean(config?.enabled && config?.baseUrl) + } catch { + return false + } +} + +function artDir() { + return path.join(core.uploads.UPLOAD_DIR, ART_SUBDIR) +} + +/** + * The operator's own art map, which wins over anything imported. + * + * Read through the atlas model rather than re-implemented, so there is one + * definition of where that file lives and what an absent one means. + */ +function operatorArt() { + // eslint-disable-next-line global-require + return require('../shardAtlas/shardAtlas.model').loadArtMap() +} + +// ── writing a sprite ─────────────────────────────────────────────────────── + +/** + * The filename one asset gets on disk. + * + * **Content-addressed on purpose.** A stable name per key (`uo-body-34.png`) + * would be overwritten in place by an Update, and every browser and CDN that had + * already cached it would keep serving last month's client's sprite — with + * nothing anywhere to notice, because the database row would be correct. Putting + * eight bytes of the hash in the name makes a changed sprite a changed URL. + * + * The old file is removed when a key's hash moves, so the directory tracks the + * catalogue rather than accumulating one file per import forever. + */ +function fileNameFor(key, sha256) { + const stem = key.replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '') + return `uo-${stem}-${String(sha256).slice(0, 8)}.png` +} + +/** + * Write one sprite and return its filename, or null if it could not be written. + * + * Never throws. A full disk or a read-only volume must degrade to "this creature + * has no picture" — which the whole site already renders correctly, because it is + * the state every install was in until this phase — rather than failing an import + * that has already fetched hundreds of others. + */ +function writeSprite(key, sha256, png) { + const name = fileNameFor(key, sha256) + + try { + fs.mkdirSync(artDir(), { recursive: true }) + fs.writeFileSync(path.join(artDir(), name), png) + return name + } catch (err) { + log.warn('could not write an imported sprite', { key, error: err.message }) + return null + } +} + +/** Best-effort removal of a sprite a key no longer points at. */ +function removeSprite(name) { + if (!name) return + + try { + fs.unlinkSync(path.join(artDir(), name)) + } catch { + // Already gone, or never written. Either way there is nothing to do, and an + // import must not fail because a file it was tidying up was tidied already. + } +} + +// ── the import ───────────────────────────────────────────────────────────── + +/** + * Import (or update) the body catalogue and the slug → body map. + * + * Returns a result rather than throwing, so a controller can render it and an + * operator can read it: + * + * `skipped` no shard configured — the file era had no equivalent here + * `unavailable` the shard could not answer (down, plane off, no libgdiplus) + * `unchanged` the client files match what was imported; nothing fetched + * `imported` fetched and applied + * `needsReview` a key we hold has vanished from the shard's manifest + * `failed` something went wrong mid-import + * + * `force` re-imports even when the client files are unchanged (which is also how + * an operator recovers from a deleted uploads directory — the database still + * holds the hashes, but the files behind them are gone). `approve` accepts a + * catalogue that no longer offers keys we hold. + */ +async function importAssets({ force = false, approve = false } = {}) { + if (!(await shardLinked())) { + return { + status: 'skipped', + reason: 'uo-link is not configured, so there is no shard to read client files from', + } + } + + let sources + + try { + sources = await bridge.sourceFingerprint() + } catch (err) { + return failure(err, 'client file manifest') + } + + // §4.4: a Linux shard host without libgdiplus cannot render a sprite at all. + // It is reported on the source gate precisely so an operator meets it while + // setting the shard up rather than from an empty bestiary weeks later. + if (sources.imaging && sources.imaging.ok === false) { + return { + status: 'unavailable', + code: 'NO_IMAGING', + reason: sources.imaging.reason || 'the shard host cannot render images', + } + } + + const meta = await db.getMeta().catch(() => null) + + if (!force && bridge.sameSources(sources, meta?.sources)) { + const counts = await db.countAssets() + const bodies = await db.countBodies() + + return { + status: 'unchanged', + assets: counts.total, + stored: counts.stored, + bodies: bodies.resolved, + hashing: sources.hashing, + importedAt: meta?.importedAt ?? null, + } + } + + let manifest + + try { + manifest = await bridge.readManifest({ family: bridge.FAMILY }) + } catch (err) { + return failure(err, 'asset manifest') + } + + const held = await db.allAssets() + const offered = new Set(manifest.rows.map((r) => r.key)) + + // A key we hold that the shard no longer offers. An unmounted client volume and + // a deliberate downgrade look identical from here, and the wrong guess deletes + // artwork, so it is staged rather than applied — the same rule, and the same + // reasoning, as a vanished cliloc overlay or a disappearing atlas facet. + const vanished = [...held.keys()].filter((key) => !offered.has(key)) + + if (vanished.length > 0 && !approve) { + return { + status: 'needsReview', + reason: + `${vanished.length} asset(s) this site holds are no longer offered by the shard; ` + + 'nothing was changed', + vanished: vanished.slice(0, 50), + vanishedCount: vanished.length, + } + } + + // The diff, and the whole reason stage 2 carries hashes and not pixels. An + // unchanged key is skipped ONLY if its file is actually still on disk: the row + // and the file can disagree (a wiped uploads volume, a restore from a database + // dump), and re-fetching a sprite is far cheaper than a creature page with a + // broken image on it. + const wanted = manifest.rows.filter((row) => { + const existing = held.get(row.key) + if (!existing || existing.sha256 !== row.sha256) return true + if (!existing.file) return true + return !fs.existsSync(path.join(artDir(), existing.file)) + }) + + let fetched = { assets: new Map(), missing: { absent: 0, unsupported: 0 } } + + if (wanted.length > 0) { + try { + fetched = await bridge.fetchAssets({ + keys: wanted.map((r) => r.key), + catalog: manifest.catalog, + }) + } catch (err) { + return failure(err, 'asset content') + } + } + + const rows = [] + let written = 0 + + for (const row of manifest.rows) { + const existing = held.get(row.key) + const got = fetched.assets.get(row.key) + + if (!got) { + // Either it was unchanged and skipped, or the shard could not serve it. The + // row is kept either way, with whatever file it already had — a key the + // shard suddenly cannot render must not lose the picture we already hold. + rows.push({ ...row, file: existing?.file ?? null }) + continue + } + + const name = writeSprite(row.key, got.sha256, got.png) + + if (name) { + written++ + if (existing?.file && existing.file !== name) removeSprite(existing.file) + } + + rows.push({ + ...row, + sha256: got.sha256 || row.sha256, + bytes: got.bytes || row.bytes, + width: got.width || row.width, + height: got.height || row.height, + body: got.body ?? row.body, + direction: got.direction ?? row.direction, + file: name ?? existing?.file ?? null, + }) + } + + const removed = [] + + if (vanished.length > 0) { + for (const key of vanished) { + removeSprite(held.get(key)?.file) + removed.push(key) + } + } + + try { + await db.saveAssets(rows, { + catalog: manifest.catalog, + extractorVersion: manifest.extractorVersion, + family: bridge.FAMILY, + playerBodies: manifest.playerBodies, + sources: { files: sources.files, extractorVersion: sources.extractorVersion }, + count: rows.length, + }) + } catch (err) { + return { status: 'failed', reason: err.message } + } + + const bodies = await resolveAtlasBodies() + const art = await applyArt() + + log.info('asset import applied', { + assets: rows.length, + fetched: fetched.assets.size, + written, + absent: fetched.missing.absent, + bodies: bodies.resolved, + art: art.applied, + }) + + return { + status: 'imported', + catalog: manifest.catalog, + extractorVersion: manifest.extractorVersion, + assets: rows.length, + fetched: fetched.assets.size, + written, + absent: fetched.missing.absent, + unsupported: fetched.missing.unsupported, + removed: removed.length, + scanned: manifest.scanned, + pages: manifest.pages, + playerBodies: manifest.playerBodies, + bodies, + art, + } +} + +function failure(err, what) { + if (err instanceof bridge.AssetBridgeError) { + return { status: 'unavailable', code: err.code, reason: err.message } + } + + log.warn(`asset import failed reading the ${what}`, { error: err.message }) + + return { status: 'failed', reason: err.message } +} + +// ── the body pass (§8) ───────────────────────────────────────────────────── + +/** + * Ask the shard for a body id for every creature the atlas knows. + * + * `shard_spawn_creatures.name` is the ServUO class name — the atlas build picks + * the winning spelling of the spawn TYPE token rather than inventing a display + * name — so this needs no new column to ask its question. + * + * Never throws: a shard that goes down between the asset fetch and this pass + * leaves the assets imported and the map as it was, which is a strictly better + * state than failing the whole import back to nothing. + */ +async function resolveAtlasBodies() { + let creatures = [] + + try { + creatures = await atlasDb.allCreatureTypes() + } catch (err) { + return { resolved: 0, asked: 0, reason: err.message } + } + + if (creatures.length === 0) { + return { resolved: 0, asked: 0, reason: 'the spawn atlas has no creatures loaded' } + } + + let rows + + try { + rows = await bridge.resolveBodies({ creatures }) + } catch (err) { + return { resolved: 0, asked: creatures.length, reason: err.message } + } + + try { + await db.replaceBodies(rows) + } catch (err) { + return { resolved: 0, asked: creatures.length, reason: err.message } + } + + const tally = { ok: 0, unknown: 0, notCreature: 0, failed: 0 } + + for (const row of rows) { + if (tally[row.status] === undefined) tally.failed++ + else tally[row.status]++ + } + + return { asked: creatures.length, answered: rows.length, resolved: tally.ok, tally } +} + +// ── the derivation (§12) ─────────────────────────────────────────────────── + +/** + * Point every atlas creature at its imported portrait. + * + * Two rules, and the second is the one worth stating: + * + * 1. The operator's `spawnAtlas.art.json` wins. Someone who drew their own + * creature portraits must not have them replaced by a sprite rip. + * 2. A slug with neither is set back to NULL rather than left alone. A creature + * whose body stopped resolving — the operator removed a script package, say + * — would otherwise keep pointing at a file that is about to be deleted, and + * a broken image is worse than no image. + */ +async function applyArt() { + const derived = await db.artBySlug() + const operator = operatorArt() + + const map = { ...derived, ...operator } + + try { + const applied = await atlasDb.setCreatureArt(map) + return { applied, derived: Object.keys(derived).length, operator: Object.keys(operator).length } + } catch (err) { + log.warn('could not apply imported creature art', { error: err.message }) + return { applied: 0, derived: Object.keys(derived).length, error: err.message } + } +} + +// ── status ───────────────────────────────────────────────────────────────── + +/** + * What the admin panel renders: what is loaded, what the shard says, and whether + * the two agree. + * + * Never throws and never fails a page: every branch that could — no shard, a + * shard that is down, an asset plane the operator switched off — is a reported + * state with a reason an operator can act on. + */ +async function getStatus() { + const counts = await db.countAssets().catch(() => ({ total: 0, stored: 0 })) + const bodies = await db.countBodies().catch(() => ({ total: 0, resolved: 0 })) + const meta = await db.getMeta().catch(() => null) + + const status = { + loaded: { + assets: counts.total, + stored: counts.stored, + creatures: bodies.total, + resolved: bodies.resolved, + catalog: meta?.catalog ?? null, + extractorVersion: meta?.extractorVersion ?? null, + importedAt: meta?.importedAt ?? null, + }, + shard: null, + drift: null, + } + + if (!(await shardLinked())) { + status.reason = 'uo-link is not configured' + return status + } + + try { + const sources = await bridge.sourceFingerprint() + + status.shard = { + files: Object.keys(sources.files).length, + extractorVersion: sources.extractorVersion, + hashing: sources.hashing, + complete: sources.complete, + imaging: sources.imaging, + } + + status.drift = meta ? !bridge.sameSources(sources, meta.sources) : true + } catch (err) { + status.reason = err.message + status.code = err instanceof bridge.AssetBridgeError ? err.code : 'UNAVAILABLE' + } + + return status +} + +module.exports = { + ART_SUBDIR, + artDir, + fileNameFor, + importAssets, + resolveAtlasBodies, + applyArt, + getStatus, +} diff --git a/server/model/shardAtlas/shardAtlas.db.js b/server/model/shardAtlas/shardAtlas.db.js index f411016..17e3754 100644 --- a/server/model/shardAtlas/shardAtlas.db.js +++ b/server/model/shardAtlas/shardAtlas.db.js @@ -452,8 +452,64 @@ function listChampions({ facet = '' } = {}) { ) } +/** + * Every creature the atlas knows, as `{ slug, name }` (docs/link/v8.md §8). + * + * `name` is the ServUO CLASS NAME, not a display string invented here: the atlas + * build picks the winning spelling of the spawn type token, so "GiantSpider" is + * what the column holds and what `ScriptCompiler.FindTypeByName` will resolve. + * That is the one property that lets the asset import ask its question without a + * new column, and it is worth knowing before anyone "tidies" this into a + * prettified label. + */ +async function allCreatureTypes() { + return query('SELECT slug, name FROM shard_spawn_creatures ORDER BY slug') +} + +/** + * Point creatures at their artwork, from a `{ slug: filename }` map. + * + * Everything NOT in the map is set back to NULL, which is deliberate: a creature + * whose body stopped resolving must lose its portrait rather than keep pointing + * at a file that is about to be deleted. A broken image is worse than no image, + * and no image is the state the whole atlas UI was designed around. + * + * One transaction, and a single `CASE` update rather than a statement per slug — + * at ~800 creatures the round trips are the cost, not the work. + */ +async function setCreatureArt(map) { + const entries = Object.entries(map ?? {}).filter( + ([slug, file]) => typeof slug === 'string' && slug !== '' && typeof file === 'string' && file !== '', + ) + + const conn = await core.pool.getConnection() + + try { + await conn.beginTransaction() + await conn.query('UPDATE shard_spawn_creatures SET art = NULL WHERE art IS NOT NULL') + + for (let i = 0; i < entries.length; i += BATCH) { + await conn.batch( + 'UPDATE shard_spawn_creatures SET art = ? WHERE slug = ?', + entries.slice(i, i + BATCH).map(([slug, file]) => [file, slug]), + ) + } + + await conn.commit() + + return entries.length + } catch (err) { + await conn.rollback().catch(() => {}) + throw err + } finally { + conn.release() + } +} + module.exports = { replaceAtlas, + allCreatureTypes, + setCreatureArt, getMeta, getFacets, getPending, diff --git a/server/model/shardAtlas/shardAtlas.model.js b/server/model/shardAtlas/shardAtlas.model.js index fb40b84..69fe879 100644 --- a/server/model/shardAtlas/shardAtlas.model.js +++ b/server/model/shardAtlas/shardAtlas.model.js @@ -107,8 +107,45 @@ function pointTypeRows(points) { return rows } +/** + * The art each creature gets when the atlas is rebuilt. + * + * **`replaceAtlas` empties `shard_spawn_creatures` and refills it**, so anything + * on that row is destroyed on every refresh — and a refresh happens on every + * boot. Before protocol 8 that cost nothing: `art` came from a file on disk and + * was simply re-read. As of phase 3 it can also come from an IMPORT, which is + * expensive to obtain and whose gate (the shard's client-file hashes) would say + * "unchanged" for weeks afterwards. So the imported values are re-derived here, + * on the way past, rather than being restored by an import that has no reason to + * run again. + * + * **The operator's map is spread last and therefore wins.** Someone who drew + * their own creature portraits must not have them replaced by a sprite rip on the + * next Update — the one property §12 states outright. + * + * Never throws: the asset tables are the newer half of this pair, and an atlas + * refresh must not start failing because an asset query did. Losing the imported + * art for one boot is recoverable by pressing Import; a boot that cannot rebuild + * the atlas is not. + */ +async function artForAtlas() { + const operator = loadArtMap() + + try { + // eslint-disable-next-line global-require + const assetsDb = require('../shardAssets/shardAssets.db') + const derived = await assetsDb.artBySlug() + return { ...derived, ...operator } + } catch (err) { + log.warn('imported creature art could not be read; using the operator map alone', { + error: err.message, + }) + return operator + } +} + async function applyAtlas(atlas) { - return db.replaceAtlas({ ...atlas, pointTypes: pointTypeRows(atlas.points) }, loadArtMap()) + return db.replaceAtlas({ ...atlas, pointTypes: pointTypeRows(atlas.points) }, await artForAtlas()) } /** diff --git a/server/router/admin/shard.router.js b/server/router/admin/shard.router.js index 04707f8..f419e88 100644 --- a/server/router/admin/shard.router.js +++ b/server/router/admin/shard.router.js @@ -28,6 +28,7 @@ const shardOps = require('./shardOps.controller') const shardVisibility = require('./shardVisibility.controller') const shardAtlas = require('./shardAtlas.controller') const shardClilocs = require('./shardClilocs.controller') +const shardAssets = require('./shardAssets.controller') const selfShard = require('../player/shard.controller') const { requireRole, validate } = core.middleware @@ -362,6 +363,42 @@ shardRouter.put( shardClilocs.setPath, ) +// ── Client assets (admin only) ──────────────────────────────────────────── +// Creature artwork, read from the shard's own UO client over the bridge +// (docs/link/v8.md §6, §8). Sits beside the cliloc routes for the same reason +// they sit beside the atlas: static content derived from the operator's own +// files, and operating it is shard administration. +// +// There is deliberately NO public counterpart. The pictures are served as +// ordinary files under `/uploads`, and `shard_spawn_creatures.art` names them on +// the atlas responses the site already returns — so nothing public needs to know +// this pipeline exists. +shardRouter.get( + '/assets', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Client asset import status: what is loaded, what the shard has, whether they differ (admin only)' + // #swagger.description = 'What the site currently holds (the imported body catalogue, how many sprites are stored, how many atlas creatures resolved to a body id) beside what the shard reports for the client files those pictures come from. `drift: true` means the client files have changed since the last import — press Import. `shard.hashing: true` means a null hash is “not computed yet”, not “changed”: the shard hashes 195 MB anim files off the request path. `shard.imaging.ok: false` is the named NO_IMAGING state — a Linux shard host without libgdiplus cannot render a sprite at all, and the reason names the package to install. A shard with no link configured, or one that is down, is a reported state with a reason rather than an error.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.responses[200] = { description: 'Asset import status', content: { "application/json": { schema: { $ref: "#/components/schemas/UoAssetStatus" } } } } */ + /* #swagger.responses[403] = { description: 'Admin role required', content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } } */ + adminOnly, + shardAssets.getStatus, +) +shardRouter.post( + '/assets/import', + // #swagger.tags = ['Admin · Shard'] + // #swagger.summary = 'Import creature artwork from the shard’s UO client (admin only)' + // #swagger.description = 'Walks the shard’s asset manifest, fetches only the sprites whose hash changed, stores them under uploads/atlas/, re-resolves every atlas creature to a body id and points each creature at its picture. This is the ONLY thing that imports — boot deliberately never calls the shard — so it is what an operator presses after patching their client. `force` re-imports even when the client files are unchanged. `approve` accepts a catalogue that no longer offers assets this site holds; refused by default, because an unmounted client volume and a deliberate downgrade are indistinguishable from the server and the wrong guess deletes artwork. An operator-supplied `spawnAtlas.art.json` always wins over an imported sprite. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane switched off, or a host that cannot render images all answer 200 with status "unavailable" and a reason naming what to fix. Assets a client simply does not have are NOT failures — two thirds of the playable ghost and gargoyle bodies have no art on a stock client.' + // #swagger.security = [{ "cookieAuth": [] }, { "bearerAuth": [] }] + /* #swagger.requestBody = { required: false, content: { "application/json": { schema: { type: "object", properties: { force: { type: "boolean", description: "Import even if the shard’s client files are unchanged." }, approve: { type: "boolean", description: "Accept a catalogue that no longer offers assets this site holds." } } } } } } */ + /* #swagger.responses[200] = { description: 'What happened', content: { "application/json": { schema: { $ref: "#/components/schemas/UoAssetImportResult" } } } } */ + adminOnly, + body('force').optional().isBoolean(), + body('approve').optional().isBoolean(), + validate, + shardAssets.importAssets, +) + // ── Feature visibility (admin only) ─────────────────────────────────── // Who can see which shard surface, and which sensitive fields within it. This // decides what ANONYMOUS visitors get, so it sits above the moderator tier. diff --git a/server/router/admin/shardAssets.controller.js b/server/router/admin/shardAssets.controller.js new file mode 100644 index 0000000..1d9db48 --- /dev/null +++ b/server/router/admin/shardAssets.controller.js @@ -0,0 +1,91 @@ +// ── Admin · Client assets ────────────────────────────────────────────────── +// +// Operating the asset import: what the site holds, what the shard's client files +// currently are, and a re-import after a client patch (docs/link/v8.md §6, §8, +// §14, protocol 8 phase 3). +// +// The policy lives in the model. This controller does three things and no more: +// it validates input, it maps an import RESULT onto an HTTP status, and it +// records the action in the admin activity log. +// +// **An import result is not an exception**, exactly as for clilocs. A shard that +// is down, an asset plane the operator has switched off, a Linux host with no +// libgdiplus, a client patched halfway through the walk — each is a 200 carrying +// `status: 'unavailable'` and a reason naming what to fix, not a 500 that says +// only "something broke". The one thing that DOES 500 is this file having a bug. +// +// **This is the only thing that imports.** Boot never calls the shard for assets, +// for the same reason it stopped calling it for clilocs: the files change when an +// operator patches their client, which is an event they know about and the site +// does not. So this endpoint is what an operator presses afterwards. +// +// The full panel — per-key review, the activity view, approve/reject as buttons — +// is phase 8. This pair is what makes phase 3 reachable at all. + +const assets = require('../../model/shardAssets/shardAssets.model') +const { activity } = require('../../core') + +const log = require('../../core').logger('admin-shard-assets') + +// GET /admin/shard/assets — what is loaded, what the shard says, whether they +// disagree. No public counterpart: the assets themselves are served as ordinary +// files under /uploads, and this is the operating view of the import. +async function getStatus(req, res) { + try { + return res.json(await assets.getStatus()) + } catch (err) { + log.error('getStatus', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +// POST /admin/shard/assets/import — import or update the body catalogue, then +// re-resolve the atlas's creatures and re-derive their artwork. +// +// `force` re-imports even when the client files are unchanged. It is also how an +// operator recovers a wiped uploads volume: the database still holds every hash, +// so the ordinary gate would report "unchanged" while every picture is missing. +// (The import checks for the file on disk per key as well, so that case usually +// heals itself — `force` is the answer when it does not.) +// +// `approve` accepts a catalogue that no longer offers keys this site holds. +// Refused by default because an unmounted client volume and a deliberate +// downgrade look identical from the server, and the wrong guess deletes artwork. +async function importAssets(req, res) { + try { + const force = !!req.body?.force + const approve = !!req.body?.approve + const result = await assets.importAssets({ force, approve }) + + await activity.log({ + req, + action: 'shard.assets.import', + detail: { + force, + approve, + status: result.status, + code: result.code ?? null, + assets: result.assets ?? null, + fetched: result.fetched ?? null, + written: result.written ?? null, + removed: result.removed ?? null, + // The body pass is logged as its own tally rather than as a single + // number: `unknown` means the spawn files name a type this shard's + // scripts do not define, which is real drift an operator should see, and + // it reads identically to `failed` if both are summed into "not resolved". + bodies: result.bodies?.tally ?? null, + vanished: result.vanishedCount ?? null, + }, + }) + + return res.json(result) + } catch (err) { + log.error('importAssets', err) + return res.status(500).json({ message: 'Internal Server Error' }) + } +} + +module.exports = { + getStatus, + importAssets, +} diff --git a/server/swagger/doc.js b/server/swagger/doc.js index 063f2ad..b5d9be0 100644 --- a/server/swagger/doc.js +++ b/server/swagger/doc.js @@ -607,6 +607,136 @@ module.exports = { }, }, }, + UoAssetStatus: { + type: 'object', + description: + 'Admin view of the client-asset import (docs/link/v8.md §6, §8). What the site holds beside what the shard’s UO client currently is. Holding nothing at all is a supported state — creature pages simply render without pictures, which is what every install did before this pipeline existed.', + properties: { + loaded: { + type: 'object', + description: 'What this site currently holds.', + properties: { + assets: { type: 'integer', description: 'Rows in the imported catalogue.', example: 787 }, + stored: { type: 'integer', description: 'How many of those have a picture on disk. Lower than `assets` when the shard listed a key it could not render.', example: 787 }, + creatures: { type: 'integer', description: 'Atlas creatures the shard has answered a body question about, resolved or not.', example: 812 }, + resolved: { type: 'integer', description: 'How many of those resolved to a body id. The rest are types this shard’s scripts do not define, or spawn entries naming an item rather than a creature.', example: 780 }, + catalog: { type: 'string', nullable: true, description: 'The shard’s catalogue id at the last import — derived from its client files, so it changes exactly when they do.', example: 'a3f9c21d4b8e0771' }, + extractorVersion: { type: 'integer', nullable: true, description: 'The version of the shard’s extraction code. A bump makes every derived byte drift even though the client files did not move.', example: 1 }, + importedAt: { type: 'string', format: 'date-time', nullable: true }, + }, + }, + shard: { + type: 'object', + nullable: true, + description: 'The shard’s own client files right now. NULL when there is no shard link or it could not be reached — see `reason`.', + properties: { + files: { type: 'integer', description: 'How many of the animation/definition files this catalogue reads the shard actually has. Few clients carry all five anim files.', example: 9 }, + extractorVersion: { type: 'integer', example: 1 }, + hashing: { type: 'boolean', description: 'A hash is being computed in the background. A null `sha256` while this is true means “not yet”, never “changed”.', example: false }, + complete: { type: 'boolean', description: 'Every client file has a content hash.', example: true }, + imaging: { + type: 'object', + nullable: true, + description: 'Whether the shard host can render an image at all. `ok: false` is the named NO_IMAGING state: ServUO under Mono needs libgdiplus, and without it a Linux shard cannot decode a sprite. Cliloc and atlas import are unaffected.', + properties: { + ok: { type: 'boolean', example: true }, + code: { type: 'string', nullable: true, example: null }, + reason: { type: 'string', nullable: true }, + }, + }, + }, + }, + drift: { + type: 'boolean', + nullable: true, + description: 'True when the shard’s client files no longer match what was imported — press Import. NULL when they could not be read.', + example: false, + }, + reason: { type: 'string', nullable: true, description: 'Why the shard could not be asked, when it could not.' }, + code: { + type: 'string', + nullable: true, + description: 'Machine-readable cause of `reason`.', + enum: ['DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'NO_IMAGING', 'SOURCE_CHANGED', 'UNAVAILABLE'], + }, + }, + }, + UoAssetImportResult: { + type: 'object', + description: + 'Outcome of an asset import. Reported rather than thrown, so a shard that is down or a host that cannot render images is an answer and not a 500.', + properties: { + status: { + type: 'string', + enum: ['skipped', 'unavailable', 'unchanged', 'imported', 'needsReview', 'failed'], + description: '`skipped`: no shard is configured. `unchanged`: the client files match what was imported and nothing was fetched. `needsReview`: assets this site holds are no longer offered by the shard, and nothing was changed — re-run with `approve` to accept it.', + example: 'imported', + }, + reason: { type: 'string', nullable: true }, + code: { + type: 'string', + nullable: true, + description: 'Machine-readable cause. `NO_IMAGING` is a shard host with no libgdiplus; `SOURCE_CHANGED` is a client patched partway through the walk, in which case nothing was applied.', + enum: ['DISABLED', 'NO_SOURCE', 'SHARD_DOWN', 'PROTOCOL', 'BUSY', 'NO_IMAGING', 'SOURCE_CHANGED', 'INCOMPLETE', 'STUCK', 'MALFORMED', 'TOO_LARGE', 'UNAVAILABLE'], + }, + catalog: { type: 'string', nullable: true, example: 'a3f9c21d4b8e0771' }, + extractorVersion: { type: 'integer', nullable: true, example: 1 }, + assets: { type: 'integer', nullable: true, description: 'Catalogue rows after the import.', example: 787 }, + fetched: { type: 'integer', nullable: true, description: 'How many sprites actually crossed the wire. On an Update after a client patch this is far smaller than `assets`, which is the point of the manifest.', example: 12 }, + written: { type: 'integer', nullable: true, description: 'How many were written to disk.', example: 12 }, + absent: { + type: 'integer', + nullable: true, + description: 'Keys the shard listed but could not render. NOT a failure: this client has no art at that key, which is the expected answer for two thirds of the playable ghost and gargoyle bodies.', + example: 0, + }, + unsupported: { type: 'integer', nullable: true, description: 'Keys the shard does not serve at all. Unlike `absent` this indicates a bug on the site’s side, not a gap in the client.', example: 0 }, + removed: { type: 'integer', nullable: true, description: 'Assets deleted because the shard no longer offers them (only with `approve`).', example: 0 }, + scanned: { type: 'integer', nullable: true, description: 'Body ids the shard walked. Far larger than `assets` — most of the addressable range has no art.', example: 2047 }, + pages: { type: 'integer', nullable: true, description: 'Manifest pages. This family pages on the shard’s scan budget rather than on bytes, so several is normal.', example: 4 }, + playerBodies: { + type: 'array', + nullable: true, + items: { type: 'integer' }, + description: 'The body ids the shard reports as player-character bodies — every registered race’s male, female and ghost bodies, asked of the shard rather than hardcoded. These render head-on; everything else renders three-quarter.', + example: [400, 401, 402, 403, 605, 606, 607, 608, 666, 667, 694, 695], + }, + vanished: { type: 'array', nullable: true, items: { type: 'string' }, description: 'On `needsReview`: up to fifty of the keys that disappeared.' }, + vanishedCount: { type: 'integer', nullable: true }, + bodies: { + type: 'object', + nullable: true, + description: 'The slug → body id pass (§8). The shard constructs each creature and reads its body id, which is the only thing correct for a shard’s own custom creatures.', + properties: { + asked: { type: 'integer', example: 812 }, + answered: { type: 'integer', example: 812 }, + resolved: { type: 'integer', example: 780 }, + tally: { + type: 'object', + description: 'Per-outcome counts. `unknown` is real drift worth acting on — a spawn file naming a type this shard’s scripts do not define. `notCreature` is a spawn entry for an item or decoration and is permanent.', + properties: { + ok: { type: 'integer', example: 780 }, + unknown: { type: 'integer', example: 20 }, + notCreature: { type: 'integer', example: 12 }, + failed: { type: 'integer', example: 0 }, + }, + }, + reason: { type: 'string', nullable: true }, + }, + }, + art: { + type: 'object', + nullable: true, + description: 'The derivation onto `shard_spawn_creatures.art`. An operator-supplied `spawnAtlas.art.json` always wins over an imported sprite.', + properties: { + applied: { type: 'integer', description: 'Creatures now pointing at a picture.', example: 763 }, + derived: { type: 'integer', description: 'From the import.', example: 763 }, + operator: { type: 'integer', description: 'From the operator’s own map.', example: 0 }, + error: { type: 'string', nullable: true }, + }, + }, + }, + }, UoShardLinkRequest: { type: 'object', required: ['code'], diff --git a/server/test/assetBridge.test.js b/server/test/assetBridge.test.js new file mode 100644 index 0000000..84cc8db --- /dev/null +++ b/server/test/assetBridge.test.js @@ -0,0 +1,368 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') + +const uoLinkClient = require('../utils/uoLinkClient') +const bridge = require('../utils/assetBridge') + +// The three walks over the asset plane, driven against a stubbed sidecar client +// (docs/link/v8.md §5, §6, §8 — protocol 8, phase 3). +// +// Two families of failure are asserted here and they are not the same shape. +// +// **The envelope failures** are ways the shard can be wrong that leave this side +// holding a catalogue it believes is complete. They are invisible downstream: a +// catalogue missing its last three hundred bodies renders as a site where some +// creatures have pictures and some do not, which is exactly what NO catalogue +// looks like. Each corresponds to a field §3.4 puts on the wire specifically so +// this side can tell the difference. +// +// **The absence failures** are the opposite mistake, and phase 3's more likely +// one: treating a body this client has no art for as an error. Two thirds of the +// playable ghost and gargoyle bodies are in that state on a stock client, and an +// import that failed — or even warned loudly — on them would teach an operator to +// ignore the panel. + +const saved = {} + +function stub({ sources, manifest = [], fetch = [], bodies = [] } = {}) { + saved.getAssetSources = uoLinkClient.getAssetSources + saved.getAssetManifest = uoLinkClient.getAssetManifest + saved.fetchAssets = uoLinkClient.fetchAssets + saved.resolveBodies = uoLinkClient.resolveBodies + + const calls = { manifest: [], fetch: [], bodies: [] } + + uoLinkClient.getAssetSources = async () => sources + uoLinkClient.getAssetManifest = async ({ family, cursor } = {}) => { + calls.manifest.push({ family: family ?? null, cursor: cursor ?? null }) + const next = manifest.shift() + if (!next) throw new Error('the walk asked for more manifest pages than the test supplied') + return next + } + uoLinkClient.fetchAssets = async ({ keys, catalog, cursor } = {}) => { + calls.fetch.push({ keys, catalog: catalog ?? null, cursor: cursor ?? null }) + const next = fetch.shift() + if (!next) throw new Error('the walk asked for more fetch pages than the test supplied') + return next + } + uoLinkClient.resolveBodies = async (types) => { + calls.bodies.push(types) + const next = bodies.shift() + if (!next) throw new Error('the walk asked for more body chunks than the test supplied') + return next + } + + return calls +} + +function restore() { + for (const [name, fn] of Object.entries(saved)) { + if (fn) uoLinkClient[name] = fn + } +} + +const ok = (data) => ({ ok: true, status: 200, data }) +const fail = (status, data) => ({ ok: false, status, data }) + +const CATALOG = 'a3f9c21d4b8e0771' + +const manifestPage = (rows, extra = {}) => + ok({ + kind: 'assets.manifest.ok', + family: 'body', + catalog: CATALOG, + extractorVersion: 1, + playerBodies: [400, 401, 402, 403], + scanned: rows.length, + rows, + more: false, + cut: 'end', + ...extra, + }) + +const fetchPage = (rows, extra = {}) => + ok({ + kind: 'assets.fetch.ok', + family: 'body', + catalog: CATALOG, + rows, + more: false, + cut: 'end', + ...extra, + }) + +const row = (body, sha = 'aa') => ({ + key: `body/${body}/a0`, + sha256: sha, + bytes: 900, + width: 24, + height: 63, + body, + direction: 1, +}) + +const png = Buffer.from([0x89, 0x50, 0x4e, 0x47]).toString('base64') + +const sourcesReply = (extra = {}) => + ok({ + kind: 'assets.sources.ok', + extractorVersion: 1, + imaging: { ok: true }, + hashing: false, + complete: true, + files: [ + { name: 'anim.idx', size: 10, mtime: 1, sha256: 'a' }, + { name: 'anim.mul', size: 20, mtime: 2, sha256: 'b' }, + { name: 'body.def', size: 30, mtime: 3, sha256: 'c' }, + // Not a source this family reads: `art.mul` decides item pictures, not + // creature ones, and folding it in would make every item-art change look + // like a reason to re-import the whole body catalogue. + { name: 'art.mul', size: 148000000, mtime: 4, sha256: 'd' }, + ], + ...extra, + }) + +// ── the source gate (§6 stage 1) ────────────────────────────────────────── + +test('the source fingerprint keeps only the files the body catalogue reads', async (t) => { + stub({ sources: sourcesReply() }) + t.after(restore) + + const fingerprint = await bridge.sourceFingerprint() + + assert.deepEqual(Object.keys(fingerprint.files).sort(), ['anim.idx', 'anim.mul', 'body.def']) + assert.equal(fingerprint.extractorVersion, 1) +}) + +test('a bumped extractor version is drift even when every client file is identical', () => { + const files = { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } } + + assert.equal( + bridge.sameSources({ files, extractorVersion: 1 }, { files, extractorVersion: 1 }), + true, + ) + // §7: a corrected frame offset changes every derived byte while every source + // file stays byte-identical. If this returned true the fix would never reach + // an install whose client never moves. + assert.equal( + bridge.sameSources({ files, extractorVersion: 2 }, { files, extractorVersion: 1 }), + false, + ) +}) + +test('a client that GAINED an anim file is drift, not a match', () => { + const before = { files: { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } }, extractorVersion: 1 } + const after = { + files: { + 'anim.mul': { size: 1, mtime: 2, sha256: 'x' }, + // A client that grows an anim5.mul is a client whose gargoyles suddenly + // resolve. Comparing only the files present in both would call that + // unchanged and never import them. + 'anim5.mul': { size: 9, mtime: 9, sha256: 'y' }, + }, + extractorVersion: 1, + } + + assert.equal(bridge.sameSources(before, after), false) +}) + +test('a null hash falls back to size and mtime rather than reading as changed', () => { + // The shard hashes 195 MB anim files off the request path, so a null sha256 is + // "not computed yet". Treating it as a difference would re-import the whole + // catalogue on every restart until the background pass finished. + const a = { files: { 'anim.mul': { size: 5, mtime: 7, sha256: null } }, extractorVersion: 1 } + const b = { files: { 'anim.mul': { size: 5, mtime: 7, sha256: 'later' } }, extractorVersion: 1 } + + assert.equal(bridge.sameSources(a, b), true) +}) + +// ── the manifest walk (§6 stage 2) ──────────────────────────────────────── + +test('the manifest walks every page and stops only on cut: end', async (t) => { + const calls = stub({ + manifest: [ + manifestPage([row(12), row(34)], { more: true, cursor: 'b:34', cut: 'limit' }), + manifestPage([row(400)]), + ], + }) + t.after(restore) + + const result = await bridge.readManifest() + + assert.equal(result.rows.length, 3) + assert.equal(result.catalog, CATALOG) + assert.deepEqual(result.playerBodies, [400, 401, 402, 403]) + assert.deepEqual( + calls.manifest.map((c) => c.cursor), + [null, 'b:34'], + ) +}) + +test('a short page that did not end the catalogue is refused', async (t) => { + // `cut: 'limit'` with `more: false` is the shard saying it stopped for its own + // reason. Importing what arrived would silently drop every body after it, and + // the result is indistinguishable from a client with fewer creatures. + stub({ manifest: [manifestPage([row(12)], { more: false, cut: 'limit' })] }) + t.after(restore) + + await assert.rejects(() => bridge.readManifest(), /stopped sending assets/) +}) + +test('a cursor that does not advance is refused rather than looped on', async (t) => { + stub({ + manifest: [ + manifestPage([row(12)], { more: true, cursor: 'b:12', cut: 'budget' }), + manifestPage([row(13)], { more: true, cursor: 'b:12', cut: 'budget' }), + ], + }) + t.after(restore) + + await assert.rejects(() => bridge.readManifest(), /without advancing its cursor/) +}) + +test('the client files changing mid-walk aborts the whole import', async (t) => { + // The catalogue id is derived from the client files themselves, so a change + // between two pages means half of what we hold describes files that no longer + // exist — and nothing later can tell which half. + stub({ + manifest: [ + manifestPage([row(12)], { more: true, cursor: 'b:12', cut: 'limit' }), + manifestPage([row(34)], { catalog: 'something-else' }), + ], + }) + t.after(restore) + + await assert.rejects(() => bridge.readManifest(), /changed while the manifest was being read/) +}) + +// ── the fetch (§5) ──────────────────────────────────────────────────────── + +test('a fetch passes the catalogue id and decodes the PNG', async (t) => { + const calls = stub({ + fetch: [fetchPage([{ key: 'body/12/a0', status: 'ok', sha256: 'aa', bytes: 4, width: 24, height: 63, body: 12, direction: 1, png }])], + }) + t.after(restore) + + const { assets } = await bridge.fetchAssets({ keys: ['body/12/a0'], catalog: CATALOG }) + + assert.equal(calls.fetch[0].catalog, CATALOG) + assert.equal(assets.get('body/12/a0').png.length, 4) + assert.equal(assets.get('body/12/a0').width, 24) +}) + +test('an absent asset is a counted row, not a failed fetch', async (t) => { + // The whole reason this is not an error: two thirds of the playable ghost and + // gargoyle bodies have no art on a stock client (§5.2), and an import that + // failed on them could never succeed. + stub({ + fetch: [ + fetchPage([ + { key: 'body/12/a0', status: 'ok', sha256: 'aa', bytes: 4, png }, + { key: 'body/666/a0', status: 'absent' }, + { key: 'body/400/a2/f3', status: 'unsupported' }, + ]), + ], + }) + t.after(restore) + + const { assets, missing } = await bridge.fetchAssets({ + keys: ['body/12/a0', 'body/666/a0', 'body/400/a2/f3'], + catalog: CATALOG, + }) + + assert.equal(assets.size, 1) + // Counted apart, because they mean different things: `absent` is a gap in the + // operator's client and `unsupported` is a bug on this side. + assert.equal(missing.absent, 1) + assert.equal(missing.unsupported, 1) +}) + +test('a busy shard is retried rather than failing the walk', async (t) => { + saved.fetchAssets = uoLinkClient.fetchAssets + t.after(restore) + + let attempts = 0 + + uoLinkClient.fetchAssets = async () => { + attempts++ + if (attempts < 3) return fail(425, { reason: 'busy' }) + return fetchPage([{ key: 'body/12/a0', status: 'ok', sha256: 'aa', bytes: 4, png }]) + } + + const { assets } = await bridge.fetchAssets({ keys: ['body/12/a0'], catalog: CATALOG }) + + assert.equal(attempts, 3) + assert.equal(assets.size, 1) +}) + +test('a shard host with no libgdiplus is named, not reported as a dead shard', async (t) => { + saved.getAssetManifest = uoLinkClient.getAssetManifest + t.after(restore) + + uoLinkClient.getAssetManifest = async () => + fail(503, { reason: "this shard host cannot render images - Mono's System.Drawing needs libgdiplus" }) + + await assert.rejects( + () => bridge.readManifest(), + (err) => err.code === 'NO_IMAGING', + ) +}) + +// ── the body pass (§8) ──────────────────────────────────────────────────── + +test('body resolution chunks to the shard cap and records every outcome', async (t) => { + const creatures = [] + + for (let i = 0; i < bridge.BODY_CHUNK + 5; i++) { + creatures.push({ slug: `c-${i}`, name: `Creature${i}` }) + } + + const reply = (types) => + ok({ + kind: 'assets.bodies.ok', + rows: types.map((type, i) => (i === 0 ? { type, status: 'unknown' } : { type, status: 'ok', body: 100 + i })), + more: false, + cut: 'end', + }) + + const calls = stub({ bodies: [] }) + t.after(restore) + + uoLinkClient.resolveBodies = async (types) => { + calls.bodies.push(types) + return reply(types) + } + + const rows = await bridge.resolveBodies({ creatures }) + + // Two chunks, and neither over the cap: the shard REFUSES an over-long list + // rather than truncating it, so a chunk size above its cap does not degrade — + // every request fails. + assert.equal(calls.bodies.length, 2) + assert.ok(calls.bodies.every((chunk) => chunk.length <= bridge.BODY_CHUNK)) + + assert.equal(rows.length, creatures.length) + // The negative answers are kept. Without them the next pass asks again, and + // the pass costs a real constructor per name on the shard's Core thread. + assert.equal(rows.filter((r) => r.status === 'unknown').length, 2) +}) + +test('two slugs sharing a class name are asked once and both get the answer', async (t) => { + const calls = stub({ + bodies: [ + ok({ kind: 'assets.bodies.ok', rows: [{ type: 'GiantSpider', status: 'ok', body: 28 }], more: false, cut: 'end' }), + ], + }) + t.after(restore) + + const rows = await bridge.resolveBodies({ + creatures: [ + { slug: 'giant-spider', name: 'GiantSpider' }, + { slug: 'giantspider', name: 'GiantSpider' }, + ], + }) + + assert.deepEqual(calls.bodies[0], ['GiantSpider']) + assert.equal(rows.length, 2) + assert.ok(rows.every((r) => r.body === 28)) +}) diff --git a/server/test/shardAssets.model.test.js b/server/test/shardAssets.model.test.js new file mode 100644 index 0000000..6f3fd99 --- /dev/null +++ b/server/test/shardAssets.model.test.js @@ -0,0 +1,334 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const core = require('../core') +const model = require('../model/shardAssets/shardAssets.model') +const db = require('../model/shardAssets/shardAssets.db') +const atlasDb = require('../model/shardAtlas/shardAtlas.db') +const atlasModel = require('../model/shardAtlas/shardAtlas.model') +const bridge = require('../utils/assetBridge') +const uoLinkConfig = require('../model/uoLinkConfig/uoLinkConfig.model') + +// The import as a decision, with the shard and the database both stubbed +// (docs/link/v8.md §6, §12 — protocol 8, phase 3). +// +// Each of these is a way the import can be wrong that an operator would either +// never notice or notice only weeks later, on a page: +// +// - Re-fetching every sprite on every Update. Correct output, and it makes the +// manifest — the entire reason stage 2 carries hashes instead of pixels — +// dead weight. +// - Silently dropping an asset the shard stopped offering. An unmounted client +// volume and a deliberate downgrade are the same thing from here, and the +// wrong guess deletes artwork nobody asked to delete. +// - Overwriting artwork the operator drew themselves. §12 states outright that +// theirs wins, and a sprite rip replacing hand-drawn portraits is not +// recoverable by pressing anything. +// - Treating a body with no art as a failure. Two thirds of the playable ghost +// and gargoyle bodies are in that state on a stock client. + +const saved = {} +let uploadDir + +function stubEverything({ manifest, fetched, held = new Map(), meta = null, sources } = {}) { + saved.sourceFingerprint = bridge.sourceFingerprint + saved.readManifest = bridge.readManifest + saved.fetchAssets = bridge.fetchAssets + saved.resolveBodies = bridge.resolveBodies + saved.allAssets = db.allAssets + saved.saveAssets = db.saveAssets + saved.getMeta = db.getMeta + saved.countAssets = db.countAssets + saved.countBodies = db.countBodies + saved.replaceBodies = db.replaceBodies + saved.artBySlug = db.artBySlug + saved.allCreatureTypes = atlasDb.allCreatureTypes + saved.setCreatureArt = atlasDb.setCreatureArt + saved.loadArtMap = atlasModel.loadArtMap + saved.getSafe = uoLinkConfig.getSafe + + const seen = { saved: null, fetchedKeys: null, art: null } + + uoLinkConfig.getSafe = async () => ({ enabled: true, baseUrl: 'http://127.0.0.1:8080' }) + + bridge.sourceFingerprint = async () => + sources ?? { + files: { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } }, + extractorVersion: 1, + hashing: false, + complete: true, + imaging: { ok: true }, + } + + bridge.readManifest = async () => manifest + bridge.fetchAssets = async ({ keys }) => { + seen.fetchedKeys = keys + return fetched ?? { assets: new Map(), missing: { absent: 0, unsupported: 0 } } + } + bridge.resolveBodies = async () => [] + + db.allAssets = async () => held + db.getMeta = async () => meta + db.countAssets = async () => ({ total: held.size, stored: held.size }) + db.countBodies = async () => ({ total: 0, resolved: 0 }) + db.saveAssets = async (rows) => { + seen.saved = rows + return rows.length + } + db.replaceBodies = async () => 0 + db.artBySlug = async () => ({}) + + atlasDb.allCreatureTypes = async () => [] + atlasDb.setCreatureArt = async (map) => { + seen.art = map + return Object.keys(map).length + } + atlasModel.loadArtMap = () => ({}) + + return seen +} + +function restore() { + for (const [name, fn] of Object.entries(saved)) { + if (!fn) continue + if (name in db) db[name] = fn + if (name in bridge) bridge[name] = fn + if (name in atlasDb) atlasDb[name] = fn + if (name === 'loadArtMap') atlasModel.loadArtMap = fn + if (name === 'getSafe') uoLinkConfig.getSafe = fn + } +} + +/** A real uploads directory, because the import checks the disk as well as the row. */ +function useTempUploads(t) { + uploadDir = fs.mkdtempSync(path.join(os.tmpdir(), 'uo-assets-')) + const previous = core.uploads + + Object.defineProperty(core, 'uploads', { + configurable: true, + get: () => ({ ...previous, UPLOAD_DIR: uploadDir }), + }) + + t.after(() => { + Object.defineProperty(core, 'uploads', { configurable: true, get: () => previous }) + fs.rmSync(uploadDir, { recursive: true, force: true }) + }) + + return uploadDir +} + +const row = (body, sha) => ({ + key: `body/${body}/a0`, + family: 'body', + sha256: sha, + bytes: 900, + width: 24, + height: 63, + body, + direction: 1, +}) + +const manifestOf = (rows) => ({ + rows, + catalog: 'cat1', + extractorVersion: 1, + playerBodies: [400], + pages: 1, + scanned: 2047, +}) + +const sprite = (sha) => ({ + sha256: sha, + bytes: 4, + width: 24, + height: 63, + body: 12, + direction: 1, + png: Buffer.from([0x89, 0x50, 0x4e, 0x47]), +}) + +// ── the gate ────────────────────────────────────────────────────────────── + +test('unchanged client files import nothing at all', async (t) => { + const sources = { + files: { 'anim.mul': { size: 1, mtime: 2, sha256: 'x' } }, + extractorVersion: 1, + hashing: false, + complete: true, + imaging: { ok: true }, + } + + stubEverything({ manifest: manifestOf([]), meta: { sources }, sources }) + t.after(restore) + + bridge.readManifest = async () => { + throw new Error('the gate should have stopped before reading a manifest') + } + + const result = await model.importAssets() + + assert.equal(result.status, 'unchanged') +}) + +test('a host that cannot render images is named rather than walked', async (t) => { + // §4.4: reported on the SOURCE gate, so an operator meets it while setting the + // shard up rather than from an empty bestiary weeks later. + stubEverything({ + manifest: manifestOf([]), + sources: { + files: {}, + extractorVersion: 1, + hashing: false, + complete: true, + imaging: { ok: false, code: 'NO_IMAGING', reason: 'needs libgdiplus' }, + }, + }) + t.after(restore) + + const result = await model.importAssets() + + assert.equal(result.status, 'unavailable') + assert.equal(result.code, 'NO_IMAGING') +}) + +// ── the diff (§6) ───────────────────────────────────────────────────────── + +test('only the keys whose hash moved are fetched', async (t) => { + const dir = useTempUploads(t) + fs.mkdirSync(path.join(dir, model.ART_SUBDIR), { recursive: true }) + fs.writeFileSync(path.join(dir, model.ART_SUBDIR, 'kept.png'), 'x') + + const held = new Map([ + ['body/12/a0', { key: 'body/12/a0', sha256: 'same', file: 'kept.png' }], + ['body/34/a0', { key: 'body/34/a0', sha256: 'old', file: 'kept.png' }], + ]) + + const seen = stubEverything({ + held, + manifest: manifestOf([row(12, 'same'), row(34, 'new')]), + fetched: { assets: new Map([['body/34/a0', sprite('new')]]), missing: { absent: 0, unsupported: 0 } }, + }) + t.after(restore) + + const result = await model.importAssets({ force: true }) + + assert.equal(result.status, 'imported') + // The whole point of a manifest that carries hashes and not pixels. + assert.deepEqual(seen.fetchedKeys, ['body/34/a0']) + assert.equal(result.written, 1) +}) + +test('an unchanged key whose file is missing from disk is fetched again', async (t) => { + // The row and the file can disagree — a wiped uploads volume, a restore from a + // database dump. Trusting the row alone leaves a broken image on a creature + // page with nothing anywhere reporting a problem, and re-fetching a sprite is + // far cheaper than that. + useTempUploads(t) + + const held = new Map([['body/12/a0', { key: 'body/12/a0', sha256: 'same', file: 'gone.png' }]]) + + const seen = stubEverything({ + held, + manifest: manifestOf([row(12, 'same')]), + fetched: { assets: new Map([['body/12/a0', sprite('same')]]), missing: { absent: 0, unsupported: 0 } }, + }) + t.after(restore) + + await model.importAssets({ force: true }) + + assert.deepEqual(seen.fetchedKeys, ['body/12/a0']) +}) + +test('a key that vanished from the manifest needs review before anything changes', async (t) => { + useTempUploads(t) + + const held = new Map([['body/99/a0', { key: 'body/99/a0', sha256: 'a', file: 'x.png' }]]) + + const seen = stubEverything({ held, manifest: manifestOf([row(12, 'a')]) }) + t.after(restore) + + const result = await model.importAssets({ force: true }) + + assert.equal(result.status, 'needsReview') + assert.equal(result.vanishedCount, 1) + // Nothing was applied. An unmounted client volume and a deliberate downgrade + // look identical from here. + assert.equal(seen.saved, null) +}) + +test('approve accepts the vanished key and removes its file', async (t) => { + const dir = useTempUploads(t) + fs.mkdirSync(path.join(dir, model.ART_SUBDIR), { recursive: true }) + fs.writeFileSync(path.join(dir, model.ART_SUBDIR, 'gone.png'), 'x') + + const held = new Map([['body/99/a0', { key: 'body/99/a0', sha256: 'a', file: 'gone.png' }]]) + + stubEverything({ + held, + manifest: manifestOf([row(12, 'a')]), + fetched: { assets: new Map([['body/12/a0', sprite('a')]]), missing: { absent: 0, unsupported: 0 } }, + }) + t.after(restore) + + const result = await model.importAssets({ force: true, approve: true }) + + assert.equal(result.status, 'imported') + assert.equal(result.removed, 1) + assert.equal(fs.existsSync(path.join(dir, model.ART_SUBDIR, 'gone.png')), false) +}) + +// ── absence is not failure (§5.2) ───────────────────────────────────────── + +test('a key the shard could not render keeps the picture already held', async (t) => { + useTempUploads(t) + + const held = new Map([['body/12/a0', { key: 'body/12/a0', sha256: 'old', file: 'existing.png' }]]) + + const seen = stubEverything({ + held, + manifest: manifestOf([row(12, 'new')]), + // Listed, asked for, and not served. A shard that suddenly cannot render one + // sprite must not cost the picture we already have. + fetched: { assets: new Map(), missing: { absent: 1, unsupported: 0 } }, + }) + t.after(restore) + + const result = await model.importAssets({ force: true }) + + assert.equal(result.status, 'imported') + assert.equal(result.absent, 1) + assert.equal(seen.saved[0].file, 'existing.png') +}) + +// ── the derivation (§12) ────────────────────────────────────────────────── + +test("the operator's own artwork wins over an imported sprite", async (t) => { + useTempUploads(t) + + const seen = stubEverything({ manifest: manifestOf([]) }) + t.after(restore) + + db.artBySlug = async () => ({ 'giant-spider': 'uo-body-28-aaaabbbb.png', wolf: 'uo-body-34-ccccdddd.png' }) + // Someone who drew their own giant spider must not have it replaced by a + // sprite rip on the next Update. §12 states this outright. + atlasModel.loadArtMap = () => ({ 'giant-spider': 'my-own-spider.png' }) + + await model.importAssets({ force: true }) + + assert.equal(seen.art['giant-spider'], 'my-own-spider.png') + assert.equal(seen.art.wolf, 'uo-body-34-ccccdddd.png') +}) + +test('a sprite filename carries its hash so a changed picture is a changed URL', () => { + const before = model.fileNameFor('body/34/a0', 'aaaaaaaabbbb') + const after = model.fileNameFor('body/34/a0', 'ccccccccdddd') + + // A stable name would be overwritten in place, and every browser and CDN that + // had cached it would keep serving last month's client's sprite — with the + // database row correct and nothing to notice. + assert.notEqual(before, after) + assert.match(before, /^uo-body-34-a0-[0-9a-f]{8}\.png$/) +}) diff --git a/server/utils/assetBridge.js b/server/utils/assetBridge.js new file mode 100644 index 0000000..432bf25 --- /dev/null +++ b/server/utils/assetBridge.js @@ -0,0 +1,539 @@ +// The Asset Bridge client (docs/link/v8.md §5, §6, §8 — protocol 8, phase 3). +// +// Three walks over the same request/reply path `clilocBridge.js` already uses, +// and everything that file says about the envelope holds here unchanged: only +// `cut: 'end'` means finished, the cursor must advance, and 425 is the ordinary +// answer during an import rather than an error. +// +// What is different is what each walk is FOR. +// +// ── `readManifest` — what the shard could serve, without the pixels ──────── +// +// §6's stage 2. Every row is `{ key, sha256, bytes, width, height }`, so the +// site can diff against what it already holds and ask for only the keys whose +// hash moved. On the ordinary case — a shard restart that changed nothing — +// that diff is empty and no pixels cross at all. +// +// This family pages on the shard's WALL CLOCK, not on bytes. Its rows are about +// ninety bytes and the whole catalogue is one page by the byte budget, but +// producing that page means decoding hundreds of sprites and the sidecar waits +// ten seconds for a reply. So `cut: 'limit'` is the normal page ending here, +// where for clilocs it would have signalled something wrong. +// +// ── `fetchAssets` — the pixels, for keys we chose ───────────────────────── +// +// Each row carries a base64 PNG. The shard encodes it: `System.Drawing` is +// already in its decode path, so PNG costs it no new dependency, and having the +// hash cover exactly the bytes we store is what makes the next Update a diff. +// +// **`catalog` is passed on every fetch and it is not optional in practice.** It +// is an id the shard derives from the client files themselves, so handing it back +// makes the shard refuse if those files moved since the manifest was read. +// Without it an operator patching their client mid-import produces one asset set +// stitched out of two, with no error anywhere — the same failure `clilocBridge` +// guards against by comparing (size, mtime) across pages. +// +// ── `resolveBodies` — the atlas's creatures, by class name ──────────────── +// +// §8. The shard constructs each type and reads `Body.BodyID`, which is the only +// thing that is correct for a shard's own custom creatures. That runs on its Core +// thread, so the batch is small and the shard REFUSES an over-long list rather +// than truncating it — hence the chunking here, and hence a chunk size that is a +// constant rather than "as many as fit". + +// Required as a namespace, not destructured: a test that stubs the sidecar +// replaces these on the module object, and a destructured copy taken at load +// time would keep calling the real one. +const uoLinkClient = require('./uoLinkClient') +const log = require('../core').logger('asset-bridge') + +/** The only family phase 3 serves. §5's key scheme covers statics and land later. */ +const FAMILY = 'body' + +// Chunk size for the body pass. The shard's own cap defaults to 100 and it +// refuses rather than truncates, so this must stay at or under it — a mismatch +// here does not degrade, it fails every chunk. +const BODY_CHUNK = 100 + +// Chunk size for a fetch request. The shard cuts the PAGE by byte budget within +// whatever it is handed, so this only bounds how large a single request is; a +// chunk of 400 one-kilobyte sprites is a couple of pages. +const FETCH_CHUNK = 400 + +// Bounds on each walk. None is expected to be reached — the catalogue is under a +// thousand rows — and each exists so that a shard answering nonsense costs a +// bounded amount of time rather than an unbounded amount of memory. +const MAX_PAGES = 200 +const MAX_ROWS = 100000 + +// 425 is flow control, not failure: the shard's asset plane serves one request at +// a time because its outbound queue is bounded in lines rather than bytes. During +// an import a page coming back busy is expected, so it is retried with a backoff +// rather than failing the walk. +const BUSY_RETRIES = 6 +const BUSY_BACKOFF_MS = [200, 400, 800, 1600, 3200, 5000] + +class AssetBridgeError extends Error { + constructor(message, code) { + super(message) + this.name = 'AssetBridgeError' + this.code = code + } +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Map a sidecar response onto one of this module's codes. + * + * Deliberately the same vocabulary `clilocBridge.describeFailure` uses, because + * the admin panel reports them side by side and an operator should not have to + * learn two names for "you have not switched this on". + * + * 422 is the one that means something different here: on the cliloc path it is a + * file the shard cannot decode, and on this one it is *also* the mid-import guard + * firing — the client files moved between the manifest and the fetch. + */ +function describeFailure(res, what) { + const reason = res?.data?.reason || res?.error || `sidecar responded ${res?.status}` + + switch (res?.status) { + case 403: + return new AssetBridgeError( + `The shard is refusing to serve client assets (Bridge.AssetsEnabled is off): ${reason}`, + 'DISABLED', + ) + case 404: + return new AssetBridgeError(`The shard has no ${what}: ${reason}`, 'NO_SOURCE') + case 409: + return new AssetBridgeError( + `The sidecar refused the protocol version this build declares: ${reason}`, + 'PROTOCOL', + ) + case 422: + return new AssetBridgeError(reason, 'SOURCE_CHANGED') + case 425: + return new AssetBridgeError( + 'The shard stayed busy serving another asset request', + 'BUSY', + ) + case 503: + // The named `NO_IMAGING` outcome arrives this way: a Linux shard host with + // no libgdiplus cannot render a sprite at all, and §4.4 requires that be an + // actionable sentence rather than a stack trace. The shard's own wording + // already names the package and the command, so it is passed through. + return new AssetBridgeError(reason, /libgdiplus/i.test(reason) ? 'NO_IMAGING' : 'SHARD_DOWN') + case 504: + return new AssetBridgeError(`The shard did not answer: ${reason}`, 'SHARD_DOWN') + default: + return new AssetBridgeError(reason, 'UNAVAILABLE') + } +} + +/** One call, with the 425 backoff. `send` returns the client's `{ ok, ... }`. */ +async function withBusyRetry(send, what) { + for (let attempt = 0; ; attempt++) { + const res = await send() + if (res.ok) return res.data + + if (res.status === 425 && attempt < BUSY_RETRIES) { + await sleep(BUSY_BACKOFF_MS[Math.min(attempt, BUSY_BACKOFF_MS.length - 1)]) + continue + } + + throw describeFailure(res, what) + } +} + +/** + * Shared page-envelope checks (§3.4). + * + * Every one of these is a way a walk can end in something that LOOKS like a + * complete import and is not, which is why they are assertions rather than + * warnings: a truncated catalogue is indistinguishable downstream from a client + * that simply has fewer creatures. + */ +function checkPage(page, { arrayName, cursor, pages }) { + if (!page || !Array.isArray(page[arrayName])) { + throw new AssetBridgeError( + `The shard sent an asset page with no ${arrayName} array`, + 'MALFORMED', + ) + } + + if (!page.more) { + if (page.cut !== 'end') { + throw new AssetBridgeError( + `The shard stopped sending assets after ${pages} page(s) (cut: ${page.cut || 'unknown'})`, + 'INCOMPLETE', + ) + } + return { done: true } + } + + if (!page.cursor || page.cursor === cursor) { + throw new AssetBridgeError( + `The shard asked for another asset page without advancing its cursor (${page.cursor || 'none'})`, + 'STUCK', + ) + } + + return { done: false, cursor: page.cursor } +} + +// The client files the body catalogue is derived from. `assets.sources` reports +// every file the shard can see; these are the ones that decide a sprite. +// +// `body.def` and `bodyconv.def` are in the list and it would be easy to leave +// them out — they hold no pixels. They decide WHICH record a body id resolves to, +// so an operator editing one changes what every affected creature looks like +// while every anim file stays byte-identical. That is precisely the drift a +// content hash of the art files cannot see. +const SOURCE_FILES = [ + 'anim.idx', 'anim.mul', + 'anim2.idx', 'anim2.mul', + 'anim3.idx', 'anim3.mul', + 'anim4.idx', 'anim4.mul', + 'anim5.idx', 'anim5.mul', + 'body.def', 'bodyconv.def', + 'verdata.mul', +] + +/** + * Stage 1 of the import gate (§6): have the client files this family reads + * changed at all? + * + * Returns `{ files, extractorVersion, hashing, complete, imaging }` where `files` + * is a `{ name: { size, mtime, sha256 } }` map over `SOURCE_FILES` — a file the + * shard does not have is simply absent, which is normal (few clients carry all + * five anim files). + * + * **A null `sha256` means "not computed yet", never "changed".** The shard hashes + * off the request path because `anim.mul` alone is 195 MB and hashing it cannot + * fit inside a reply, and it reports `hashing: true` while that runs. + * `sameSources` below falls back to (size, mtime) in that case, which is the same + * gate the shard itself applies. + */ +async function sourceFingerprint() { + const res = await uoLinkClient.getAssetSources() + if (!res.ok) throw describeFailure(res, 'client file manifest') + + const wanted = new Set(SOURCE_FILES) + const files = {} + + for (const entry of res.data?.files ?? []) { + const name = String(entry?.name || '').toLowerCase() + if (!wanted.has(name)) continue + + files[name] = { + size: Number(entry.size) || 0, + mtime: Number(entry.mtime) || 0, + sha256: entry.sha256 ?? null, + } + } + + return { + files, + extractorVersion: Number(res.data?.extractorVersion) || 0, + hashing: Boolean(res.data?.hashing), + complete: Boolean(res.data?.complete), + imaging: res.data?.imaging ?? null, + } +} + +/** + * True when two source fingerprints describe the same client files. + * + * The file SET has to match as well as each file's contents: a client that gained + * an `anim5.mul` it did not have before is a client whose gargoyles suddenly + * resolve, and comparing only the files present in both would call that + * unchanged. + */ +function sameSources(a, b) { + if (!a || !b) return false + if (a.extractorVersion !== b.extractorVersion) return false + + const names = new Set([...Object.keys(a.files ?? {}), ...Object.keys(b.files ?? {})]) + + for (const name of names) { + const left = a.files?.[name] + const right = b.files?.[name] + + if (!left || !right) return false + + if (left.sha256 && right.sha256) { + if (left.sha256 !== right.sha256) return false + continue + } + + if (left.size !== right.size || left.mtime !== right.mtime || left.size <= 0) return false + } + + return names.size > 0 +} + +/** + * Stage 2: the whole manifest for the body family. + * + * Returns `{ rows, catalog, extractorVersion, playerBodies, pages, scanned }`. + * No pixels — `rows` is `[{ key, sha256, bytes, width, height, body, direction }]`. + */ +async function readManifest({ family = FAMILY } = {}) { + const started = Date.now() + const rows = [] + + let cursor = null + let pages = 0 + let catalog = null + let extractorVersion = 0 + let playerBodies = [] + let scanned = 0 + let finished = false + + while (pages < MAX_PAGES) { + const page = await withBusyRetry( + () => uoLinkClient.getAssetManifest({ family, cursor }), + `${family} asset manifest`, + ) + pages++ + + if (catalog === null) { + catalog = page.catalog ?? null + extractorVersion = Number(page.extractorVersion) || 0 + playerBodies = Array.isArray(page.playerBodies) ? page.playerBodies.map(Number) : [] + } else if (page.catalog !== catalog) { + // The client files moved between two pages of one walk. Refusing is the + // only honest answer: half of what we hold describes files that no longer + // exist, and nothing later can tell which half. + throw new AssetBridgeError( + "The shard's client files changed while the manifest was being read; nothing was imported", + 'SOURCE_CHANGED', + ) + } + + scanned += Number(page.scanned) || 0 + + for (const row of page.rows) { + const key = String(row?.key ?? '') + if (key === '') continue + + rows.push({ + key, + family, + sha256: String(row?.sha256 ?? ''), + bytes: Number(row?.bytes) || 0, + width: Number(row?.width) || 0, + height: Number(row?.height) || 0, + body: Number.isFinite(Number(row?.body)) ? Number(row.body) : null, + direction: Number.isFinite(Number(row?.direction)) ? Number(row.direction) : null, + }) + } + + if (rows.length > MAX_ROWS) { + throw new AssetBridgeError( + `The shard listed more than ${MAX_ROWS} assets; refusing to keep reading`, + 'TOO_LARGE', + ) + } + + const state = checkPage(page, { arrayName: 'rows', cursor, pages }) + + if (state.done) { + finished = true + break + } + + cursor = state.cursor + } + + if (!finished) { + throw new AssetBridgeError( + `The asset manifest did not end within ${MAX_PAGES} pages; nothing was imported`, + 'TOO_LARGE', + ) + } + + log.info('asset manifest read from the shard', { + family, + rows: rows.length, + scanned, + pages, + ms: Date.now() - started, + }) + + return { rows, catalog, extractorVersion, playerBodies, pages, scanned } +} + +/** + * The bytes for an explicit list of keys. + * + * Returns a Map of key → `{ sha256, bytes, width, height, body, direction, png }` + * where `png` is a Buffer. A key the shard could not serve is **absent from the + * map** rather than present with a null — the caller then decides what that means + * for its own row, and the two ways it happens (`absent`, `unsupported`) are + * counted separately in the returned tallies so an operator can tell "this client + * has no art for that body" from "the site asked for a key shape this shard does + * not serve", which is a bug rather than a gap. + */ +async function fetchAssets({ keys, catalog } = {}) { + const started = Date.now() + const out = new Map() + const missing = { absent: 0, unsupported: 0 } + + const list = Array.isArray(keys) ? keys.filter((k) => typeof k === 'string' && k !== '') : [] + + if (list.length === 0) return { assets: out, missing, pages: 0 } + + let pages = 0 + + for (let i = 0; i < list.length; i += FETCH_CHUNK) { + const chunk = list.slice(i, i + FETCH_CHUNK) + + let cursor = null + let finished = false + let walked = 0 + + while (walked < MAX_PAGES) { + const page = await withBusyRetry( + () => uoLinkClient.fetchAssets({ keys: chunk, catalog, cursor }), + 'asset content', + ) + pages++ + walked++ + + for (const row of page.rows ?? []) { + const key = String(row?.key ?? '') + if (key === '') continue + + if (row?.status !== 'ok') { + if (row?.status === 'unsupported') missing.unsupported++ + else missing.absent++ + continue + } + + if (typeof row.png !== 'string' || row.png === '') { + missing.absent++ + continue + } + + out.set(key, { + sha256: String(row.sha256 ?? ''), + bytes: Number(row.bytes) || 0, + width: Number(row.width) || 0, + height: Number(row.height) || 0, + body: Number.isFinite(Number(row.body)) ? Number(row.body) : null, + direction: Number.isFinite(Number(row.direction)) ? Number(row.direction) : null, + png: Buffer.from(row.png, 'base64'), + }) + } + + const state = checkPage(page, { arrayName: 'rows', cursor, pages: walked }) + + if (state.done) { + finished = true + break + } + + cursor = state.cursor + } + + if (!finished) { + throw new AssetBridgeError( + `An asset fetch did not end within ${MAX_PAGES} pages; nothing was imported`, + 'TOO_LARGE', + ) + } + } + + log.info('asset content fetched from the shard', { + asked: list.length, + got: out.size, + absent: missing.absent, + unsupported: missing.unsupported, + pages, + ms: Date.now() - started, + }) + + return { assets: out, missing, pages } +} + +/** + * Slug → body id, for the atlas's own creature list (§8). + * + * `creatures` is `[{ slug, name }]` where `name` is the ServUO class name — which + * `shard_spawn_creatures.name` already holds, because the atlas build picks the + * winning spelling of the spawn TYPE token rather than inventing a display name. + * That is why this needs no new column to ask its question. + * + * Returns `[{ slug, typeName, body, status }]`, one row per creature asked, with + * every outcome recorded — including the negative ones. A creature the shard says + * it does not have is a fact worth keeping: without it, the next pass asks again, + * and the pass costs a real constructor per name on the shard's Core thread. + */ +async function resolveBodies({ creatures } = {}) { + const started = Date.now() + const list = Array.isArray(creatures) ? creatures : [] + const out = [] + + for (let i = 0; i < list.length; i += BODY_CHUNK) { + const chunk = list.slice(i, i + BODY_CHUNK) + const bySlug = new Map() + + for (const creature of chunk) { + const typeName = String(creature?.name ?? '').trim() + if (typeName === '') continue + // Several slugs can share a type name only if the atlas slugified two + // spellings to one slug, in which case they ARE one creature; asking once + // per distinct name is what keeps the batch inside the shard's cap. + if (!bySlug.has(typeName)) bySlug.set(typeName, []) + bySlug.get(typeName).push(String(creature.slug)) + } + + const types = [...bySlug.keys()] + if (types.length === 0) continue + + const page = await withBusyRetry(() => uoLinkClient.resolveBodies(types), 'body resolution') + + if (!page || !Array.isArray(page.rows)) { + throw new AssetBridgeError('The shard sent a body resolution with no rows array', 'MALFORMED') + } + + for (const row of page.rows) { + const typeName = String(row?.type ?? '') + const slugs = bySlug.get(typeName) + + if (!slugs) continue + + const status = String(row?.status ?? 'failed') + const body = status === 'ok' && Number.isFinite(Number(row?.body)) ? Number(row.body) : null + + for (const slug of slugs) out.push({ slug, typeName, body, status }) + } + } + + const resolved = out.filter((r) => r.status === 'ok').length + + log.info('creature bodies resolved by the shard', { + asked: list.length, + answered: out.length, + resolved, + ms: Date.now() - started, + }) + + return out +} + +module.exports = { + AssetBridgeError, + FAMILY, + BODY_CHUNK, + FETCH_CHUNK, + MAX_PAGES, + MAX_ROWS, + SOURCE_FILES, + sourceFingerprint, + sameSources, + readManifest, + fetchAssets, + resolveBodies, +} diff --git a/server/utils/uoLinkClient.js b/server/utils/uoLinkClient.js index c35e318..9de2683 100644 --- a/server/utils/uoLinkClient.js +++ b/server/utils/uoLinkClient.js @@ -209,6 +209,43 @@ const getClilocTable = ({ lang, cursor } = {}) => { return call(`/cliloc${qs ? `?${qs}` : ''}`) } +// Stage 2 of the import gate, PAGED: every asset the shard could serve, with a +// hash and a size and no pixels. The website diffs it against what it holds and +// fetches only the keys whose hash moved — which on an ordinary restart is none +// of them, and is the whole difference between an Update and a re-download. +// +// This family pages on the shard's WALL CLOCK rather than on bytes: its rows are +// ~90 bytes, but building one means decoding a sprite, so a page ends when the +// shard's scan budget is spent (`cut: 'limit'`) far more often than when the byte +// budget is (`cut: 'budget'`). Neither means finished; only `cut: 'end'` does. +const getAssetManifest = ({ family, cursor } = {}) => { + const params = new URLSearchParams() + if (family) params.set('family', family) + if (cursor) params.set('cursor', cursor) + const qs = params.toString() + return call(`/assets/manifest${qs ? `?${qs}` : ''}`) +} + +// The pixels, for keys the caller names. POST because the key list IS the request. +// +// `catalog` is the mid-import guard and should always be passed: it is an id the +// manifest derived from the client files themselves, and handing it back makes the +// shard refuse (422) if those files moved in between. Without it, an operator who +// patched their client halfway through an import gets one asset set stitched out +// of two, with nothing anywhere reporting a problem. +const fetchAssets = ({ keys, catalog, cursor } = {}) => + call('/assets/fetch', { method: 'POST', body: { keys, catalog, cursor } }) + +// Slug → body id (docs/link/v8.md §8). The atlas knows a creature by its ServUO +// class name; the client knows it by a body id; nothing in the ServUO tree +// declares the mapping, so the shard answers it by constructing the creature and +// reading `Body.BodyID`. +// +// That runs on the shard's CORE THREAD, so the batch is small and the shard +// refuses an over-long list rather than truncating it. `assetBridge.js` chunks; +// nothing else should call this directly. +const resolveBodies = (types) => call('/assets/bodies', { method: 'POST', body: { types } }) + // ── Commands ────────────────────────────────────────────────────────────── const confirmLink = (code, websiteUserId) => call('/link/confirm', { method: 'POST', body: { code, websiteUserId: String(websiteUserId) } }) @@ -431,6 +468,9 @@ module.exports = { getMarket, getAssetSources, getClilocTable, + getAssetManifest, + fetchAssets, + resolveBodies, confirmLink, linkLookup, createAccount, diff --git a/swagger-fragment.json b/swagger-fragment.json index 03ba544..4049d38 100644 --- a/swagger-fragment.json +++ b/swagger-fragment.json @@ -110,6 +110,100 @@ ] } }, + "/api/v1/admin/shard/assets": { + "get": { + "tags": [ + "Admin · Shard" + ], + "summary": "Client asset import status: what is loaded, what the shard has, whether they differ (admin only)", + "description": "What the site currently holds (the imported body catalogue, how many sprites are stored, how many atlas creatures resolved to a body id) beside what the shard reports for the client files those pictures come from. `drift: true` means the client files have changed since the last import — press Import. `shard.hashing: true` means a null hash is “not computed yet”, not “changed”: the shard hashes 195 MB anim files off the request path. `shard.imaging.ok: false` is the named NO_IMAGING state — a Linux shard host without libgdiplus cannot render a sprite at all, and the reason names the package to install. A shard with no link configured, or one that is down, is a reported state with a reason rather than an error.", + "responses": { + "200": { + "description": "Asset import status", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UoAssetStatus" + } + } + } + }, + "403": { + "description": "Admin role required", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "500": { + "description": "Internal Server Error" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ] + } + }, + "/api/v1/admin/shard/assets/import": { + "post": { + "tags": [ + "Admin · Shard" + ], + "summary": "Import creature artwork from the shard’s UO client (admin only)", + "description": "Walks the shard’s asset manifest, fetches only the sprites whose hash changed, stores them under uploads/atlas/, re-resolves every atlas creature to a body id and points each creature at its picture. This is the ONLY thing that imports — boot deliberately never calls the shard — so it is what an operator presses after patching their client. `force` re-imports even when the client files are unchanged. `approve` accepts a catalogue that no longer offers assets this site holds; refused by default, because an unmounted client volume and a deliberate downgrade are indistinguishable from the server and the wrong guess deletes artwork. An operator-supplied `spawnAtlas.art.json` always wins over an imported sprite. Nothing here throws for an operator-visible problem: a shard that is down, an asset plane switched off, or a host that cannot render images all answer 200 with status \"unavailable\" and a reason naming what to fix. Assets a client simply does not have are NOT failures — two thirds of the playable ghost and gargoyle bodies have no art on a stock client.", + "responses": { + "200": { + "description": "What happened", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UoAssetImportResult" + } + } + } + }, + "500": { + "description": "Internal Server Error" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "force": { + "type": "boolean", + "description": "Import even if the shard’s client files are unchanged." + }, + "approve": { + "type": "boolean", + "description": "Accept a catalogue that no longer offers assets this site holds." + } + } + } + } + } + } + } + }, "/api/v1/admin/shard/atlas": { "get": { "tags": [ @@ -8229,6 +8323,1004 @@ } } }, + "UoAssetStatus": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "Admin view of the client-asset import (docs/link/v8.md §6, §8). What the site holds beside what the shard’s UO client currently is. Holding nothing at all is a supported state — creature pages simply render without pictures, which is what every install did before this pipeline existed." + }, + "properties": { + "type": "object", + "properties": { + "loaded": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "What this site currently holds." + }, + "properties": { + "type": "object", + "properties": { + "assets": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "Rows in the imported catalogue." + }, + "example": { + "type": "number", + "example": 787 + } + } + }, + "stored": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "How many of those have a picture on disk. Lower than `assets` when the shard listed a key it could not render." + }, + "example": { + "type": "number", + "example": 787 + } + } + }, + "creatures": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "Atlas creatures the shard has answered a body question about, resolved or not." + }, + "example": { + "type": "number", + "example": 812 + } + } + }, + "resolved": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "How many of those resolved to a body id. The rest are types this shard’s scripts do not define, or spawn entries naming an item rather than a creature." + }, + "example": { + "type": "number", + "example": 780 + } + } + }, + "catalog": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The shard’s catalogue id at the last import — derived from its client files, so it changes exactly when they do." + }, + "example": { + "type": "string", + "example": "a3f9c21d4b8e0771" + } + } + }, + "extractorVersion": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The version of the shard’s extraction code. A bump makes every derived byte drift even though the client files did not move." + }, + "example": { + "type": "number", + "example": 1 + } + } + }, + "importedAt": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "format": { + "type": "string", + "example": "date-time" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + } + } + } + } + }, + "shard": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The shard’s own client files right now. NULL when there is no shard link or it could not be reached — see `reason`." + }, + "properties": { + "type": "object", + "properties": { + "files": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "How many of the animation/definition files this catalogue reads the shard actually has. Few clients carry all five anim files." + }, + "example": { + "type": "number", + "example": 9 + } + } + }, + "extractorVersion": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 1 + } + } + }, + "hashing": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "description": { + "type": "string", + "example": "A hash is being computed in the background. A null `sha256` while this is true means “not yet”, never “changed”." + }, + "example": { + "type": "boolean", + "example": false + } + } + }, + "complete": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "description": { + "type": "string", + "example": "Every client file has a content hash." + }, + "example": { + "type": "boolean", + "example": true + } + } + }, + "imaging": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Whether the shard host can render an image at all. `ok: false` is the named NO_IMAGING state: ServUO under Mono needs libgdiplus, and without it a Linux shard cannot decode a sprite. Cliloc and atlas import are unaffected." + }, + "properties": { + "type": "object", + "properties": { + "ok": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "example": { + "type": "boolean", + "example": true + } + } + }, + "code": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "example": {} + } + }, + "reason": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + } + } + } + } + } + } + } + } + }, + "drift": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "boolean" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "True when the shard’s client files no longer match what was imported — press Import. NULL when they could not be read." + }, + "example": { + "type": "boolean", + "example": false + } + } + }, + "reason": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Why the shard could not be asked, when it could not." + } + } + }, + "code": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Machine-readable cause of `reason`." + }, + "enum": { + "type": "array", + "example": [ + "DISABLED", + "NO_SOURCE", + "SHARD_DOWN", + "PROTOCOL", + "BUSY", + "NO_IMAGING", + "SOURCE_CHANGED", + "UNAVAILABLE" + ], + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "UoAssetImportResult": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "Outcome of an asset import. Reported rather than thrown, so a shard that is down or a host that cannot render images is an answer and not a 500." + }, + "properties": { + "type": "object", + "properties": { + "status": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "enum": { + "type": "array", + "example": [ + "skipped", + "unavailable", + "unchanged", + "imported", + "needsReview", + "failed" + ], + "items": { + "type": "string" + } + }, + "description": { + "type": "string", + "example": "`skipped`: no shard is configured. `unchanged`: the client files match what was imported and nothing was fetched. `needsReview`: assets this site holds are no longer offered by the shard, and nothing was changed — re-run with `approve` to accept it." + }, + "example": { + "type": "string", + "example": "imported" + } + } + }, + "reason": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "code": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Machine-readable cause. `NO_IMAGING` is a shard host with no libgdiplus; `SOURCE_CHANGED` is a client patched partway through the walk, in which case nothing was applied." + }, + "enum": { + "type": "array", + "example": [ + "DISABLED", + "NO_SOURCE", + "SHARD_DOWN", + "PROTOCOL", + "BUSY", + "NO_IMAGING", + "SOURCE_CHANGED", + "INCOMPLETE", + "STUCK", + "MALFORMED", + "TOO_LARGE", + "UNAVAILABLE" + ], + "items": { + "type": "string" + } + } + } + }, + "catalog": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "example": { + "type": "string", + "example": "a3f9c21d4b8e0771" + } + } + }, + "extractorVersion": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "example": { + "type": "number", + "example": 1 + } + } + }, + "assets": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Catalogue rows after the import." + }, + "example": { + "type": "number", + "example": 787 + } + } + }, + "fetched": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "How many sprites actually crossed the wire. On an Update after a client patch this is far smaller than `assets`, which is the point of the manifest." + }, + "example": { + "type": "number", + "example": 12 + } + } + }, + "written": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "How many were written to disk." + }, + "example": { + "type": "number", + "example": 12 + } + } + }, + "absent": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Keys the shard listed but could not render. NOT a failure: this client has no art at that key, which is the expected answer for two thirds of the playable ghost and gargoyle bodies." + }, + "example": { + "type": "number", + "example": 0 + } + } + }, + "unsupported": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Keys the shard does not serve at all. Unlike `absent` this indicates a bug on the site’s side, not a gap in the client." + }, + "example": { + "type": "number", + "example": 0 + } + } + }, + "removed": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Assets deleted because the shard no longer offers them (only with `approve`)." + }, + "example": { + "type": "number", + "example": 0 + } + } + }, + "scanned": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Body ids the shard walked. Far larger than `assets` — most of the addressable range has no art." + }, + "example": { + "type": "number", + "example": 2047 + } + } + }, + "pages": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "Manifest pages. This family pages on the shard’s scan budget rather than on bytes, so several is normal." + }, + "example": { + "type": "number", + "example": 4 + } + } + }, + "playerBodies": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "array" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + } + } + }, + "description": { + "type": "string", + "example": "The body ids the shard reports as player-character bodies — every registered race’s male, female and ghost bodies, asked of the shard rather than hardcoded. These render head-on; everything else renders three-quarter." + }, + "example": { + "type": "array", + "example": [ + 400, + 401, + 402, + 403, + 605, + 606, + 607, + 608, + 666, + 667, + 694, + 695 + ], + "items": { + "type": "number" + } + } + } + }, + "vanished": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "array" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + } + } + }, + "description": { + "type": "string", + "example": "On `needsReview`: up to fifty of the keys that disappeared." + } + } + }, + "vanishedCount": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + }, + "bodies": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The slug → body id pass (§8). The shard constructs each creature and reads its body id, which is the only thing correct for a shard’s own custom creatures." + }, + "properties": { + "type": "object", + "properties": { + "asked": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 812 + } + } + }, + "answered": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 812 + } + } + }, + "resolved": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 780 + } + } + }, + "tally": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "description": { + "type": "string", + "example": "Per-outcome counts. `unknown` is real drift worth acting on — a spawn file naming a type this shard’s scripts do not define. `notCreature` is a spawn entry for an item or decoration and is permanent." + }, + "properties": { + "type": "object", + "properties": { + "ok": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 780 + } + } + }, + "unknown": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 20 + } + } + }, + "notCreature": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 12 + } + } + }, + "failed": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "example": { + "type": "number", + "example": 0 + } + } + } + } + } + } + }, + "reason": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + } + } + } + } + }, + "art": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "object" + }, + "nullable": { + "type": "boolean", + "example": true + }, + "description": { + "type": "string", + "example": "The derivation onto `shard_spawn_creatures.art`. An operator-supplied `spawnAtlas.art.json` always wins over an imported sprite." + }, + "properties": { + "type": "object", + "properties": { + "applied": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "Creatures now pointing at a picture." + }, + "example": { + "type": "number", + "example": 763 + } + } + }, + "derived": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "From the import." + }, + "example": { + "type": "number", + "example": 763 + } + } + }, + "operator": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "integer" + }, + "description": { + "type": "string", + "example": "From the operator’s own map." + }, + "example": { + "type": "number", + "example": 0 + } + } + }, + "error": { + "type": "object", + "properties": { + "type": { + "type": "string", + "example": "string" + }, + "nullable": { + "type": "boolean", + "example": true + } + } + } + } + } + } + } + } + } + } + }, "UoShardLinkRequest": { "type": "object", "properties": {