From 64c0ec00b1a3386abb33d7ac56fa57171bc79cbf Mon Sep 17 00:00:00 2001 From: wtclaude Date: Thu, 10 Sep 2026 18:40:12 -0500 Subject: [PATCH] feat(asset-bridge): the body catalogue and slug to body id (Phase 3) Two request families, and they run on opposite threads on purpose. `assets.bodies` (BridgeBodies) answers the question only code inside ServUO can: the atlas knows a creature by the class name in Spawns/*.xml, the client knows it by a body id, and nothing in the tree declares the mapping. Construct the type, read Body.BodyID, Delete(). That is world mutation, so it answers on the CORE thread and is the one family here that does not take the asset worker's slot -- and the batch is capped at 100 names, REFUSED rather than truncated, because a truncated answer is indistinguishable from a complete one from the website's side. `assets.manifest` / `assets.fetch` (BridgeCatalog) are the catalogue, on the worker. The manifest carries { key, sha256, bytes, width, height } and no pixels, so an Update fetches only what moved; the fetch carries base64 PNG. The scan keeps the bytes it hashed rather than decoding all 787 sprites twice. Three things worth stating about the shapes: - It pages on the WALL CLOCK as well as on bytes. The rows are ~90 bytes and the whole catalogue is one page by the byte budget, but building it means decoding hundreds of sprites against a 10 s reply timeout. - `catalog` is derived from the client files (sizes, mtimes, both direction settings, EXTRACTOR_VERSION), not minted per build -- the cache is released when idle, and a fresh id per build would force a restart mid-import although nothing about the client moved. - ARGB1555 is expanded to 32bpp here rather than handed to GDI+, because what it does with a one-bit alpha channel varies by platform and a black rectangle behind every sprite would pass any test that only checked the bytes decoded. Nothing trusts the library's success. Every body goes through CheckEntry and AnimationRecordSane before it is decoded, which is what keeps the 357 bodies whose index entry reads `length 0` -- and which the decoder hands back the PREVIOUS creature's bitmap for -- out of the catalogue. Walked on a live shard: 787 rows in one 734 ms page; bodies 320, 607, 666 all absent rather than wrong; 783 at direction 1 and 4 at direction 0; all 455 stock creature classes resolved at ~190 ms per 100 with zero mobiles leaked. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016wDDVXWMDz82WqE1i969r4 --- overlay/Config/Bridge.cfg | 33 + overlay/Scripts/Custom/Bridge/BridgeAssets.cs | 15 + overlay/Scripts/Custom/Bridge/BridgeBodies.cs | 254 +++++ overlay/Scripts/Custom/Bridge/BridgeBoot.cs | 1 + .../Scripts/Custom/Bridge/BridgeCatalog.cs | 1010 +++++++++++++++++ overlay/Scripts/Custom/Bridge/BridgeConfig.cs | 62 + 6 files changed, 1375 insertions(+) create mode 100644 overlay/Scripts/Custom/Bridge/BridgeBodies.cs create mode 100644 overlay/Scripts/Custom/Bridge/BridgeCatalog.cs diff --git a/overlay/Config/Bridge.cfg b/overlay/Config/Bridge.cfg index 5be83c3..9fc1141 100644 --- a/overlay/Config/Bridge.cfg +++ b/overlay/Config/Bridge.cfg @@ -307,6 +307,39 @@ AssetsEnabled=true # still fits. AssetBatchBytes=524288 +# How many ServUO class names one `assets.bodies` request may carry (phase 3). The only +# bound on this plane counted in items rather than bytes, because what it bounds is not +# reply size -- it is constructing and deleting that many real mobiles ON THE CORE +# THREAD, between two ticks of the world. A larger request is refused, never truncated. +# Clamped to [1, 500]. +AssetBodyBatch=100 + +# How many keys one `assets.fetch` request may name. The byte budget above still decides +# where a page is cut; this only bounds how large a request the shard will parse at all. +# Clamped to [1, 10000]. +AssetFetchKeys=2000 + +# The wall-clock budget for one catalogue page, in milliseconds. The catalogue's manifest +# rows are ~90 bytes so the byte budget never stops it -- but building them means +# decoding hundreds of animations, and the sidecar waits 10 s for a reply. Kept well +# under that, because the page still has to be serialised and written afterwards. +# Clamped to [250, 5000]. +AssetScanMs=3000 + +# Which direction the catalogue renders. NOT part of the asset key: five directions +# would five-fold every count in the working set to express a choice nobody varies. +# +# The split was found by RENDERING all five, not from a table. 0 is head-on, facing the +# viewer -- what a character portrait wants, and the least legible view there is of a +# four-legged creature (a wolf seen from the front is a dark blob). 1 is the front +# three-quarter, where the same wolf is unmistakably a wolf. +# +# Which bodies count as player bodies is asked of the shard (every registered race's +# male/female/ghost ids), never hardcoded. Clamped to [0, 4]: 5-7 are the client +# mirroring 1-3 through a decode branch this overlay has not verified. +AssetPlayerDirection=0 +AssetCreatureDirection=1 + # The test scaffolding in tools/scaffolding/ reads its own flags from this file # (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose: # Config.Get returns the default of false when a key is missing, so a deployed diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs index 3cca03e..d809aa6 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs @@ -473,6 +473,21 @@ namespace Server.Custom.Bridge sb.Append('}'); } + /// + /// Whether this host can produce a picture, for the families that produce pictures. + /// + /// reports this on the source gate so an operator learns it + /// while setting the shard up. The catalogue needs the same answer as a *decision* — + /// it must refuse rather than throw a DllNotFoundException out of the middle of + /// a decode loop — so the check itself is shared and this is its one accessor. + /// + internal static bool ImagingOk(out string reason) + { + CheckImaging(); + reason = _imagingReason; + return _imagingOk; + } + private static void CheckImaging() { if (_imagingChecked) diff --git a/overlay/Scripts/Custom/Bridge/BridgeBodies.cs b/overlay/Scripts/Custom/Bridge/BridgeBodies.cs new file mode 100644 index 0000000..00b599f --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeBodies.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +using Server.Mobiles; + +namespace Server.Custom.Bridge +{ + /// + /// **Slug to body id — the part only the shard can do** (docs/link/v8.md §8, protocol 8, + /// phase 3). + /// + /// The spawn atlas knows a creature by a **slug** derived from the class name it found in + /// `Spawns/*.xml` ("giant-spider"). The client knows the same creature by a **body id** + /// (28). Nothing in the ServUO tree declares that mapping as data. Today an operator + /// bridges it by hand, grepping `Scripts/Mobiles/Normal/<Name>.cs` for `Body =`, + /// which appears as a decimal, as hex (`0xD1`), as `Utility.RandomList(35, 36)` and as an + /// `m_IDs[]` table — a parse that is wrong on the shard's own custom creatures, which is + /// precisely the set an operator most wants pictures for. + /// + /// Inside ServUO the problem does not exist: construct the type, read `Body.BodyID`, + /// delete it. BridgeWorld.cs already does exactly that for a different feature. + /// + /// **This is the one asset-plane family that does NOT run on the asset worker**, and the + /// reason is the whole point of §8. Constructing and deleting a mobile is world mutation, + /// so it must happen on the Core thread — while the decode in + /// must happen off it, because it reads hundreds of megabytes and would stop the world for + /// every player on the shard. That split is why body resolution is its own request kind + /// rather than a step inside asset extraction. + /// + /// Two consequences follow from answering on the Core thread, and both are bounds: + /// + /// **The batch is small and the shard enforces the cap itself.** Every type constructed + /// here runs a real constructor — packing items, rolling skills, starting AI timers — and + /// all of that happens between two ticks of the world. The website chunks its own list; + /// a request over names is **refused** rather + /// than truncated, so the two sides cannot quietly disagree about what was answered. + /// + /// **It does not take the asset plane's single slot.** The slot exists to stop several + /// large replies queueing at once (§3.2); this reply is a few kilobytes and the work is + /// not on the worker, so claiming the slot would only make a body pass and a catalogue + /// page refuse each other for no benefit. + /// + /// **A creature whose constructor randomises its body reports one of its variants**, not + /// an error and not a set. Constructing twice to detect that would double every side + /// effect above to learn something the bestiary does not render differently — both ids are + /// the same creature. The answer is stable enough to cache and cheap enough to redo. + /// + public static class BridgeBodies + { + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("assets.bodies", OnBodies); + } + + // ── the request ────────────────────────────────────────────────────────────────────── + + private static void OnBodies(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + // Rule 1 of the asset plane: without a correlation id this reply lands on the + // event path, is persisted to the sidecar's store and broadcast to every + // subscriber. Refuse rather than answer. + BridgeAssets.Fail(null, "BAD_REQUEST", "assets.bodies requires a reqId"); + return; + } + + if (!BridgeConfig.AssetsEnabled) + { + BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return; + } + + var types = BridgeJson.GetStringList(o, "types"); + + if (types.Count == 0) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.bodies requires a non-empty `types` array of ServUO class names"); + return; + } + + if (types.Count > BridgeConfig.AssetBodyBatch) + { + // Refuse, never truncate. A silently shortened answer looks identical to a + // complete one from the website's side, and the types that fell off the end would + // be recorded as "asked and unanswerable" rather than "never asked". + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.bodies takes at most " + BridgeConfig.AssetBodyBatch + + " types per request (asked for " + types.Count + "); send them in chunks"); + return; + } + + Reply(reqId, types); + } + + /// + /// Core thread. Constructs each type once, reads its body, deletes it. + /// + /// Every outcome is a **row**, never a failed request: a shard is expected to be asked + /// about types it does not have (an atlas built from a tree that has since changed, a + /// spawn file naming a creature from a script package the operator removed), and a + /// status screen that fails the whole pass over one of those teaches an operator to + /// stop pressing the button. + /// + private static void Reply(string reqId, List types) + { + var sb = BridgeJson.Begin("assets.bodies.ok"); + + sb.Str("reqId", reqId) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", types.Count); + + // The envelope is shared with every other family (§3.4) even though this one never + // pages: the website drives the chunking, so `more` is always false and `cut` always + // "end". Writing it anyway means one reader shape on the other side rather than two. + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int resolved = 0; + + foreach (var name in types) + { + string status; + int body; + + Resolve(name, out body, out status); + + if (status == "ok") + resolved++; + + var item = new StringBuilder(96); + + item.Append("{\"type\":"); + BridgeJson.Text(item, name); + item.Append(",\"status\":\"").Append(status).Append('"'); + + if (status == "ok") + item.Append(",\"body\":").Append(body.ToString(CultureInfo.InvariantCulture)); + + item.Append('}'); + + // A chunk this small cannot spend the budget — the cap above is a hundred names + // and the budget is half a megabyte — but the check costs nothing and the day + // someone raises `AssetBodyBatch` it is the difference between a short page and a + // line the sidecar drops. + if (!page.TryAdd(item.ToString(), null)) + break; + } + + page.Close(); + + sb.Num("resolved", resolved); + + BridgeLink.Emit(sb.End()); + } + + /// + /// One type name to one body id. + /// + /// `status` is the field the website records, and the four values are four different + /// things an operator can act on: + /// + /// ok — constructed, body read. + /// unknown — no such type on this shard. The spawn file names something the + /// scripts do not define, which is a real drift an operator wants to see. + /// notCreature — the type exists but is not a `BaseCreature`. Spawn files + /// legitimately name items and static decorations; those have no body and never will, + /// so this is a permanent answer rather than a retryable failure. + /// failed — the constructor threw, or the type has none that takes no + /// arguments. Caught per type, because one creature whose constructor depends on a + /// script package the operator removed must not cost the other ninety-nine. + /// + private static void Resolve(string name, out int body, out string status) + { + body = 0; + status = "failed"; + + Type type; + + try + { + // `true` is ignoreCase — spawn files are hand-edited and their casing drifts from + // the class it names far more often than the name itself does. + type = ScriptCompiler.FindTypeByName(name, true); + } + catch + { + status = "failed"; + return; + } + + if (type == null) + { + status = "unknown"; + return; + } + + if (!typeof(BaseCreature).IsAssignableFrom(type) || type.IsAbstract) + { + status = "notCreature"; + return; + } + + BaseCreature creature = null; + + try + { + creature = Activator.CreateInstance(type) as BaseCreature; + + if (creature == null) + { + status = "failed"; + return; + } + + body = creature.Body.BodyID; + status = body > 0 ? "ok" : "failed"; + } + catch (Exception e) + { + Console.WriteLine("[Bridge] assets.bodies: {0}: {1}: {2}", + name, e.GetType().Name, e.Message); + + status = "failed"; + } + finally + { + if (creature != null) + { + try + { + // Deleting the mobile deletes the items it packed — `Mobile.Delete` walks + // `Items`, and `Item.Delete` walks what each contains — and stops its AI + // timer. A creature left alive here is a creature standing at (0,0,0) on + // the internal map forever, saved with the world, once per import. + creature.Delete(); + } + catch + { + // Nothing useful is left to do, and throwing out of `finally` would lose + // whatever the try block was already reporting. + } + } + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs index d09dc43..1511e8f 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs @@ -261,6 +261,7 @@ namespace Server.Custom.Bridge e.Mobile.SendMessage("Bridge: {0}", BridgeWorld.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeOneShots.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeAssets.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeCatalog.Status()); break; } } diff --git a/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs new file mode 100644 index 0000000..00b3ad0 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs @@ -0,0 +1,1010 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The body catalogue** (docs/link/v8.md §4.8, §5, §6, §11 — protocol 8, phase 3). + /// + /// One thumbnail per creature body: the working set that makes a bestiary, a marketplace + /// listing and a character sheet render. Everything deeper — every action, every frame — + /// is the same addressing scheme at a deeper key, fetched on demand in a later phase; this + /// is the set that is worth importing before anything asks for it, because on this + /// machine's client it is **787 sprites at about a kilobyte each**. + /// + /// Two request kinds, which are §6's two stages for assets rather than for sources: + /// + /// assets.manifest — `[{ key, sha256, bytes, width, height }]`, no pixels. The + /// website diffs it against what it already holds and asks only for what changed. That is + /// the whole difference between an Update and a re-download. + /// + /// assets.fetch — the pixels, for an explicit list of keys. + /// + /// ── **Why the manifest builds the pictures it refuses to send** ── + /// + /// A manifest row carries a hash of the bytes, and the only way to hash bytes is to have + /// them. So the scan decodes, encodes to PNG and hashes, then sends the row and **keeps + /// the bytes** — a megabyte for the whole catalogue, against re-decoding all 787 sprites a + /// second time when the fetch arrives moments later. + /// + /// ── **Why the manifest pages on TIME rather than on bytes** ── + /// + /// Every other family on this plane pages because its rows are large. This one's rows are + /// ninety bytes and the whole catalogue is one page by the byte budget — but producing + /// that page means decoding 787 animations, and the sidecar gives a reply ten seconds + /// (§3.3). So the scan carries a **wall-clock budget** as well + /// () and cuts the page `limit` when it is spent, + /// resuming from its cursor on the next call. The byte budget is still enforced, because + /// the day a family's rows grow is not the day to discover only one of the two bounds was + /// real. + /// + /// ── **Why `catalog` is derived from the sources and not minted per build** ── + /// + /// A manifest walk and the fetch that follows it must be talking about the same client + /// files, or the website stitches one catalogue out of two. The obvious answer is a fresh + /// id per build, and it is wrong: this cache is released when it goes idle, so a rebuild + /// halfway through a slow import would change the id and force a restart although nothing + /// about the client moved. So the id is a hash of what actually decides the bytes — every + /// anim file's size and mtime, both direction settings and + /// . It is stable across a rebuild and it + /// changes exactly when an operator patches their client. + /// + /// ── **The never-sweep rule, and the 357** ── + /// + /// Nothing here asks a file type for an index it does not own, and nothing here trusts the + /// library's own success. takes + /// `BodyConverter.Convert`'s answer and reports nothing if it leads nowhere (sweeping + /// instead puts a giant spider on the gargoyle page, decoding cleanly); and every body is + /// put through and + /// RecordReader.AnimationSane **before** it is decoded, because a body whose index + /// entry reads `length 0` gets a bitmap back anyway — the previously-decoded creature's, + /// from the library's reused stream buffer. That is 357 of the 1,144 bodies the library + /// claims on a stock client, and importing them would have written 357 duplicate + /// portraits whose subject depended on the order this walk happened to run in. + /// + public static class BridgeCatalog + { + /// The only family this phase serves. §5's key scheme covers the rest. + private const string Family = "body"; + + /// Bodies are addressable to 2047; the sweep behind §4.8 covered exactly this. + private const int MaxBody = 2047; + + /// The catalogue is first frames only. Deep keys are phase 6. + private const int CatalogAction = 0; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("assets.manifest", OnManifest); + BridgeBoot.RegisterHandler("assets.fetch", OnFetch); + } + + // ── the cache ──────────────────────────────────────────────────────────────────────── + + private sealed class Sprite + { + public string Key; + public int Body; + public int Direction; + public int FileType; + public string Sha256; + public byte[] Png; + public int Width; + public int Height; + } + + private sealed class Catalog + { + public string Id; + public readonly Dictionary ByKey = + new Dictionary(StringComparer.Ordinal); + public readonly List Order = new List(); + + /// The next body the scan has yet to look at. + public int Next = 1; + + public bool Complete; + public DateTime LastUsed; + } + + private static readonly object _sync = new object(); + private static Catalog _catalog; + + private static readonly TimeSpan IdleFor = TimeSpan.FromMinutes(5); + + // ── assets.manifest ────────────────────────────────────────────────────────────────── + + private static void OnManifest(Dictionary o) + { + string reqId; + + if (!Admit(o, "assets.manifest", out reqId)) + return; + + var family = BridgeJson.GetString(o, "family") ?? Family; + + if (!String.Equals(family, Family, StringComparison.Ordinal)) + { + // Named rather than ignored: `family` exists so §5's statics and land can join + // this envelope in phase 5 without a second request kind, and a website that + // asked for one of those against a phase-3 overlay must be told it asked too + // early rather than handed a body catalogue it did not request. + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "this shard serves the '" + Family + "' asset family only (asked for '" + + family + "')"); + return; + } + + var cursor = BridgeJson.GetString(o, "cursor"); + + BridgeAssets.Accept(reqId, "assets.manifest", () => ReplyManifest(reqId, cursor)); + } + + /// + /// Worker thread. Scans forward from the cursor until the byte budget or the time + /// budget is spent, hashing what it decodes and keeping the bytes for the fetch. + /// + private static void ReplyManifest(string reqId, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + // Never a stack trace and never a 500: on a Linux host without libgdiplus this is + // the expected outcome, and it is actionable in one line (§4.4). + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. Install it (apt-get install libgdiplus) and re-run the import. " + + "Cliloc and atlas import are unaffected. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + Catalog catalog; + + lock (_sync) + { + if (_catalog == null || _catalog.Id != id) + _catalog = new Catalog { Id = id }; + + catalog = _catalog; + catalog.LastUsed = DateTime.UtcNow; + } + + int from = ParseBodyCursor(cursor); + + var sb = BridgeJson.Begin("assets.manifest.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + // What the website compares across pages, and across the fetch that follows. A + // change means the operator patched their client mid-import and the half already + // read describes files that no longer exist. + .Str("catalog", catalog.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("maxBody", MaxBody) + .Num("from", from); + + WritePlayerBodies(sb); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int scanned = 0; + int last = from - 1; + bool timedOut = false; + bool budgetCut = false; + + var deadline = DateTime.UtcNow.AddMilliseconds(BridgeConfig.AssetScanMs); + + using (var readers = new Readers()) + { + int body = from; + + for (; body <= MaxBody; body++) + { + // Checked before the body rather than after it, so the budget bounds the reply + // rather than the reply plus one more decode. One sprite is milliseconds; the + // ceiling this lives under is ten seconds and the cost of overshooting it is + // the whole page, retried. + if (body > from && DateTime.UtcNow >= deadline) + { + timedOut = true; + break; + } + + scanned++; + last = body; + + Sprite sprite = Resolve(catalog, readers, body); + + if (sprite == null) + continue; + + var item = new StringBuilder(128); + + item.Append("{\"key\":"); + BridgeJson.Text(item, sprite.Key); + item.Append(",\"sha256\":\"").Append(sprite.Sha256).Append('"'); + item.Append(",\"bytes\":").Append(sprite.Png.Length.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"width\":").Append(sprite.Width.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append('}'); + + if (!page.TryAdd(item.ToString(), "b:" + body.ToString(CultureInfo.InvariantCulture))) + { + // The budget stopped this page BEFORE this body's row went on it, so the + // next page must resume AT this body rather than after it. Getting this + // one line wrong drops exactly one creature from the catalogue per page, + // which nothing downstream could ever notice. + budgetCut = true; + last = body - 1; + scanned--; + break; + } + } + } + + if (timedOut) + page.Cut("limit"); + + // The walk reached the end of the addressable range without either budget stopping it. + // Derived from the two flags rather than from the row count, because a page that ends + // exactly on a boundary is indistinguishable from a finished one by count alone — + // §3.4's whole argument for `cut` existing. + bool finished = !timedOut && !budgetCut && last >= MaxBody; + + int held; + + lock (_sync) + { + if (_catalog == catalog) + { + catalog.Next = Math.Max(catalog.Next, last + 1); + catalog.LastUsed = DateTime.UtcNow; + + if (finished) + catalog.Complete = true; + } + + held = catalog.Order.Count; + } + + page.Close(); + + // Past Close(), which is normally the mistake BridgeCliloc's `from` comment warns + // about — but these three are not knowable until the scan has run, and they cost + // about fifty bytes against PageBuilder's 256-byte reserve, of which Close() itself + // spends around forty. Anything larger than this belongs before the page opens. + sb.Num("scanned", scanned) + .Num("held", held) + .Bool("complete", finished); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + // ── assets.fetch ───────────────────────────────────────────────────────────────────── + + private static void OnFetch(Dictionary o) + { + string reqId; + + if (!Admit(o, "assets.fetch", out reqId)) + return; + + var keys = BridgeJson.GetStringList(o, "keys"); + + if (keys.Count == 0) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.fetch requires a non-empty `keys` array"); + return; + } + + if (keys.Count > BridgeConfig.AssetFetchKeys) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.fetch takes at most " + BridgeConfig.AssetFetchKeys + + " keys per request (asked for " + keys.Count + ")"); + return; + } + + var catalog = BridgeJson.GetString(o, "catalog"); + var cursor = BridgeJson.GetString(o, "cursor"); + + BridgeAssets.Accept(reqId, "assets.fetch", () => ReplyFetch(reqId, keys, catalog, cursor)); + } + + private static void ReplyFetch(string reqId, List keys, string expected, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + if (expected != null && expected != id) + { + // The client files moved between the manifest and this fetch. Refusing is the only + // honest answer: the keys were chosen against a catalogue that no longer describes + // what is on disk, and serving them would mix two clients in one import with no + // error anywhere. + BridgeAssets.Fail(reqId, "UNREADABLE", + "the shard's client files changed since that manifest was read (catalog " + + expected + " is now " + id + "); start the import again"); + return; + } + + Catalog catalog; + + lock (_sync) + { + if (_catalog == null || _catalog.Id != id) + _catalog = new Catalog { Id = id }; + + catalog = _catalog; + catalog.LastUsed = DateTime.UtcNow; + } + + int from = ParseKeyCursor(cursor); + + var sb = BridgeJson.Begin("assets.fetch.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + .Str("catalog", catalog.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", keys.Count) + .Num("from", from); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int i = from; + + using (var readers = new Readers()) + { + for (; i < keys.Count; i++) + { + var item = Render(catalog, readers, keys[i]); + + if (!page.TryAdd(item, "k:" + (i + 1).ToString(CultureInfo.InvariantCulture))) + break; + } + } + + page.Close(); + + sb.Num("sent", page.Count); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + /// + /// One key to one row, with the bytes. + /// + /// A key this shard cannot serve is a **row**, not a failed request: the website asked + /// for a list, and one key naming a body whose art this client does not carry must not + /// cost the other four hundred. `status` distinguishes the two ways that happens — + /// `absent` (this client has no art at that key, the expected answer for two thirds of + /// the player bodies) and `unsupported` (a key shape this phase does not serve, which + /// is a website bug rather than a client gap). + /// + private static string Render(Catalog catalog, Readers readers, string key) + { + int body; + + if (!TryParseKey(key, out body)) + { + var bad = new StringBuilder(96); + bad.Append("{\"key\":"); + BridgeJson.Text(bad, key); + bad.Append(",\"status\":\"unsupported\"}"); + return bad.ToString(); + } + + Sprite sprite = Resolve(catalog, readers, body); + + var item = new StringBuilder(2048); + + item.Append("{\"key\":"); + BridgeJson.Text(item, key); + + if (sprite == null) + { + item.Append(",\"status\":\"absent\"}"); + return item.ToString(); + } + + item.Append(",\"status\":\"ok\""); + item.Append(",\"sha256\":\"").Append(sprite.Sha256).Append('"'); + item.Append(",\"bytes\":").Append(sprite.Png.Length.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"width\":").Append(sprite.Width.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"png\":\"").Append(Convert.ToBase64String(sprite.Png)).Append("\"}"); + + return item.ToString(); + } + + // ── decode ─────────────────────────────────────────────────────────────────────────── + + /// + /// The catalogue entry for one body, decoded and hashed on first sight and cached + /// after. Returns null when this client has no art for it — which is an ordinary + /// answer for well over half of the addressable range, not a failure. + /// + private static Sprite Resolve(Catalog catalog, Readers readers, int body) + { + string key = Key(body); + + lock (_sync) + { + Sprite cached; + + if (catalog.ByKey.TryGetValue(key, out cached)) + return cached; + } + + int direction = IsPlayerBody(body) + ? BridgeConfig.AssetPlayerDirection + : BridgeConfig.AssetCreatureDirection; + + int fileType, at; + string reason; + + if (!BridgeAssetValidator.ResolveAnimation(body, CatalogAction, direction, + out fileType, out at, out reason)) + return null; + + FileIndex index = readers.Index(fileType); + + if (index == null) + return null; + + if (BridgeAssetValidator.CheckEntry(index, at, readers.MulLength(fileType), + readers.VerdataLength, out reason) != BridgeAssetValidator.Verdict.Ok) + { + // The `length 0` case lands here, and it is the 357. The library would hand back + // the previously-decoded body's bitmap for every one of them. + return null; + } + + var reader = readers.Reader(fileType); + + if (reader == null) + return null; + + // `maxFrames: 1` because that is what `firstFrame: true` decodes. Checking frames + // nobody reads would invent refusals, and a checker that refuses real art is worse + // than no checker at all. + if (!reader.AnimationSane(index, at, 1, out reason)) + return null; + + Sprite sprite; + + try + { + sprite = Decode(key, body, direction, fileType); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: body {0}: {1}: {2}", + body, e.GetType().Name, e.Message); + return null; + } + + if (sprite == null) + return null; + + lock (_sync) + { + if (!catalog.ByKey.ContainsKey(key)) + { + catalog.ByKey[key] = sprite; + catalog.Order.Add(sprite); + } + + return catalog.ByKey[key]; + } + } + + private static Sprite Decode(string key, int body, int direction, int fileType) + { + int hue = 0; + + // `preserveHue: false` — the catalogue is the creature's own art, and a body-level hue + // from Body.def belongs to a specific mob rather than to the species. §5's key scheme + // is where a hued variant is expressed (`static/3922/h33`), not here. + Frame[] frames = Animations.GetAnimation(body, CatalogAction, direction, ref hue, false, true); + + if (frames == null || frames.Length == 0 || frames[0] == null) + return null; + + Bitmap bitmap = frames[0].Bitmap; + + if (bitmap == null || bitmap.Width <= 0 || bitmap.Height <= 0) + return null; + + byte[] png = ToPng(bitmap); + + if (png == null) + return null; + + return new Sprite + { + Key = key, + Body = body, + Direction = direction, + FileType = fileType, + Png = png, + Width = bitmap.Width, + Height = bitmap.Height, + Sha256 = Hash(png) + }; + } + + /// + /// ARGB1555 to a PNG with a transparent background. + /// + /// Frame writes 16-bit ARGB1555: a pixel the sprite does not cover is left as + /// zero and a pixel it does cover carries the top bit set. Saving that format straight + /// to PNG asks GDI+ to make the conversion, and what it does with a one-bit alpha + /// channel varies by platform — on Mono it is a different implementation entirely. A + /// sprite that came back with a black rectangle behind it would look fine in a test + /// that only checked the bytes decoded, and wrong on every page that showed it. + /// + /// So the expansion is done here, explicitly: alpha bit clear becomes fully + /// transparent, and each 5-bit channel is widened to 8 bits by repeating its high bits + /// ((c << 3) | (c >> 2)) rather than by shifting alone, which would + /// cap white at 248 and tint the whole catalogue. + /// + private static byte[] ToPng(Bitmap source) + { + var rect = new Rectangle(0, 0, source.Width, source.Height); + + if (source.PixelFormat != PixelFormat.Format16bppArgb1555) + { + // Not what this library has ever produced. Save it rather than reinterpret it: + // guessing at an unknown layout is how a catalogue fills with confident nonsense. + using (var ms = new MemoryStream()) + { + source.Save(ms, ImageFormat.Png); + return ms.ToArray(); + } + } + + using (var target = new Bitmap(source.Width, source.Height, PixelFormat.Format32bppArgb)) + { + BitmapData src = source.LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format16bppArgb1555); + BitmapData dst = null; + + try + { + dst = target.LockBits(rect, ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb); + + var line = new short[source.Width]; + var outLine = new int[source.Width]; + + for (int y = 0; y < source.Height; y++) + { + Marshal.Copy(new IntPtr(src.Scan0.ToInt64() + ((long)y * src.Stride)), + line, 0, source.Width); + + for (int x = 0; x < source.Width; x++) + { + int p = line[x] & 0xFFFF; + + if ((p & 0x8000) == 0) + { + outLine[x] = 0; + continue; + } + + int r = (p >> 10) & 0x1F; + int g = (p >> 5) & 0x1F; + int b = p & 0x1F; + + outLine[x] = unchecked((int)0xFF000000) + | (((r << 3) | (r >> 2)) << 16) + | (((g << 3) | (g >> 2)) << 8) + | ((b << 3) | (b >> 2)); + } + + Marshal.Copy(outLine, 0, new IntPtr(dst.Scan0.ToInt64() + ((long)y * dst.Stride)), + source.Width); + } + } + finally + { + if (dst != null) + target.UnlockBits(dst); + + source.UnlockBits(src); + } + + using (var ms = new MemoryStream()) + { + target.Save(ms, ImageFormat.Png); + return ms.ToArray(); + } + } + } + + private static string Hash(byte[] bytes) + { + using (var sha = SHA256.Create()) + { + byte[] digest = sha.ComputeHash(bytes); + var sb = new StringBuilder(digest.Length * 2); + + foreach (byte b in digest) + sb.Append(b.ToString("x2", CultureInfo.InvariantCulture)); + + return sb.ToString(); + } + } + + // ── player bodies (§5.2) ───────────────────────────────────────────────────────────── + + /// + /// Asked of the shard, never hardcoded. + /// + /// Every registered race carries four body ids, and a shard that calls `RegisterRace` + /// adds ids no table of ours could contain. Even on stock ServUO a hardcoded list + /// would already be wrong in a way that is easy to miss: `RaceDefinitions.cs` passes + /// the gargoyle's ghost bodies in the OPPOSITE order to the other two races. + /// + /// This set is the whole of what §5.1 gives direction 0 — head-on, facing the viewer, + /// because a character is a portrait and should look at you. Everything else takes + /// direction 1, the front three-quarter, because head-on is the least legible view of + /// a four-legged creature: a wolf seen from the front is a dark blob. + /// + /// + /// Cached for the life of the process: `RegisterRace` runs at Configure time, before + /// anything on this plane can be asked a question, so the set cannot change under a + /// running shard. Rebuilding it per body would enumerate every race 2,047 times per + /// scan to answer a question whose answer is twelve integers. + /// + private static HashSet _playerBodies; + + private static HashSet PlayerBodies() + { + var cached = _playerBodies; + + if (cached != null) + return cached; + + var set = new HashSet(); + + try + { + foreach (var race in Race.AllRaces) + { + if (race == null) + continue; + + set.Add(race.MaleBody); + set.Add(race.FemaleBody); + set.Add(race.MaleGhostBody); + set.Add(race.FemaleGhostBody); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: cannot enumerate races: {0}", e.Message); + } + + set.Remove(0); + + _playerBodies = set; + + return set; + } + + private static bool IsPlayerBody(int body) + { + return PlayerBodies().Contains(body); + } + + private static void WritePlayerBodies(StringBuilder sb) + { + var bodies = new List(PlayerBodies()); + bodies.Sort(); + + sb.Append(",\"playerBodies\":["); + + for (int i = 0; i < bodies.Count; i++) + { + if (i > 0) + sb.Append(','); + + sb.Append(bodies[i].ToString(CultureInfo.InvariantCulture)); + } + + sb.Append(']'); + } + + // ── keys, cursors and the source id ────────────────────────────────────────────────── + + private static string Key(int body) + { + return "body/" + body.ToString(CultureInfo.InvariantCulture) + + "/a" + CatalogAction.ToString(CultureInfo.InvariantCulture); + } + + /// + /// `body/<id>/a0`, and nothing else in this phase. A deeper key + /// (`body/400/a2/f3`) is well-formed under §5 and simply not served yet, so it comes + /// back `unsupported` rather than being silently read as its own first frame. + /// + private static bool TryParseKey(string key, out int body) + { + body = 0; + + if (key == null) + return false; + + string[] parts = key.Split('/'); + + if (parts.Length != 3 || parts[0] != "body") + return false; + + if (!Int32.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out body)) + return false; + + if (body < 1 || body > MaxBody) + return false; + + return parts[2] == "a" + CatalogAction.ToString(CultureInfo.InvariantCulture); + } + + private static int ParseBodyCursor(string cursor) + { + if (cursor == null) + return 1; + + int value; + + if (cursor.StartsWith("b:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(1, value + 1); + + return 1; + } + + private static int ParseKeyCursor(string cursor) + { + if (cursor == null) + return 0; + + int value; + + if (cursor.StartsWith("k:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(0, value); + + return 0; + } + + /// + /// Everything that decides the bytes, hashed into one short id. + /// + /// Deliberately (size, mtime) rather than content: §6 makes exactly the same choice + /// for the source gate, and for the same reason — the anim files are 195 MB and + /// hashing them on every page of a walk would turn a manifest into a minute. + /// `assets.sources` is where an operator gets content hashes, computed off the request + /// path; this is a "did it move while I was reading" check, which (size, mtime) + /// answers. + /// + private static string SourceId() + { + var sb = new StringBuilder(256); + + sb.Append(BridgeAssets.EXTRACTOR_VERSION) + .Append(':').Append(BridgeConfig.AssetPlayerDirection) + .Append(':').Append(BridgeConfig.AssetCreatureDirection); + + for (int fileType = 1; fileType <= 5; fileType++) + { + sb.Append('|'); + + string path = BridgeAssetValidator.AnimDataPath(fileType); + + if (path == null) + continue; + + try + { + var info = new FileInfo(path); + + if (!info.Exists) + continue; + + sb.Append(info.Length).Append(',').Append(info.LastWriteTimeUtc.Ticks); + } + catch + { + // An unreadable file is itself a state, and one that must not change from page + // to page without being noticed. Leaving the slot empty does that. + } + } + + return Hash(Encoding.UTF8.GetBytes(sb.ToString())).Substring(0, 16); + } + + // ── shared plumbing ────────────────────────────────────────────────────────────────── + + /// + /// The two gates every request on this plane passes: a correlation id, and the + /// operator's consent. Both refuse rather than answer. + /// + private static bool Admit(Dictionary o, string kind, out string reqId) + { + reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + BridgeAssets.Fail(null, "BAD_REQUEST", kind + " requires a reqId"); + return false; + } + + if (!BridgeConfig.AssetsEnabled) + { + BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return false; + } + + return true; + } + + /// + /// The five anim files' index and record readers, opened for one reply and closed with + /// it. Holding them across replies would keep handles on the operator's client files + /// for as long as the cache lives, for no gain: opening five is microseconds and a + /// page decodes hundreds of sprites through them. + /// + private sealed class Readers : IDisposable + { + private readonly FileIndex[] _index = new FileIndex[6]; + private readonly BridgeAssetValidator.RecordReader[] _reader = + new BridgeAssetValidator.RecordReader[6]; + private readonly long[] _length = new long[6]; + private readonly bool[] _open = new bool[6]; + + public readonly long VerdataLength; + + public Readers() + { + VerdataLength = BridgeAssetValidator.MulLength(VerdataPath()); + } + + private static string VerdataPath() + { + try + { + return Files.GetFilePath("verdata.mul"); + } + catch + { + return null; + } + } + + private void Ensure(int fileType) + { + if (fileType < 1 || fileType > 5 || _open[fileType]) + return; + + _open[fileType] = true; + + string path = BridgeAssetValidator.AnimDataPath(fileType); + + if (path == null) + return; + + try + { + _index[fileType] = BridgeAssetValidator.OpenAnimIndex(fileType); + _length[fileType] = BridgeAssetValidator.MulLength(path); + _reader[fileType] = new BridgeAssetValidator.RecordReader(path, VerdataPath()); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: anim file type {0}: {1}", + fileType, e.Message); + } + } + + public FileIndex Index(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _index[fileType] : null; + } + + public long MulLength(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _length[fileType] : 0; + } + + public BridgeAssetValidator.RecordReader Reader(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _reader[fileType] : null; + } + + public void Dispose() + { + for (int i = 1; i <= 5; i++) + { + if (_reader[i] == null) + continue; + + try + { + _reader[i].Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + } + } + + /// + /// Lets a megabyte of the operator's client art go once nothing has asked for it in + /// five minutes. A rebuild costs one scan and, because is + /// derived from the files rather than minted per build, it produces the same catalogue + /// id — so an import that spans the drop does not have to start over. + /// + private static void Sweep() + { + lock (_sync) + { + if (_catalog == null) + return; + + if (DateTime.UtcNow - _catalog.LastUsed > IdleFor) + _catalog = null; + } + } + + public static string Status() + { + lock (_sync) + { + if (_catalog == null) + return "catalog(empty)"; + + return String.Format("catalog(id={0} held={1} next={2} complete={3})", + _catalog.Id, _catalog.Order.Count, _catalog.Next, _catalog.Complete); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs index 0f140f4..dede1bd 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs @@ -97,6 +97,33 @@ namespace Server.Custom.Bridge public static bool AssetsEnabled { get; private set; } public static int AssetBatchBytes { get; private set; } + // How many types one `assets.bodies` request may name (§8, phase 3). This is the ONLY + // asset-plane bound counted in items rather than bytes, and deliberately so: the cost it + // bounds is not the size of the reply, it is constructing and deleting that many real + // mobiles ON THE CORE THREAD, between two ticks of the world. + public static int AssetBodyBatch { get; private set; } + + // How many keys one `assets.fetch` request may name. Bytes still cut the page; this only + // bounds how large a request the shard will parse and walk at all. + public static int AssetFetchKeys { get; private set; } + + // The wall-clock budget for one catalogue page (§4.8, phase 3). The catalogue's rows are + // ninety bytes, so the byte budget never stops it -- but building them means decoding + // hundreds of animations, and the sidecar gives a reply ten seconds. Kept well under that, + // because the reply still has to be built, serialised and cross the wire afterwards. + public static int AssetScanMs { get; private set; } + + // Which direction the catalogue renders (§5.1). Both are settings and neither is in the + // asset key, because five directions would five-fold every count in §11 to express a + // choice nobody is going to vary. + // + // The split is not arbitrary and was found by RENDERING all five rather than from a table: + // index 0 is head-on, which is what a character portrait wants and the least legible view + // there is of a four-legged creature. A wolf seen from the front is a dark blob; at index + // 1, the front three-quarter, it is unmistakably a wolf. + public static int AssetPlayerDirection { get; private set; } + public static int AssetCreatureDirection { get; private set; } + public static int LeaseMaxDurationSec { get; private set; } public static int LeaseGraceSec { get; private set; } @@ -168,6 +195,33 @@ namespace Server.Custom.Bridge if (AssetBatchBytes > 512 * 1024) AssetBatchBytes = 512 * 1024; + AssetBodyBatch = Config.Get("Bridge.AssetBodyBatch", 100); + if (AssetBodyBatch < 1) + AssetBodyBatch = 1; + if (AssetBodyBatch > 500) + AssetBodyBatch = 500; + + AssetFetchKeys = Config.Get("Bridge.AssetFetchKeys", 2000); + if (AssetFetchKeys < 1) + AssetFetchKeys = 1; + if (AssetFetchKeys > 10000) + AssetFetchKeys = 10000; + + AssetScanMs = Config.Get("Bridge.AssetScanMs", 3000); + if (AssetScanMs < 250) + AssetScanMs = 250; + // Half the sidecar's 10 s reply timeout, so the page still has time to be serialised + // and written after the scan stops. A budget set at the timeout would produce replies + // that are always thrown away. + if (AssetScanMs > 5000) + AssetScanMs = 5000; + + // Clamped to 0-4: 5-7 are the client MIRRORING 1-3, which `Frame` decodes through a + // different pointer-arithmetic branch that nothing in BridgeAssetValidator has + // checked. Accepting one would hand an unverified write path a bitmap to fill. + AssetPlayerDirection = Clamp(Config.Get("Bridge.AssetPlayerDirection", 0), 0, 4); + AssetCreatureDirection = Clamp(Config.Get("Bridge.AssetCreatureDirection", 1), 0, 4); + StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30); DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60); EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300); @@ -518,6 +572,14 @@ namespace Server.Custom.Bridge return fallback; } + private static int Clamp(int value, int min, int max) + { + if (value < min) + return min; + + return value > max ? max : value; + } + public static string Describe() { return String.Format(