diff --git a/overlay/Config/Bridge.cfg b/overlay/Config/Bridge.cfg index 5be83c3..9fc1141 100644 --- a/overlay/Config/Bridge.cfg +++ b/overlay/Config/Bridge.cfg @@ -307,6 +307,39 @@ AssetsEnabled=true # still fits. AssetBatchBytes=524288 +# How many ServUO class names one `assets.bodies` request may carry (phase 3). The only +# bound on this plane counted in items rather than bytes, because what it bounds is not +# reply size -- it is constructing and deleting that many real mobiles ON THE CORE +# THREAD, between two ticks of the world. A larger request is refused, never truncated. +# Clamped to [1, 500]. +AssetBodyBatch=100 + +# How many keys one `assets.fetch` request may name. The byte budget above still decides +# where a page is cut; this only bounds how large a request the shard will parse at all. +# Clamped to [1, 10000]. +AssetFetchKeys=2000 + +# The wall-clock budget for one catalogue page, in milliseconds. The catalogue's manifest +# rows are ~90 bytes so the byte budget never stops it -- but building them means +# decoding hundreds of animations, and the sidecar waits 10 s for a reply. Kept well +# under that, because the page still has to be serialised and written afterwards. +# Clamped to [250, 5000]. +AssetScanMs=3000 + +# Which direction the catalogue renders. NOT part of the asset key: five directions +# would five-fold every count in the working set to express a choice nobody varies. +# +# The split was found by RENDERING all five, not from a table. 0 is head-on, facing the +# viewer -- what a character portrait wants, and the least legible view there is of a +# four-legged creature (a wolf seen from the front is a dark blob). 1 is the front +# three-quarter, where the same wolf is unmistakably a wolf. +# +# Which bodies count as player bodies is asked of the shard (every registered race's +# male/female/ghost ids), never hardcoded. Clamped to [0, 4]: 5-7 are the client +# mirroring 1-3 through a decode branch this overlay has not verified. +AssetPlayerDirection=0 +AssetCreatureDirection=1 + # The test scaffolding in tools/scaffolding/ reads its own flags from this file # (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose: # Config.Get returns the default of false when a key is missing, so a deployed diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs index 3cca03e..d809aa6 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs @@ -473,6 +473,21 @@ namespace Server.Custom.Bridge sb.Append('}'); } + /// + /// Whether this host can produce a picture, for the families that produce pictures. + /// + /// reports this on the source gate so an operator learns it + /// while setting the shard up. The catalogue needs the same answer as a *decision* — + /// it must refuse rather than throw a DllNotFoundException out of the middle of + /// a decode loop — so the check itself is shared and this is its one accessor. + /// + internal static bool ImagingOk(out string reason) + { + CheckImaging(); + reason = _imagingReason; + return _imagingOk; + } + private static void CheckImaging() { if (_imagingChecked) diff --git a/overlay/Scripts/Custom/Bridge/BridgeBodies.cs b/overlay/Scripts/Custom/Bridge/BridgeBodies.cs new file mode 100644 index 0000000..00b599f --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeBodies.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +using Server.Mobiles; + +namespace Server.Custom.Bridge +{ + /// + /// **Slug to body id — the part only the shard can do** (docs/link/v8.md §8, protocol 8, + /// phase 3). + /// + /// The spawn atlas knows a creature by a **slug** derived from the class name it found in + /// `Spawns/*.xml` ("giant-spider"). The client knows the same creature by a **body id** + /// (28). Nothing in the ServUO tree declares that mapping as data. Today an operator + /// bridges it by hand, grepping `Scripts/Mobiles/Normal/<Name>.cs` for `Body =`, + /// which appears as a decimal, as hex (`0xD1`), as `Utility.RandomList(35, 36)` and as an + /// `m_IDs[]` table — a parse that is wrong on the shard's own custom creatures, which is + /// precisely the set an operator most wants pictures for. + /// + /// Inside ServUO the problem does not exist: construct the type, read `Body.BodyID`, + /// delete it. BridgeWorld.cs already does exactly that for a different feature. + /// + /// **This is the one asset-plane family that does NOT run on the asset worker**, and the + /// reason is the whole point of §8. Constructing and deleting a mobile is world mutation, + /// so it must happen on the Core thread — while the decode in + /// must happen off it, because it reads hundreds of megabytes and would stop the world for + /// every player on the shard. That split is why body resolution is its own request kind + /// rather than a step inside asset extraction. + /// + /// Two consequences follow from answering on the Core thread, and both are bounds: + /// + /// **The batch is small and the shard enforces the cap itself.** Every type constructed + /// here runs a real constructor — packing items, rolling skills, starting AI timers — and + /// all of that happens between two ticks of the world. The website chunks its own list; + /// a request over names is **refused** rather + /// than truncated, so the two sides cannot quietly disagree about what was answered. + /// + /// **It does not take the asset plane's single slot.** The slot exists to stop several + /// large replies queueing at once (§3.2); this reply is a few kilobytes and the work is + /// not on the worker, so claiming the slot would only make a body pass and a catalogue + /// page refuse each other for no benefit. + /// + /// **A creature whose constructor randomises its body reports one of its variants**, not + /// an error and not a set. Constructing twice to detect that would double every side + /// effect above to learn something the bestiary does not render differently — both ids are + /// the same creature. The answer is stable enough to cache and cheap enough to redo. + /// + public static class BridgeBodies + { + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("assets.bodies", OnBodies); + } + + // ── the request ────────────────────────────────────────────────────────────────────── + + private static void OnBodies(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + // Rule 1 of the asset plane: without a correlation id this reply lands on the + // event path, is persisted to the sidecar's store and broadcast to every + // subscriber. Refuse rather than answer. + BridgeAssets.Fail(null, "BAD_REQUEST", "assets.bodies requires a reqId"); + return; + } + + if (!BridgeConfig.AssetsEnabled) + { + BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return; + } + + var types = BridgeJson.GetStringList(o, "types"); + + if (types.Count == 0) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.bodies requires a non-empty `types` array of ServUO class names"); + return; + } + + if (types.Count > BridgeConfig.AssetBodyBatch) + { + // Refuse, never truncate. A silently shortened answer looks identical to a + // complete one from the website's side, and the types that fell off the end would + // be recorded as "asked and unanswerable" rather than "never asked". + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.bodies takes at most " + BridgeConfig.AssetBodyBatch + + " types per request (asked for " + types.Count + "); send them in chunks"); + return; + } + + Reply(reqId, types); + } + + /// + /// Core thread. Constructs each type once, reads its body, deletes it. + /// + /// Every outcome is a **row**, never a failed request: a shard is expected to be asked + /// about types it does not have (an atlas built from a tree that has since changed, a + /// spawn file naming a creature from a script package the operator removed), and a + /// status screen that fails the whole pass over one of those teaches an operator to + /// stop pressing the button. + /// + private static void Reply(string reqId, List types) + { + var sb = BridgeJson.Begin("assets.bodies.ok"); + + sb.Str("reqId", reqId) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", types.Count); + + // The envelope is shared with every other family (§3.4) even though this one never + // pages: the website drives the chunking, so `more` is always false and `cut` always + // "end". Writing it anyway means one reader shape on the other side rather than two. + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int resolved = 0; + + foreach (var name in types) + { + string status; + int body; + + Resolve(name, out body, out status); + + if (status == "ok") + resolved++; + + var item = new StringBuilder(96); + + item.Append("{\"type\":"); + BridgeJson.Text(item, name); + item.Append(",\"status\":\"").Append(status).Append('"'); + + if (status == "ok") + item.Append(",\"body\":").Append(body.ToString(CultureInfo.InvariantCulture)); + + item.Append('}'); + + // A chunk this small cannot spend the budget — the cap above is a hundred names + // and the budget is half a megabyte — but the check costs nothing and the day + // someone raises `AssetBodyBatch` it is the difference between a short page and a + // line the sidecar drops. + if (!page.TryAdd(item.ToString(), null)) + break; + } + + page.Close(); + + sb.Num("resolved", resolved); + + BridgeLink.Emit(sb.End()); + } + + /// + /// One type name to one body id. + /// + /// `status` is the field the website records, and the four values are four different + /// things an operator can act on: + /// + /// ok — constructed, body read. + /// unknown — no such type on this shard. The spawn file names something the + /// scripts do not define, which is a real drift an operator wants to see. + /// notCreature — the type exists but is not a `BaseCreature`. Spawn files + /// legitimately name items and static decorations; those have no body and never will, + /// so this is a permanent answer rather than a retryable failure. + /// failed — the constructor threw, or the type has none that takes no + /// arguments. Caught per type, because one creature whose constructor depends on a + /// script package the operator removed must not cost the other ninety-nine. + /// + private static void Resolve(string name, out int body, out string status) + { + body = 0; + status = "failed"; + + Type type; + + try + { + // `true` is ignoreCase — spawn files are hand-edited and their casing drifts from + // the class it names far more often than the name itself does. + type = ScriptCompiler.FindTypeByName(name, true); + } + catch + { + status = "failed"; + return; + } + + if (type == null) + { + status = "unknown"; + return; + } + + if (!typeof(BaseCreature).IsAssignableFrom(type) || type.IsAbstract) + { + status = "notCreature"; + return; + } + + BaseCreature creature = null; + + try + { + creature = Activator.CreateInstance(type) as BaseCreature; + + if (creature == null) + { + status = "failed"; + return; + } + + body = creature.Body.BodyID; + status = body > 0 ? "ok" : "failed"; + } + catch (Exception e) + { + Console.WriteLine("[Bridge] assets.bodies: {0}: {1}: {2}", + name, e.GetType().Name, e.Message); + + status = "failed"; + } + finally + { + if (creature != null) + { + try + { + // Deleting the mobile deletes the items it packed — `Mobile.Delete` walks + // `Items`, and `Item.Delete` walks what each contains — and stops its AI + // timer. A creature left alive here is a creature standing at (0,0,0) on + // the internal map forever, saved with the world, once per import. + creature.Delete(); + } + catch + { + // Nothing useful is left to do, and throwing out of `finally` would lose + // whatever the try block was already reporting. + } + } + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs index d09dc43..1511e8f 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs @@ -261,6 +261,7 @@ namespace Server.Custom.Bridge e.Mobile.SendMessage("Bridge: {0}", BridgeWorld.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeOneShots.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeAssets.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeCatalog.Status()); break; } } diff --git a/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs new file mode 100644 index 0000000..00b3ad0 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs @@ -0,0 +1,1010 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The body catalogue** (docs/link/v8.md §4.8, §5, §6, §11 — protocol 8, phase 3). + /// + /// One thumbnail per creature body: the working set that makes a bestiary, a marketplace + /// listing and a character sheet render. Everything deeper — every action, every frame — + /// is the same addressing scheme at a deeper key, fetched on demand in a later phase; this + /// is the set that is worth importing before anything asks for it, because on this + /// machine's client it is **787 sprites at about a kilobyte each**. + /// + /// Two request kinds, which are §6's two stages for assets rather than for sources: + /// + /// assets.manifest — `[{ key, sha256, bytes, width, height }]`, no pixels. The + /// website diffs it against what it already holds and asks only for what changed. That is + /// the whole difference between an Update and a re-download. + /// + /// assets.fetch — the pixels, for an explicit list of keys. + /// + /// ── **Why the manifest builds the pictures it refuses to send** ── + /// + /// A manifest row carries a hash of the bytes, and the only way to hash bytes is to have + /// them. So the scan decodes, encodes to PNG and hashes, then sends the row and **keeps + /// the bytes** — a megabyte for the whole catalogue, against re-decoding all 787 sprites a + /// second time when the fetch arrives moments later. + /// + /// ── **Why the manifest pages on TIME rather than on bytes** ── + /// + /// Every other family on this plane pages because its rows are large. This one's rows are + /// ninety bytes and the whole catalogue is one page by the byte budget — but producing + /// that page means decoding 787 animations, and the sidecar gives a reply ten seconds + /// (§3.3). So the scan carries a **wall-clock budget** as well + /// () and cuts the page `limit` when it is spent, + /// resuming from its cursor on the next call. The byte budget is still enforced, because + /// the day a family's rows grow is not the day to discover only one of the two bounds was + /// real. + /// + /// ── **Why `catalog` is derived from the sources and not minted per build** ── + /// + /// A manifest walk and the fetch that follows it must be talking about the same client + /// files, or the website stitches one catalogue out of two. The obvious answer is a fresh + /// id per build, and it is wrong: this cache is released when it goes idle, so a rebuild + /// halfway through a slow import would change the id and force a restart although nothing + /// about the client moved. So the id is a hash of what actually decides the bytes — every + /// anim file's size and mtime, both direction settings and + /// . It is stable across a rebuild and it + /// changes exactly when an operator patches their client. + /// + /// ── **The never-sweep rule, and the 357** ── + /// + /// Nothing here asks a file type for an index it does not own, and nothing here trusts the + /// library's own success. takes + /// `BodyConverter.Convert`'s answer and reports nothing if it leads nowhere (sweeping + /// instead puts a giant spider on the gargoyle page, decoding cleanly); and every body is + /// put through and + /// RecordReader.AnimationSane **before** it is decoded, because a body whose index + /// entry reads `length 0` gets a bitmap back anyway — the previously-decoded creature's, + /// from the library's reused stream buffer. That is 357 of the 1,144 bodies the library + /// claims on a stock client, and importing them would have written 357 duplicate + /// portraits whose subject depended on the order this walk happened to run in. + /// + public static class BridgeCatalog + { + /// The only family this phase serves. §5's key scheme covers the rest. + private const string Family = "body"; + + /// Bodies are addressable to 2047; the sweep behind §4.8 covered exactly this. + private const int MaxBody = 2047; + + /// The catalogue is first frames only. Deep keys are phase 6. + private const int CatalogAction = 0; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("assets.manifest", OnManifest); + BridgeBoot.RegisterHandler("assets.fetch", OnFetch); + } + + // ── the cache ──────────────────────────────────────────────────────────────────────── + + private sealed class Sprite + { + public string Key; + public int Body; + public int Direction; + public int FileType; + public string Sha256; + public byte[] Png; + public int Width; + public int Height; + } + + private sealed class Catalog + { + public string Id; + public readonly Dictionary ByKey = + new Dictionary(StringComparer.Ordinal); + public readonly List Order = new List(); + + /// The next body the scan has yet to look at. + public int Next = 1; + + public bool Complete; + public DateTime LastUsed; + } + + private static readonly object _sync = new object(); + private static Catalog _catalog; + + private static readonly TimeSpan IdleFor = TimeSpan.FromMinutes(5); + + // ── assets.manifest ────────────────────────────────────────────────────────────────── + + private static void OnManifest(Dictionary o) + { + string reqId; + + if (!Admit(o, "assets.manifest", out reqId)) + return; + + var family = BridgeJson.GetString(o, "family") ?? Family; + + if (!String.Equals(family, Family, StringComparison.Ordinal)) + { + // Named rather than ignored: `family` exists so §5's statics and land can join + // this envelope in phase 5 without a second request kind, and a website that + // asked for one of those against a phase-3 overlay must be told it asked too + // early rather than handed a body catalogue it did not request. + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "this shard serves the '" + Family + "' asset family only (asked for '" + + family + "')"); + return; + } + + var cursor = BridgeJson.GetString(o, "cursor"); + + BridgeAssets.Accept(reqId, "assets.manifest", () => ReplyManifest(reqId, cursor)); + } + + /// + /// Worker thread. Scans forward from the cursor until the byte budget or the time + /// budget is spent, hashing what it decodes and keeping the bytes for the fetch. + /// + private static void ReplyManifest(string reqId, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + // Never a stack trace and never a 500: on a Linux host without libgdiplus this is + // the expected outcome, and it is actionable in one line (§4.4). + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. Install it (apt-get install libgdiplus) and re-run the import. " + + "Cliloc and atlas import are unaffected. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + Catalog catalog; + + lock (_sync) + { + if (_catalog == null || _catalog.Id != id) + _catalog = new Catalog { Id = id }; + + catalog = _catalog; + catalog.LastUsed = DateTime.UtcNow; + } + + int from = ParseBodyCursor(cursor); + + var sb = BridgeJson.Begin("assets.manifest.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + // What the website compares across pages, and across the fetch that follows. A + // change means the operator patched their client mid-import and the half already + // read describes files that no longer exist. + .Str("catalog", catalog.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("maxBody", MaxBody) + .Num("from", from); + + WritePlayerBodies(sb); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int scanned = 0; + int last = from - 1; + bool timedOut = false; + bool budgetCut = false; + + var deadline = DateTime.UtcNow.AddMilliseconds(BridgeConfig.AssetScanMs); + + using (var readers = new Readers()) + { + int body = from; + + for (; body <= MaxBody; body++) + { + // Checked before the body rather than after it, so the budget bounds the reply + // rather than the reply plus one more decode. One sprite is milliseconds; the + // ceiling this lives under is ten seconds and the cost of overshooting it is + // the whole page, retried. + if (body > from && DateTime.UtcNow >= deadline) + { + timedOut = true; + break; + } + + scanned++; + last = body; + + Sprite sprite = Resolve(catalog, readers, body); + + if (sprite == null) + continue; + + var item = new StringBuilder(128); + + item.Append("{\"key\":"); + BridgeJson.Text(item, sprite.Key); + item.Append(",\"sha256\":\"").Append(sprite.Sha256).Append('"'); + item.Append(",\"bytes\":").Append(sprite.Png.Length.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"width\":").Append(sprite.Width.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append('}'); + + if (!page.TryAdd(item.ToString(), "b:" + body.ToString(CultureInfo.InvariantCulture))) + { + // The budget stopped this page BEFORE this body's row went on it, so the + // next page must resume AT this body rather than after it. Getting this + // one line wrong drops exactly one creature from the catalogue per page, + // which nothing downstream could ever notice. + budgetCut = true; + last = body - 1; + scanned--; + break; + } + } + } + + if (timedOut) + page.Cut("limit"); + + // The walk reached the end of the addressable range without either budget stopping it. + // Derived from the two flags rather than from the row count, because a page that ends + // exactly on a boundary is indistinguishable from a finished one by count alone — + // §3.4's whole argument for `cut` existing. + bool finished = !timedOut && !budgetCut && last >= MaxBody; + + int held; + + lock (_sync) + { + if (_catalog == catalog) + { + catalog.Next = Math.Max(catalog.Next, last + 1); + catalog.LastUsed = DateTime.UtcNow; + + if (finished) + catalog.Complete = true; + } + + held = catalog.Order.Count; + } + + page.Close(); + + // Past Close(), which is normally the mistake BridgeCliloc's `from` comment warns + // about — but these three are not knowable until the scan has run, and they cost + // about fifty bytes against PageBuilder's 256-byte reserve, of which Close() itself + // spends around forty. Anything larger than this belongs before the page opens. + sb.Num("scanned", scanned) + .Num("held", held) + .Bool("complete", finished); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + // ── assets.fetch ───────────────────────────────────────────────────────────────────── + + private static void OnFetch(Dictionary o) + { + string reqId; + + if (!Admit(o, "assets.fetch", out reqId)) + return; + + var keys = BridgeJson.GetStringList(o, "keys"); + + if (keys.Count == 0) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.fetch requires a non-empty `keys` array"); + return; + } + + if (keys.Count > BridgeConfig.AssetFetchKeys) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.fetch takes at most " + BridgeConfig.AssetFetchKeys + + " keys per request (asked for " + keys.Count + ")"); + return; + } + + var catalog = BridgeJson.GetString(o, "catalog"); + var cursor = BridgeJson.GetString(o, "cursor"); + + BridgeAssets.Accept(reqId, "assets.fetch", () => ReplyFetch(reqId, keys, catalog, cursor)); + } + + private static void ReplyFetch(string reqId, List keys, string expected, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + if (expected != null && expected != id) + { + // The client files moved between the manifest and this fetch. Refusing is the only + // honest answer: the keys were chosen against a catalogue that no longer describes + // what is on disk, and serving them would mix two clients in one import with no + // error anywhere. + BridgeAssets.Fail(reqId, "UNREADABLE", + "the shard's client files changed since that manifest was read (catalog " + + expected + " is now " + id + "); start the import again"); + return; + } + + Catalog catalog; + + lock (_sync) + { + if (_catalog == null || _catalog.Id != id) + _catalog = new Catalog { Id = id }; + + catalog = _catalog; + catalog.LastUsed = DateTime.UtcNow; + } + + int from = ParseKeyCursor(cursor); + + var sb = BridgeJson.Begin("assets.fetch.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + .Str("catalog", catalog.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", keys.Count) + .Num("from", from); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int i = from; + + using (var readers = new Readers()) + { + for (; i < keys.Count; i++) + { + var item = Render(catalog, readers, keys[i]); + + if (!page.TryAdd(item, "k:" + (i + 1).ToString(CultureInfo.InvariantCulture))) + break; + } + } + + page.Close(); + + sb.Num("sent", page.Count); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + /// + /// One key to one row, with the bytes. + /// + /// A key this shard cannot serve is a **row**, not a failed request: the website asked + /// for a list, and one key naming a body whose art this client does not carry must not + /// cost the other four hundred. `status` distinguishes the two ways that happens — + /// `absent` (this client has no art at that key, the expected answer for two thirds of + /// the player bodies) and `unsupported` (a key shape this phase does not serve, which + /// is a website bug rather than a client gap). + /// + private static string Render(Catalog catalog, Readers readers, string key) + { + int body; + + if (!TryParseKey(key, out body)) + { + var bad = new StringBuilder(96); + bad.Append("{\"key\":"); + BridgeJson.Text(bad, key); + bad.Append(",\"status\":\"unsupported\"}"); + return bad.ToString(); + } + + Sprite sprite = Resolve(catalog, readers, body); + + var item = new StringBuilder(2048); + + item.Append("{\"key\":"); + BridgeJson.Text(item, key); + + if (sprite == null) + { + item.Append(",\"status\":\"absent\"}"); + return item.ToString(); + } + + item.Append(",\"status\":\"ok\""); + item.Append(",\"sha256\":\"").Append(sprite.Sha256).Append('"'); + item.Append(",\"bytes\":").Append(sprite.Png.Length.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"width\":").Append(sprite.Width.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"png\":\"").Append(Convert.ToBase64String(sprite.Png)).Append("\"}"); + + return item.ToString(); + } + + // ── decode ─────────────────────────────────────────────────────────────────────────── + + /// + /// The catalogue entry for one body, decoded and hashed on first sight and cached + /// after. Returns null when this client has no art for it — which is an ordinary + /// answer for well over half of the addressable range, not a failure. + /// + private static Sprite Resolve(Catalog catalog, Readers readers, int body) + { + string key = Key(body); + + lock (_sync) + { + Sprite cached; + + if (catalog.ByKey.TryGetValue(key, out cached)) + return cached; + } + + int direction = IsPlayerBody(body) + ? BridgeConfig.AssetPlayerDirection + : BridgeConfig.AssetCreatureDirection; + + int fileType, at; + string reason; + + if (!BridgeAssetValidator.ResolveAnimation(body, CatalogAction, direction, + out fileType, out at, out reason)) + return null; + + FileIndex index = readers.Index(fileType); + + if (index == null) + return null; + + if (BridgeAssetValidator.CheckEntry(index, at, readers.MulLength(fileType), + readers.VerdataLength, out reason) != BridgeAssetValidator.Verdict.Ok) + { + // The `length 0` case lands here, and it is the 357. The library would hand back + // the previously-decoded body's bitmap for every one of them. + return null; + } + + var reader = readers.Reader(fileType); + + if (reader == null) + return null; + + // `maxFrames: 1` because that is what `firstFrame: true` decodes. Checking frames + // nobody reads would invent refusals, and a checker that refuses real art is worse + // than no checker at all. + if (!reader.AnimationSane(index, at, 1, out reason)) + return null; + + Sprite sprite; + + try + { + sprite = Decode(key, body, direction, fileType); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: body {0}: {1}: {2}", + body, e.GetType().Name, e.Message); + return null; + } + + if (sprite == null) + return null; + + lock (_sync) + { + if (!catalog.ByKey.ContainsKey(key)) + { + catalog.ByKey[key] = sprite; + catalog.Order.Add(sprite); + } + + return catalog.ByKey[key]; + } + } + + private static Sprite Decode(string key, int body, int direction, int fileType) + { + int hue = 0; + + // `preserveHue: false` — the catalogue is the creature's own art, and a body-level hue + // from Body.def belongs to a specific mob rather than to the species. §5's key scheme + // is where a hued variant is expressed (`static/3922/h33`), not here. + Frame[] frames = Animations.GetAnimation(body, CatalogAction, direction, ref hue, false, true); + + if (frames == null || frames.Length == 0 || frames[0] == null) + return null; + + Bitmap bitmap = frames[0].Bitmap; + + if (bitmap == null || bitmap.Width <= 0 || bitmap.Height <= 0) + return null; + + byte[] png = ToPng(bitmap); + + if (png == null) + return null; + + return new Sprite + { + Key = key, + Body = body, + Direction = direction, + FileType = fileType, + Png = png, + Width = bitmap.Width, + Height = bitmap.Height, + Sha256 = Hash(png) + }; + } + + /// + /// ARGB1555 to a PNG with a transparent background. + /// + /// Frame writes 16-bit ARGB1555: a pixel the sprite does not cover is left as + /// zero and a pixel it does cover carries the top bit set. Saving that format straight + /// to PNG asks GDI+ to make the conversion, and what it does with a one-bit alpha + /// channel varies by platform — on Mono it is a different implementation entirely. A + /// sprite that came back with a black rectangle behind it would look fine in a test + /// that only checked the bytes decoded, and wrong on every page that showed it. + /// + /// So the expansion is done here, explicitly: alpha bit clear becomes fully + /// transparent, and each 5-bit channel is widened to 8 bits by repeating its high bits + /// ((c << 3) | (c >> 2)) rather than by shifting alone, which would + /// cap white at 248 and tint the whole catalogue. + /// + private static byte[] ToPng(Bitmap source) + { + var rect = new Rectangle(0, 0, source.Width, source.Height); + + if (source.PixelFormat != PixelFormat.Format16bppArgb1555) + { + // Not what this library has ever produced. Save it rather than reinterpret it: + // guessing at an unknown layout is how a catalogue fills with confident nonsense. + using (var ms = new MemoryStream()) + { + source.Save(ms, ImageFormat.Png); + return ms.ToArray(); + } + } + + using (var target = new Bitmap(source.Width, source.Height, PixelFormat.Format32bppArgb)) + { + BitmapData src = source.LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format16bppArgb1555); + BitmapData dst = null; + + try + { + dst = target.LockBits(rect, ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb); + + var line = new short[source.Width]; + var outLine = new int[source.Width]; + + for (int y = 0; y < source.Height; y++) + { + Marshal.Copy(new IntPtr(src.Scan0.ToInt64() + ((long)y * src.Stride)), + line, 0, source.Width); + + for (int x = 0; x < source.Width; x++) + { + int p = line[x] & 0xFFFF; + + if ((p & 0x8000) == 0) + { + outLine[x] = 0; + continue; + } + + int r = (p >> 10) & 0x1F; + int g = (p >> 5) & 0x1F; + int b = p & 0x1F; + + outLine[x] = unchecked((int)0xFF000000) + | (((r << 3) | (r >> 2)) << 16) + | (((g << 3) | (g >> 2)) << 8) + | ((b << 3) | (b >> 2)); + } + + Marshal.Copy(outLine, 0, new IntPtr(dst.Scan0.ToInt64() + ((long)y * dst.Stride)), + source.Width); + } + } + finally + { + if (dst != null) + target.UnlockBits(dst); + + source.UnlockBits(src); + } + + using (var ms = new MemoryStream()) + { + target.Save(ms, ImageFormat.Png); + return ms.ToArray(); + } + } + } + + private static string Hash(byte[] bytes) + { + using (var sha = SHA256.Create()) + { + byte[] digest = sha.ComputeHash(bytes); + var sb = new StringBuilder(digest.Length * 2); + + foreach (byte b in digest) + sb.Append(b.ToString("x2", CultureInfo.InvariantCulture)); + + return sb.ToString(); + } + } + + // ── player bodies (§5.2) ───────────────────────────────────────────────────────────── + + /// + /// Asked of the shard, never hardcoded. + /// + /// Every registered race carries four body ids, and a shard that calls `RegisterRace` + /// adds ids no table of ours could contain. Even on stock ServUO a hardcoded list + /// would already be wrong in a way that is easy to miss: `RaceDefinitions.cs` passes + /// the gargoyle's ghost bodies in the OPPOSITE order to the other two races. + /// + /// This set is the whole of what §5.1 gives direction 0 — head-on, facing the viewer, + /// because a character is a portrait and should look at you. Everything else takes + /// direction 1, the front three-quarter, because head-on is the least legible view of + /// a four-legged creature: a wolf seen from the front is a dark blob. + /// + /// + /// Cached for the life of the process: `RegisterRace` runs at Configure time, before + /// anything on this plane can be asked a question, so the set cannot change under a + /// running shard. Rebuilding it per body would enumerate every race 2,047 times per + /// scan to answer a question whose answer is twelve integers. + /// + private static HashSet _playerBodies; + + private static HashSet PlayerBodies() + { + var cached = _playerBodies; + + if (cached != null) + return cached; + + var set = new HashSet(); + + try + { + foreach (var race in Race.AllRaces) + { + if (race == null) + continue; + + set.Add(race.MaleBody); + set.Add(race.FemaleBody); + set.Add(race.MaleGhostBody); + set.Add(race.FemaleGhostBody); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: cannot enumerate races: {0}", e.Message); + } + + set.Remove(0); + + _playerBodies = set; + + return set; + } + + private static bool IsPlayerBody(int body) + { + return PlayerBodies().Contains(body); + } + + private static void WritePlayerBodies(StringBuilder sb) + { + var bodies = new List(PlayerBodies()); + bodies.Sort(); + + sb.Append(",\"playerBodies\":["); + + for (int i = 0; i < bodies.Count; i++) + { + if (i > 0) + sb.Append(','); + + sb.Append(bodies[i].ToString(CultureInfo.InvariantCulture)); + } + + sb.Append(']'); + } + + // ── keys, cursors and the source id ────────────────────────────────────────────────── + + private static string Key(int body) + { + return "body/" + body.ToString(CultureInfo.InvariantCulture) + + "/a" + CatalogAction.ToString(CultureInfo.InvariantCulture); + } + + /// + /// `body/<id>/a0`, and nothing else in this phase. A deeper key + /// (`body/400/a2/f3`) is well-formed under §5 and simply not served yet, so it comes + /// back `unsupported` rather than being silently read as its own first frame. + /// + private static bool TryParseKey(string key, out int body) + { + body = 0; + + if (key == null) + return false; + + string[] parts = key.Split('/'); + + if (parts.Length != 3 || parts[0] != "body") + return false; + + if (!Int32.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out body)) + return false; + + if (body < 1 || body > MaxBody) + return false; + + return parts[2] == "a" + CatalogAction.ToString(CultureInfo.InvariantCulture); + } + + private static int ParseBodyCursor(string cursor) + { + if (cursor == null) + return 1; + + int value; + + if (cursor.StartsWith("b:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(1, value + 1); + + return 1; + } + + private static int ParseKeyCursor(string cursor) + { + if (cursor == null) + return 0; + + int value; + + if (cursor.StartsWith("k:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(0, value); + + return 0; + } + + /// + /// Everything that decides the bytes, hashed into one short id. + /// + /// Deliberately (size, mtime) rather than content: §6 makes exactly the same choice + /// for the source gate, and for the same reason — the anim files are 195 MB and + /// hashing them on every page of a walk would turn a manifest into a minute. + /// `assets.sources` is where an operator gets content hashes, computed off the request + /// path; this is a "did it move while I was reading" check, which (size, mtime) + /// answers. + /// + private static string SourceId() + { + var sb = new StringBuilder(256); + + sb.Append(BridgeAssets.EXTRACTOR_VERSION) + .Append(':').Append(BridgeConfig.AssetPlayerDirection) + .Append(':').Append(BridgeConfig.AssetCreatureDirection); + + for (int fileType = 1; fileType <= 5; fileType++) + { + sb.Append('|'); + + string path = BridgeAssetValidator.AnimDataPath(fileType); + + if (path == null) + continue; + + try + { + var info = new FileInfo(path); + + if (!info.Exists) + continue; + + sb.Append(info.Length).Append(',').Append(info.LastWriteTimeUtc.Ticks); + } + catch + { + // An unreadable file is itself a state, and one that must not change from page + // to page without being noticed. Leaving the slot empty does that. + } + } + + return Hash(Encoding.UTF8.GetBytes(sb.ToString())).Substring(0, 16); + } + + // ── shared plumbing ────────────────────────────────────────────────────────────────── + + /// + /// The two gates every request on this plane passes: a correlation id, and the + /// operator's consent. Both refuse rather than answer. + /// + private static bool Admit(Dictionary o, string kind, out string reqId) + { + reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + BridgeAssets.Fail(null, "BAD_REQUEST", kind + " requires a reqId"); + return false; + } + + if (!BridgeConfig.AssetsEnabled) + { + BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return false; + } + + return true; + } + + /// + /// The five anim files' index and record readers, opened for one reply and closed with + /// it. Holding them across replies would keep handles on the operator's client files + /// for as long as the cache lives, for no gain: opening five is microseconds and a + /// page decodes hundreds of sprites through them. + /// + private sealed class Readers : IDisposable + { + private readonly FileIndex[] _index = new FileIndex[6]; + private readonly BridgeAssetValidator.RecordReader[] _reader = + new BridgeAssetValidator.RecordReader[6]; + private readonly long[] _length = new long[6]; + private readonly bool[] _open = new bool[6]; + + public readonly long VerdataLength; + + public Readers() + { + VerdataLength = BridgeAssetValidator.MulLength(VerdataPath()); + } + + private static string VerdataPath() + { + try + { + return Files.GetFilePath("verdata.mul"); + } + catch + { + return null; + } + } + + private void Ensure(int fileType) + { + if (fileType < 1 || fileType > 5 || _open[fileType]) + return; + + _open[fileType] = true; + + string path = BridgeAssetValidator.AnimDataPath(fileType); + + if (path == null) + return; + + try + { + _index[fileType] = BridgeAssetValidator.OpenAnimIndex(fileType); + _length[fileType] = BridgeAssetValidator.MulLength(path); + _reader[fileType] = new BridgeAssetValidator.RecordReader(path, VerdataPath()); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: anim file type {0}: {1}", + fileType, e.Message); + } + } + + public FileIndex Index(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _index[fileType] : null; + } + + public long MulLength(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _length[fileType] : 0; + } + + public BridgeAssetValidator.RecordReader Reader(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _reader[fileType] : null; + } + + public void Dispose() + { + for (int i = 1; i <= 5; i++) + { + if (_reader[i] == null) + continue; + + try + { + _reader[i].Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + } + } + + /// + /// Lets a megabyte of the operator's client art go once nothing has asked for it in + /// five minutes. A rebuild costs one scan and, because is + /// derived from the files rather than minted per build, it produces the same catalogue + /// id — so an import that spans the drop does not have to start over. + /// + private static void Sweep() + { + lock (_sync) + { + if (_catalog == null) + return; + + if (DateTime.UtcNow - _catalog.LastUsed > IdleFor) + _catalog = null; + } + } + + public static string Status() + { + lock (_sync) + { + if (_catalog == null) + return "catalog(empty)"; + + return String.Format("catalog(id={0} held={1} next={2} complete={3})", + _catalog.Id, _catalog.Order.Count, _catalog.Next, _catalog.Complete); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs index 0f140f4..dede1bd 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs @@ -97,6 +97,33 @@ namespace Server.Custom.Bridge public static bool AssetsEnabled { get; private set; } public static int AssetBatchBytes { get; private set; } + // How many types one `assets.bodies` request may name (§8, phase 3). This is the ONLY + // asset-plane bound counted in items rather than bytes, and deliberately so: the cost it + // bounds is not the size of the reply, it is constructing and deleting that many real + // mobiles ON THE CORE THREAD, between two ticks of the world. + public static int AssetBodyBatch { get; private set; } + + // How many keys one `assets.fetch` request may name. Bytes still cut the page; this only + // bounds how large a request the shard will parse and walk at all. + public static int AssetFetchKeys { get; private set; } + + // The wall-clock budget for one catalogue page (§4.8, phase 3). The catalogue's rows are + // ninety bytes, so the byte budget never stops it -- but building them means decoding + // hundreds of animations, and the sidecar gives a reply ten seconds. Kept well under that, + // because the reply still has to be built, serialised and cross the wire afterwards. + public static int AssetScanMs { get; private set; } + + // Which direction the catalogue renders (§5.1). Both are settings and neither is in the + // asset key, because five directions would five-fold every count in §11 to express a + // choice nobody is going to vary. + // + // The split is not arbitrary and was found by RENDERING all five rather than from a table: + // index 0 is head-on, which is what a character portrait wants and the least legible view + // there is of a four-legged creature. A wolf seen from the front is a dark blob; at index + // 1, the front three-quarter, it is unmistakably a wolf. + public static int AssetPlayerDirection { get; private set; } + public static int AssetCreatureDirection { get; private set; } + public static int LeaseMaxDurationSec { get; private set; } public static int LeaseGraceSec { get; private set; } @@ -168,6 +195,33 @@ namespace Server.Custom.Bridge if (AssetBatchBytes > 512 * 1024) AssetBatchBytes = 512 * 1024; + AssetBodyBatch = Config.Get("Bridge.AssetBodyBatch", 100); + if (AssetBodyBatch < 1) + AssetBodyBatch = 1; + if (AssetBodyBatch > 500) + AssetBodyBatch = 500; + + AssetFetchKeys = Config.Get("Bridge.AssetFetchKeys", 2000); + if (AssetFetchKeys < 1) + AssetFetchKeys = 1; + if (AssetFetchKeys > 10000) + AssetFetchKeys = 10000; + + AssetScanMs = Config.Get("Bridge.AssetScanMs", 3000); + if (AssetScanMs < 250) + AssetScanMs = 250; + // Half the sidecar's 10 s reply timeout, so the page still has time to be serialised + // and written after the scan stops. A budget set at the timeout would produce replies + // that are always thrown away. + if (AssetScanMs > 5000) + AssetScanMs = 5000; + + // Clamped to 0-4: 5-7 are the client MIRRORING 1-3, which `Frame` decodes through a + // different pointer-arithmetic branch that nothing in BridgeAssetValidator has + // checked. Accepting one would hand an unverified write path a bitmap to fill. + AssetPlayerDirection = Clamp(Config.Get("Bridge.AssetPlayerDirection", 0), 0, 4); + AssetCreatureDirection = Clamp(Config.Get("Bridge.AssetCreatureDirection", 1), 0, 4); + StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30); DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60); EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300); @@ -518,6 +572,14 @@ namespace Server.Custom.Bridge return fallback; } + private static int Clamp(int value, int min, int max) + { + if (value < min) + return min; + + return value > max ? max : value; + } + public static string Describe() { return String.Format(