diff --git a/README.md b/README.md index a35e457..91d11b7 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,26 @@ Runtime script compilation therefore had no effect, silently. `overlay/Scripts/S | `BridgeAccountLink.cs` | `[link` account linking (Phase 5): one-time code, `link.confirm`, `WebsiteUserId` account tag. | | `BridgeTownCrier.cs` | Town-crier news (Phase 6): inbound `towncrier.add` / `remove` into the global crier list, with abuse caps. | +The Asset Bridge's own files (protocol 8, `docs/link/v8.md`) — the shard reading the operator's UO +client and its own ServUO tree, and the only part of this plugin that touches files rather than the +world: + +| File | Responsibility | +|------|----------------| +| `BridgeAssets.cs` | The plane's front door: the single-slot gate that answers `bridge.busy`, the 512 KiB batch budget, the paging envelope, the family registry, and the background hashing pass that fingerprints the client files without holding the slot. | +| `BridgeAssetValidator.cs` | Judges an index entry (and, for statics, the record behind it) **before** handing an id to `Ultima`. The boundary that turns 22,102 confident wrong pictures on a stock client into honest absences (§4.5). | +| `BridgeCatalog.cs` | The body catalogue: which bodies have art, at which action, and the per-body action ceiling that stops the fallback walk serving the next body's picture (§4.10). | +| `BridgeArt.cs` | Item statics and land tiles on demand, hued on the shard from `tiledata.mul`, behind a byte-bounded cache. | +| `BridgeUop.cs` | The narrow UOP animation reader, written without `System.Drawing` — the one decoder here that is not ServUO's (§4.3). | +| `BridgePng.cs` | Our own PNG encoder, for the same reason. | +| `BridgeBodies.cs` | Slug → body id, on the **Core thread**: construct the type, read `Body.BodyID`, delete it. The one question no code outside ServUO can answer (§8). | +| `BridgeCliloc.cs` | The Mythic cliloc decompressor, ported from UOFiddler (Beerware) — ServUO's own `Ultima.StringList` cannot read a modern client's compressed table (§9). | +| `BridgeTree.cs` | The shard's own `Spawns/*.xml` and friends as a `tree` key family, in gzipped 512 KiB chunks, behind its own consent `Bridge.TreeEnabled` (§10). | + +**The table above is the transport plus the Asset Bridge, not all 38 files** in that directory — +the streams added by protocols 3 through 7 (visibility, leases, participation, the event plane's +world verbs) are documented in their own design docs rather than here. + `Emit()` is called from the Core thread. It enqueues and returns — it never touches the socket, never blocks, never allocates a syscall. **A wedged or absent sidecar cannot stall the shard**, and that is the property everything else depends on. ## Testing diff --git a/overlay.toml b/overlay.toml index 7bc1fcc..5643385 100644 --- a/overlay.toml +++ b/overlay.toml @@ -23,8 +23,9 @@ # manual duty: when the protocol changes, bump it here in the same PR that # changes the emitters, exactly as link bumps PROTOCOL_VERSION. # -# Current: 6 — see docs/link/v6.md (idempotency keys on inbound commands, champ.boss.killed). -protocol = 7 +# Current: 8 — see docs/link/v8.md (the Asset Bridge: client assets over the loopback link +# instead of a converter on somebody's desktop). +protocol = 8 # ── ServUO compatibility ───────────────────────────────────────────────────── # diff --git a/overlay/Config/Bridge.cfg b/overlay/Config/Bridge.cfg index 5da1626..5e662d3 100644 --- a/overlay/Config/Bridge.cfg +++ b/overlay/Config/Bridge.cfg @@ -295,6 +295,75 @@ EventsMaxGrantStack=1000 # would land at a moment nobody chose. Set to 0 to allow a save at any time. EventsMinSaveIntervalSec=300 +# The asset plane (docs/link/v8.md, protocol 8). Its own switch, deliberately: turning +# this on is consenting to the website reading this host's UO CLIENT FILES -- art, +# animations, the string table -- over the link. Nothing on this plane writes anything. +AssetsEnabled=true + +# The largest reply the asset plane will build, in encoded bytes. Not an item count: +# the ceiling it lives inside is the sidecar's 1 MiB inbound line cap, and base64 adds +# 33% to every payload. Clamped to [64 KiB, 512 KiB] -- half the wire cap, so that a +# single oversized item (always admitted, or its family could never make progress) +# still fits. +AssetBatchBytes=524288 + +# How many ServUO class names one `assets.bodies` request may carry (phase 3). The only +# bound on this plane counted in items rather than bytes, because what it bounds is not +# reply size -- it is constructing and deleting that many real mobiles ON THE CORE +# THREAD, between two ticks of the world. A larger request is refused, never truncated. +# Clamped to [1, 500]. +AssetBodyBatch=100 + +# How many keys one `assets.fetch` request may name. The byte budget above still decides +# where a page is cut; this only bounds how large a request the shard will parse at all. +# Clamped to [1, 10000]. +AssetFetchKeys=2000 + +# The wall-clock budget for one catalogue page, in milliseconds. The catalogue's manifest +# rows are ~90 bytes so the byte budget never stops it -- but building them means +# decoding hundreds of animations, and the sidecar waits 10 s for a reply. Kept well +# under that, because the page still has to be serialised and written afterwards. +# Clamped to [250, 5000]. +AssetScanMs=3000 + +# Which direction the catalogue renders. NOT part of the asset key: five directions +# would five-fold every count in the working set to express a choice nobody varies. +# +# The split was found by RENDERING all five, not from a table. 0 is head-on, facing the +# viewer -- what a character portrait wants, and the least legible view there is of a +# four-legged creature (a wolf seen from the front is a dark blob). 1 is the front +# three-quarter, where the same wolf is unmistakably a wolf. +# +# Which bodies count as player bodies is asked of the shard (every registered race's +# male/female/ghost ids), never hardcoded. Clamped to [0, 4]: 5-7 are the client +# mirroring 1-3 through a decode branch this overlay has not verified. +AssetPlayerDirection=0 +AssetCreatureDirection=1 + +# The tree plane (docs/link/v8.md §10, phase 7). A THIRD switch, for a third consent: +# the asset switch above is about this host's UO client, which came from EA. This one is +# about the shard's own configuration -- Spawns/*.xml, Data/Regions.xml, +# Data/Locations/*.xml, Config/ChampionSpawns.xml and Data/Decoration/**.cfg -- which is +# the operator's own work and is what the website's spawn atlas is built from. Before +# protocol 8 the website read those files off a shared filesystem; that was the one place +# the platform's own rule (only the sidecar bridges the shard) was broken, and broken by +# the component that faces the internet. Turning this off closes the bridge route and +# leaves that shared-filesystem path as the only way an atlas can be built. +# +# Reads only, and only those five groups. Nothing here joins a path the website sent: a +# request names a label this shard itself enumerated, or it is refused. +TreeEnabled=true + +# How much of a tree file one chunk carries, BEFORE compression. Chunking is not an +# optimisation here, it is what makes a spawn file transferable: a stock trammel.xml is +# 4.03 MB, the sidecar discards any inbound line over 1 MiB, and the whole file as one +# base64 row would time out and be re-requested forever with no error anywhere. Each +# chunk is gzipped (a spawn file compresses ~18x, so a chunk is typically 40 KB on the +# wire), but the BOUND comes from the chunk rather than the compression, because nothing +# guarantees input compresses at all. Clamped to [64 KiB, 512 KiB]: at the ceiling a +# worst-case incompressible chunk is ~683 KiB of base64, which still fits the wire. +TreeChunkBytes=524288 + # The test scaffolding in tools/scaffolding/ reads its own flags from this file # (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose: # Config.Get returns the default of false when a key is missing, so a deployed diff --git a/overlay/Scripts/Custom/Bridge/BridgeArt.cs b/overlay/Scripts/Custom/Bridge/BridgeArt.cs new file mode 100644 index 0000000..c00ef8c --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeArt.cs @@ -0,0 +1,842 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Globalization; +using System.IO; +using System.Text; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **Item and land art, on demand** (docs/link/v8.md §5, §11 — protocol 8, phase 5). + /// + /// The body catalogue is a *set*: 1,022 sprites, enumerated, hashed and imported in one + /// pass because a bestiary needs all of them. This is the opposite shape. This client + /// addresses **49,152 static ids** and has real art for **39,189** of them, plus 4,244 land + /// tiles of 16,384 — and then there are hues, which multiply the statics by three thousand. + /// Nothing enumerates that. So there is no manifest here and no scan: the website asks for + /// the handful of keys its own data actually names, and this answers them. + /// + /// (49,152 rather than the 81,884 entries `artidx.mul` declares: FileIndex sizes its + /// table from the **length argument it is constructed with**, `0x10000`, not from the idx + /// file — so the addressable range is `0x10000 - 0x4000`. Reading the ceiling off the file + /// instead would invent 16,348 ids, every one of them answered out of an array nobody + /// bounded.) + /// + /// ── **The keys** (§5) ── + /// + /// + /// static/3922 one item graphic, as the client files hold it + /// static/3922/h33 the same graphic with hue 33 applied + /// land/3 one land tile + /// + /// + /// ── **Why the hue is applied HERE and not on the website** ── + /// + /// Because it cannot be applied correctly anywhere else, and the incorrect version looks + /// fine. + /// + /// A hue is not a tint. It is a 32-entry colour ramp out of `hues.mul` indexed by a + /// pixel's own red channel — and whether it replaces *every* pixel or only the grey ones + /// is decided by the PartialHue flag in tiledata.mul, per item id. On this + /// client **13,259 of 65,536 item ids carry that flag**. Get it wrong on one of them and + /// you do not get an error: item 597 is a wooden screen with painted flowers, and hued red + /// the right way the flowers turn red, the wrong way the whole screen turns red. Both + /// decode. Both are the right size. One is wrong. + /// + /// The website has neither file and never will — shipping `Hues.mul` semantics and a + /// 65,536-row flag table into Node to answer a question the shard can answer for free is + /// the same trade §2.1 already refused. So hue is part of the key, and the key is resolved + /// where the files are. + /// + /// ── **The trap this phase existed to find** ── + /// + /// Art.GetStatic memoises into a static Bitmap[0xFFFF] and returns **the same + /// instance** every time; Hue.ApplyTo repaints a bitmap **in place**. Hue a static + /// once and the library's own copy is hued from then on — the plain key comes back hued, + /// and the next hue stacks on the last. It is §4.5's failure mode (a confident, plausible, + /// wrong picture that every success count agrees with) reached through a door §4.5 never + /// looked at, because phase 0 was auditing *records* and this is the library's *cache*. + /// + /// turns Files.CacheData off for the life of + /// the process, which makes every bitmap this file receives its own. That invariant is + /// load-bearing enough that **re-checks it** before applying a hue and + /// refuses rather than risk it: an invariant nothing verifies is a comment. + /// + /// ── **What is validated, and against what** ── + /// + /// Everything §4.5 built, reused as-is. An index entry is judged before the id is handed to + /// Ultima (), a static's record header + /// and row table are walked bounded (StaticSane), a land record is checked against + /// the 2,024 bytes LoadLand reads whatever the length says (LandLengthSane), + /// and the bound is taken against **whichever file FileIndex actually opened** — + /// artLegacyMUL.uop on every current client, never art.mul (§4.6). + /// + /// Two of §4.5's measurements are this family's, not the catalogue's, and they are the + /// reason all of it is here: on a **stock** client **9,963 static ids and 12,140 land ids** + /// have an index entry reading `lookup 0, length 0`, which FileIndex.Seek treats as + /// a hit and the decoder answers with whatever was decoded last. Measured through this + /// reader over the whole range, those are the ONLY refusals — every one of the 39,189 + /// statics and 4,244 land tiles that carries art is served, which is the half of the + /// measurement that says the boundary is in the right place (§4.5). + /// + public static class BridgeArt + { + /// Item graphics. static/<id>, optionally /h<hue>. + private const string StaticFamily = "static"; + + /// Land tiles. land/<id>, and no hue segment — see . + private const string LandFamily = "land"; + + /// The art index addresses land at its own id and statics at 0x4000 + id. + private const int StaticBase = 0x4000; + + /// Land is addressed with index & 0x3FFF by the library itself. + private const int LandCount = 0x4000; + + /// hues.mul holds 3,000 slots; the wire's hue 1 is slot 0. + private const int MaxHue = 3000; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeAssets.RegisterFamily(StaticFamily, ReplyFetch); + BridgeAssets.RegisterFamily(LandFamily, ReplyFetch); + } + + // ── the cache (§11) ────────────────────────────────────────────────────────────────── + + private sealed class Rendered + { + public string Key; + public string Status; + public string Reason; + public string Sha256; + public byte[] Png; + public int Width; + public int Height; + public int Hue; + public bool PartialHue; + public string Source; + + public int Weight + { + get { return Png == null ? 128 : Png.Length + 128; } + } + } + + private sealed class Cache + { + public string Id; + + public readonly Dictionary ByKey = + new Dictionary(StringComparer.Ordinal); + + /// Insertion order, for eviction. See . + public readonly Queue Order = new Queue(); + + public long Bytes; + public DateTime LastUsed; + } + + private static readonly object _sync = new object(); + private static Cache _cache; + + private static readonly TimeSpan IdleFor = TimeSpan.FromMinutes(5); + + // ── assets.fetch, the static and land half ─────────────────────────────────────────── + + /// + /// Both families' answer to assets.fetch. The correlation id, the operator's + /// consent, the key ceiling and the family decision were made by + /// ; every key here belongs to this reader. + /// + /// The paging envelope, the byte budget and the `catalog` guard are §3.4's and + /// phase 3's, unchanged — a caller that already walks the body catalogue walks this + /// with the same loop. + /// + private static void ReplyFetch(string reqId, List keys, string expected, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + // §17.9: a flat refusal, not a partial answer. Every picture in this family needs + // a decoder that goes through GDI+, so there is no half of it to serve. + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + if (BridgeAssets.CatalogMismatch(expected, id)) + { + BridgeAssets.Fail(reqId, "UNREADABLE", + "the shard's client files changed since that catalogue was read (catalog " + + expected + " is now " + id + "); ask again"); + return; + } + + Cache cache; + + lock (_sync) + { + if (_cache == null || _cache.Id != id) + _cache = new Cache { Id = id }; + + cache = _cache; + cache.LastUsed = DateTime.UtcNow; + } + + int from = ParseKeyCursor(cursor); + + var sb = BridgeJson.Begin("assets.fetch.ok"); + + sb.Str("reqId", reqId) + .Str("family", BridgeAssets.FamilyOfKey(keys[0])) + .Str("catalog", cache.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", keys.Count) + .Num("from", from); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + using (var readers = new Readers()) + { + for (int i = from; i < keys.Count; i++) + { + string row = Row(cache, readers, keys[i]); + + if (!page.TryAdd(row, "k:" + (i + 1).ToString(CultureInfo.InvariantCulture))) + break; + } + } + + page.Close(); + + sb.Num("sent", page.Count); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + /// + /// One key to one JSON row. + /// + /// A key this shard cannot serve is a **row**, never a failed request: an item id with + /// no art must not cost the other three hundred on the page. The three outcomes are the + /// ones phase 3 defined, and this family adds a `reason` beside them — additive, and + /// the only way an operator learns that eight of their records are damaged rather than + /// simply absent, which is a difference §4.5 spent a whole phase establishing. + /// + private static string Row(Cache cache, Readers readers, string key) + { + Rendered item = Resolve(cache, readers, key); + + var sb = new StringBuilder(2048); + + sb.Append("{\"key\":"); + BridgeJson.Text(sb, key); + + sb.Append(",\"status\":\"").Append(item.Status).Append('"'); + + if (item.Reason != null) + { + sb.Append(",\"reason\":"); + BridgeJson.Text(sb, item.Reason); + } + + if (item.Status != "ok") + { + sb.Append('}'); + return sb.ToString(); + } + + sb.Append(",\"sha256\":\"").Append(item.Sha256).Append('"'); + sb.Append(",\"bytes\":").Append(item.Png.Length.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"width\":").Append(item.Width.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"height\":").Append(item.Height.ToString(CultureInfo.InvariantCulture)); + + if (item.Hue > 0) + { + sb.Append(",\"hue\":").Append(item.Hue.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"partialHue\":").Append(item.PartialHue ? "true" : "false"); + } + + sb.Append(",\"source\":\"").Append(item.Source).Append('"'); + sb.Append(",\"png\":\"").Append(Convert.ToBase64String(item.Png)).Append("\"}"); + + return sb.ToString(); + } + + private static Rendered Resolve(Cache cache, Readers readers, string key) + { + lock (_sync) + { + Rendered cached; + + if (cache.ByKey.TryGetValue(key, out cached)) + return cached; + } + + Rendered item; + + try + { + item = Render(readers, key); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] art: {0}: {1}: {2}", key, e.GetType().Name, e.Message); + + item = new Rendered + { + Key = key, + Status = "absent", + Reason = e.GetType().Name + }; + } + + if (item.Status == "ok") + Remember(cache, item); + + return item; + } + + /// + /// Holds the encoded bytes against a byte budget, evicting oldest-first. + /// + /// **Oldest-first rather than least-recently-used, deliberately.** The access pattern + /// this serves is a warm pass: the website asks for the keys it has never held, stores + /// them permanently, and does not ask again. What this cache is actually for is the + /// second page of a batch, a retry after a 425, and the same picture appearing in two + /// of a page's rows — all of which insertion order serves exactly as well as recency, + /// and with no bookkeeping on the hot path. A cache whose hit pattern has no recency in + /// it should not pretend to rank by it. + /// + /// Only successes are held. An absent key costs one index lookup, which is cheaper than + /// the dictionary entry that would remember it. + /// + private static void Remember(Cache cache, Rendered item) + { + lock (_sync) + { + if (cache.ByKey.ContainsKey(item.Key)) + return; + + cache.ByKey[item.Key] = item; + cache.Order.Enqueue(item.Key); + cache.Bytes += item.Weight; + + while (cache.Bytes > BridgeConfig.AssetArtCacheBytes && cache.Order.Count > 0) + { + string oldest = cache.Order.Dequeue(); + + Rendered evicted; + + if (!cache.ByKey.TryGetValue(oldest, out evicted)) + continue; + + cache.ByKey.Remove(oldest); + cache.Bytes -= evicted.Weight; + } + } + } + + // ── decode ─────────────────────────────────────────────────────────────────────────── + + /// + /// Validate, decode, hue, encode. In that order, and the order is the point. + /// + private static Rendered Render(Readers readers, string key) + { + bool land; + int id, hue; + + if (!TryParseKey(key, out land, out id, out hue)) + return Unsupported(key, "not a key this shard serves"); + + FileIndex index = readers.Index; + + if (index == null || index.Index == null) + return Absent(key, "this shard has no art file"); + + int at = land ? id : StaticBase + id; + + if (at < 0 || at >= index.Index.Length) + return Unsupported(key, "id " + id + " is past the end of this client's art index"); + + string reason; + + BridgeAssetValidator.Verdict verdict = + BridgeAssetValidator.CheckEntry(index, at, readers.DataLength, readers.VerdataLength, out reason); + + if (verdict == BridgeAssetValidator.Verdict.Absent) + { + // The 9,962 statics and 12,140 land tiles of §4.5: an index entry that reads + // `lookup 0, length 0`, which the library treats as a hit and answers with the + // previous asset's pixels. Absent is the true answer and the only safe one. + return Absent(key, reason); + } + + if (verdict != BridgeAssetValidator.Verdict.Ok) + { + // A damaged record rather than a missing one. Still absent to the website — there + // is no picture either way — but the reason is worth carrying, because this one an + // operator can act on. + Console.WriteLine("[Bridge] art: {0} refused: {1}", key, reason); + return Absent(key, reason); + } + + if (land) + { + if (!BridgeAssetValidator.LandLengthSane(index, at, out reason)) + { + Console.WriteLine("[Bridge] art: {0} refused: {1}", key, reason); + return Absent(key, reason); + } + } + else if (readers.Reader == null || !readers.Reader.StaticSane(index, at, out reason)) + { + Console.WriteLine("[Bridge] art: {0} refused: {1}", + key, reason ?? "the art record could not be read"); + + return Absent(key, reason ?? "the art record could not be read"); + } + + // A hue is resolved BEFORE anything is decoded, so a bad one costs no pixels and, more + // to the point, cannot half-apply to a picture that then gets cached and served. + Ultima.Hue applied = null; + bool partial = false; + + if (hue > 0) + { + if (!TryHue(id, hue, out applied, out partial, out reason)) + return Unsupported(key, reason); + } + + Bitmap bitmap = land + ? Art.GetLand(id) + // `checkmaxid: false` on purpose (§4.5): the default maps an out-of-range id to 0 + // and returns ITEM ZERO'S PICTURE. The id is already bounded against the index + // that was actually opened, so this can only be loud. + : Art.GetStatic(id, false); + + // **Whether this bitmap is ours to dispose is the same question as whether it is ours + // to hue**, and it has the same answer. With the library's cache off — which + // `BridgeAssets.Initialize` guarantees and `TryHue` re-checks — every call decodes a + // fresh instance that nothing else holds, so not disposing it would leak one bitmap per + // fetched key. With the cache on, that instance is the library's own copy and disposing + // it would leave a disposed `Bitmap` in a static array for the next caller to fault on. + // Both mistakes are silent; the flag decides, once, here. + bool owned = !Files.CacheData; + + try + { + if (bitmap == null || bitmap.Width <= 0 || bitmap.Height <= 0) + return Absent(key, "the decoder returned no picture"); + + if (applied != null) + applied.ApplyTo(bitmap, partial); + + byte[] png = BridgeAssets.BitmapToPng(bitmap); + + if (png == null) + return Absent(key, "the picture could not be encoded"); + + return new Rendered + { + Key = key, + Status = "ok", + Sha256 = BridgeAssets.Sha256Hex(png), + Png = png, + Width = bitmap.Width, + Height = bitmap.Height, + Hue = hue, + PartialHue = partial, + Source = readers.Source + }; + } + finally + { + if (owned && bitmap != null) + bitmap.Dispose(); + } + } + + /// + /// Resolves one wire hue onto a ramp, and decides whether it repaints the whole sprite + /// or only its grey pixels. + /// + /// Four things have to hold, and every one of them has a way of not holding that + /// produces a picture rather than an error: + /// + /// **The library's cache is off.** Re-checked here because ApplyTo repaints in + /// place: with the cache on, this would edit the copy Art hands to everyone + /// else. turns it off at boot and this refuses + /// if it somehow did not, because the failure is invisible and permanent. + /// + /// **`hues.mul` is present.** When it is missing Hues.Initialize does not throw + /// — it fills all 3,000 slots with a new Hue(index) whose ramp is **all zeroes**, + /// and applying one of those paints the sprite black. An all-zero ramp is therefore + /// refused whatever the reason for it; on this client there are none. + /// + /// **The index is in range.** The wire's hue is 1-based — Ultima.Map does the + /// same GetHue(hue - 1) at line 450 — and GetHue itself masks with + /// `0x3FFF` and falls back to slot 0 rather than failing, so an out-of-range hue would + /// silently become a different colour. Bound it here instead. + /// + /// **The PartialHue flag decides the mode**, per item id, out of + /// tiledata.mul. This is the one that is invisible: both modes decode, both are + /// the right size, and 13,259 of this client's item ids need the grey-only one. + /// **Land has no such flag**, which is why does not accept a + /// hue on a land key at all rather than guessing a mode for it. + /// + private static bool TryHue(int id, int hue, out Ultima.Hue applied, out bool partial, out string reason) + { + applied = null; + partial = false; + reason = null; + + if (Files.CacheData) + { + reason = "this shard's art cache is on, so a hue cannot be applied safely"; + Console.WriteLine("[Bridge] art: refusing hue {0}: {1}", hue, reason); + return false; + } + + if (hue < 1 || hue > MaxHue) + { + reason = "hue " + hue + " is outside 1-" + MaxHue; + return false; + } + + Ultima.Hue[] list = Ultima.Hues.List; + + if (list == null || hue - 1 >= list.Length || list[hue - 1] == null) + { + reason = "this client has no hue table"; + return false; + } + + Ultima.Hue candidate = list[hue - 1]; + + if (candidate.Colors == null || AllZero(candidate.Colors)) + { + reason = "hue " + hue + " has no colours in this client's hues.mul"; + return false; + } + + if (!TryPartialHue(id, out partial, out reason)) + return false; + + applied = candidate; + return true; + } + + private static bool AllZero(short[] colors) + { + for (int i = 0; i < colors.Length; i++) + { + if (colors[i] != 0) + return false; + } + + return true; + } + + /// + /// The PartialHue flag for one item id. + /// + /// Refuses rather than defaults when tiledata.mul cannot be read. Defaulting + /// either way would be a coin flip on 13,259 ids, and the losing side of it is a + /// picture that looks deliberate. + /// + /// **Every type here is spelled Ultima. on purpose, and it is not style.** + /// ServUO declares its own Server.TileData, Server.ItemData and + /// Server.TileFlag — with a PartialHue member — in + /// Server/TileData.cs. This file lives in Server.Custom.Bridge, so the + /// enclosing namespace beats the using Ultima; and the unqualified spelling + /// silently binds to the *server's* table: it compiles, the flag exists, and the answer + /// comes from a file resolved through Core.DataDirectories rather than through + /// Ultima.Files, which is the one thing §4.6 says never to do — decide a picture + /// with a file other than the one the pixels came out of. The first run of this reader + /// did exactly that and refused every hued key with a TypeInitializationException + /// from a class this code never meant to name. + /// + private static bool TryPartialHue(int id, out bool partial, out string reason) + { + partial = false; + reason = null; + + Ultima.ItemData[] table; + + try + { + table = Ultima.TileData.ItemTable; + } + catch (Exception e) + { + reason = "this client's tiledata could not be read (" + e.GetType().Name + ")"; + return false; + } + + if (table == null || id < 0 || id >= table.Length) + { + reason = "this client's tiledata does not describe item " + id; + return false; + } + + partial = (table[id].Flags & Ultima.TileFlag.PartialHue) != 0; + return true; + } + + private static Rendered Absent(string key, string reason) + { + return new Rendered { Key = key, Status = "absent", Reason = reason }; + } + + private static Rendered Unsupported(string key, string reason) + { + return new Rendered { Key = key, Status = "unsupported", Reason = reason }; + } + + // ── keys, cursors and the source id ────────────────────────────────────────────────── + + /// + /// static/<id>, static/<id>/h<hue> and + /// land/<id>. + /// + /// **A land key takes no hue segment.** The client can hue a land tile, but the mode + /// that decides how is an *item* flag and land has no equivalent — so the honest answer + /// to `land/3/h33` is that this shard does not serve it, rather than a picture produced + /// by guessing. Nothing on the wire carries a hued land tile today; if something ever + /// does, it arrives with a reason to choose. + /// + private static bool TryParseKey(string key, out bool land, out int id, out int hue) + { + land = false; + id = 0; + hue = 0; + + if (key == null) + return false; + + string[] parts = key.Split('/'); + + if (parts.Length < 2 || parts.Length > 3) + return false; + + if (parts[0] == LandFamily) + land = true; + else if (parts[0] != StaticFamily) + return false; + + if (!Int32.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out id)) + return false; + + if (id < 0) + return false; + + if (land && id >= LandCount) + return false; + + if (parts.Length == 2) + return true; + + if (land) + return false; + + string segment = parts[2]; + + if (segment.Length < 2 || segment[0] != 'h') + return false; + + if (!Int32.TryParse(segment.Substring(1), NumberStyles.None, + CultureInfo.InvariantCulture, out hue)) + return false; + + // **`h0` is not a key.** Hue 0 on the wire means "this item is not hued", so the plain + // key already names its picture. Accepting `static/3922/h0` as a synonym would have + // the website store the identical PNG twice under two names, diff them separately on + // every Update, and show whichever row it happened to join against -- for a distinction + // that does not exist. The caller drops the segment instead. + return hue > 0; + } + + private static int ParseKeyCursor(string cursor) + { + if (cursor == null) + return 0; + + int value; + + if (cursor.StartsWith("k:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(0, value); + + return 0; + } + + /// + /// Everything that decides these bytes, hashed into one short id — the same guard + /// phase 3 built, over this family's inputs. + /// + /// Four files, and each earns its place: the art data file holds the pixels, + /// `hues.mul` holds the ramps, `tiledata.mul` decides which of the two hue modes an + /// item gets, and `verdata.mul` can patch any record in any of them. Leaving + /// `tiledata.mul` out would be the subtle one — a client patch that only flipped + /// PartialHue flags changes no pixel in any source file and every hued picture + /// derived from them. + /// + private static string SourceId() + { + var sb = new StringBuilder(256); + + sb.Append(BridgeAssets.EXTRACTOR_VERSION); + + foreach (string path in new[] + { + BridgeAssetValidator.ArtDataPath(), + FilePath("hues.mul"), + FilePath("tiledata.mul"), + FilePath("verdata.mul") + }) + { + sb.Append('|'); + + if (path == null) + continue; + + try + { + var info = new FileInfo(path); + + if (!info.Exists) + continue; + + sb.Append(info.Length).Append(',').Append(info.LastWriteTimeUtc.Ticks); + } + catch + { + // An unreadable file is itself a state, and one that must not change from page + // to page without being noticed. Leaving the slot empty does that. + } + } + + return BridgeAssets.Sha256Hex(Encoding.UTF8.GetBytes(sb.ToString())).Substring(0, 16); + } + + private static string FilePath(string name) + { + try + { + return Files.GetFilePath(name); + } + catch + { + return null; + } + } + + // ── shared plumbing ────────────────────────────────────────────────────────────────── + + /// + /// The art index and its record reader, opened for one reply and closed with it — the + /// same lifetime rule phase 3's Readers follows, and for the same reason: a page + /// decodes hundreds of sprites through them and opening them is microseconds, so + /// holding handles on the operator's client files for the life of a cache buys nothing. + /// + private sealed class Readers : IDisposable + { + public readonly FileIndex Index; + public readonly BridgeAssetValidator.RecordReader Reader; + public readonly long DataLength; + public readonly long VerdataLength; + + /// + /// Which file the pixels came out of — `uop` or `legacy` — carried on every row + /// beside the body catalogue's own `source` (§4.9). On this plane it answers §4.6's + /// operator question: art added to `art.mul` while `artLegacyMUL.uop` is present is + /// never read, and a row that says `uop` is what says so. + /// + public readonly string Source; + + public Readers() + { + string data = BridgeAssetValidator.ArtDataPath(); + string verdata = FilePath("verdata.mul"); + + DataLength = BridgeAssetValidator.MulLength(data); + VerdataLength = BridgeAssetValidator.MulLength(verdata); + + Source = data != null && data.EndsWith(".uop", StringComparison.OrdinalIgnoreCase) + ? "uop" + : "legacy"; + + try + { + Index = BridgeAssetValidator.OpenArtIndex(); + + if (data != null) + Reader = new BridgeAssetValidator.RecordReader(data, verdata); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] art: could not open the art files: {0}", e.Message); + } + } + + public void Dispose() + { + if (Reader == null) + return; + + try + { + Reader.Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + } + + /// + /// Lets the held pictures go once nothing has asked for one in five minutes. The id is + /// derived from the client files rather than minted per build, so a walk that spans the + /// drop resumes against the same catalogue instead of starting over. + /// + private static void Sweep() + { + lock (_sync) + { + if (_cache == null) + return; + + if (DateTime.UtcNow - _cache.LastUsed > IdleFor) + _cache = null; + } + } + + public static string Status() + { + lock (_sync) + { + if (_cache == null) + return "art(empty)"; + + return String.Format("art(id={0} held={1} bytes={2} cap={3})", + _cache.Id, _cache.ByKey.Count, _cache.Bytes, BridgeConfig.AssetArtCacheBytes); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs b/overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs new file mode 100644 index 0000000..e1ecacc --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs @@ -0,0 +1,863 @@ +using System; +using System.IO; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **Validate before calling** (docs/link/v8.md §4.5) — the boundary between this protocol + /// and ServUO's vendored Ultima decoders. Phase 0 prototyped it in + /// tools/scaffolding/BridgeAssetProbe.cs and measured it both ways; phase 1 promoted + /// it here, into the overlay, and extended it to animations. + /// + /// The principle: `Ultima`'s decoders take their bounds from the file they are reading, so + /// the extractor must decide whether a record is worth handing over *before* handing it + /// over. Every check below is against the index entry and the record header — cheap, and + /// enough to turn an uncatchable corrupted-state exception into a skipped asset. + /// + /// **The failure this exists for is a wrong picture, not a crash.** `LoadStatic`, + /// `LoadLand` and `GetAnimation` all decode out of a shared m_StreamBuffer that is + /// reused, only ever grown, and filled by a stream.Read whose return value is + /// discarded. A record that is short, absent or out of bounds therefore renders **whatever + /// the previously-decoded asset left behind**, reports success, and is undetectable by + /// anything downstream. On the stock client on the machine phase 0 ran on that is 22,102 + /// ids whose index entry reads lookup 0, length 0. + /// + /// It cannot be complete and does not claim to be. It closes the shapes that reading the + /// source showed are reachable. What says the boundary is in the right place is the second + /// measurement rather than the first: against a client patched 21 ways it refused all eight + /// record-level defects, and against the **stock** client it refused **nothing** across + /// 49,151 statics and 16,384 land tiles. A checker that refuses real art would be worse + /// than no checker. + /// + public static class BridgeAssetValidator + { + public enum Verdict + { + /// Nothing at this id, and the index says so honestly. + Absent, + + /// The entry is self-consistent and inside its file. + Ok, + + /// The entry claims something the file cannot support. Do not decode it. + Refused + } + + /// Land tiles decode a fixed 44×44 diamond: 2 × (2+4+…+44) ushorts. + public const int LandRecordBytes = 2024; + + /// + /// A ceiling on decoded art dimensions. `LoadStatic` allocates + /// new Bitmap(width, height) straight from two ushorts in the record, so a + /// corrupt header asks for up to 65535×65535 — an 8 GB allocation, from a file. Real + /// art is a couple of hundred pixels at most. + /// + public const int MaxArtDimension = 1024; + + /// + /// Builds our own index over the same files, with the same constructor arguments + /// Art uses — including hasExtra: false, which is the whole reason the + /// art path is safe where the gump path is not (§4.1). + /// + public static FileIndex OpenArtIndex() + { + if (ArtDataPath() == null) + return null; + + return new FileIndex("Artidx.mul", "Art.mul", "artLegacyMUL.uop", 0x10000, 4, ".tga", 0x13FDC, false); + } + + /// + /// The file an art index entry's lookup is an offset **into** — which is not + /// art.mul on any current client. + /// + /// This cost a whole probe run to learn and it is the single most important thing + /// phase 1 must not get wrong. FileIndex's UOP constructor ends with a bare + /// MulPath = uopPath: **when artLegacyMUL.uop exists it wins outright**, + /// and art.mul / artidx.mul are never opened at all. A validator that + /// bounds offsets against art.mul while the index holds UOP offsets is not + /// merely approximate, it is nonsense — the first run of this probe refused 34,299 + /// perfectly good statics for "declaring 10533x2085" because it was reading UOP + /// offsets into the wrong file. + /// + /// So the resolution order here mirrors FileIndex's exactly, and anything that + /// needs the bytes behind an entry must ask this rather than assume. + /// + public static string ArtDataPath() + { + var uop = Files.GetFilePath("artlegacymul.uop"); + + if (uop != null) + return uop; + + return Files.GetFilePath("art.mul"); + } + + public static long MulLength(string path) + { + if (path == null) + return 0; + + try + { + return new FileInfo(path).Length; + } + catch + { + return 0; + } + } + + /// + /// Judges one index entry. + /// + /// The check FileIndex.Seek is missing is the last one: it tests + /// Stream.Length < e.lookup — that the record *starts* inside the file — and + /// never that it *ends* inside it. A record that begins two bytes before EOF and + /// declares a length of 4,000 passes, and stream.Read then returns a short count + /// that the decoders discard, leaving the previous asset's bytes in the shared buffer. + /// + public static Verdict CheckEntry(FileIndex index, int at, long mulLength, long verdataLength, out string reason) + { + reason = null; + + if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) + { + reason = "index " + at + " out of range"; + return Verdict.Absent; + } + + Entry3D e = index.Index[at]; + + if (e.lookup < 0) + { + reason = "lookup " + e.lookup; + return Verdict.Absent; + } + + bool patched = (e.length & (1 << 31)) != 0; + int length = e.length & 0x7FFFFFFF; + + if (!patched && e.length < 0) + { + reason = "length " + e.length; + return Verdict.Absent; + } + + if (length == 0) + { + reason = "lookup " + e.lookup + ", length 0"; + return Verdict.Absent; + } + + long ceiling = patched ? verdataLength : mulLength; + + if (ceiling <= 0) + { + reason = (patched ? "verdata.mul" : "the art data file") + " has no length"; + return Verdict.Refused; + } + + if (e.lookup >= ceiling) + { + reason = "lookup " + e.lookup + " past the end of " + + (patched ? "verdata.mul" : "the mul") + " (" + ceiling + ")"; + return Verdict.Refused; + } + + // The missing check. A short read is silent, and its consequence is the PREVIOUS + // asset's picture served under this id. + if (e.lookup + (long)length > ceiling) + { + reason = "record runs " + (e.lookup + (long)length - ceiling) + " bytes past the end of " + + (patched ? "verdata.mul" : "the mul"); + return Verdict.Refused; + } + + return Verdict.Ok; + } + + /// + /// `LoadLand` reads 2,024 bytes regardless of the declared length, so a shorter record + /// reads past the end of a buffer sized from that length. + /// + public static bool LandLengthSane(FileIndex index, int at, out string reason) + { + reason = null; + + if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) + return true; + + int length = index.Index[at].length & 0x7FFFFFFF; + + if (length > 0 && length < LandRecordBytes) + { + reason = "land record is " + length + " bytes; LoadLand always reads " + LandRecordBytes; + return false; + } + + return true; + } + + /// + /// Walks a static record's own row table the way LoadStatic will, and refuses + /// it if that walk would read outside the record. + /// + /// This is the check with teeth. LoadStatic's inner loop guards the write into + /// the bitmap (xOffset > delta, xOffset + xRun > delta) and does + /// nothing at all about the read cursor, which advances until it happens to find a + /// zero pair — potentially far outside a pinned array. Simulating the same walk with + /// a bound is the cheapest way to know whether handing the id over is safe. + /// + public static bool StaticRecordSane(byte[] record, int length, out string reason) + { + reason = null; + + if (length < 8) + { + reason = "record is " + length + " bytes; a static header needs 8"; + return false; + } + + int words = length / 2; + int width = ReadUInt16(record, 4); + int height = ReadUInt16(record, 6); + + // LoadStatic returns null for these rather than misbehaving, so it is not a refusal. + if (width <= 0 || height <= 0) + return true; + + if (width > MaxArtDimension || height > MaxArtDimension) + { + reason = "declares " + width + "x" + height + ", past the " + MaxArtDimension + "px ceiling"; + return false; + } + + // The row-lookup table: height ushorts starting at word 4. + if (4 + height > words) + { + reason = "row table (" + height + " entries) does not fit in a " + length + "-byte record"; + return false; + } + + int start = height + 4; + + for (int y = 0; y < height; y++) + { + int cursor = start + ReadUInt16(record, (4 + y) * 2); + + while (true) + { + // Two ushorts for the run header, and they must both be inside the record. + if (cursor < 0 || cursor + 1 >= words) + { + reason = "row " + y + " reads at word " + cursor + ", past the record's " + words; + return false; + } + + int xOffset = ReadUInt16(record, cursor * 2); + int xRun = ReadUInt16(record, (cursor + 1) * 2); + cursor += 2; + + if (xOffset + xRun == 0) + break; + + // LoadStatic stops the row here, so the read cursor stops with it. + if (xOffset > width || xOffset + xRun > width) + break; + + if (cursor + xRun > words) + { + reason = "row " + y + " declares a " + xRun + "-pixel run running past the record"; + return false; + } + + cursor += xRun; + } + } + + return true; + } + + // ── animations (phase 1) ───────────────────────────────────────────────────────────── + // + // Phase 0 measured the art path and left this half unbuilt, and then proved it was + // needed: the patched client's verdata entry for body 34 points past verdata.mul's own + // end, and the wolf still "decoded" — counted among the 1,144 successes while rendering + // something else entirely. `GetAnimation` has every weakness `LoadStatic` has and one + // more, because the buffer it decodes from is longer than the record it read. + + /// The palette every animation record opens with: 0x100 ushorts. + public const int AnimPaletteBytes = 0x100 * 2; + + /// + /// A ceiling on an animation's declared frame count. GetAnimation does + /// new int[frameCount] straight from four bytes in the file, before it has + /// looked at anything else. Real actions are tens of frames. + /// + public const int MaxAnimFrames = 1024; + + /// The xor Frame applies to every run header before decoding it. + private const int DoubleXor = (0x200 << 22) | (0x200 << 12); + + /// + /// The anim*.mul an animation index entry's lookup is an offset into. + /// + /// Unlike art (§4.6) there is no UOP precedence to get wrong here, and that is not + /// luck: Animations constructs its five FileIndexes with the four-argument + /// constructor, which passes uopFile: null. It never reads + /// AnimationFrame*.uop at all — which is the same fact that leaves six of the + /// twelve player-character bodies undecodable until §4.3's reader lands in phase 4. + /// + public static string AnimDataPath(int fileType) + { + switch (fileType) + { + case 1: return Files.GetFilePath("anim.mul"); + case 2: return Files.GetFilePath("anim2.mul"); + case 3: return Files.GetFilePath("anim3.mul"); + case 4: return Files.GetFilePath("anim4.mul"); + case 5: return Files.GetFilePath("anim5.mul"); + default: return null; + } + } + + /// + /// Builds our own index over one anim file, with the same constructor arguments + /// Animations uses — the entry lengths especially, since they decide how far + /// into the file an index runs. + /// + public static FileIndex OpenAnimIndex(int fileType) + { + if (AnimDataPath(fileType) == null) + return null; + + switch (fileType) + { + case 1: return new FileIndex("Anim.idx", "Anim.mul", 0x40000, 6); + case 2: return new FileIndex("Anim2.idx", "Anim2.mul", 0x10000, -1); + case 3: return new FileIndex("Anim3.idx", "Anim3.mul", 0x20000, -1); + case 4: return new FileIndex("Anim4.idx", "Anim4.mul", 0x20000, -1); + case 5: return new FileIndex("Anim5.idx", "Anim5.mul", 0x20000, -1); + default: return null; + } + } + + /// + /// Where a body's animation actually lives: which anim file, and which index in it. + /// + /// **This is the never-sweep-file-types rule, written as code** (§4.3). It asks + /// BodyConverter.Convert once, takes its answer, and if that answer leads + /// nowhere it reports nowhere. There is deliberately no loop here and no fallback, + /// because asking the *other* anim files for an index they do not own does not fail — + /// it returns 175 decodable action/direction combinations of **a giant spider** for + /// gargoyle 666, and misaligned colour fragments for the other two. Every one of those + /// reads reports success, and nothing downstream can tell them from art. + /// + /// A false return with set is the ordinary, expected answer + /// for a body this client has no art for — the caller reports absent, not an error. + /// + public static bool ResolveAnimation( + int body, int action, int direction, out int fileType, out int index, out string reason) + { + reason = null; + fileType = 0; + index = -1; + + if (body <= 0 || action < 0) + { + reason = "body " + body + " action " + action + " is not addressable"; + return false; + } + + // Directions 5-7 are the client mirroring 1-3, and `Frame` decodes them through its + // flip branch — different pointer arithmetic, which nothing below has checked. + // §5.1 fixed this protocol at direction 0 or 1, so refusing the rest costs nothing + // and keeps the validator honest about what it has actually verified. + if (direction < 0 || direction > 4) + { + reason = "direction " + direction + " is mirrored; this protocol reads 0-4 only"; + return false; + } + + int translated = body; + int hue = 0; + + try + { + // Exactly what GetAnimation(..., preserveHue: false, ...) does first. + Animations.Translate(ref translated, ref hue); + fileType = BodyConverter.Convert(ref translated); + } + catch (Exception e) + { + reason = "body.def/bodyconv.def lookup failed: " + e.GetType().Name; + return false; + } + + if (AnimDataPath(fileType) == null) + { + // Gargoyle 666 lands here: Bodyconv.def maps it to anim5, and this client has no + // anim5. Absent is the correct answer and the ONLY safe one. + reason = "bodyconv sends body " + body + " to file type " + fileType + + ", which this client does not have"; + return false; + } + + int actions = ActionsOf(translated, fileType); + + if (action >= actions) + { + // §4.10, measured in phase 6: this is the never-sweep rule again, one axis over. + // A body's slots are contiguous and the next body's begin immediately after them, + // so `index + action * 5` past the ceiling addresses ANOTHER BODY'S action — a + // real record, at a real offset, that every check below passes. Measured on this + // client: of 795 legacy bodies, 643 return a fully validated picture one action + // past their band and **452 of those are byte-identical to body+1's action 0**. + // Body 1 action 22 is an ettin; body 3 action 22 is an imp. Nothing downstream + // can tell, which is why the refusal has to be here. + reason = "body " + body + " has " + actions + " actions in file type " + fileType + + "; action " + action + " belongs to the next body"; + return false; + } + + index = AnimIndexOf(translated, fileType) + (action * 5) + direction; + + return true; + } + + /// + /// How many actions the index reserves for a body — the only safe ceiling, and it is + /// the banding rather than the library's own answer. + /// + /// Animations.GetAnimLength exists and looks like the right source. It is not: + /// for a body reaching file type 5 as id 34 it answers **22** while + /// puts that body in the 65-slot band, which is **13**. The + /// two disagree on exactly one body of this client (reached by translation from body + /// 276), and taking the larger number is nine actions of somebody else's art. So the + /// count is derived from the same arithmetic that produces the offset, in the same + /// file, where the two cannot drift apart. + /// + public static bool ActionCount(int body, out int actions, out int fileType, out string reason) + { + reason = null; + actions = 0; + fileType = 0; + + if (body <= 0) + { + reason = "body " + body + " is not addressable"; + return false; + } + + int translated = body; + int hue = 0; + + try + { + Animations.Translate(ref translated, ref hue); + fileType = BodyConverter.Convert(ref translated); + } + catch (Exception e) + { + reason = "body.def/bodyconv.def lookup failed: " + e.GetType().Name; + return false; + } + + if (AnimDataPath(fileType) == null) + { + reason = "bodyconv sends body " + body + " to file type " + fileType + + ", which this client does not have"; + return false; + } + + actions = ActionsOf(translated, fileType); + + return true; + } + + /// + /// The banding of , read as an action count: a body's slots + /// are five directions per action, so the band size divided by five is how many + /// actions it owns. + /// + private static int ActionsOf(int body, int fileType) + { + return SlotsOf(body, fileType) / 5; + } + + /// + /// How many index slots 's arithmetic gives this body. The + /// bands are transcribed there and their sizes here, from the same source and in the + /// same order, because a ceiling that disagrees with an offset is worse than no + /// ceiling at all. + /// + private static int SlotsOf(int body, int fileType) + { + switch (fileType) + { + case 2: + return body < 200 ? 110 : 65; + + case 3: + if (body < 300) + return 65; + + return body < 400 ? 110 : 175; + + case 5: + // Body 34's exclusion again — it is in the second band here, so it owns 13 + // actions and not 22. This is the one body `GetAnimLength` is wrong about. + if (body < 200 && body != 34) + return 110; + + return body < 400 ? 65 : 175; + + default: // 1 and 4 share their banding + if (body < 200) + return 110; + + return body < 400 ? 65 : 175; + } + } + + /// + /// Animations.GetFileIndex's own arithmetic, which is private. The banding is + /// per file type and the boundaries differ between them, so this is transcribed rather + /// than generalised — an index that disagrees with the library's by one is a picture + /// of the wrong creature, validated. + /// + private static int AnimIndexOf(int body, int fileType) + { + switch (fileType) + { + case 2: + return body < 200 ? body * 110 : 22000 + ((body - 200) * 65); + + case 3: + if (body < 300) + return body * 65; + + return body < 400 ? 33000 + ((body - 300) * 110) : 35000 + ((body - 400) * 175); + + case 5: + // "looks strange, though it works" — the library's own comment. Body 34 is + // excluded from the first band here and nowhere else. + if (body < 200 && body != 34) + return body * 110; + + return body < 400 ? 22000 + ((body - 200) * 65) : 35000 + ((body - 400) * 175); + + default: // 1 and 4 share their banding + if (body < 200) + return body * 110; + + return body < 400 ? 22000 + ((body - 200) * 65) : 35000 + ((body - 400) * 175); + } + } + + /// + /// Walks an animation record the way GetAnimation and Frame will, and + /// refuses it if that walk would read outside the record or write outside the bitmap. + /// + /// Two things make this stricter than the static walk, and both come from the library: + /// + /// GetAnimation decodes through new MemoryStream(m_StreamBuffer, false) — + /// the whole shared buffer, not the length bytes it just read into it. So a + /// truncated record does not hit end-of-stream and throw; the reader sails on into the + /// **previous** animation's bytes and returns a plausible frame. Bounding against + /// rather than against the buffer is the entire point. + /// + /// And Frame's run loop is a *write* through a LockBits pointer whose + /// origin comes from two signed shorts in the file (xCenter, yCenter), + /// with no bound of any kind. LoadStatic at least guards its writes; this does + /// not, so the destination of every run is checked against the bitmap it locked. + /// + /// is how many frames the caller will actually decode — + /// 1 for the catalogue's thumbnail (FirstFrame: true), 0 for all of them. + /// Checking frames nobody decodes would invent refusals, which §4.5 costs more than + /// it saves. + /// + public static bool AnimationRecordSane(byte[] record, int length, int maxFrames, out string reason) + { + reason = null; + + if (length < AnimPaletteBytes + 4) + { + reason = "record is " + length + " bytes; an animation needs " + + (AnimPaletteBytes + 4) + " for its palette and frame count"; + return false; + } + + int start = AnimPaletteBytes; + int frameCount = ReadInt32(record, start); + + if (frameCount <= 0) + { + reason = "declares " + frameCount + " frames"; + return false; + } + + if (frameCount > MaxAnimFrames) + { + reason = "declares " + frameCount + " frames, past the " + MaxAnimFrames + " ceiling"; + return false; + } + + // The lookup table is read in full whatever FirstFrame says, so it is bounded in full. + long tableEnd = (long)start + 4 + ((long)frameCount * 4); + + if (tableEnd > length) + { + reason = "frame table (" + frameCount + " entries) does not fit in a " + + length + "-byte record"; + return false; + } + + int check = maxFrames > 0 && maxFrames < frameCount ? maxFrames : frameCount; + + for (int i = 0; i < check; i++) + { + int at = start + ReadInt32(record, start + 4 + (i * 4)); + + if (!FrameSane(record, length, at, i, out reason)) + return false; + } + + return true; + } + + private static bool FrameSane(byte[] record, int length, int at, int frame, out string reason) + { + reason = null; + + if (at < 0 || at + 8 > length) + { + reason = "frame " + frame + " starts at " + at + ", outside the " + + length + "-byte record"; + return false; + } + + int xCenter = ReadInt16(record, at); + int yCenter = ReadInt16(record, at + 2); + int width = ReadUInt16(record, at + 4); + int height = ReadUInt16(record, at + 6); + + // Frame's constructor returns before locking anything for these, so they are empty + // rather than dangerous — and an empty frame is a real thing in this format. + if (width == 0 || height == 0) + return true; + + if (width > MaxArtDimension || height > MaxArtDimension) + { + reason = "frame " + frame + " declares " + width + "x" + height + ", past the " + + MaxArtDimension + "px ceiling"; + return false; + } + + // Settings.PixelFormat is 16bpp and GDI+ pads each scanline to four bytes, so a row + // is `delta` ushorts wide and the locked region is height*delta of them. This is the + // same `bd.Stride >> 1` Frame computes. + int delta = (((width * 2) + 3) & ~3) >> 1; + long pixels = (long)height * delta; + + long origin = (xCenter - 0x200) + ((long)((yCenter + height) - 0x200) * delta); + int cursor = at + 8; + + while (true) + { + if (cursor + 4 > length) + { + reason = "frame " + frame + + " runs off the end of the record looking for its terminator"; + return false; + } + + int header = ReadInt32(record, cursor); + cursor += 4; + + if (header == 0x7FFF7FFF) + break; + + header ^= DoubleXor; + + long dy = (header >> 12) & 0x3FF; + long dx = (header >> 22) & 0x3FF; + int run = header & 0xFFF; + + long first = origin + (dy * delta) + dx; + + if (first < 0 || first + run > pixels) + { + reason = "frame " + frame + " writes pixels " + first + ".." + (first + run) + + " outside its own " + pixels + "-pixel bitmap"; + return false; + } + + // One palette byte per pixel, read straight out of the record. + if (cursor + run > length) + { + reason = "frame " + frame + " declares a " + run + + "-pixel run running past the record"; + return false; + } + + cursor += run; + } + + return true; + } + + private static int ReadUInt16(byte[] b, int at) + { + return b[at] | (b[at + 1] << 8); + } + + private static int ReadInt16(byte[] b, int at) + { + return (short)(b[at] | (b[at + 1] << 8)); + } + + private static int ReadInt32(byte[] b, int at) + { + return b[at] | (b[at + 1] << 8) | (b[at + 2] << 16) | (b[at + 3] << 24); + } + + /// + /// Reads a record's actual bytes so or + /// can walk it. + /// + /// Holds its own handles rather than borrowing the library's, because FileIndex + /// hands out the stream it decodes from and moving that stream's position underneath + /// the decoder would be its own bug. Opened FileShare.ReadWrite to match how + /// FileIndex opens the same files. + /// + /// One reader serves one data file, so an animation sweep wants one per file type, + /// built from . + /// + public sealed class RecordReader : IDisposable + { + private readonly FileStream _mul; + private readonly FileStream _verdata; + private byte[] _scratch = new byte[64 * 1024]; + + public RecordReader(string mulPath, string verdataPath) + { + _mul = Open(mulPath); + _verdata = Open(verdataPath); + } + + private static FileStream Open(string path) + { + if (path == null || !File.Exists(path)) + return null; + + try + { + return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + catch + { + return null; + } + } + + /// + /// True when the record at is safe to hand to + /// Art.GetStatic. A record that cannot be read at all is reported sane — + /// has already judged the entry, and this must not + /// invent a second reason to refuse. + /// + public bool StaticSane(FileIndex index, int at, out string reason) + { + int length = ReadRecord(index, at, out reason); + + if (length < 0) + return true; + + if (length == 0) + return false; + + return StaticRecordSane(_scratch, length, out reason); + } + + /// + /// True when the record at is safe to hand to + /// Animations.GetAnimation. is how many frames + /// the caller will decode — 1 for a FirstFrame call, 0 for all of them. + /// + public bool AnimationSane(FileIndex index, int at, int maxFrames, out string reason) + { + int length = ReadRecord(index, at, out reason); + + if (length < 0) + return true; + + if (length == 0) + return false; + + return AnimationRecordSane(_scratch, length, maxFrames, out reason); + } + + /// + /// Reads one record into . Returns its length, 0 for a + /// failure (with set), or -1 when there is nothing to + /// read at all — has already judged the entry, and this + /// must not invent a second reason to refuse. + /// + private int ReadRecord(FileIndex index, int at, out string reason) + { + reason = null; + + if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) + return -1; + + Entry3D e = index.Index[at]; + bool patched = (e.length & (1 << 31)) != 0; + int length = e.length & 0x7FFFFFFF; + + var stream = patched ? _verdata : _mul; + + if (stream == null || length <= 0 || e.lookup < 0) + return -1; + + if (_scratch.Length < length) + _scratch = new byte[length]; + + int read; + + try + { + stream.Seek(e.lookup, SeekOrigin.Begin); + read = stream.Read(_scratch, 0, length); + } + catch (Exception ex) + { + reason = "cannot read the record: " + ex.GetType().Name; + return 0; + } + + // The short read the decoders discard. Refusing here is the whole point: the + // library would decode whatever the shared buffer happened to hold. + if (read < length) + { + reason = "short read — " + read + " of " + length + " bytes available"; + return 0; + } + + return length; + } + + public void Dispose() + { + if (_mul != null) + _mul.Dispose(); + + if (_verdata != null) + _verdata.Dispose(); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs new file mode 100644 index 0000000..1490de0 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs @@ -0,0 +1,1305 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The Asset Bridge's transport** (docs/link/v8.md §3, §6, §7 — protocol 8, phase 1). + /// + /// Everything else on this link answers on the Core thread, reads live world state, and + /// replies in microseconds. The asset plane cannot: it reads hundreds of megabytes of the + /// operator's client files and decodes pictures out of them, and doing either on the Core + /// thread would stop the world for every player on the shard. So this class is the one + /// genuinely new shape in protocol 8 — a handler that accepts on the Core thread, hands + /// the work to a **dedicated asset worker**, and returns immediately. + /// + /// Three rules hold it together, and each of them is answering a specific way this could + /// go wrong. + /// + /// **1. Replies, never events.** Every asset frame carries the caller's `reqId`, so + /// `rpc.rs`'s `try_route` consumes it before `app.rs` can persist it to SQLite and + /// broadcast it to every WebSocket subscriber. An asset stream on the event path would + /// grow the sidecar's store without bound and fan megabytes at every connected client, + /// forever. Nothing here emits an unsolicited frame — if a request has no `reqId` it is + /// refused rather than answered. + /// + /// **2. One request outstanding, always.** 's queue is bounded + /// drop-oldest in **lines, not bytes** — a design that is right for live events and + /// dangerous for bulk transfer, because 10,000 queued 200 KB replies is 2 GB of shard + /// memory. The bound that actually holds is flow control, not a bigger queue: this plane + /// has **one slot**, and a second asset request arriving while one is in flight is + /// answered `bridge.busy` (which the sidecar already maps to 425) rather than queued. + /// Queue depth therefore stays at approximately one by construction. A dropped or lost + /// reply just times out and is re-requested, which is safe because reading a client file + /// is idempotent and touches no world state. + /// + /// Note what that costs, deliberately: a status poll shares the slot with a batch, so + /// polling during a long import gets 425 until the batch lands. That is honest — this + /// plane really is doing one thing at a time — and the admin surface (phase 8) is where a + /// separate status lane would have to argue for itself. + /// + /// **3. Byte budgets, not counts.** Batches are cut by encoded size + /// (, 512 KB), not by item count, because the + /// ceilings this has to live inside are byte ceilings: the sidecar refuses an inbound line + /// over 1 MiB, and base64 costs 33% on top of whatever the payload measures. + /// is that budget, and every asset family shares it so the + /// envelope cannot drift apart between them. + /// + /// **Emitting from off the Core thread is safe here, and it is worth saying why.** + /// BridgeLink.Emit enqueues onto a ConcurrentQueue and never touches the + /// socket, so the enqueue itself is fine. The subtle part is + /// BridgeIdempotency.Observe, which Emit calls while a keyed command is in + /// flight: it captures a line only when that line's correlation field **exactly equals** + /// the open command's correlation value, and correlation values come from one monotonic + /// counter in the sidecar. An asset reply therefore cannot be mistaken for a keyed + /// command's reply, whatever the interleaving. + /// + public static class BridgeAssets + { + /// + /// What version of *our derivation* produced these bytes (§7). + /// + /// The source gate hashes the operator's client files, which answers "did the inputs + /// change". It cannot answer "did the way we read them change" — and that is the case + /// that bites, because a corrected frame offset or a fixed hue application changes + /// every derived byte while every source file stays identical. So this is folded into + /// stage 1 alongside the hashes, and bumping it makes the whole working set drift, + /// which is the intended and correct blast radius. + /// + /// Bump it whenever extraction changes what it produces from unchanged input. It is + /// the same rule spawnAtlasSource.js's `PARSER_VERSION` follows, and it applies + /// here more rather than less: this pipeline derives far more from far less. + /// + /// **2** — phase 4 (§4.3, §4.9). The catalogue now falls back to + /// AnimationFrame*.uop for bodies the legacy path has nothing for, which on a + /// stock client is 235 new sprites and two of them player-character bodies; and the + /// player-body set no longer carries ghost ids. Every client file is byte-identical + /// and the answer is different, which is precisely what this number exists to say. + /// + /// **3** — phase 6 (§4.10, §11.2). A body with no art at action 0 is catalogued at + /// the first action that has any, and its key names that action. 74 more bodies on a + /// stock client, no existing key's bytes changed — but a body that was absent is now + /// a row, which is the same "unchanged input, different answer" this number covers. + /// + public const int EXTRACTOR_VERSION = 3; + + // ── the one slot (§3.2) ────────────────────────────────────────────────────────────── + + private static readonly object _sync = new object(); + private static Thread _worker; + private static readonly AutoResetEvent _wake = new AutoResetEvent(false); + private static Action _job; + private static string _inFlight; + private static DateTime _inFlightSince; + private static bool _running; + + private static long _served, _busied, _failed; + + // ── the hash cache (§6) ────────────────────────────────────────────────────────────── + + private static readonly Dictionary _hashes = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + private static Thread _hasher; + private static volatile bool _hashing; + + private sealed class CachedHash + { + public long Size; + public long MTime; + public string Sha256; + } + + // ── imaging (§4.4) ─────────────────────────────────────────────────────────────────── + + private static bool _imagingChecked; + private static bool _imagingOk; + private static string _imagingReason; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + DisableTheLibraryCache(); + + BridgeBoot.RegisterHandler("assets.sources", OnSources); + BridgeBoot.RegisterHandler("assets.fetch", OnFetch); + + // Owned here since phase 7, for the same reason `assets.fetch` moved here in phase 5: + // it is the transport, and more than one family has something to enumerate. + BridgeBoot.RegisterHandler("assets.manifest", OnManifest); + } + + /// + /// **Turns Ultima.Files.CacheData off for the life of the process** (phase 5, + /// §17.10). One line, and it answers two separate problems that both end in a + /// confident wrong picture or an out-of-memory shard. + /// + /// **The poisoning.** Art.GetStatic and Art.GetLand memoise into a + /// Bitmap[0xFFFF] and hand back **the same instance** on every call, while + /// Hue.ApplyTo repaints a bitmap **in place**. So hueing a static edits the + /// library's cached copy: measured on this client, hue item 3922 once and every later + /// request for the *plain* 3922 comes back hued, and a second hue stacks on the first. + /// Nothing downstream can see it — the row is the right size, the right shape and the + /// right id. It is §4.5's failure mode arriving through a completely different door. + /// + /// **The retention.** That array is never trimmed. Decoding this client's 39,189 + /// statics once would leave 74 MB of Bitmap in a static field of a game server, + /// kept for as long as the process lives, to serve pictures nobody asked for twice. + /// + /// The obvious alternative — copy each bitmap before hueing — was rejected, and not + /// only for the retention: new Bitmap(src) **throws** on the + /// Format16bppArgb1555 these decoders produce, so the copy has to name the + /// source pixel format explicitly, which is a subtlety on the wrong side of a + /// correctness boundary. + /// + /// **What it costs is nothing measurable here.** Animations — the whole of the + /// body catalogue — does not consult this flag at all, and + /// and each keep their own cache of + /// *encoded PNG bytes*, which is the thing worth holding: a tenth of the size, already + /// hashed, and released when it goes idle. + /// + /// It is a process-global on a library nothing else in this overlay reads, which is why + /// setting it here rather than saving and restoring it around each decode is safe — + /// and a save/restore would not be, because the asset worker is a thread. + /// + private static void DisableTheLibraryCache() + { + try + { + Files.CacheData = false; + } + catch (Exception e) + { + // A client this library cannot even open. The families report that for themselves, + // per key, with a reason; it must not stop the plugin booting. + Console.WriteLine("[Bridge] assets: could not disable the Ultima bitmap cache: {0}", + e.Message); + } + } + + public static string Status() + { + int cached; + + lock (_hashes) + { + cached = _hashes.Count; + } + + lock (_sync) + { + return String.Format( + "assets(served={0} busied={1} failed={2} inFlight={3} hashing={4} cached={5})", + _served, _busied, _failed, _inFlight ?? "-", _hashing, cached); + } + } + + // ── the request plane ──────────────────────────────────────────────────────────────── + + /// + /// Stage 1 of §6: what the shard's client files currently are. No pixels, no assets — + /// just the gate that lets the website decide whether anything needs importing at all, + /// because the normal case is a restart that changed nothing and it must cost nothing. + /// + private static void OnSources(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + // Rule 1. Without a correlation id this would land on the event path, be persisted + // to the sidecar's store and broadcast to every subscriber. Refuse instead. + Fail(null, "BAD_REQUEST", "assets.sources requires a reqId"); + return; + } + + // Stage 1 answers for the whole plane, not for the client files alone: since phase 7 + // an operator can serve the shard's own configuration tree while declining to serve + // their UO client, and `families` is where a website discovers which. Refused only + // when there is nothing at all to report. + if (Families().Count == 0) + { + Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return; + } + + Accept(reqId, "assets.sources", () => ReplySources(reqId)); + } + + /// + /// Claims the single slot and hands the work to the worker, or answers `bridge.busy`. + /// Runs on the Core thread and does nothing expensive; runs on + /// the worker and must touch no world state. + /// + internal static void Accept(string reqId, string kind, Action job) + { + lock (_sync) + { + if (_inFlight != null) + { + _busied++; + Busy(reqId, kind); + return; + } + + _inFlight = kind; + _inFlightSince = DateTime.UtcNow; + _job = job; + + try + { + EnsureWorker(); + } + catch (Exception e) + { + // The slot is claimed and there is now nothing that will ever free it. Give it + // back here or this plane answers `bridge.busy` for the life of the process. + _inFlight = null; + _job = null; + + Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message); + Fail(reqId, "UNAVAILABLE", "the shard could not start its asset worker"); + return; + } + } + + _wake.Set(); + } + + private static void Busy(string reqId, string kind) + { + var held = (DateTime.UtcNow - _inFlightSince).TotalSeconds; + + var sb = BridgeJson.Begin("bridge.busy"); + + sb.Str("reqId", reqId) + // `busyKind`, never a second `kind` — `Begin` has already written this frame's own, + // and a JSON object carrying two makes every parser take the last. Protocol 6 shipped + // that bug once and it made the sidecar answer 200 for a refusal. + .Str("busyKind", kind) + .Num("heldForSec", (long)held) + .Str("reason", "the asset plane serves one request at a time"); + + BridgeLink.Emit(sb.End()); + } + + // ── assets.fetch, and the families behind it (§5, phase 5) ─────────────────────────── + + /// + /// One family's answer to a fetch. Runs on the asset worker, never the Core thread. + /// + internal delegate void FamilyFetch(string reqId, List keys, string catalog, string cursor); + + /// + /// One family's answer to a manifest walk — everything it can serve, no payload. + /// Runs on the asset worker, never the Core thread. A family with nothing to + /// enumerate (statics and land are addressed, not listed) registers none. + /// + internal delegate void FamilyManifest(string reqId, string cursor); + + /// + /// What one §5 key family registered: how to serve it, how to list it, and — since + /// phase 7 — which operator consent it answers to. + /// + /// The gate is per family rather than per plane because the planes are not one + /// consent. `body`, `static` and `land` are the operator's UO CLIENT, licensed from + /// EA and read off their disk; `tree` is the shard's OWN configuration, which they + /// wrote. An operator can reasonably want the second published and not the first, and + /// before this the atlas would have been what silently disappeared when they said so. + /// + private sealed class FamilyReader + { + public FamilyFetch Fetch; + public FamilyManifest Manifest; + public Func Enabled; + public string DisabledReason; + } + + private static readonly Dictionary _families = + new Dictionary(StringComparer.Ordinal); + + /// + /// Claims one §5 key family for a reader. + /// + /// Phase 3 gave assets.fetch to the body catalogue outright, which was right + /// while there was one family and wrong the moment there were three: the command is + /// the *transport*, and the family is a property of the key. So the shared parts — the + /// correlation id, the operator's consent, the key-count ceiling, and deciding which + /// reader a request belongs to — live here once, and a family only ever sees a list of + /// keys it owns. + /// + /// Registration is order-independent on purpose: ServUO calls every + /// Initialize in an order nothing here controls, and this fills a dictionary the + /// handler does not read until a request arrives. + /// + internal static void RegisterFamily(string name, FamilyFetch fetch) + { + RegisterFamily(name, fetch, null, null, null); + } + + /// + /// The full registration: a fetch reader, an optional manifest reader, and the + /// consent this family answers to. + /// + /// null means the asset plane's own gate + /// (Bridge.AssetsEnabled), which is what every client-file family wants. + /// A family that reads something else entirely passes its own. + /// + internal static void RegisterFamily(string name, FamilyFetch fetch, FamilyManifest manifest, + Func enabled, string disabledReason) + { + lock (_families) + { + _families[name] = new FamilyReader + { + Fetch = fetch, + Manifest = manifest, + Enabled = enabled, + DisabledReason = disabledReason + }; + } + } + + /// + /// The families this shard can serve **right now**, for §6's stage 1 and for + /// diagnostics. + /// + /// Filtered by consent rather than by registration, because that is the question the + /// website is actually asking: a family it can see in this list is one it can fetch. + /// Listing a family the operator has switched off would turn one clear refusal at + /// import time into a per-key refusal on every pass, forever — which is exactly the + /// failure `families` was added in phase 5 to prevent. + /// + internal static List Families() + { + var names = new List(); + + lock (_families) + { + foreach (var pair in _families) + { + if (EnabledFor(pair.Value)) + names.Add(pair.Key); + } + } + + names.Sort(StringComparer.Ordinal); + return names; + } + + private static bool EnabledFor(FamilyReader reader) + { + if (reader == null) + return false; + + try + { + return reader.Enabled == null ? BridgeConfig.AssetsEnabled : reader.Enabled(); + } + catch + { + // A gate that throws is a gate that has not consented. + return false; + } + } + + private static FamilyReader FamilyFor(string name) + { + lock (_families) + { + FamilyReader reader; + return _families.TryGetValue(name, out reader) ? reader : null; + } + } + + /// + /// Resolves a named family and answers the request itself when it cannot. + /// + /// Shared by assets.fetch and assets.manifest so the two cannot drift + /// apart about what "this shard does not serve that" means — and so the consent check + /// happens in exactly one place for both. + /// + private static bool Resolve(string reqId, string family, out FamilyReader reader) + { + reader = FamilyFor(family); + + if (reader == null) + { + Fail(reqId, "BAD_REQUEST", + "this shard serves no '" + family + "' asset family (it serves " + + String.Join(", ", Families().ToArray()) + ")"); + return false; + } + + if (!EnabledFor(reader)) + { + Fail(reqId, "DISABLED", reader.DisabledReason + ?? "asset extraction is disabled on this shard"); + return false; + } + + return true; + } + + /// + /// The family segment of a §5 key: everything before the first `/`. + /// + internal static string FamilyOfKey(string key) + { + if (String.IsNullOrEmpty(key)) + return null; + + int slash = key.IndexOf('/'); + + return slash <= 0 ? null : key.Substring(0, slash); + } + + /// + /// §14's `assets.fetch`, for every family. + /// + /// **The family is derived from the keys and is not a request field.** §5 made the key + /// the address of an asset, so a request that had to name its family as well would have + /// two places to be wrong and one of them silent. A batch must be of one family — + /// mixing them is refused rather than split — because the reply carries a single + /// `catalog` id, and that id is what stops an operator patching their client mid-import + /// from stitching one asset set out of two. Two families, two fingerprints, and a reply + /// that claimed one of them would be lying about the other. + /// + private static void OnFetch(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + Fail(null, "BAD_REQUEST", "assets.fetch requires a reqId"); + return; + } + + // The consent check is NOT here any more (phase 7). It cannot be: which consent this + // request needs is a property of the keys, and the keys have not been read yet. So the + // shape checks come first and the gate happens in `Resolve`, once the family is known. + var keys = BridgeJson.GetStringList(o, "keys"); + + if (keys.Count == 0) + { + Fail(reqId, "BAD_REQUEST", "assets.fetch requires a non-empty `keys` array"); + return; + } + + if (keys.Count > BridgeConfig.AssetFetchKeys) + { + Fail(reqId, "BAD_REQUEST", + "assets.fetch takes at most " + BridgeConfig.AssetFetchKeys + + " keys per request (asked for " + keys.Count + ")"); + return; + } + + string family = FamilyOfKey(keys[0]); + + for (int i = 1; i < keys.Count; i++) + { + if (String.Equals(FamilyOfKey(keys[i]), family, StringComparison.Ordinal)) + continue; + + Fail(reqId, "BAD_REQUEST", + "assets.fetch takes keys of one family per request; this one mixes '" + + family + "' with '" + FamilyOfKey(keys[i]) + "'"); + return; + } + + FamilyReader reader; + + if (!Resolve(reqId, family, out reader)) + return; + + if (reader.Fetch == null) + { + Fail(reqId, "BAD_REQUEST", + "the '" + family + "' family cannot be fetched by key on this shard"); + return; + } + + var catalog = BridgeJson.GetString(o, "catalog"); + var cursor = BridgeJson.GetString(o, "cursor"); + FamilyFetch fetch = reader.Fetch; + + Accept(reqId, "assets.fetch", () => fetch(reqId, keys, catalog, cursor)); + } + + /// + /// §14's `assets.manifest`, for every family that has one. + /// + /// Phase 3 gave this command to the body catalogue outright and phase 5 learned, for + /// `assets.fetch`, that the command is the transport and the family is a property of + /// the key. Phase 7 is where the same lesson lands one level up: the tree family + /// enumerates its files exactly the way the catalogue enumerates its bodies, and + /// nothing about the envelope, the cursor or the consent differs between them. + /// + /// **`family` still defaults to `body`.** A phase-3 website asks without naming one + /// and must keep getting the catalogue it asked for. + /// + private static void OnManifest(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + Fail(null, "BAD_REQUEST", "assets.manifest requires a reqId"); + return; + } + + var family = BridgeJson.GetString(o, "family") ?? "body"; + + FamilyReader reader; + + if (!Resolve(reqId, family, out reader)) + return; + + if (reader.Manifest == null) + { + // Named rather than defaulted: statics and land are ADDRESSED (§11.1) rather than + // listed, and a website that asked for a list of 49,152 item graphics has made a + // mistake it needs told about rather than an empty page it will read as "none". + Fail(reqId, "BAD_REQUEST", + "the '" + family + "' family is fetched by key and has no manifest"); + return; + } + + var cursor = BridgeJson.GetString(o, "cursor"); + FamilyManifest manifest = reader.Manifest; + + Accept(reqId, "assets.manifest", () => manifest(reqId, cursor)); + } + + /// + /// ARGB1555 to a PNG with a transparent background. + /// + /// Frame writes 16-bit ARGB1555: a pixel the sprite does not cover is left as + /// zero and a pixel it does cover carries the top bit set. Saving that format straight + /// to PNG asks GDI+ to make the conversion, and what it does with a one-bit alpha + /// channel varies by platform — on Mono it is a different implementation entirely. A + /// sprite that came back with a black rectangle behind it would look fine in a test + /// that only checked the bytes decoded, and wrong on every page that showed it. + /// + /// So the expansion is done here, explicitly: alpha bit clear becomes fully + /// transparent, and each 5-bit channel is widened to 8 bits by repeating its high bits + /// ((c << 3) | (c >> 2)) rather than by shifting alone, which would + /// cap white at 248 and tint the whole catalogue. + /// + internal static byte[] BitmapToPng(Bitmap source) + { + var rect = new Rectangle(0, 0, source.Width, source.Height); + + if (source.PixelFormat != PixelFormat.Format16bppArgb1555) + { + // Not what this library has ever produced. Save it rather than reinterpret it: + // guessing at an unknown layout is how a catalogue fills with confident nonsense. + using (var ms = new MemoryStream()) + { + source.Save(ms, ImageFormat.Png); + return ms.ToArray(); + } + } + + using (var target = new Bitmap(source.Width, source.Height, PixelFormat.Format32bppArgb)) + { + BitmapData src = source.LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format16bppArgb1555); + BitmapData dst = null; + + try + { + dst = target.LockBits(rect, ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb); + + var line = new short[source.Width]; + var outLine = new int[source.Width]; + + for (int y = 0; y < source.Height; y++) + { + Marshal.Copy(new IntPtr(src.Scan0.ToInt64() + ((long)y * src.Stride)), + line, 0, source.Width); + + for (int x = 0; x < source.Width; x++) + { + int p = line[x] & 0xFFFF; + + if ((p & 0x8000) == 0) + { + outLine[x] = 0; + continue; + } + + int r = (p >> 10) & 0x1F; + int g = (p >> 5) & 0x1F; + int b = p & 0x1F; + + outLine[x] = unchecked((int)0xFF000000) + | (((r << 3) | (r >> 2)) << 16) + | (((g << 3) | (g >> 2)) << 8) + | ((b << 3) | (b >> 2)); + } + + Marshal.Copy(outLine, 0, new IntPtr(dst.Scan0.ToInt64() + ((long)y * dst.Stride)), + source.Width); + } + } + finally + { + if (dst != null) + target.UnlockBits(dst); + + source.UnlockBits(src); + } + + using (var ms = new MemoryStream()) + { + target.Save(ms, ImageFormat.Png); + return ms.ToArray(); + } + } + } + + /// + /// Does a caller's asserted catalog id disagree with what this shard holds? + /// + /// **An absent fingerprint and an empty one mean the same thing**, and that is the + /// whole reason this is a function rather than an inline `expected != null`. A caller + /// with nothing to assert sends the field absent or empty depending on how its own + /// client serialises a missing value, and treating `""` as a real id refuses **every** + /// fetch it makes — with a sentence naming no catalog at all ("catalog is now + /// 8159778b"), which reads as a shard fault rather than a caller one. + /// + /// Phase 7 found this on the tree family, where a probe passed an empty string by + /// accident. It was inline in three places by then; it is one function now, because + /// three copies of a comparison are three chances for the next family to get it wrong + /// in a way only a differently-written client would ever reveal. + /// + /// Note this is deliberately NOT the shape `BridgeLeases` uses for its own `expected`: + /// there the value is a world property, where an empty string is a legitimate thing to + /// expect and `!= null` is correct. + /// + internal static bool CatalogMismatch(string expected, string actual) + { + return !String.IsNullOrEmpty(expected) && !String.Equals(expected, actual, StringComparison.Ordinal); + } + + /// + /// SHA-256, lowercase hex. Shared because the hash in a manifest row, the hash in a + /// fetch row and the hash the website stores must be one function. + /// + internal static string Sha256Hex(byte[] bytes) + { + using (var sha = SHA256.Create()) + { + byte[] digest = sha.ComputeHash(bytes); + var sb = new StringBuilder(digest.Length * 2); + + foreach (byte b in digest) + sb.Append(b.ToString("x2", CultureInfo.InvariantCulture)); + + return sb.ToString(); + } + } + + /// + /// The asset plane's one refusal frame, shared by every family on it. + /// + /// is what the sidecar maps to a status, and it exists because + /// the alternative it replaced — matching on the words in — + /// makes an operator-facing sentence load-bearing. Rewording "disabled" would silently + /// turn a 403 into a 400. The codes are `DISABLED` (the operator switched this plane + /// off), `NOT_FOUND` (the shard has no such file), `UNREADABLE` (it has it and cannot + /// decode it), `UNAVAILABLE` (the shard cannot do this right now) and `BAD_REQUEST` + /// (the default, and the caller's fault). + /// + internal static void Fail(string reqId, string code, string reason) + { + var sb = BridgeJson.Begin("assets.error"); + + if (reqId != null) + sb.Str("reqId", reqId); + + sb.Str("code", code) + .Str("reason", reason); + + BridgeLink.Emit(sb.End()); + } + + // ── the worker ─────────────────────────────────────────────────────────────────────── + + /// + /// Started on first use rather than at boot, so a shard that never imports an asset + /// never carries the thread. Caller must hold . + /// + private static void EnsureWorker() + { + if (_worker != null) + return; + + _running = true; + + _worker = new Thread(WorkLoop) + { + Name = "BridgeAssets", + IsBackground = true + }; + + _worker.Start(); + } + + private static void WorkLoop() + { + while (_running) + { + _wake.WaitOne(1000); + + Action job; + + lock (_sync) + { + job = _job; + _job = null; + } + + if (job == null) + continue; + + try + { + job(); + Interlocked.Increment(ref _served); + } + catch (Exception e) + { + // A handler that throws must still free the slot, or this plane is wedged for + // the life of the process and every later request answers `bridge.busy`. + Interlocked.Increment(ref _failed); + Console.WriteLine("[Bridge] asset worker: {0}: {1}", e.GetType().Name, e.Message); + } + finally + { + lock (_sync) + { + _inFlight = null; + } + } + } + } + + // ── assets.sources ─────────────────────────────────────────────────────────────────── + + /// + /// The client files whose bytes decide everything downstream. + /// + /// Resolved through Ultima.Files rather than by joining a configured directory, + /// because that is what the decoders themselves do — a file this reports is a file + /// they would actually open. + /// + private static readonly string[] SourceFiles = + { + "cliloc.enu", + "artlegacymul.uop", "art.mul", "artidx.mul", + "anim.idx", "anim.mul", + "anim2.idx", "anim2.mul", + "anim3.idx", "anim3.mul", + "anim4.idx", "anim4.mul", + "anim5.idx", "anim5.mul", + "animationframe1.uop", "animationframe2.uop", "animationframe3.uop", + "animationframe4.uop", "animationframe6.uop", + "body.def", "bodyconv.def", + "hues.mul", + "verdata.mul" + }; + + private static void ReplySources(string reqId) + { + var sb = BridgeJson.Begin("assets.sources.ok"); + + sb.Str("reqId", reqId) + .Num("extractorVersion", EXTRACTOR_VERSION) + // Which of the two consents this shard has given (phase 7). Without it a website + // whose operator switched client-file extraction off would read an empty `files` + // array as "your client has no cliloc.enu" — a sentence that sends them looking at + // their client install for a setting that lives on their shard. + .Bool("assetsEnabled", BridgeConfig.AssetsEnabled) + .Bool("treeEnabled", BridgeConfig.TreeEnabled); + + WriteImaging(sb); + + // §4.6: whichever of art.mul / artLegacyMUL.uop `FileIndex` would actually open. An + // operator who added custom graphics to art.mul while the UOP is present is getting + // nothing, silently, and this is the only place that can tell them so. + string artData = BridgeAssetValidator.ArtDataPath(); + + sb.Str("artDataFile", artData == null ? null : Path.GetFileName(artData)); + + // Which §5 families this shard can be asked for. Additive, so the protocol stays 8: a + // consumer that does not read it behaves exactly as it did. One that does can tell an + // older overlay (bodies only) from this one without discovering it as a refused fetch + // halfway through a warm pass. + var families = Families(); + + sb.Append(",\"families\":["); + + for (int i = 0; i < families.Count; i++) + { + if (i > 0) + sb.Append(','); + + BridgeJson.Text(sb, families[i]); + } + + sb.Append(']'); + + var page = new PageBuilder(sb, "files", BridgeConfig.AssetBatchBytes); + bool anyMissingHash = false; + + // The client files are the asset plane's own subject, so they are listed under the + // asset plane's own consent. A tree-only shard answers this call — that is how its + // website learns the `tree` family exists — and reports no client files at all, + // which is the truthful answer to "what may I read here". + for (int i = 0; BridgeConfig.AssetsEnabled && i < SourceFiles.Length; i++) + { + string name = SourceFiles[i]; + string path = ResolvePath(name); + + if (path == null) + continue; + + var item = new StringBuilder(256); + + item.Append("{\"name\":"); + BridgeJson.Text(item, name); + item.Append(",\"path\":"); + BridgeJson.Text(item, path); + + long size = 0, mtime = 0; + + try + { + var info = new FileInfo(path); + size = info.Length; + mtime = ToUnixMs(info.LastWriteTimeUtc); + } + catch (Exception e) + { + item.Append(",\"unreadable\":"); + BridgeJson.Text(item, e.GetType().Name); + } + + item.Append(",\"size\":").Append(size.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"mtime\":").Append(mtime.ToString(CultureInfo.InvariantCulture)); + + string hash = CachedHashFor(path, size, mtime); + + if (hash == null) + anyMissingHash = true; + + item.Append(",\"sha256\":"); + BridgeJson.Text(item, hash); + + // The one diagnostic §4.6 asks for: art.mul is present, and unread. + if (artData != null + && (name == "art.mul" || name == "artidx.mul") + && !artData.EndsWith(".mul", StringComparison.OrdinalIgnoreCase)) + { + item.Append(",\"shadowedBy\":"); + BridgeJson.Text(item, Path.GetFileName(artData)); + } + + item.Append('}'); + + if (!page.TryAdd(item.ToString(), name)) + break; + } + + page.Close(); + + // §6's gate is (size, mtime) first and a content hash only when those differ, because + // anim.mul and art.mul are 195 MB and 148 MB and a full hash on every status poll + // would make the admin panel feel broken. It would also blow the sidecar's 10 s reply + // timeout outright on the first call. So a hash that is not cached is reported `null` + // and computed in the background: this reply is always fast, and the next poll — after + // `hashing` goes false — carries the answer. + if (anyMissingHash) + StartHashing(); + + sb.Bool("hashing", _hashing); + sb.Bool("complete", !anyMissingHash); + + BridgeLink.Emit(sb.End()); + } + + private static string ResolvePath(string name) + { + try + { + string path = Files.GetFilePath(name); + + if (path != null) + return path; + } + catch + { + // Ultima's lookup reads the registry on Windows; a host where that throws still + // has the directories ServUO itself booted from, which is what the fallback uses. + } + + // `Ultima.Files` has a fixed table of file names that predates UOP animations, so it + // answers null for every `AnimationFrame*.uop` however present they are (§4.3). Phase + // 4 added those to this list, so the fallback is what makes their size, mtime and hash + // reachable at all. + return BridgeUop.FindClientFile(name); + } + + private static long ToUnixMs(DateTime utc) + { + return (long)(utc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; + } + + // ── imaging (§4.4) ─────────────────────────────────────────────────────────────────── + + /// + /// Whether this host can turn a record into a picture at all. + /// + /// ServUO targets net48, so a Linux shard runs it under Mono, where + /// System.Drawing is a thin layer over **libgdiplus** — and §4.2 put + /// System.Drawing in the *decode* path, not merely the encode: Frame + /// writes ARGB1555 through a LockBits pointer. Without that library a Linux + /// shard cannot read a sprite at all, while clilocs and the atlas are unaffected + /// because neither touches pixels. + /// + /// It must never present as a stack trace or a 500. It is a named, actionable outcome + /// in the same family as the cliloc reader's `COMPRESSED`, and it is reported here — + /// on the *source gate*, the first call any import makes — so an operator learns it + /// while setting the shard up rather than from an empty bestiary weeks later. + /// + private static void WriteImaging(StringBuilder sb) + { + CheckImaging(); + + sb.Append(",\"imaging\":{\"ok\":").Append(_imagingOk ? "true" : "false"); + + if (!_imagingOk) + { + sb.Append(",\"code\":\"NO_IMAGING\",\"reason\":"); + BridgeJson.Text(sb, + "This shard host cannot render images — Mono's System.Drawing needs libgdiplus. " + + "Install it (apt-get install libgdiplus) and re-run the import. Cliloc and atlas " + + "import are unaffected. (" + _imagingReason + ")"); + } + + sb.Append('}'); + } + + /// + /// Whether this host can produce a picture, for the families that produce pictures. + /// + /// reports this on the source gate so an operator learns it + /// while setting the shard up. The catalogue needs the same answer as a *decision* — + /// it must refuse rather than throw a DllNotFoundException out of the middle of + /// a decode loop — so the check itself is shared and this is its one accessor. + /// + internal static bool ImagingOk(out string reason) + { + CheckImaging(); + reason = _imagingReason; + return _imagingOk; + } + + private static void CheckImaging() + { + if (_imagingChecked) + return; + + _imagingChecked = true; + + try + { + TouchImaging(); + _imagingOk = true; + } + catch (Exception e) + { + // On a host with no libgdiplus this is a TypeInitializationException wrapping a + // DllNotFoundException, and it can surface as the method failing to JIT rather + // than as a throw from inside it — which is why the construction lives in its own + // method, so the failure is contained here instead of taking this class's + // static initialisation with it. + _imagingOk = false; + _imagingReason = e.GetType().Name + ": " + e.Message; + } + } + + private static void TouchImaging() + { + using (var bmp = new System.Drawing.Bitmap(1, 1)) + { + bmp.SetPixel(0, 0, System.Drawing.Color.Black); + } + } + + // ── the hash cache (§6) ────────────────────────────────────────────────────────────── + + private static string CachedHashFor(string path, long size, long mtime) + { + lock (_hashes) + { + CachedHash cached; + + if (_hashes.TryGetValue(path, out cached) + && cached.Size == size + && cached.MTime == mtime) + { + return cached.Sha256; + } + } + + return null; + } + + /// + /// Rehashes whatever the cache is missing, on its own thread. + /// + /// Deliberately **not** a job on the asset worker: hashing 343 MB takes seconds to + /// tens of seconds, and holding the single slot for that long would answer every + /// status poll `bridge.busy` for the whole pass — which is exactly the moment an + /// operator is watching the panel. It emits nothing and correlates with nothing; it + /// only fills the cache that the next `assets.sources` reads. + /// + private static void StartHashing() + { + lock (_sync) + { + if (_hashing) + return; + + _hashing = true; + + _hasher = new Thread(HashLoop) + { + Name = "BridgeAssetHash", + IsBackground = true + }; + + _hasher.Start(); + } + } + + private static void HashLoop() + { + try + { + for (int i = 0; i < SourceFiles.Length; i++) + { + string path = ResolvePath(SourceFiles[i]); + + if (path == null) + continue; + + long size, mtime; + + try + { + var info = new FileInfo(path); + size = info.Length; + mtime = ToUnixMs(info.LastWriteTimeUtc); + } + catch + { + continue; + } + + if (CachedHashFor(path, size, mtime) != null) + continue; + + string hash = HashFile(path); + + if (hash == null) + continue; + + lock (_hashes) + { + _hashes[path] = new CachedHash { Size = size, MTime = mtime, Sha256 = hash }; + } + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] asset hash pass: {0}: {1}", e.GetType().Name, e.Message); + } + finally + { + // Under _sync, matching StartHashing: cleared outside it, two passes could both + // pass the guard and hash the same 343 MB twice. + lock (_sync) + { + _hashing = false; + } + } + } + + private static string HashFile(string path) + { + try + { + using (var sha = SHA256.Create()) + using (var stream = new FileStream( + path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1 << 20)) + { + var buffer = new byte[1 << 20]; + int read; + + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + sha.TransformBlock(buffer, 0, read, null, 0); + + sha.TransformFinalBlock(buffer, 0, 0); + + return Hex(sha.Hash); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] cannot hash {0}: {1}", path, e.Message); + return null; + } + } + + private static string Hex(byte[] bytes) + { + var sb = new StringBuilder(bytes.Length * 2); + + for (int i = 0; i < bytes.Length; i++) + sb.Append(bytes[i].ToString("x2", CultureInfo.InvariantCulture)); + + return sb.ToString(); + } + + // ── the paging envelope (§3.3) ─────────────────────────────────────────────────────── + + /// + /// **One envelope for every asset family**, defined here in phase 1 so that clilocs + /// (phase 2), the body catalogue (3), statics and land (5), deep animation keys (6) + /// and the ServUO tree files (7) all page the same way. They are otherwise five + /// chances to invent five slightly different shapes, and the website would have to + /// learn each one. + /// + /// The envelope a reply closes with: + /// + /// + /// "items": [ … ], + /// "more": true, // ask again with this cursor + /// "cursor": "s:4104", // opaque to everyone but the shard; absent when more:false + /// "cut": "budget" // budget | end | limit — WHY this page stopped + /// + /// + /// **The budget is bytes, and it is UTF-8 bytes.** Not item count, because the ceiling + /// this lives inside is the sidecar's inbound line cap; and not chars, because a + /// cliloc row is real text and a `StringBuilder`'s Length would undercount every + /// non-ASCII character in it. + /// + /// `cut` exists because "the page is short" has three different meanings and the + /// website must not have to guess which: the source ran out (`end`), the byte budget + /// was spent (`budget`), or the family stopped at its own limit (`limit`). Only the + /// first means the import is finished. + /// + /// **The first item is always admitted**, even if it alone exceeds the budget. + /// Otherwise an oversized item would make its family unable to make any progress at + /// all — it would be skipped for the budget on every page, forever. That is safe + /// precisely because the budget is set to half the sidecar's line cap + /// (), so one such item still fits the wire. + /// + public sealed class PageBuilder + { + private readonly StringBuilder _sb; + private readonly int _budget; + private int _bytes; + private int _count; + private string _cursor; + private string _cut = "end"; + + /// + /// Room kept back for the fields the envelope must still be able to write after + /// the last item — `more`, `cursor`, `cut` and the closing brace. Without it a + /// page could fill the budget exactly and then overrun it closing itself. + /// + private const int Reserve = 256; + + public PageBuilder(StringBuilder sb, string arrayName, int budget) + { + _sb = sb; + _budget = budget; + + sb.Append(",\"").Append(arrayName).Append("\":["); + + // The prefix is already written, and it counts: the cap the sidecar enforces is + // on the whole line, not on the array. + _bytes = Encoding.UTF8.GetByteCount(sb.ToString()); + } + + public int Count { get { return _count; } } + + /// + /// Adds one already-serialised item. is where the + /// family should resume if this turns out to be the last item on the page. + /// Returns false when the budget is spent — the caller stops, and `more` is true. + /// + public bool TryAdd(string item, string cursorAfter) + { + if (item == null) + return true; + + int cost = Encoding.UTF8.GetByteCount(item) + (_count > 0 ? 1 : 0); + + if (_count > 0 && _bytes + cost + Reserve > _budget) + { + _cut = "budget"; + return false; + } + + if (_count > 0) + _sb.Append(','); + + _sb.Append(item); + + _bytes += cost; + _count++; + _cursor = cursorAfter; + + return true; + } + + /// + /// Stops the page for a reason of the family's own — a per-request limit, say — + /// rather than because the budget ran out. + /// + public void Cut(string why) + { + _cut = why; + } + + public void Close() + { + bool more = _cut != "end"; + + _sb.Append(']'); + _sb.Bool("more", more); + _sb.Str("cut", _cut); + + if (more && _cursor != null) + _sb.Str("cursor", _cursor); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeBodies.cs b/overlay/Scripts/Custom/Bridge/BridgeBodies.cs new file mode 100644 index 0000000..00b599f --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeBodies.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +using Server.Mobiles; + +namespace Server.Custom.Bridge +{ + /// + /// **Slug to body id — the part only the shard can do** (docs/link/v8.md §8, protocol 8, + /// phase 3). + /// + /// The spawn atlas knows a creature by a **slug** derived from the class name it found in + /// `Spawns/*.xml` ("giant-spider"). The client knows the same creature by a **body id** + /// (28). Nothing in the ServUO tree declares that mapping as data. Today an operator + /// bridges it by hand, grepping `Scripts/Mobiles/Normal/<Name>.cs` for `Body =`, + /// which appears as a decimal, as hex (`0xD1`), as `Utility.RandomList(35, 36)` and as an + /// `m_IDs[]` table — a parse that is wrong on the shard's own custom creatures, which is + /// precisely the set an operator most wants pictures for. + /// + /// Inside ServUO the problem does not exist: construct the type, read `Body.BodyID`, + /// delete it. BridgeWorld.cs already does exactly that for a different feature. + /// + /// **This is the one asset-plane family that does NOT run on the asset worker**, and the + /// reason is the whole point of §8. Constructing and deleting a mobile is world mutation, + /// so it must happen on the Core thread — while the decode in + /// must happen off it, because it reads hundreds of megabytes and would stop the world for + /// every player on the shard. That split is why body resolution is its own request kind + /// rather than a step inside asset extraction. + /// + /// Two consequences follow from answering on the Core thread, and both are bounds: + /// + /// **The batch is small and the shard enforces the cap itself.** Every type constructed + /// here runs a real constructor — packing items, rolling skills, starting AI timers — and + /// all of that happens between two ticks of the world. The website chunks its own list; + /// a request over names is **refused** rather + /// than truncated, so the two sides cannot quietly disagree about what was answered. + /// + /// **It does not take the asset plane's single slot.** The slot exists to stop several + /// large replies queueing at once (§3.2); this reply is a few kilobytes and the work is + /// not on the worker, so claiming the slot would only make a body pass and a catalogue + /// page refuse each other for no benefit. + /// + /// **A creature whose constructor randomises its body reports one of its variants**, not + /// an error and not a set. Constructing twice to detect that would double every side + /// effect above to learn something the bestiary does not render differently — both ids are + /// the same creature. The answer is stable enough to cache and cheap enough to redo. + /// + public static class BridgeBodies + { + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("assets.bodies", OnBodies); + } + + // ── the request ────────────────────────────────────────────────────────────────────── + + private static void OnBodies(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + // Rule 1 of the asset plane: without a correlation id this reply lands on the + // event path, is persisted to the sidecar's store and broadcast to every + // subscriber. Refuse rather than answer. + BridgeAssets.Fail(null, "BAD_REQUEST", "assets.bodies requires a reqId"); + return; + } + + if (!BridgeConfig.AssetsEnabled) + { + BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return; + } + + var types = BridgeJson.GetStringList(o, "types"); + + if (types.Count == 0) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.bodies requires a non-empty `types` array of ServUO class names"); + return; + } + + if (types.Count > BridgeConfig.AssetBodyBatch) + { + // Refuse, never truncate. A silently shortened answer looks identical to a + // complete one from the website's side, and the types that fell off the end would + // be recorded as "asked and unanswerable" rather than "never asked". + BridgeAssets.Fail(reqId, "BAD_REQUEST", + "assets.bodies takes at most " + BridgeConfig.AssetBodyBatch + + " types per request (asked for " + types.Count + "); send them in chunks"); + return; + } + + Reply(reqId, types); + } + + /// + /// Core thread. Constructs each type once, reads its body, deletes it. + /// + /// Every outcome is a **row**, never a failed request: a shard is expected to be asked + /// about types it does not have (an atlas built from a tree that has since changed, a + /// spawn file naming a creature from a script package the operator removed), and a + /// status screen that fails the whole pass over one of those teaches an operator to + /// stop pressing the button. + /// + private static void Reply(string reqId, List types) + { + var sb = BridgeJson.Begin("assets.bodies.ok"); + + sb.Str("reqId", reqId) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", types.Count); + + // The envelope is shared with every other family (§3.4) even though this one never + // pages: the website drives the chunking, so `more` is always false and `cut` always + // "end". Writing it anyway means one reader shape on the other side rather than two. + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int resolved = 0; + + foreach (var name in types) + { + string status; + int body; + + Resolve(name, out body, out status); + + if (status == "ok") + resolved++; + + var item = new StringBuilder(96); + + item.Append("{\"type\":"); + BridgeJson.Text(item, name); + item.Append(",\"status\":\"").Append(status).Append('"'); + + if (status == "ok") + item.Append(",\"body\":").Append(body.ToString(CultureInfo.InvariantCulture)); + + item.Append('}'); + + // A chunk this small cannot spend the budget — the cap above is a hundred names + // and the budget is half a megabyte — but the check costs nothing and the day + // someone raises `AssetBodyBatch` it is the difference between a short page and a + // line the sidecar drops. + if (!page.TryAdd(item.ToString(), null)) + break; + } + + page.Close(); + + sb.Num("resolved", resolved); + + BridgeLink.Emit(sb.End()); + } + + /// + /// One type name to one body id. + /// + /// `status` is the field the website records, and the four values are four different + /// things an operator can act on: + /// + /// ok — constructed, body read. + /// unknown — no such type on this shard. The spawn file names something the + /// scripts do not define, which is a real drift an operator wants to see. + /// notCreature — the type exists but is not a `BaseCreature`. Spawn files + /// legitimately name items and static decorations; those have no body and never will, + /// so this is a permanent answer rather than a retryable failure. + /// failed — the constructor threw, or the type has none that takes no + /// arguments. Caught per type, because one creature whose constructor depends on a + /// script package the operator removed must not cost the other ninety-nine. + /// + private static void Resolve(string name, out int body, out string status) + { + body = 0; + status = "failed"; + + Type type; + + try + { + // `true` is ignoreCase — spawn files are hand-edited and their casing drifts from + // the class it names far more often than the name itself does. + type = ScriptCompiler.FindTypeByName(name, true); + } + catch + { + status = "failed"; + return; + } + + if (type == null) + { + status = "unknown"; + return; + } + + if (!typeof(BaseCreature).IsAssignableFrom(type) || type.IsAbstract) + { + status = "notCreature"; + return; + } + + BaseCreature creature = null; + + try + { + creature = Activator.CreateInstance(type) as BaseCreature; + + if (creature == null) + { + status = "failed"; + return; + } + + body = creature.Body.BodyID; + status = body > 0 ? "ok" : "failed"; + } + catch (Exception e) + { + Console.WriteLine("[Bridge] assets.bodies: {0}: {1}: {2}", + name, e.GetType().Name, e.Message); + + status = "failed"; + } + finally + { + if (creature != null) + { + try + { + // Deleting the mobile deletes the items it packed — `Mobile.Delete` walks + // `Items`, and `Item.Delete` walks what each contains — and stops its AI + // timer. A creature left alive here is a creature standing at (0,0,0) on + // the internal map forever, saved with the world, once per import. + creature.Delete(); + } + catch + { + // Nothing useful is left to do, and throwing out of `finally` would lose + // whatever the try block was already reporting. + } + } + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs index 3d6c131..8ce6170 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs @@ -260,6 +260,10 @@ namespace Server.Custom.Bridge e.Mobile.SendMessage("Bridge: {0}", BridgeParticipation.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeWorld.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeOneShots.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeAssets.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeCatalog.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeArt.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeTree.Status()); break; } } diff --git a/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs new file mode 100644 index 0000000..859bc89 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs @@ -0,0 +1,1137 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Globalization; +using System.IO; +using System.Text; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The body catalogue** (docs/link/v8.md §4.8, §5, §6, §11 — protocol 8, phase 3). + /// + /// One thumbnail per creature body: the working set that makes a bestiary, a marketplace + /// listing and a character sheet render. Everything deeper — every action, every frame — + /// is the same addressing scheme at a deeper key and is **not served** (§11.2, phase 6: + /// the site shows still pictures, so frames wait for a consumer that wants them). This is + /// the set that is worth importing before anything asks for it, because on this machine's + /// client it is **1,096 sprites at about a kilobyte each** — 787 out of the legacy + /// `anim*.mul` files, 235 more out of `AnimationFrame*.uop` since phase 4 (§4.3, §4.9), + /// and 74 more since phase 6, which have no art at action 0 and real art deeper. + /// + /// ── **One picture per body, at the first action that has one** ── + /// + /// A key carries the action it came from — `body/820/a23` for a horse whose action 0 is + /// empty — so the catalogue is still exactly one row per body, and the row says which + /// picture it is. What it never does is decode an action the walk did not choose: a fetch + /// for `body/820/a0` is `unsupported`, not a second attempt, because the slots past a + /// body's band belong to the next body and every check passes on them (§4.10). + /// + /// Two request kinds, which are §6's two stages for assets rather than for sources: + /// + /// assets.manifest — `[{ key, sha256, bytes, width, height }]`, no pixels. The + /// website diffs it against what it already holds and asks only for what changed. That is + /// the whole difference between an Update and a re-download. + /// + /// assets.fetch — the pixels, for an explicit list of keys. + /// + /// ── **Why the manifest builds the pictures it refuses to send** ── + /// + /// A manifest row carries a hash of the bytes, and the only way to hash bytes is to have + /// them. So the scan decodes, encodes to PNG and hashes, then sends the row and **keeps + /// the bytes** — a megabyte for the whole catalogue, against re-decoding all 787 sprites a + /// second time when the fetch arrives moments later. + /// + /// ── **Why the manifest pages on TIME rather than on bytes** ── + /// + /// Every other family on this plane pages because its rows are large. This one's rows are + /// ninety bytes and the whole catalogue is one page by the byte budget — but producing + /// that page means decoding 787 animations, and the sidecar gives a reply ten seconds + /// (§3.3). So the scan carries a **wall-clock budget** as well + /// () and cuts the page `limit` when it is spent, + /// resuming from its cursor on the next call. The byte budget is still enforced, because + /// the day a family's rows grow is not the day to discover only one of the two bounds was + /// real. + /// + /// ── **Why `catalog` is derived from the sources and not minted per build** ── + /// + /// A manifest walk and the fetch that follows it must be talking about the same client + /// files, or the website stitches one catalogue out of two. The obvious answer is a fresh + /// id per build, and it is wrong: this cache is released when it goes idle, so a rebuild + /// halfway through a slow import would change the id and force a restart although nothing + /// about the client moved. So the id is a hash of what actually decides the bytes — every + /// anim file's size and mtime, both direction settings and + /// . It is stable across a rebuild and it + /// changes exactly when an operator patches their client. + /// + /// ── **The never-sweep rule, and the 357** ── + /// + /// Nothing here asks a file type for an index it does not own, and nothing here trusts the + /// library's own success. takes + /// `BodyConverter.Convert`'s answer and reports nothing if it leads nowhere (sweeping + /// instead puts a giant spider on the gargoyle page, decoding cleanly); and every body is + /// put through and + /// RecordReader.AnimationSane **before** it is decoded, because a body whose index + /// entry reads `length 0` gets a bitmap back anyway — the previously-decoded creature's, + /// from the library's reused stream buffer. That is 357 of the 1,144 bodies the library + /// claims on a stock client, and importing them would have written 357 duplicate + /// portraits whose subject depended on the order this walk happened to run in. + /// + /// ── **The UOP fallback, and why it cannot reintroduce that** ── + /// + /// Phase 4 added beneath the legacy reader: a body the vendored + /// path has nothing for is looked for in the UOP packages before it is reported absent. + /// That is where two of the six player-character bodies live — `Bodyconv.def` sends + /// gargoyles 666 and 667 to `anim5`, at an index past the end of `anim5.idx` — and 233 + /// other bodies besides. It cannot produce a wrong picture the way a legacy sweep would, + /// because a UOP entry is addressed by the hash of a name that contains the body id and + /// the payload then declares that id again, which + /// checks. + /// + public static class BridgeCatalog + { + /// The only family this phase serves. §5's key scheme covers the rest. + private const string Family = "body"; + + /// Bodies are addressable to 2047; the sweep behind §4.8 covered exactly this. + private const int MaxBody = 2047; + + /// + /// The action a thumbnail comes from when the body has one, which is nearly always. + /// Everything deeper than a first frame is deferred — see §11.2. + /// + private const int PreferredAction = 0; + + /// + /// How far the fallback looks for a body with no art at . + /// + /// 35 because that is the largest band any file type gives a body, so an action beyond + /// it is not something the client's own layout can name. The legacy arm is bounded + /// tighter still and per body, by — + /// this is only the scan's outer stop, and it is the UOP arm's real one, where an + /// action is a named entry rather than an offset. + /// + private const int MaxAction = 35; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + // Both commands are shared plumbing: `assets.fetch` since phase 5 and + // `assets.manifest` since phase 7 (§5, §10). BridgeAssets owns the correlation id, the + // operator's consent, the key ceiling and the family decision; what is registered here + // is only this family's two readers, and each is called on the asset worker with work + // it owns. + BridgeAssets.RegisterFamily(Family, ReplyFetch, ReplyManifest, null, null); + } + + // ── the cache ──────────────────────────────────────────────────────────────────────── + + private sealed class Sprite + { + public string Key; + public int Body; + + /// + /// Which action this body's thumbnail came from — + /// for all but 74 bodies on this client, and on the wire because the key names it + /// (§5, §11.2). A consumer that assumes `a0` would build a dead URL for a horse. + /// + public int Action; + + public int Direction; + public int FileType; + public string Sha256; + public byte[] Png; + public int Width; + public int Height; + + /// + /// Which reader produced it: `legacy` for ServUO's vendored Animations over + /// anim*.mul, `uop` for phase 4's own reader over + /// AnimationFrame*.uop (§4.3, §4.9). On the wire so that an operator + /// looking at a wrong picture can tell which half of the extractor to doubt, and + /// so the acceptance walk can prove the fallback fired at all. + /// + public string Source; + } + + private sealed class Catalog + { + public string Id; + + /// + /// Keyed by **body**, not by asset key, since phase 6: a body's key now carries + /// the action its picture came from, so the key cannot be spelled until the body + /// has been resolved. A fetch arrives holding a key and has to reach the same + /// sprite, which it does by parsing the body out of it and comparing. + /// + public readonly Dictionary ByBody = new Dictionary(); + + public readonly List Order = new List(); + + /// The next body the scan has yet to look at. + public int Next = 1; + + public bool Complete; + public DateTime LastUsed; + } + + private static readonly object _sync = new object(); + private static Catalog _catalog; + + private static readonly TimeSpan IdleFor = TimeSpan.FromMinutes(5); + + // ── assets.manifest ────────────────────────────────────────────────────────────────── + + /// + /// Worker thread. Scans forward from the cursor until the byte budget or the time + /// budget is spent, hashing what it decodes and keeping the bytes for the fetch. + /// + private static void ReplyManifest(string reqId, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + // Never a stack trace and never a 500: on a Linux host without libgdiplus this is + // the expected outcome, and it is actionable in one line (§4.4). + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. Install it (apt-get install libgdiplus) and re-run the import. " + + "Cliloc and atlas import are unaffected. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + Catalog catalog; + + lock (_sync) + { + if (_catalog == null || _catalog.Id != id) + _catalog = new Catalog { Id = id }; + + catalog = _catalog; + catalog.LastUsed = DateTime.UtcNow; + } + + int from = ParseBodyCursor(cursor); + + var sb = BridgeJson.Begin("assets.manifest.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + // What the website compares across pages, and across the fetch that follows. A + // change means the operator patched their client mid-import and the half already + // read describes files that no longer exist. + .Str("catalog", catalog.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("maxBody", MaxBody) + .Num("from", from); + + WritePlayerBodies(sb); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int scanned = 0; + int last = from - 1; + bool timedOut = false; + bool budgetCut = false; + + var deadline = DateTime.UtcNow.AddMilliseconds(BridgeConfig.AssetScanMs); + + using (var readers = new Readers()) + { + int body = from; + + for (; body <= MaxBody; body++) + { + // Checked before the body rather than after it, so the budget bounds the reply + // rather than the reply plus one more decode. One sprite is milliseconds; the + // ceiling this lives under is ten seconds and the cost of overshooting it is + // the whole page, retried. + if (body > from && DateTime.UtcNow >= deadline) + { + timedOut = true; + break; + } + + scanned++; + last = body; + + Sprite sprite = Resolve(catalog, readers, body); + + if (sprite == null) + continue; + + var item = new StringBuilder(128); + + item.Append("{\"key\":"); + BridgeJson.Text(item, sprite.Key); + item.Append(",\"sha256\":\"").Append(sprite.Sha256).Append('"'); + item.Append(",\"bytes\":").Append(sprite.Png.Length.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"width\":").Append(sprite.Width.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"action\":").Append(sprite.Action.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"source\":\"").Append(sprite.Source).Append('"'); + item.Append('}'); + + if (!page.TryAdd(item.ToString(), "b:" + body.ToString(CultureInfo.InvariantCulture))) + { + // The budget stopped this page BEFORE this body's row went on it, so the + // next page must resume AT this body rather than after it. Getting this + // one line wrong drops exactly one creature from the catalogue per page, + // which nothing downstream could ever notice. + budgetCut = true; + last = body - 1; + scanned--; + break; + } + } + } + + if (timedOut) + page.Cut("limit"); + + // The walk reached the end of the addressable range without either budget stopping it. + // Derived from the two flags rather than from the row count, because a page that ends + // exactly on a boundary is indistinguishable from a finished one by count alone — + // §3.4's whole argument for `cut` existing. + bool finished = !timedOut && !budgetCut && last >= MaxBody; + + int held; + + lock (_sync) + { + if (_catalog == catalog) + { + catalog.Next = Math.Max(catalog.Next, last + 1); + catalog.LastUsed = DateTime.UtcNow; + + if (finished) + catalog.Complete = true; + } + + held = catalog.Order.Count; + } + + page.Close(); + + // Past Close(), which is normally the mistake BridgeCliloc's `from` comment warns + // about — but these three are not knowable until the scan has run, and they cost + // about fifty bytes against PageBuilder's 256-byte reserve, of which Close() itself + // spends around forty. Anything larger than this belongs before the page opens. + sb.Num("scanned", scanned) + .Num("held", held) + .Bool("complete", finished); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + // ── assets.fetch ───────────────────────────────────────────────────────────────────── + + /// + /// The `body` family's half of assets.fetch. The correlation id, the operator's + /// consent, the key ceiling and the family decision have already been made by + /// 's caller; every key here is this family's. + /// + private static void ReplyFetch(string reqId, List keys, string expected, string cursor) + { + string imagingReason; + + if (!BridgeAssets.ImagingOk(out imagingReason)) + { + BridgeAssets.Fail(reqId, "UNAVAILABLE", + "this shard host cannot render images - Mono's System.Drawing needs " + + "libgdiplus. (" + imagingReason + ")"); + return; + } + + string id = SourceId(); + + if (BridgeAssets.CatalogMismatch(expected, id)) + { + // The client files moved between the manifest and this fetch. Refusing is the only + // honest answer: the keys were chosen against a catalogue that no longer describes + // what is on disk, and serving them would mix two clients in one import with no + // error anywhere. + BridgeAssets.Fail(reqId, "UNREADABLE", + "the shard's client files changed since that manifest was read (catalog " + + expected + " is now " + id + "); start the import again"); + return; + } + + Catalog catalog; + + lock (_sync) + { + if (_catalog == null || _catalog.Id != id) + _catalog = new Catalog { Id = id }; + + catalog = _catalog; + catalog.LastUsed = DateTime.UtcNow; + } + + int from = ParseKeyCursor(cursor); + + var sb = BridgeJson.Begin("assets.fetch.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + .Str("catalog", catalog.Id) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Num("asked", keys.Count) + .Num("from", from); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int i = from; + + using (var readers = new Readers()) + { + for (; i < keys.Count; i++) + { + var item = Render(catalog, readers, keys[i]); + + if (!page.TryAdd(item, "k:" + (i + 1).ToString(CultureInfo.InvariantCulture))) + break; + } + } + + page.Close(); + + sb.Num("sent", page.Count); + + BridgeLink.Emit(sb.End()); + + Sweep(); + } + + /// + /// One key to one row, with the bytes. + /// + /// A key this shard cannot serve is a **row**, not a failed request: the website asked + /// for a list, and one key naming a body whose art this client does not carry must not + /// cost the other four hundred. `status` distinguishes the two ways that happens — + /// `absent` (this client has no art at that key, the expected answer for two thirds of + /// the player bodies) and `unsupported` (a key shape this phase does not serve, which + /// is a website bug rather than a client gap). + /// + private static string Render(Catalog catalog, Readers readers, string key) + { + int body, action; + + if (!TryParseKey(key, out body, out action)) + { + var bad = new StringBuilder(96); + bad.Append("{\"key\":"); + BridgeJson.Text(bad, key); + bad.Append(",\"status\":\"unsupported\"}"); + return bad.ToString(); + } + + Sprite sprite = Resolve(catalog, readers, body); + + var item = new StringBuilder(2048); + + item.Append("{\"key\":"); + BridgeJson.Text(item, key); + + if (sprite == null) + { + item.Append(",\"status\":\"absent\"}"); + return item.ToString(); + } + + if (sprite.Action != action) + { + // The body has a picture, but not at the action this key names. Two ways to get + // here and both are the caller's: an old manifest that catalogued this body at + // `a0` before the client was patched, or a key someone built by assuming the + // action. Neither is served — decoding the asked-for action instead would be + // §4.10's wrong picture, arrived at politely. + item.Append(",\"status\":\"unsupported\""); + item.Append(",\"action\":").Append(sprite.Action.ToString(CultureInfo.InvariantCulture)); + item.Append('}'); + return item.ToString(); + } + + item.Append(",\"status\":\"ok\""); + item.Append(",\"sha256\":\"").Append(sprite.Sha256).Append('"'); + item.Append(",\"bytes\":").Append(sprite.Png.Length.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"width\":").Append(sprite.Width.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"action\":").Append(sprite.Action.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"source\":\"").Append(sprite.Source).Append('"'); + item.Append(",\"png\":\"").Append(Convert.ToBase64String(sprite.Png)).Append("\"}"); + + return item.ToString(); + } + + // ── decode ─────────────────────────────────────────────────────────────────────────── + + /// + /// The catalogue entry for one body, decoded and hashed on first sight and cached + /// after. Returns null when this client has no art for it — which is an ordinary + /// answer for well over half of the addressable range, not a failure. + /// + private static Sprite Resolve(Catalog catalog, Readers readers, int body) + { + lock (_sync) + { + Sprite cached; + + if (catalog.ByBody.TryGetValue(body, out cached)) + return cached; + } + + int direction = IsPlayerBody(body) + ? BridgeConfig.AssetPlayerDirection + : BridgeConfig.AssetCreatureDirection; + + Sprite sprite = ResolveAny(readers, body, direction); + + if (sprite == null) + return null; + + lock (_sync) + { + if (!catalog.ByBody.ContainsKey(body)) + { + catalog.ByBody[body] = sprite; + catalog.Order.Add(sprite); + } + + return catalog.ByBody[body]; + } + } + + /// + /// One body's thumbnail: action 0 if it has one, otherwise the first action that does. + /// + /// ── **Why there is a fallback at all** ── + /// + /// Through phase 5 a body with no art at action 0 was simply absent, and on this + /// client **74 bodies are in exactly that state while carrying real art deeper** — 66 + /// of them UOP, 8 legacy. Body 820's first drawn action is 23, and it is a horse. + /// They rendered as text on the bestiary for want of looking one action further. + /// + /// ── **Why the key says which action it is** ── + /// + /// The fallback's picture is `body/820/a23`, not `body/820/a0`. Naming it `a0` would + /// have been fewer changes downstream and a key that lies about its content, which is + /// the failure this protocol keeps meeting from other directions (§4.5, §4.8, §11.1). + /// + /// ── **Why the ceiling is not a detail** ── + /// + /// Scanning actions is the one thing that can walk off the end of a body's slots, and + /// the slots immediately after a body's are the **next body's**. Measured in phase 6: + /// 643 of 795 legacy bodies return a fully validated, correctly-sized picture one + /// action past their band, and 452 of those are byte-identical to body+1's action 0. + /// refuses past the ceiling, so + /// this walk cannot produce one — see §4.10. + /// + private static Sprite ResolveAny(Readers readers, int body, int direction) + { + int actions, fileType; + string reason; + + // The legacy ceiling. A body the legacy path cannot place at all still gets the UOP + // arm below, where an action is a named entry rather than an offset into a band. + if (!BridgeAssetValidator.ActionCount(body, out actions, out fileType, out reason)) + actions = 0; + + for (int action = PreferredAction; action < MaxAction; action++) + { + // Legacy first, always. The vendored decoder is what 787 of this client's bodies + // come out of, it is what phase 3 measured, and the UOP packages hold a different + // and mostly disjoint set (measured: of the 244 bodies they carry, 8 also have + // legacy art). So this is a fallback rather than a choice, and no body changes + // reader while a client sits still. + Sprite sprite = action < actions + ? ResolveLegacy(Key(body, action), readers, body, action, direction) + : null; + + if (sprite == null) + sprite = ResolveUop(Key(body, action), readers, body, action, direction); + + if (sprite != null) + return sprite; + } + + return null; + } + + /// + /// ServUO's vendored Animations over anim*.mul, behind §4.5's validator. + /// + private static Sprite ResolveLegacy(string key, Readers readers, int body, int action, int direction) + { + int fileType, at; + string reason; + + if (!BridgeAssetValidator.ResolveAnimation(body, action, direction, + out fileType, out at, out reason)) + return null; + + FileIndex index = readers.Index(fileType); + + if (index == null) + return null; + + if (BridgeAssetValidator.CheckEntry(index, at, readers.MulLength(fileType), + readers.VerdataLength, out reason) != BridgeAssetValidator.Verdict.Ok) + { + // The `length 0` case lands here, and it is the 357. The library would hand back + // the previously-decoded body's bitmap for every one of them. + return null; + } + + var reader = readers.Reader(fileType); + + if (reader == null) + return null; + + // `maxFrames: 1` because that is what `firstFrame: true` decodes. Checking frames + // nobody reads would invent refusals, and a checker that refuses real art is worse + // than no checker at all. + if (!reader.AnimationSane(index, at, 1, out reason)) + return null; + + try + { + return Decode(key, body, action, direction, fileType); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: body {0} action {1}: {2}: {3}", + body, action, e.GetType().Name, e.Message); + return null; + } + } + + /// + /// Phase 4's own reader over AnimationFrame*.uop (§4.3, §4.9), for the bodies + /// the legacy path has nothing for. + /// + /// On this machine's client that is **235 bodies** the catalogue could not reach + /// before, including the two gargoyle player bodies — `Bodyconv.def` sends 666 and 667 + /// to `anim5`, at an index past the end of `anim5.idx`, and the art has been in + /// `AnimationFrame3.uop` all along. + /// + /// Nothing here can produce §4.8's failure. A legacy index is addressed by position, + /// so a wrong lookup is another creature's picture; a UOP entry is addressed by the + /// hash of a name carrying the body id, and the payload repeats that id in its own + /// header for to check. A miss is a miss. + /// + private static Sprite ResolveUop(string key, Readers readers, int body, int action, int direction) + { + ulong hash = BridgeUop.HashOf(body, action); + + byte[] payload = null; + string reason = null; + + foreach (int n in BridgeUop.Packages) + { + BridgeUop.Package package = readers.Package(n); + + if (package == null || !package.Has(hash)) + continue; + + if (!package.TryRead(hash, out payload, out reason)) + { + Console.WriteLine("[Bridge] catalogue: body {0} action {1} in {2}: {3}", + body, action, BridgeUop.PackageName(n), reason); + return null; + } + + break; + } + + if (payload == null) + return null; + + BridgeUop.Group group; + + if (!BridgeUop.Group.TryOpen(payload, body, out group, out reason)) + { + Console.WriteLine("[Bridge] catalogue: body {0} action {1} uop: {2}", + body, action, reason); + return null; + } + + int frame = group.DirectionAt(direction); + + if (frame < 0) + return null; + + BridgeUop.Pixels pixels; + bool empty; + + if (!group.TryDecode(frame, out pixels, out empty, out reason)) + { + // A 0x0 frame is no art rather than damage — the vendored decoder returns early on + // exactly the same condition — so it is absent, silently. Anything else is a + // record this reader refused, and that is worth a line. + if (!empty) + Console.WriteLine("[Bridge] catalogue: body {0} action {1} uop: {2}", + body, action, reason); + + return null; + } + + byte[] png = BridgePng.FromArgb1555(pixels.Argb1555, pixels.Width, pixels.Height); + + if (png == null) + return null; + + return new Sprite + { + Key = key, + Body = body, + Action = action, + Direction = direction, + FileType = 0, + Png = png, + Width = pixels.Width, + Height = pixels.Height, + Sha256 = BridgeAssets.Sha256Hex(png), + Source = "uop" + }; + } + + private static Sprite Decode(string key, int body, int action, int direction, int fileType) + { + int hue = 0; + + // `preserveHue: false` — the catalogue is the creature's own art, and a body-level hue + // from Body.def belongs to a specific mob rather than to the species. §5's key scheme + // is where a hued variant is expressed (`static/3922/h33`), not here. + Frame[] frames = Animations.GetAnimation(body, action, direction, ref hue, false, true); + + if (frames == null || frames.Length == 0 || frames[0] == null) + return null; + + Bitmap bitmap = frames[0].Bitmap; + + if (bitmap == null || bitmap.Width <= 0 || bitmap.Height <= 0) + return null; + + byte[] png = BridgeAssets.BitmapToPng(bitmap); + + if (png == null) + return null; + + return new Sprite + { + Key = key, + Body = body, + Action = action, + Direction = direction, + FileType = fileType, + Png = png, + Width = bitmap.Width, + Height = bitmap.Height, + Sha256 = BridgeAssets.Sha256Hex(png), + Source = "legacy" + }; + } + + // ── player bodies (§5.2) ───────────────────────────────────────────────────────────── + + /// + /// Asked of the shard, never hardcoded. + /// + /// Every registered race carries its living male and female body ids, and a shard that + /// calls `RegisterRace` adds ids no table of ours could contain — which is the whole + /// argument against a hardcoded list, and it was never hypothetical: stock ServUO's + /// own `RaceDefinitions.cs` passes the gargoyle's ghost bodies in the OPPOSITE order + /// to the other two races. + /// + /// This set is the whole of what §5.1 gives direction 0 — head-on, facing the viewer, + /// because a character is a portrait and should look at you. Everything else takes + /// direction 1, the front three-quarter, because head-on is the least legible view of + /// a four-legged creature: a wolf seen from the front is a dark blob. + /// + /// **Ghost bodies are deliberately not in it** (§5.2, decided 2026-09-10 in phase 4). + /// A race declares four ids and two of them are its ghosts, and no UO client has art + /// for any of them: 402/403 and 694/695 read `lookup -1` in `anim.idx`, 607/608 read + /// `length 0` — the §4.8 shape, where the library hands back the previously-decoded + /// body's picture — and none of the six is in any `AnimationFrame*.uop`, which phase 4 + /// established by claiming all 10,724 entries of the five packages with the one name + /// scheme. Listing them only advertised keys that cannot exist. A shard whose client + /// does ship ghost art still gets it: the body is catalogued like any other, at + /// direction 1 rather than 0. + /// + /// + /// Cached for the life of the process: `RegisterRace` runs at Configure time, before + /// anything on this plane can be asked a question, so the set cannot change under a + /// running shard. Rebuilding it per body would enumerate every race 2,047 times per + /// scan to answer a question whose answer is six integers. + /// + private static HashSet _playerBodies; + + private static HashSet PlayerBodies() + { + var cached = _playerBodies; + + if (cached != null) + return cached; + + var set = new HashSet(); + + try + { + foreach (var race in Race.AllRaces) + { + if (race == null) + continue; + + set.Add(race.MaleBody); + set.Add(race.FemaleBody); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: cannot enumerate races: {0}", e.Message); + } + + set.Remove(0); + + _playerBodies = set; + + return set; + } + + private static bool IsPlayerBody(int body) + { + return PlayerBodies().Contains(body); + } + + private static void WritePlayerBodies(StringBuilder sb) + { + var bodies = new List(PlayerBodies()); + bodies.Sort(); + + sb.Append(",\"playerBodies\":["); + + for (int i = 0; i < bodies.Count; i++) + { + if (i > 0) + sb.Append(','); + + sb.Append(bodies[i].ToString(CultureInfo.InvariantCulture)); + } + + sb.Append(']'); + } + + // ── keys, cursors and the source id ────────────────────────────────────────────────── + + private static string Key(int body, int action) + { + return "body/" + body.ToString(CultureInfo.InvariantCulture) + + "/a" + action.ToString(CultureInfo.InvariantCulture); + } + + /// + /// `body/<id>/a<n>`, and nothing else in this phase. A deeper key + /// (`body/400/a2/f3`) is well-formed under §5 and simply not served, so it comes back + /// `unsupported` rather than being silently read as its own first frame. + /// + /// The action is parsed rather than required to be zero — 74 of this client's bodies + /// are catalogued at a different one (§11.2) — but a parsed action is not an accepted + /// one. serves a key only when it is the key the catalogue itself + /// chose for that body, which is what keeps §4.10's ceiling from being reachable + /// through a request: nothing the website can ask makes this decode an action the + /// catalogue did not already pick. + /// + private static bool TryParseKey(string key, out int body, out int action) + { + body = 0; + action = -1; + + if (key == null) + return false; + + string[] parts = key.Split('/'); + + if (parts.Length != 3 || parts[0] != "body") + return false; + + if (!Int32.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out body)) + return false; + + if (body < 1 || body > MaxBody) + return false; + + if (parts[2].Length < 2 || parts[2][0] != 'a') + return false; + + if (!Int32.TryParse(parts[2].Substring(1), NumberStyles.None, + CultureInfo.InvariantCulture, out action)) + return false; + + return action >= 0 && action < MaxAction; + } + + private static int ParseBodyCursor(string cursor) + { + if (cursor == null) + return 1; + + int value; + + if (cursor.StartsWith("b:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(1, value + 1); + + return 1; + } + + private static int ParseKeyCursor(string cursor) + { + if (cursor == null) + return 0; + + int value; + + if (cursor.StartsWith("k:", StringComparison.Ordinal) + && Int32.TryParse(cursor.Substring(2), NumberStyles.None, CultureInfo.InvariantCulture, out value)) + return Math.Max(0, value); + + return 0; + } + + /// + /// Everything that decides the bytes, hashed into one short id. + /// + /// Deliberately (size, mtime) rather than content: §6 makes exactly the same choice + /// for the source gate, and for the same reason — the anim files are 195 MB and + /// hashing them on every page of a walk would turn a manifest into a minute. + /// `assets.sources` is where an operator gets content hashes, computed off the request + /// path; this is a "did it move while I was reading" check, which (size, mtime) + /// answers. + /// + private static string SourceId() + { + var sb = new StringBuilder(256); + + sb.Append(BridgeAssets.EXTRACTOR_VERSION) + .Append(':').Append(BridgeConfig.AssetPlayerDirection) + .Append(':').Append(BridgeConfig.AssetCreatureDirection); + + var paths = new List(); + + for (int fileType = 1; fileType <= 5; fileType++) + paths.Add(BridgeAssetValidator.AnimDataPath(fileType)); + + // Since phase 4 the catalogue's bytes depend on the UOP packages too — 235 of its + // bodies come out of them — so patching one has to change the catalogue id, exactly as + // patching an anim*.mul does. Leaving them out would let an operator replace a + // gargoyle and have an Update find nothing to do. + foreach (int n in BridgeUop.Packages) + paths.Add(BridgeUop.PackagePath(n)); + + foreach (string path in paths) + { + sb.Append('|'); + + if (path == null) + continue; + + try + { + var info = new FileInfo(path); + + if (!info.Exists) + continue; + + sb.Append(info.Length).Append(',').Append(info.LastWriteTimeUtc.Ticks); + } + catch + { + // An unreadable file is itself a state, and one that must not change from page + // to page without being noticed. Leaving the slot empty does that. + } + } + + return BridgeAssets.Sha256Hex(Encoding.UTF8.GetBytes(sb.ToString())).Substring(0, 16); + } + + // ── shared plumbing ────────────────────────────────────────────────────────────────── + + /// + /// The five anim files' index and record readers — and, since phase 4, the five UOP + /// packages beside them — opened for one reply and closed with it. Holding them across + /// replies would keep handles on the operator's client files for as long as the cache + /// lives, for no gain: opening them is microseconds and a page decodes hundreds of + /// sprites through them. + /// + private sealed class Readers : IDisposable + { + private readonly FileIndex[] _index = new FileIndex[6]; + private readonly BridgeAssetValidator.RecordReader[] _reader = + new BridgeAssetValidator.RecordReader[6]; + private readonly long[] _length = new long[6]; + private readonly bool[] _open = new bool[6]; + + private readonly Dictionary _packages = + new Dictionary(); + + public readonly long VerdataLength; + + public Readers() + { + VerdataLength = BridgeAssetValidator.MulLength(VerdataPath()); + } + + private static string VerdataPath() + { + try + { + return Files.GetFilePath("verdata.mul"); + } + catch + { + return null; + } + } + + private void Ensure(int fileType) + { + if (fileType < 1 || fileType > 5 || _open[fileType]) + return; + + _open[fileType] = true; + + string path = BridgeAssetValidator.AnimDataPath(fileType); + + if (path == null) + return; + + try + { + _index[fileType] = BridgeAssetValidator.OpenAnimIndex(fileType); + _length[fileType] = BridgeAssetValidator.MulLength(path); + _reader[fileType] = new BridgeAssetValidator.RecordReader(path, VerdataPath()); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] catalogue: anim file type {0}: {1}", + fileType, e.Message); + } + } + + public FileIndex Index(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _index[fileType] : null; + } + + public long MulLength(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _length[fileType] : 0; + } + + public BridgeAssetValidator.RecordReader Reader(int fileType) + { + Ensure(fileType); + return fileType >= 1 && fileType <= 5 ? _reader[fileType] : null; + } + + /// + /// One AnimationFrame*.uop, opened on first use. A package this client does + /// not ship is a null that is cached as one: the miss must not be re-resolved and + /// re-opened once per body across a 2,047-body walk. + /// + public BridgeUop.Package Package(int n) + { + BridgeUop.Package package; + + if (_packages.TryGetValue(n, out package)) + return package; + + package = BridgeUop.Package.Open(BridgeUop.PackagePath(n)); + + _packages[n] = package; + + return package; + } + + public void Dispose() + { + foreach (var package in _packages.Values) + { + if (package == null) + continue; + + try + { + package.Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + + for (int i = 1; i <= 5; i++) + { + if (_reader[i] == null) + continue; + + try + { + _reader[i].Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + } + } + + /// + /// Lets a megabyte of the operator's client art go once nothing has asked for it in + /// five minutes. A rebuild costs one scan and, because is + /// derived from the files rather than minted per build, it produces the same catalogue + /// id — so an import that spans the drop does not have to start over. + /// + private static void Sweep() + { + lock (_sync) + { + if (_catalog == null) + return; + + if (DateTime.UtcNow - _catalog.LastUsed > IdleFor) + _catalog = null; + } + } + + public static string Status() + { + lock (_sync) + { + if (_catalog == null) + return "catalog(empty)"; + + return String.Format("catalog(id={0} held={1} next={2} complete={3})", + _catalog.Id, _catalog.Order.Count, _catalog.Next, _catalog.Complete); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeCliloc.cs b/overlay/Scripts/Custom/Bridge/BridgeCliloc.cs new file mode 100644 index 0000000..8d53b9b --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeCliloc.cs @@ -0,0 +1,772 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The cliloc table, over the bridge** (docs/link/v8.md §9 — protocol 8, phase 2). + /// + /// A "cliloc" is UO's localization table: an integer id mapped to a display string. Items + /// on the wire carry a `LabelNumber`, never a name, so without this table the website can + /// only render `id 1023721` where the game renders "quarter staff". The number was never + /// the missing piece; the table was. + /// + /// Until this phase the operator supplied it by hand: install UOFiddler, build a converter + /// against its `Ultima.dll`, run it over their own `Cliloc.enu`, copy a 5 MB file to the + /// web host and point a setting at it. That whole pipeline existed for one reason — the + /// file is compressed and **nothing in this stack could read it**. ServUO's own bundled + /// `Ultima.StringList` implements the plain layout only and throws on a modern client's + /// file, which is also why the shard's `VendorSearch.GetItemName` has always been inert. + /// + /// So this class is the one decoder protocol 8 **writes** rather than calls (§4): a port + /// of UOFiddler's Mythic decompressor into the overlay, after which the shard can read its + /// own client's table and hand it to the website over the same request/reply path as + /// everything else. The operator installs nothing. + /// + /// **Attribution.** The decompression below is a port of `Ultima/Helpers/MythicDecompress` + /// and `MoveToFront` from UOFiddler (https://github.com/polserver/UOFiddler), which is + /// released under the **Beerware** licence — compatible with this tree's GPL-3.0-or-later. + /// It is rewritten for .NET Framework 4.8: the original is written against `Span<T>`, + /// `ArrayPool<T>` and `BinaryPrimitives`, none of which ServUO's `net48` target has. + /// + /// **What is NOT here, deliberately.** Shard-added items carry cliloc ids no client table + /// contains, and ServUO has no server-side notion of a custom cliloc — there is nothing in + /// the tree to read. That gap is in the *game*, not in this pipeline, so the website keeps + /// its `custom/` overlay directory and merges it over whatever arrives here. This class + /// answers exactly one question: what does the client's own table say. + /// + public static class BridgeCliloc + { + /// + /// Languages this can serve. + /// + /// Not an arbitrary code: Ultima.Files resolves only the names in its own file + /// table, and cliloc files are represented there by these four. Asking for anything + /// else cannot resolve to a path however the client is laid out, so it is refused by + /// name rather than answered with an empty table. + /// + /// `custom1` / `custom2` are the *client-side* custom cliloc files a shard ships to + /// its players. Nothing on the website imports them today — its `custom/` overlay + /// directory is the supported answer — but they are the shard's files and they are + /// readable, so they are not artificially excluded. + /// + private static readonly string[] Languages = { "enu", "deu", "custom1", "custom2" }; + + private const string DefaultLanguage = "enu"; + + /// + /// How long a decoded table is kept in memory after its last page. + /// + /// A stock `Cliloc.enu` decodes to ~67,000 live strings; holding that forever on a + /// shard that imports once a month is rude, and decoding it again costs about a + /// second. So it is cached only for as long as an import is plausibly still running: + /// freed when the last page is served, and expired on the next request if one never + /// comes (an import abandoned halfway leaves nothing behind). + /// + private static readonly TimeSpan CacheIdle = TimeSpan.FromMinutes(5); + + private static readonly object _sync = new object(); + private static Table _cached; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("cliloc.table", OnTable); + } + + // ── the request plane ──────────────────────────────────────────────────────────────── + + /// + /// Core thread. Validates, then hands the decode to the asset worker — reading and + /// decompressing five megabytes is emphatically not something to do while the world + /// is waiting, and 's single slot is what keeps the shard's + /// outbound queue at a depth of about one while it happens. + /// + private static void OnTable(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + // Without a correlation id this reply would land on the event path, be persisted + // to the sidecar's store and broadcast to every subscriber — a megabyte of + // strings to every connected client, forever. Refuse instead (§3.1). + BridgeAssets.Fail(null, "BAD_REQUEST", "cliloc.table requires a reqId"); + return; + } + + if (!BridgeConfig.AssetsEnabled) + { + BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard"); + return; + } + + var lang = BridgeJson.GetString(o, "lang"); + + if (String.IsNullOrEmpty(lang)) + lang = DefaultLanguage; + + lang = lang.ToLowerInvariant(); + + if (Array.IndexOf(Languages, lang) < 0) + { + BridgeAssets.Fail(reqId, "NOT_FOUND", + "no cliloc file for language '" + lang + "' (this shard can serve: " + + String.Join(", ", Languages) + ")"); + return; + } + + // The cursor is this family's own resume point and it is a cliloc NUMBER, not an + // offset into anything. That matters: the cache behind it can be dropped and rebuilt + // between two pages of the same import (idle expiry, a second import, a restart), and + // an index into a list would silently mean something different afterwards. "Resume + // after id N" survives all of it, because the table is served in id order. + int after = -1; + var cursor = BridgeJson.GetString(o, "cursor"); + + if (!String.IsNullOrEmpty(cursor)) + { + if (!TryParseCursor(cursor, out after)) + { + BridgeAssets.Fail(reqId, "BAD_REQUEST", "malformed cursor: " + cursor); + return; + } + } + + string language = lang; + int resumeAfter = after; + + BridgeAssets.Accept(reqId, "cliloc.table", () => ReplyTable(reqId, language, resumeAfter)); + } + + private static bool TryParseCursor(string cursor, out int after) + { + after = -1; + + if (!cursor.StartsWith("n:", StringComparison.Ordinal)) + return false; + + return Int32.TryParse( + cursor.Substring(2), NumberStyles.Integer, CultureInfo.InvariantCulture, out after); + } + + /// + /// Asset worker. Decodes (or reuses) the table and writes one page of it. + /// + private static void ReplyTable(string reqId, string lang, int after) + { + string path = ResolvePath(lang); + + if (path == null) + { + BridgeAssets.Fail(reqId, "NOT_FOUND", + "this shard's client has no cliloc." + lang + " (looked where ServUO's own " + + "data path points)"); + return; + } + + Table table; + string code, reason; + + if (!TryLoad(lang, path, out table, out code, out reason)) + { + BridgeAssets.Fail(reqId, code, reason); + return; + } + + var sb = BridgeJson.Begin("cliloc.table.ok"); + + sb.Str("reqId", reqId) + .Str("lang", lang) + .Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION) + .Str("file", Path.GetFileName(path)) + // The website pages this table over several round trips and must be able to tell + // that the file changed underneath it — an operator patching their client mid-import + // would otherwise produce one table stitched from two, with no error anywhere. It + // compares these two fields across pages and starts over if they move. + .Num("size", table.Size) + .Num("mtime", table.MTime) + .Num("total", table.Count) + .Bool("compressed", table.Compressed); + + // Before the page opens, not after it closes: PageBuilder reserves room for the + // envelope it still has to write, and a field appended past Close() is spent outside + // that reserve. It fits today by a wide margin, and it is the kind of thing the next + // family copies. + int start = table.IndexAfter(after); + sb.Num("from", start); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int i = start; + + for (; i < table.Count; i++) + { + var item = new StringBuilder(96); + + item.Append("{\"n\":").Append(table.Numbers[i].ToString(CultureInfo.InvariantCulture)); + item.Append(",\"f\":").Append(table.Flags[i].ToString(CultureInfo.InvariantCulture)); + item.Append(",\"t\":"); + BridgeJson.Text(item, table.Texts[i]); + item.Append('}'); + + if (!page.TryAdd(item.ToString(), "n:" + table.Numbers[i].ToString(CultureInfo.InvariantCulture))) + break; + } + + page.Close(); + + bool finished = i >= table.Count; + + BridgeLink.Emit(sb.End()); + + // The last page is also the end of the import, so let the strings go. A retry of that + // page re-decodes, which costs a second and happens approximately never; holding ~67k + // strings against that is the wrong trade. + if (finished) + Release(lang); + } + + private static string ResolvePath(string lang) + { + try + { + // ServUO's own `Scripts/Misc/DataPath.cs` calls `Files.SetMulPath` for every + // configured data directory at Configure time, so this resolves against the + // client the SHARD is running on — including on Linux, where `Ultima.Files`'s + // registry lookup finds nothing on its own. + return Files.GetFilePath("cliloc." + lang); + } + catch + { + return null; + } + } + + // ── the decoded table ──────────────────────────────────────────────────────────────── + + private sealed class Table + { + public string Lang; + public long Size; + public long MTime; + public bool Compressed; + public int[] Numbers; + public byte[] Flags; + public string[] Texts; + public DateTime LastUsed; + + public int Count { get { return Numbers.Length; } } + + /// + /// Index of the first row with a number greater than . + /// Binary search, because the rows are in id order by construction and a page + /// deep into the table would otherwise walk everything before it. + /// + public int IndexAfter(int after) + { + if (after < 0) + return 0; + + int lo = 0, hi = Numbers.Length; + + while (lo < hi) + { + int mid = lo + ((hi - lo) >> 1); + + if (Numbers[mid] <= after) + lo = mid + 1; + else + hi = mid; + } + + return lo; + } + } + + private static bool TryLoad(string lang, string path, out Table table, out string code, out string reason) + { + code = null; + reason = null; + + long size, mtime; + + try + { + var info = new FileInfo(path); + size = info.Length; + mtime = (long)(info.LastWriteTimeUtc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)) + .TotalMilliseconds; + } + catch (Exception e) + { + table = null; + code = "UNREADABLE"; + reason = "cannot stat " + Path.GetFileName(path) + ": " + e.Message; + return false; + } + + lock (_sync) + { + if (_cached != null) + { + bool stale = _cached.Lang != lang + || _cached.Size != size + || _cached.MTime != mtime + || DateTime.UtcNow - _cached.LastUsed > CacheIdle; + + if (stale) + _cached = null; + } + + if (_cached != null) + { + _cached.LastUsed = DateTime.UtcNow; + table = _cached; + return true; + } + } + + byte[] raw; + + try + { + raw = File.ReadAllBytes(path); + } + catch (Exception e) + { + table = null; + code = "UNREADABLE"; + reason = "cannot read " + Path.GetFileName(path) + ": " + e.Message; + return false; + } + + bool compressed = IsCompressed(raw); + byte[] plain; + + if (compressed) + { + try + { + plain = Mythic.Decompress(raw); + } + catch (Exception e) + { + table = null; + code = "UNREADABLE"; + reason = "cannot decompress " + Path.GetFileName(path) + ": " + e.Message; + return false; + } + } + else + { + plain = raw; + } + + var built = new Table + { + Lang = lang, + Size = size, + MTime = mtime, + Compressed = compressed, + LastUsed = DateTime.UtcNow + }; + + if (!TryParseRecords(plain, built, out reason)) + { + table = null; + code = "UNREADABLE"; + return false; + } + + lock (_sync) + { + _cached = built; + } + + table = built; + return true; + } + + private static void Release(string lang) + { + lock (_sync) + { + if (_cached != null && _cached.Lang == lang) + _cached = null; + } + } + + /// + /// Every compressed cliloc begins with a DWORD whose high byte is 0x8E — the + /// top byte of UOFiddler's `HeaderXorKey`, showing through because the value it hides + /// (a length) is far smaller than the key. That single byte is what tells a modern + /// client's file from the pre-2010 plain layout, and both are accepted here: a shard + /// running an old or hand-built client is not a broken shard. + /// + private static bool IsCompressed(byte[] buffer) + { + return buffer.Length >= 4 && buffer[3] == 0x8E; + } + + // ── the plain layout ───────────────────────────────────────────────────────────────── + + private const int HeaderBytes = 6; // int32 version + int16 language marker + private const int RecordHeaderBytes = 7; // int32 number + byte flag + uint16 length + + /// + /// Parses the plain layout into the sorted, blank-free arrays the wire wants. + /// + /// **Strict about truncation**, and that strictness is the point: a half-decoded table + /// is indistinguishable from a complete one downstream — you would simply see some + /// items named and some not, which is exactly what "no table at all" looks like. So a + /// record running past the end of the buffer is an error naming its offset, never a + /// short table. + /// + /// **Blanks are dropped here rather than on the website.** Roughly 56,000 of a stock + /// table's 123,490 entries are empty strings the client reserves and never uses, the + /// website discards them at import already, and a row that resolves to no name is + /// indistinguishable from no row at all to every caller. Dropping them halves what + /// crosses the wire for data that would be thrown away on arrival. + /// + /// **A repeated id is resolved last-wins**, matching the client's own loader (its + /// dictionary assignment overwrites). The plain format permits it, so a file the game + /// itself would load happily must not fail here. + /// + private static bool TryParseRecords(byte[] data, Table into, out string reason) + { + reason = null; + + if (data.Length < HeaderBytes) + { + reason = "cliloc file is shorter than its 6-byte header"; + return false; + } + + var byNumber = new Dictionary(140000); + int offset = HeaderBytes; + int read = 0; + + while (offset < data.Length) + { + if (offset + RecordHeaderBytes > data.Length) + { + reason = "truncated record header at byte " + offset + " (" + read + " entries read)"; + return false; + } + + int number = ReadInt32(data, offset); + byte flag = data[offset + 4]; + // Unsigned: reading this signed (as ServUO's own SDK does) turns any string over + // 32 KB into a negative length. Real tables top out around 12 KB, so it changes + // nothing today and costs nothing to get right. + int length = data[offset + 5] | (data[offset + 6] << 8); + + offset += RecordHeaderBytes; + + if (offset + length > data.Length) + { + reason = "truncated record body at byte " + offset + " (" + read + " entries read)"; + return false; + } + + string text; + + try + { + text = Encoding.UTF8.GetString(data, offset, length); + } + catch (Exception e) + { + reason = "entry " + number + " at byte " + offset + " is not valid UTF-8: " + e.Message; + return false; + } + + offset += length; + read++; + + byNumber[number] = new Entry { Flag = flag, Text = text }; + } + + var numbers = new List(byNumber.Count); + + foreach (var pair in byNumber) + { + if (IsBlank(pair.Value.Text)) + continue; + + numbers.Add(pair.Key); + } + + numbers.Sort(); + + into.Numbers = numbers.ToArray(); + into.Flags = new byte[numbers.Count]; + into.Texts = new string[numbers.Count]; + + for (int i = 0; i < numbers.Count; i++) + { + var entry = byNumber[numbers[i]]; + + into.Flags[i] = entry.Flag; + into.Texts[i] = entry.Text; + } + + return true; + } + + private struct Entry + { + public byte Flag; + public string Text; + } + + private static bool IsBlank(string text) + { + if (String.IsNullOrEmpty(text)) + return true; + + for (int i = 0; i < text.Length; i++) + { + if (!Char.IsWhiteSpace(text[i])) + return false; + } + + return true; + } + + private static int ReadInt32(byte[] data, int at) + { + return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24); + } + + // ── the Mythic container ───────────────────────────────────────────────────────────── + + /// + /// The decompressor, ported from UOFiddler (Beerware; see this class's summary). + /// + /// The container is two stages over the plain cliloc bytes, undone in reverse: + /// + /// 1. A 4-byte header holding the decompressed length, XORed with `0x8E2C9A3D` — + /// which is where the `0x8E` sniff byte comes from. + /// 2. A **move-to-front** coding of… + /// 3. …a Burrows-Wheeler-style transform whose 1 KB frequency header (256 little-endian + /// counts, one per byte value) is both the table sizes and the total output length. + /// + /// Rewritten against plain arrays: the upstream is `Span<T>`/`ArrayPool<T>` + /// code and ServUO targets `net48`, which has neither without a package this tree does + /// not vendor. The algorithm is unchanged, including the parts that read oddly — the + /// three-region `partial` table (counts, cursors, ends) and the symbol-table shifts are + /// the original's, deliberately, because this is a format decoder and a tidier + /// rewrite is a chance to be subtly wrong about someone else's bytes. + /// + private static class Mythic + { + private const uint HeaderXorKey = 0x8E2C9A3D; + private const int FrequencyHeaderSize = 1024; // 256 little-endian ints + + public static byte[] Decompress(byte[] source) + { + if (source.Length < 4) + throw new InvalidDataException("compressed cliloc is shorter than its header"); + + uint declared = (uint)ReadInt32(source, 0) ^ HeaderXorKey; + + if (declared == 0 || declared > Int32.MaxValue) + throw new InvalidDataException("compressed cliloc declares an impossible length"); + + var mtf = new byte[source.Length - 4]; + MoveToFrontDecode(source, 4, mtf); + + var output = new byte[(int)declared]; + int written = InverseTransform(mtf, output); + + if (written != (int)declared) + { + throw new InvalidDataException( + "decompressed length " + written + " does not match the declared " + declared); + } + + return output; + } + + private static void MoveToFrontDecode(byte[] input, int from, byte[] output) + { + var symbols = new byte[256]; + + for (int i = 0; i < 256; i++) + symbols[i] = (byte)i; + + for (int i = 0; i < output.Length; i++) + { + int index = input[from + i]; + byte symbol = symbols[index]; + + output[i] = symbol; + + for (int j = index; j > 0; j--) + symbols[j] = symbols[j - 1]; + + symbols[0] = symbol; + } + } + + private static int InverseTransform(byte[] input, byte[] destination) + { + if (input.Length < FrequencyHeaderSize) + throw new InvalidDataException("compressed cliloc is smaller than its frequency header"); + + // Three regions of 256: [0..255] the counts read from the header, [256..511] a + // moving cursor per symbol, [512..767] where that symbol's run ends. + var partial = new int[256 * 3]; + + for (int i = 0; i < 256; i++) + partial[i] = ReadInt32(input, i * 4); + + int sum = 0; + + for (int i = 0; i < 256; i++) + { + if (partial[i] < 0) + throw new InvalidDataException("compressed cliloc has a negative symbol count"); + + sum += partial[i]; + } + + if (sum == 0) + return 0; + + if (destination.Length < sum) + throw new InvalidDataException("compressed cliloc's frequency header outruns its declared length"); + + int nonZero = 0; + + for (int i = 0; i < 256; i++) + { + if (partial[i] != 0) + nonZero++; + } + + var frequency = new byte[256]; + Frequency(partial, frequency); + + var symbols = new byte[256]; + + for (int i = 0; i < 256; i++) + symbols[i] = (byte)i; + + for (int i = 0, m = 0; i < nonZero; ++i) + { + byte freq = frequency[i]; + + Need(input, m + FrequencyHeaderSize); + + symbols[input[m + FrequencyHeaderSize]] = freq; + partial[freq + 256] = m + 1; + m += partial[freq]; + partial[freq + 512] = m; + } + + byte val = symbols[0]; + int count = 0; + + do + { + destination[count] = val; + + if (partial[val + 256] < partial[val + 512]) + { + Need(input, partial[val + 256] + FrequencyHeaderSize); + + byte idx = input[partial[val + 256] + FrequencyHeaderSize]; + partial[val + 256]++; + + if (idx != 0) + { + ShiftLeft(symbols, idx); + + symbols[idx] = val; + val = symbols[0]; + } + } + else if (nonZero-- > 0) + { + ShiftLeft(symbols, nonZero); + + val = symbols[0]; + } + + count++; + } + while (count < sum); + + return sum; + } + + /// + /// The upstream indexes the payload without bounds-checking it, which is safe for + /// a file the client wrote and is not safe for a file this shard was handed. A + /// truncated or hand-edited container would otherwise read whatever follows the + /// buffer in memory — or, on .NET, throw an `IndexOutOfRangeException` from inside + /// a decoder, which says nothing useful to an operator. This turns both into one + /// named, reportable failure. + /// + private static void Need(byte[] input, int at) + { + if (at < 0 || at >= input.Length) + throw new InvalidDataException("compressed cliloc ends mid-stream (wanted byte " + at + ")"); + } + + /// + /// Symbol values ordered by descending count — the order the coder assigned its + /// runs in. Repeated max-finding rather than a sort, as upstream: 256 passes over + /// 256 entries is nothing, and it reproduces the original's tie-breaking (the + /// lowest index wins), which a comparison sort would not. + /// + private static void Frequency(int[] counts, byte[] output) + { + var tmp = new int[256]; + Array.Copy(counts, tmp, 256); + + for (int i = 0; i < 256; i++) + { + int value = 0; + byte index = 0; + + for (int j = 0; j < 256; j++) + { + if (tmp[j] > value) + { + index = (byte)j; + value = tmp[j]; + } + } + + if (value == 0) + break; + + output[i] = index; + tmp[index] = 0; + } + } + + private static void ShiftLeft(byte[] symbols, int upTo) + { + for (int i = 0; i < upTo; ++i) + symbols[i] = symbols[i + 1]; + } + + private static int ReadInt32(byte[] data, int at) + { + return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs index dedf4bc..74988c0 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs @@ -87,6 +87,80 @@ namespace Server.Custom.Bridge // morning. Those are different consents, and one switch cannot express both. public static bool EventsEnabled { get; private set; } + // ---- the asset plane (docs/link/v8.md §3, protocol 8) ---- + // + // Its own gate again, and for the same reason the event plane got one: enabling this is + // an operator consenting to the WEBSITE READING THEIR CLIENT FILES -- art, animations and + // the string table, off the host's disk, over the link. That is a different consent from + // publishing world state, and one switch cannot express both. Reads only: nothing on this + // plane writes anything, anywhere. + public static bool AssetsEnabled { get; private set; } + public static int AssetBatchBytes { get; private set; } + + // How many types one `assets.bodies` request may name (§8, phase 3). This is the ONLY + // asset-plane bound counted in items rather than bytes, and deliberately so: the cost it + // bounds is not the size of the reply, it is constructing and deleting that many real + // mobiles ON THE CORE THREAD, between two ticks of the world. + public static int AssetBodyBatch { get; private set; } + + // How many keys one `assets.fetch` request may name. Bytes still cut the page; this only + // bounds how large a request the shard will parse and walk at all. + public static int AssetFetchKeys { get; private set; } + + // The wall-clock budget for one catalogue page (§4.8, phase 3). The catalogue's rows are + // ninety bytes, so the byte budget never stops it -- but building them means decoding + // hundreds of animations, and the sidecar gives a reply ten seconds. Kept well under that, + // because the reply still has to be built, serialised and cross the wire afterwards. + public static int AssetScanMs { get; private set; } + + // Which direction the catalogue renders (§5.1). Both are settings and neither is in the + // asset key, because five directions would five-fold every count in §11 to express a + // choice nobody is going to vary. + // + // The split is not arbitrary and was found by RENDERING all five rather than from a table: + // index 0 is head-on, which is what a character portrait wants and the least legible view + // there is of a four-legged creature. A wolf seen from the front is a dark blob; at index + // 1, the front three-quarter, it is unmistakably a wolf. + public static int AssetPlayerDirection { get; private set; } + public static int AssetCreatureDirection { get; private set; } + + // ---- the tree plane (docs/link/v8.md §10, phase 7) ---- + // + // Its OWN gate, and the third one on this link for the third kind of consent. The asset + // gate above is the operator agreeing that the website may read THEIR UO CLIENT -- art + // and animations and a string table that came from EA. This one is the operator agreeing + // that it may read THE SHARD'S OWN CONFIGURATION: the spawn files, the region and + // location definitions, the champion table, the decoration lists. Those are the + // operator's own work rather than a licensed client, and they are what the spawn atlas is + // built out of -- so a shard that declines to serve client art must still be able to + // publish where its creatures live. One switch could not have expressed both, and the + // atlas would have been the thing that silently disappeared. + // + // Reads only, and only the five labelled groups SPAWN_ATLAS.md already names. Nothing + // here joins a path the website sent: a request names a label this shard enumerated, or + // it is refused. + public static bool TreeEnabled { get; private set; } + + // How much of a tree file one chunk carries, BEFORE compression (§10). The chunk is the + // thing that makes this transferable at all: a stock Spawns/trammel.xml is 4.03 MB and + // the sidecar discards any inbound line over 1 MiB, so the file as a single base64 row + // could never arrive -- it would time out and be re-requested forever, which is a failure + // with no error in it anywhere. + // + // Compression is what makes it cheap (a spawn file gzips ~18x, so a chunk is typically + // 40 KB on the wire) and the chunk is what makes it BOUNDED: gzip cannot be relied on to + // shrink anything, so the ceiling has to hold for input that does not compress at all. + // At 512 KiB a worst-case incompressible chunk is ~683 KiB of base64, which still fits + // the wire under AssetBatchBytes' deliberate factor of two. + public static int TreeChunkBytes { get; private set; } + + // How many bytes of rendered item and land art the shard holds between requests (§11, + // phase 5). This is a convenience, not a store: the website keeps every picture it fetches + // and does not ask twice, so what this actually buys is the second page of a batch, a + // retry after a 425, and the same item appearing in two rows of one page. Sized so a + // full 512 KB batch and the one before it both fit with room over. + public static int AssetArtCacheBytes { get; private set; } + public static int LeaseMaxDurationSec { get; private set; } public static int LeaseGraceSec { get; private set; } @@ -144,6 +218,70 @@ namespace Server.Custom.Bridge Port = Config.Get("Bridge.Port", 7788); QueueCap = Config.Get("Bridge.QueueCap", 10000); + AssetsEnabled = Config.Get("Bridge.AssetsEnabled", true); + + // The largest reply this plane will build, in ENCODED bytes -- not items, because the + // ceiling it has to live inside is a byte ceiling. Clamped to half the sidecar's 1 MiB + // inbound line cap, and the halving is load-bearing rather than cautious: a page + // always admits its first item even when that item alone exceeds the budget (the + // alternative is an oversized item being skipped forever and its family never making + // progress), so the wire must still have room for one such overshoot. + AssetBatchBytes = Config.Get("Bridge.AssetBatchBytes", 512 * 1024); + if (AssetBatchBytes < 64 * 1024) + AssetBatchBytes = 64 * 1024; + if (AssetBatchBytes > 512 * 1024) + AssetBatchBytes = 512 * 1024; + + AssetBodyBatch = Config.Get("Bridge.AssetBodyBatch", 100); + if (AssetBodyBatch < 1) + AssetBodyBatch = 1; + if (AssetBodyBatch > 500) + AssetBodyBatch = 500; + + AssetFetchKeys = Config.Get("Bridge.AssetFetchKeys", 2000); + if (AssetFetchKeys < 1) + AssetFetchKeys = 1; + if (AssetFetchKeys > 10000) + AssetFetchKeys = 10000; + + AssetScanMs = Config.Get("Bridge.AssetScanMs", 3000); + if (AssetScanMs < 250) + AssetScanMs = 250; + // Half the sidecar's 10 s reply timeout, so the page still has time to be serialised + // and written after the scan stops. A budget set at the timeout would produce replies + // that are always thrown away. + if (AssetScanMs > 5000) + AssetScanMs = 5000; + + // Clamped to 0-4: 5-7 are the client MIRRORING 1-3, which `Frame` decodes through a + // different pointer-arithmetic branch that nothing in BridgeAssetValidator has + // checked. Accepting one would hand an unverified write path a bitmap to fill. + AssetPlayerDirection = Clamp(Config.Get("Bridge.AssetPlayerDirection", 0), 0, 4); + AssetCreatureDirection = Clamp(Config.Get("Bridge.AssetCreatureDirection", 1), 0, 4); + + // The floor is one batch: a cache that cannot hold the page being built evicts rows + // while they are still being written, which is a cache that costs and never pays. The + // ceiling is a game server's memory, and 64 MB of PNG is already ~34,000 sprites -- + // most of this client's art, held for a working set that is measured in hundreds. + AssetArtCacheBytes = Config.Get("Bridge.AssetArtCacheBytes", 16 * 1024 * 1024); + if (AssetArtCacheBytes < AssetBatchBytes) + AssetArtCacheBytes = AssetBatchBytes; + if (AssetArtCacheBytes > 64 * 1024 * 1024) + AssetArtCacheBytes = 64 * 1024 * 1024; + + TreeEnabled = Config.Get("Bridge.TreeEnabled", true); + + // Floor and ceiling both matter. Below 64 KiB a stock tree is thousands of chunks and + // the per-row overhead starts to dominate the payload; above 512 KiB an incompressible + // chunk stops fitting inside the sidecar's inbound line cap, which is the one bound + // this number exists to respect. Kept equal to AssetBatchBytes' own ceiling so the two + // budgets cannot drift into disagreeing about the same wire. + TreeChunkBytes = Config.Get("Bridge.TreeChunkBytes", 512 * 1024); + if (TreeChunkBytes < 64 * 1024) + TreeChunkBytes = 64 * 1024; + if (TreeChunkBytes > 512 * 1024) + TreeChunkBytes = 512 * 1024; + StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30); DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60); EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300); @@ -494,6 +632,14 @@ namespace Server.Custom.Bridge return fallback; } + private static int Clamp(int value, int min, int max) + { + if (value < min) + return min; + + return value > max ? max : value; + } + public static string Describe() { return String.Format( diff --git a/overlay/Scripts/Custom/Bridge/BridgePng.cs b/overlay/Scripts/Custom/Bridge/BridgePng.cs new file mode 100644 index 0000000..47b89e1 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgePng.cs @@ -0,0 +1,231 @@ +using System; +using System.IO; +using System.IO.Compression; + +namespace Server.Custom.Bridge +{ + /// + /// **A PNG encoder that does not go through GDI+** (docs/link/v8.md §4.4, §4.9 — phase 4). + /// + /// decodes into a ushort[] of ARGB1555 rather than into a + /// Bitmap, which is the whole point of §4.4's note that the UOP reader is written + /// without System.Drawing: libgdiplus was archived in March 2025, and every line of + /// extraction that does not depend on it is a line that survives its absence. That leaves + /// the encode, and Bitmap.Save(…, ImageFormat.Png) is GDI+ too — so this is the + /// other half. + /// + /// It is deliberately the smallest thing that produces a correct file: 8-bit RGBA, one + /// IDAT, filter type 0 on every row. No interlacing, no palette, no colour-type choice, no + /// filter heuristics. A sprite is a few hundred pixels across and the bytes go straight + /// into a base64 field; the compression difference between this and a tuned encoder is a + /// rounding error against the wire, and every knob not turned is a way this cannot be + /// subtly wrong. + /// + /// Phase 3's BridgeCatalog.ToPng is left exactly as it is. It is measured, shipped, + /// and its input really is a Bitmap from the vendored decoder — a path that needs + /// GDI+ to produce the pixels in the first place, so encoding them without it buys nothing. + /// + public static class BridgePng + { + private static readonly byte[] Signature = + { + 0x89, (byte)'P', (byte)'N', (byte)'G', 0x0D, 0x0A, 0x1A, 0x0A + }; + + private static readonly uint[] CrcTable = BuildCrcTable(); + + private static readonly byte[] Empty = new byte[0]; + + /// + /// ARGB1555 to an RGBA8 PNG with a transparent background. + /// + /// The expansion is the same one BridgeCatalog.ToPng documents and for the same + /// reason: alpha bit clear is fully transparent, and each 5-bit channel is widened by + /// repeating its high bits — (c << 3) | (c >> 2), not a plain shift, + /// which would cap white at 248 and tint every sprite. + /// + public static byte[] FromArgb1555(ushort[] pixels, int width, int height) + { + if (pixels == null || width <= 0 || height <= 0) + return null; + + if ((long)width * height > pixels.Length) + return null; + + // One filter byte per row, then RGBA per pixel. This is the PNG "raw" stream, the + // thing that gets deflated. Bounded by the caller's dimension ceiling + // (BridgeAssetValidator.MaxArtDimension), so the arithmetic cannot overflow an int — + // the check is here anyway, because that ceiling lives in another file. + long size = (((long)width * 4) + 1) * height; + + if (size > Int32.MaxValue / 2) + return null; + + var raw = new byte[size]; + + int at = 0; + + for (int y = 0; y < height; y++) + { + raw[at++] = 0; // filter: None + + int row = y * width; + + for (int x = 0; x < width; x++) + { + int p = pixels[row + x]; + + if ((p & 0x8000) == 0) + { + at += 4; // already zero: transparent black + continue; + } + + int r = (p >> 10) & 0x1F; + int g = (p >> 5) & 0x1F; + int b = p & 0x1F; + + raw[at++] = (byte)((r << 3) | (r >> 2)); + raw[at++] = (byte)((g << 3) | (g >> 2)); + raw[at++] = (byte)((b << 3) | (b >> 2)); + raw[at++] = 0xFF; + } + } + + using (var ms = new MemoryStream(raw.Length / 2)) + { + ms.Write(Signature, 0, Signature.Length); + + var header = new byte[13]; + + WriteBigEndian(header, 0, (uint)width); + WriteBigEndian(header, 4, (uint)height); + + header[8] = 8; // bit depth + header[9] = 6; // colour type: truecolour with alpha + header[10] = 0; // compression: deflate + header[11] = 0; // filter method 0 + header[12] = 0; // no interlace + + WriteChunk(ms, "IHDR", header, 0, header.Length); + + byte[] deflated = Zlib(raw); + + WriteChunk(ms, "IDAT", deflated, 0, deflated.Length); + WriteChunk(ms, "IEND", Empty, 0, 0); + + return ms.ToArray(); + } + } + + /// + /// A zlib stream around .NET Framework's raw-deflate-only DeflateStream: the + /// two-byte header PNG requires, the deflate data, and the adler32 trailer computed + /// here because nothing in the framework will do it. Written by hand for exactly the + /// same reason reads one by hand — net48 exposes deflate and + /// calls it zlib, and the two are not the same format. + /// + private static byte[] Zlib(byte[] data) + { + using (var ms = new MemoryStream(data.Length / 2)) + { + // CMF 0x78 (deflate, 32K window) and FLG 0x9C (default level, no dictionary): + // 0x789C is the pair whose value is divisible by 31, which is the check a decoder + // applies. + ms.WriteByte(0x78); + ms.WriteByte(0x9C); + + using (var deflate = new DeflateStream(ms, CompressionMode.Compress, true)) + deflate.Write(data, 0, data.Length); + + uint adler = Adler32(data); + + ms.WriteByte((byte)(adler >> 24)); + ms.WriteByte((byte)(adler >> 16)); + ms.WriteByte((byte)(adler >> 8)); + ms.WriteByte((byte)adler); + + return ms.ToArray(); + } + } + + private static void WriteChunk(Stream to, string type, byte[] data, int offset, int length) + { + var head = new byte[8]; + + WriteBigEndian(head, 0, (uint)length); + + head[4] = (byte)type[0]; + head[5] = (byte)type[1]; + head[6] = (byte)type[2]; + head[7] = (byte)type[3]; + + to.Write(head, 0, head.Length); + + if (length > 0) + to.Write(data, offset, length); + + // The CRC covers the type and the data, and not the length. + uint crc = Crc32(head, 4, 4, 0xFFFFFFFF); + + if (length > 0) + crc = Crc32(data, offset, length, crc); + + crc ^= 0xFFFFFFFF; + + var tail = new byte[4]; + + WriteBigEndian(tail, 0, crc); + + to.Write(tail, 0, tail.Length); + } + + private static void WriteBigEndian(byte[] into, int at, uint value) + { + into[at] = (byte)(value >> 24); + into[at + 1] = (byte)(value >> 16); + into[at + 2] = (byte)(value >> 8); + into[at + 3] = (byte)value; + } + + private static uint[] BuildCrcTable() + { + var table = new uint[256]; + + for (uint n = 0; n < 256; n++) + { + uint c = n; + + for (int k = 0; k < 8; k++) + c = (c & 1) != 0 ? 0xEDB88320 ^ (c >> 1) : c >> 1; + + table[n] = c; + } + + return table; + } + + private static uint Crc32(byte[] data, int offset, int length, uint crc) + { + for (int i = 0; i < length; i++) + crc = CrcTable[(crc ^ data[offset + i]) & 0xFF] ^ (crc >> 8); + + return crc; + } + + private static uint Adler32(byte[] data) + { + const uint Mod = 65521; + + uint a = 1, b = 0; + + for (int i = 0; i < data.Length; i++) + { + a = (a + data[i]) % Mod; + b = (b + a) % Mod; + } + + return (b << 16) | a; + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeTree.cs b/overlay/Scripts/Custom/Bridge/BridgeTree.cs new file mode 100644 index 0000000..0727cdd --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeTree.cs @@ -0,0 +1,775 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Text; + +namespace Server.Custom.Bridge +{ + /// + /// **The shard's own configuration, over the bridge** (docs/link/v8.md §10 — protocol 8, + /// phase 7). + /// + /// Everything else on the asset plane reads the operator's UO CLIENT. This family reads + /// the shard's own files: the spawn tables, the region and location definitions, the + /// champion list and the decoration lists. The website parses those into its spawn atlas — + /// where every creature lives, which regions exist, what this shard calls scenery — and + /// until protocol 8 it did so by **reading the ServUO tree off a shared filesystem**: + /// same host, a bind mount, or a shared volume. + /// + /// That was the one place the platform's own rule was broken, and broken by the component + /// that faces the internet. This closes it. The parsers do not move — `spawnAtlasParse.js` + /// is pure, fs-free and covered by CI without a ServUO tree anywhere near it, and every + /// quirk it handles stays exactly where it is. The shard sends bytes; the website still + /// decides what they mean. + /// + /// ── What phase 7 measured, and the shape it forced ──────────────────────────────── + /// + /// §10 said "the shard serves `tree/<label>` → bytes". Measured against a stock 57.4 + /// tree, it cannot: `Spawns/trammel.xml` is **4.03 MB**, the sidecar discards any inbound + /// line over **1 MiB** (`shard.rs` `MAX_INBOUND_LINE_BYTES`), and that file as a single + /// base64 row is 5.4 MiB. It would never arrive — the reply would be discarded, the + /// request would time out, and the import would retry forever with no error anywhere in + /// it. Two files on a *stock* tree are in that state; a shard with hand-built spawn tables + /// has more. + /// + /// So a file crosses as **chunks, each gzipped**: + /// + /// + /// tree/Spawns/trammel.xml the manifest row — size, hash, chunk count + /// tree/Spawns/trammel.xml/c0 the first 512 KiB of it, gzipped + /// tree/Spawns/trammel.xml/c1 the next + /// + /// + /// which is §5's depth scheme at work a second time, exactly as `body/400/a0/f0` is — + /// and, as there, nothing about it needed a protocol change. + /// + /// **The chunk is the bound and the compression is the saving**, and it matters which is + /// which. Compression is what makes this cheap: the stock tree is 11.34 MB and gzips to + /// 927 KB, so the whole atlas source arrives in about three pages instead of thirty-one. + /// But nothing guarantees that an operator's files compress at all, so the ceiling has to + /// hold when they do not — and it does, because a 512 KiB chunk that refuses to compress + /// is still only ~683 KiB of base64, inside the wire cap that + /// ' deliberate factor of two leaves room for. + /// A design that leaned on the ratio would work on every tree anyone tested and fail on + /// the first one nobody did. + /// + /// ── Two rules that are not negotiable here ──────────────────────────────────────── + /// + /// **1. The label set is this shard's, never the caller's.** This is the only family on + /// this link whose keys look like paths, and the website is the internet-facing component. + /// So nothing here joins a path that arrived on the wire: a fetch resolves its label + /// against the set itself produced, and a label that is not in it + /// is refused — before any file is opened, and whatever it spells. The five groups are + /// fixed in code, the extensions are fixed in code, and the resolved path is checked to be + /// under the tree root even after all of that. + /// + /// **2. A row re-declares its own address.** Each chunk carries its label, its index, its + /// byte offset and the hash of its own (uncompressed) bytes, and the manifest carries the + /// hash of the whole file. That is the §4.10 lesson on a fourth axis: a reassembly that + /// silently put chunk 3 where chunk 4 belongs would produce a file that parses — XML is + /// forgiving about what it skips — and a spawn atlas subtly missing a facet. Per-chunk + /// hashes make it a named error instead. + /// + public static class BridgeTree + { + /// The §5 key family this serves. + private const string Family = "tree"; + + /// + /// The five labelled groups `spawnAtlasSource.js` reads, and nothing else. + /// + /// Fixed in code rather than configured, because a configurable list is a way for the + /// website to ask for a file this shard never meant to publish. An operator who wants + /// a different tree served wants a different feature. + /// + private static readonly string[] SingleFiles = + { + "Data/Regions.xml", + "Config/ChampionSpawns.xml" + }; + + private const string LocationsDir = "Data/Locations"; + private const string SpawnsDir = "Spawns"; + private const string DecorationDir = "Data/Decoration"; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + // Its own consent, not the asset plane's (§10, phase 7). An operator who declines to + // serve their UO client still gets a spawn atlas, because these are their own files. + BridgeAssets.RegisterFamily(Family, ReplyFetch, ReplyManifest, + () => BridgeConfig.TreeEnabled, + "the shard's configuration tree is not served (Bridge.TreeEnabled is off)"); + } + + // ── the file set ───────────────────────────────────────────────────────────────────── + + private sealed class TreeFile + { + public string Label; + public string Path; + public long Bytes; + public long MTime; + } + + /// + /// Every atlas source file this shard has, tree-relative and forward-slashed. + /// + /// The labels are `spawnAtlasSource.js`'s own, character for character, because they + /// are what the website keys its stored fingerprint on: the same tree read here and + /// read there has to produce the same label or every import looks like a change. + /// Forward slashes for the same reason — a Windows shard and a Linux one must agree. + /// + private static List Enumerate() + { + string root = Core.BaseDirectory; + var files = new List(); + + foreach (string label in SingleFiles) + Add(files, root, label); + + foreach (string label in ListByExtension(root, LocationsDir, ".xml")) + Add(files, root, label); + + foreach (string label in ListByExtension(root, SpawnsDir, ".xml")) + Add(files, root, label); + + foreach (string label in ListTree(root, DecorationDir, ".cfg")) + Add(files, root, label); + + return files; + } + + private static void Add(List files, string root, string label) + { + string path = Resolve(root, label); + + if (path == null) + return; + + try + { + var info = new FileInfo(path); + + if (!info.Exists) + return; + + files.Add(new TreeFile + { + Label = label, + Path = path, + Bytes = info.Length, + MTime = ToUnixMs(info.LastWriteTimeUtc) + }); + } + catch (Exception e) + { + // A file the shard cannot stat is a file it cannot serve. Say so once, here, + // rather than as a refused row on every import pass forever. + Console.WriteLine("[Bridge] tree: cannot read {0}: {1}", label, e.Message); + } + } + + /// One directory's files with the given extension, sorted, as labels. + private static List ListByExtension(string root, string dir, string extension) + { + var labels = new List(); + string full = Path.Combine(root, dir.Replace('/', Path.DirectorySeparatorChar)); + + try + { + if (!Directory.Exists(full)) + return labels; + + foreach (string path in Directory.GetFiles(full)) + { + string name = Path.GetFileName(path); + + if (name.EndsWith(extension, StringComparison.OrdinalIgnoreCase)) + labels.Add(dir + "/" + name); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] tree: cannot list {0}: {1}", dir, e.Message); + } + + labels.Sort(StringComparer.Ordinal); + return labels; + } + + /// + /// One directory tree's files with the given extension, recursively. + /// + /// Recursive because `Data/Decoration` nests two deep in places (`Magincia/Trammel`, + /// `Stygian Abyss/Ter Mur`, `Old/Britannia`), and the website's own reader says why + /// that matters: a flat read indexes a third of what the shard has, and the failure is + /// an authoring dropdown quietly missing whole expansions rather than an error anyone + /// would notice. + /// + private static List ListTree(string root, string dir, string extension) + { + var labels = new List(); + string full = Path.Combine(root, dir.Replace('/', Path.DirectorySeparatorChar)); + + try + { + if (!Directory.Exists(full)) + return labels; + + foreach (string path in Directory.GetFiles(full, "*", SearchOption.AllDirectories)) + { + if (!path.EndsWith(extension, StringComparison.OrdinalIgnoreCase)) + continue; + + string rel = path.Substring(full.Length).Replace('\\', '/').TrimStart('/'); + + if (rel.Length > 0) + labels.Add(dir + "/" + rel); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] tree: cannot walk {0}: {1}", dir, e.Message); + } + + labels.Sort(StringComparer.Ordinal); + return labels; + } + + /// + /// A label to a path on this host, or null if it is not one this shard serves. + /// + /// Rule 1 of the class doc lives here. The label has already been matched against the + /// enumerated set by the time a fetch calls this, and this still refuses anything with + /// a traversal segment, a drive or a root in it, and still checks that what + /// Path.GetFullPath produced is under the tree root. Three checks for one rule + /// because the cost of being wrong once is the website reading an arbitrary file off a + /// game server's disk. + /// + private static string Resolve(string root, string label) + { + if (String.IsNullOrEmpty(label) || label.IndexOf('\\') >= 0) + return null; + + string[] segments = label.Split('/'); + + foreach (string segment in segments) + { + if (segment.Length == 0 || segment == "." || segment == "..") + return null; + } + + if (Path.IsPathRooted(label)) + return null; + + try + { + string rootFull = Path.GetFullPath(root); + string full = Path.GetFullPath(Path.Combine(rootFull, + label.Replace('/', Path.DirectorySeparatorChar))); + + if (!rootFull.EndsWith(Path.DirectorySeparatorChar.ToString(CultureInfo.InvariantCulture), + StringComparison.Ordinal)) + { + rootFull += Path.DirectorySeparatorChar; + } + + return full.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase) ? full : null; + } + catch + { + return null; + } + } + + // ── the fingerprint ────────────────────────────────────────────────────────────────── + + /// + /// What the whole tree currently is, in sixteen hex characters. + /// + /// The same job BridgeCatalog.SourceId does for client files, and the same + /// reason: it goes on every page of a walk, and a page whose id differs from the + /// first's means the operator edited a spawn file while it was being read. Half of + /// what arrived then describes a tree that no longer exists and nothing later can tell + /// which half, so the website refuses the import outright rather than stitching one. + /// + /// Built from (label, size, mtime) rather than from content hashes, because it is + /// computed on every page and hashing the tree's contents each time would spend a + /// tenth of a second per page to answer a question (size, mtime) answers for free. + /// The CONTENT hashes are still sent — once, per file, on the manifest — which is + /// where the website's own drift gate reads them from. + /// + private static string FingerprintOf(List files) + { + var sb = new StringBuilder(256); + + sb.Append(files.Count); + + foreach (TreeFile file in files) + { + sb.Append('|').Append(file.Label) + .Append(':').Append(file.Bytes.ToString(CultureInfo.InvariantCulture)) + .Append(':').Append(file.MTime.ToString(CultureInfo.InvariantCulture)); + } + + return BridgeAssets.Sha256Hex(Encoding.UTF8.GetBytes(sb.ToString())).Substring(0, 16); + } + + // ── assets.manifest, for this family ───────────────────────────────────────────────── + + /// + /// Worker thread. Every file this shard would serve, with its size, its content hash + /// and how many chunks it takes — and no bytes. + /// + /// That separation is what makes the normal case free. The website stores these + /// hashes; on the next import it asks for this list again, compares, and fetches + /// nothing at all when nothing moved — which on a shard whose maps are not being + /// edited is every import. + /// + /// A stock tree is 141 rows and fits in one page comfortably. It pages anyway, by the + /// same envelope as every other family, because the day a shard has three thousand + /// decoration files is not the day to discover this was the one walk that could not + /// end. + /// + private static void ReplyManifest(string reqId, string cursor) + { + List files = Enumerate(); + string fingerprint = FingerprintOf(files); + + int from = ParseCursor(cursor); + + if (from < 0 || from > files.Count) + from = 0; + + var sb = BridgeJson.Begin("assets.manifest.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + .Str("catalog", fingerprint) + .Num("chunkBytes", BridgeConfig.TreeChunkBytes) + .Num("total", files.Count) + .Num("from", from); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int i = from; + + for (; i < files.Count; i++) + { + TreeFile file = files[i]; + string hash = HashFile(file.Path); + + var item = new StringBuilder(256); + + item.Append("{\"key\":"); + BridgeJson.Text(item, Family + "/" + file.Label); + item.Append(",\"label\":"); + BridgeJson.Text(item, file.Label); + item.Append(",\"bytes\":").Append(file.Bytes.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"mtime\":").Append(file.MTime.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"chunks\":").Append( + ChunkCount(file.Bytes).ToString(CultureInfo.InvariantCulture)); + item.Append(",\"sha256\":"); + BridgeJson.Text(item, hash); + item.Append('}'); + + if (!page.TryAdd(item.ToString(), "t:" + (i + 1).ToString(CultureInfo.InvariantCulture))) + break; + } + + page.Close(); + + sb.Num("sent", page.Count); + + BridgeLink.Emit(sb.End()); + } + + /// + /// How many chunks a file of this size takes. + /// + /// **An empty file is one chunk, not none.** `Data/Locations` can legitimately hold an + /// empty file, and zero chunks would make it a manifest row the website could never + /// fetch: it would wait for content that has no address, and report the import + /// incomplete forever. + /// + private static int ChunkCount(long bytes) + { + long chunk = BridgeConfig.TreeChunkBytes; + long count = (bytes + chunk - 1) / chunk; + + return count < 1 ? 1 : (int)count; + } + + // ── assets.fetch, for this family ──────────────────────────────────────────────────── + + /// + /// Worker thread. The bytes for an explicit list of chunk keys. + /// + /// Chunks are read with a seek rather than by holding the file, so the memory this + /// costs a running game server is one chunk regardless of how large an operator's + /// spawn tables are. A 4 MB file served eight times over is eight seeks and eight + /// 512 KiB reads — cheaper than caching it would be, and with no cache to invalidate + /// when the operator edits it mid-pass. + /// + private static void ReplyFetch(string reqId, List keys, string expected, string cursor) + { + List files = Enumerate(); + string fingerprint = FingerprintOf(files); + + // Shared with every other family on this plane, because an absent fingerprint and an + // empty one have to mean the same thing here and there — see + // `BridgeAssets.CatalogMismatch` for what treating them differently costs. + if (BridgeAssets.CatalogMismatch(expected, fingerprint)) + { + // The tree moved between the manifest and this fetch. The same refusal the + // catalogue makes for a patched client, and for the same reason: these keys were + // chosen against a listing that no longer describes what is on disk. + BridgeAssets.Fail(reqId, "UNREADABLE", + "the shard's configuration tree changed since that manifest was read (catalog " + + expected + " is now " + fingerprint + "); start the import again"); + return; + } + + var byLabel = new Dictionary(StringComparer.Ordinal); + + foreach (TreeFile file in files) + byLabel[file.Label] = file; + + int from = ParseCursor(cursor); + + if (from < 0 || from > keys.Count) + from = 0; + + var sb = BridgeJson.Begin("assets.fetch.ok"); + + sb.Str("reqId", reqId) + .Str("family", Family) + .Str("catalog", fingerprint) + .Num("chunkBytes", BridgeConfig.TreeChunkBytes) + .Num("asked", keys.Count) + .Num("from", from); + + var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes); + + int i = from; + + for (; i < keys.Count; i++) + { + string item = Render(byLabel, keys[i]); + + if (!page.TryAdd(item, "t:" + (i + 1).ToString(CultureInfo.InvariantCulture))) + break; + } + + page.Close(); + + sb.Num("sent", page.Count); + + BridgeLink.Emit(sb.End()); + } + + /// + /// One key to one row. + /// + /// A key this shard cannot serve is a row rather than a failed request, exactly as in + /// every other family, and `status` keeps the two kinds apart: `absent` is a file this + /// shard does not have (a tree with no `ChampionSpawns.xml` is a normal tree), and + /// `unsupported` is a key shape this family does not serve — which is a website bug, + /// and is counted separately so it cannot hide inside the expected gaps. + /// + private static string Render(Dictionary byLabel, string key) + { + string label; + int chunk; + + if (!ParseKey(key, out label, out chunk)) + return Refusal(key, "unsupported", "not a tree chunk key (tree/