diff --git a/overlay.toml b/overlay.toml index 7bc1fcc..5643385 100644 --- a/overlay.toml +++ b/overlay.toml @@ -23,8 +23,9 @@ # manual duty: when the protocol changes, bump it here in the same PR that # changes the emitters, exactly as link bumps PROTOCOL_VERSION. # -# Current: 6 — see docs/link/v6.md (idempotency keys on inbound commands, champ.boss.killed). -protocol = 7 +# Current: 8 — see docs/link/v8.md (the Asset Bridge: client assets over the loopback link +# instead of a converter on somebody's desktop). +protocol = 8 # ── ServUO compatibility ───────────────────────────────────────────────────── # diff --git a/overlay/Config/Bridge.cfg b/overlay/Config/Bridge.cfg index 5da1626..5be83c3 100644 --- a/overlay/Config/Bridge.cfg +++ b/overlay/Config/Bridge.cfg @@ -295,6 +295,18 @@ EventsMaxGrantStack=1000 # would land at a moment nobody chose. Set to 0 to allow a save at any time. EventsMinSaveIntervalSec=300 +# The asset plane (docs/link/v8.md, protocol 8). Its own switch, deliberately: turning +# this on is consenting to the website reading this host's UO CLIENT FILES -- art, +# animations, the string table -- over the link. Nothing on this plane writes anything. +AssetsEnabled=true + +# The largest reply the asset plane will build, in encoded bytes. Not an item count: +# the ceiling it lives inside is the sidecar's 1 MiB inbound line cap, and base64 adds +# 33% to every payload. Clamped to [64 KiB, 512 KiB] -- half the wire cap, so that a +# single oversized item (always admitted, or its family could never make progress) +# still fits. +AssetBatchBytes=524288 + # The test scaffolding in tools/scaffolding/ reads its own flags from this file # (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose: # Config.Get returns the default of false when a key is missing, so a deployed diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs b/overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs new file mode 100644 index 0000000..e4ed921 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs @@ -0,0 +1,751 @@ +using System; +using System.IO; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **Validate before calling** (docs/link/v8.md §4.5) — the boundary between this protocol + /// and ServUO's vendored Ultima decoders. Phase 0 prototyped it in + /// tools/scaffolding/BridgeAssetProbe.cs and measured it both ways; phase 1 promoted + /// it here, into the overlay, and extended it to animations. + /// + /// The principle: `Ultima`'s decoders take their bounds from the file they are reading, so + /// the extractor must decide whether a record is worth handing over *before* handing it + /// over. Every check below is against the index entry and the record header — cheap, and + /// enough to turn an uncatchable corrupted-state exception into a skipped asset. + /// + /// **The failure this exists for is a wrong picture, not a crash.** `LoadStatic`, + /// `LoadLand` and `GetAnimation` all decode out of a shared m_StreamBuffer that is + /// reused, only ever grown, and filled by a stream.Read whose return value is + /// discarded. A record that is short, absent or out of bounds therefore renders **whatever + /// the previously-decoded asset left behind**, reports success, and is undetectable by + /// anything downstream. On the stock client on the machine phase 0 ran on that is 22,102 + /// ids whose index entry reads lookup 0, length 0. + /// + /// It cannot be complete and does not claim to be. It closes the shapes that reading the + /// source showed are reachable. What says the boundary is in the right place is the second + /// measurement rather than the first: against a client patched 21 ways it refused all eight + /// record-level defects, and against the **stock** client it refused **nothing** across + /// 49,151 statics and 16,384 land tiles. A checker that refuses real art would be worse + /// than no checker. + /// + public static class BridgeAssetValidator + { + public enum Verdict + { + /// Nothing at this id, and the index says so honestly. + Absent, + + /// The entry is self-consistent and inside its file. + Ok, + + /// The entry claims something the file cannot support. Do not decode it. + Refused + } + + /// Land tiles decode a fixed 44×44 diamond: 2 × (2+4+…+44) ushorts. + public const int LandRecordBytes = 2024; + + /// + /// A ceiling on decoded art dimensions. `LoadStatic` allocates + /// new Bitmap(width, height) straight from two ushorts in the record, so a + /// corrupt header asks for up to 65535×65535 — an 8 GB allocation, from a file. Real + /// art is a couple of hundred pixels at most. + /// + public const int MaxArtDimension = 1024; + + /// + /// Builds our own index over the same files, with the same constructor arguments + /// Art uses — including hasExtra: false, which is the whole reason the + /// art path is safe where the gump path is not (§4.1). + /// + public static FileIndex OpenArtIndex() + { + if (ArtDataPath() == null) + return null; + + return new FileIndex("Artidx.mul", "Art.mul", "artLegacyMUL.uop", 0x10000, 4, ".tga", 0x13FDC, false); + } + + /// + /// The file an art index entry's lookup is an offset **into** — which is not + /// art.mul on any current client. + /// + /// This cost a whole probe run to learn and it is the single most important thing + /// phase 1 must not get wrong. FileIndex's UOP constructor ends with a bare + /// MulPath = uopPath: **when artLegacyMUL.uop exists it wins outright**, + /// and art.mul / artidx.mul are never opened at all. A validator that + /// bounds offsets against art.mul while the index holds UOP offsets is not + /// merely approximate, it is nonsense — the first run of this probe refused 34,299 + /// perfectly good statics for "declaring 10533x2085" because it was reading UOP + /// offsets into the wrong file. + /// + /// So the resolution order here mirrors FileIndex's exactly, and anything that + /// needs the bytes behind an entry must ask this rather than assume. + /// + public static string ArtDataPath() + { + var uop = Files.GetFilePath("artlegacymul.uop"); + + if (uop != null) + return uop; + + return Files.GetFilePath("art.mul"); + } + + public static long MulLength(string path) + { + if (path == null) + return 0; + + try + { + return new FileInfo(path).Length; + } + catch + { + return 0; + } + } + + /// + /// Judges one index entry. + /// + /// The check FileIndex.Seek is missing is the last one: it tests + /// Stream.Length < e.lookup — that the record *starts* inside the file — and + /// never that it *ends* inside it. A record that begins two bytes before EOF and + /// declares a length of 4,000 passes, and stream.Read then returns a short count + /// that the decoders discard, leaving the previous asset's bytes in the shared buffer. + /// + public static Verdict CheckEntry(FileIndex index, int at, long mulLength, long verdataLength, out string reason) + { + reason = null; + + if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) + { + reason = "index " + at + " out of range"; + return Verdict.Absent; + } + + Entry3D e = index.Index[at]; + + if (e.lookup < 0) + { + reason = "lookup " + e.lookup; + return Verdict.Absent; + } + + bool patched = (e.length & (1 << 31)) != 0; + int length = e.length & 0x7FFFFFFF; + + if (!patched && e.length < 0) + { + reason = "length " + e.length; + return Verdict.Absent; + } + + if (length == 0) + { + reason = "lookup " + e.lookup + ", length 0"; + return Verdict.Absent; + } + + long ceiling = patched ? verdataLength : mulLength; + + if (ceiling <= 0) + { + reason = (patched ? "verdata.mul" : "the art data file") + " has no length"; + return Verdict.Refused; + } + + if (e.lookup >= ceiling) + { + reason = "lookup " + e.lookup + " past the end of " + + (patched ? "verdata.mul" : "the mul") + " (" + ceiling + ")"; + return Verdict.Refused; + } + + // The missing check. A short read is silent, and its consequence is the PREVIOUS + // asset's picture served under this id. + if (e.lookup + (long)length > ceiling) + { + reason = "record runs " + (e.lookup + (long)length - ceiling) + " bytes past the end of " + + (patched ? "verdata.mul" : "the mul"); + return Verdict.Refused; + } + + return Verdict.Ok; + } + + /// + /// `LoadLand` reads 2,024 bytes regardless of the declared length, so a shorter record + /// reads past the end of a buffer sized from that length. + /// + public static bool LandLengthSane(FileIndex index, int at, out string reason) + { + reason = null; + + if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) + return true; + + int length = index.Index[at].length & 0x7FFFFFFF; + + if (length > 0 && length < LandRecordBytes) + { + reason = "land record is " + length + " bytes; LoadLand always reads " + LandRecordBytes; + return false; + } + + return true; + } + + /// + /// Walks a static record's own row table the way LoadStatic will, and refuses + /// it if that walk would read outside the record. + /// + /// This is the check with teeth. LoadStatic's inner loop guards the write into + /// the bitmap (xOffset > delta, xOffset + xRun > delta) and does + /// nothing at all about the read cursor, which advances until it happens to find a + /// zero pair — potentially far outside a pinned array. Simulating the same walk with + /// a bound is the cheapest way to know whether handing the id over is safe. + /// + public static bool StaticRecordSane(byte[] record, int length, out string reason) + { + reason = null; + + if (length < 8) + { + reason = "record is " + length + " bytes; a static header needs 8"; + return false; + } + + int words = length / 2; + int width = ReadUInt16(record, 4); + int height = ReadUInt16(record, 6); + + // LoadStatic returns null for these rather than misbehaving, so it is not a refusal. + if (width <= 0 || height <= 0) + return true; + + if (width > MaxArtDimension || height > MaxArtDimension) + { + reason = "declares " + width + "x" + height + ", past the " + MaxArtDimension + "px ceiling"; + return false; + } + + // The row-lookup table: height ushorts starting at word 4. + if (4 + height > words) + { + reason = "row table (" + height + " entries) does not fit in a " + length + "-byte record"; + return false; + } + + int start = height + 4; + + for (int y = 0; y < height; y++) + { + int cursor = start + ReadUInt16(record, (4 + y) * 2); + + while (true) + { + // Two ushorts for the run header, and they must both be inside the record. + if (cursor < 0 || cursor + 1 >= words) + { + reason = "row " + y + " reads at word " + cursor + ", past the record's " + words; + return false; + } + + int xOffset = ReadUInt16(record, cursor * 2); + int xRun = ReadUInt16(record, (cursor + 1) * 2); + cursor += 2; + + if (xOffset + xRun == 0) + break; + + // LoadStatic stops the row here, so the read cursor stops with it. + if (xOffset > width || xOffset + xRun > width) + break; + + if (cursor + xRun > words) + { + reason = "row " + y + " declares a " + xRun + "-pixel run running past the record"; + return false; + } + + cursor += xRun; + } + } + + return true; + } + + // ── animations (phase 1) ───────────────────────────────────────────────────────────── + // + // Phase 0 measured the art path and left this half unbuilt, and then proved it was + // needed: the patched client's verdata entry for body 34 points past verdata.mul's own + // end, and the wolf still "decoded" — counted among the 1,144 successes while rendering + // something else entirely. `GetAnimation` has every weakness `LoadStatic` has and one + // more, because the buffer it decodes from is longer than the record it read. + + /// The palette every animation record opens with: 0x100 ushorts. + public const int AnimPaletteBytes = 0x100 * 2; + + /// + /// A ceiling on an animation's declared frame count. GetAnimation does + /// new int[frameCount] straight from four bytes in the file, before it has + /// looked at anything else. Real actions are tens of frames. + /// + public const int MaxAnimFrames = 1024; + + /// The xor Frame applies to every run header before decoding it. + private const int DoubleXor = (0x200 << 22) | (0x200 << 12); + + /// + /// The anim*.mul an animation index entry's lookup is an offset into. + /// + /// Unlike art (§4.6) there is no UOP precedence to get wrong here, and that is not + /// luck: Animations constructs its five FileIndexes with the four-argument + /// constructor, which passes uopFile: null. It never reads + /// AnimationFrame*.uop at all — which is the same fact that leaves six of the + /// twelve player-character bodies undecodable until §4.3's reader lands in phase 4. + /// + public static string AnimDataPath(int fileType) + { + switch (fileType) + { + case 1: return Files.GetFilePath("anim.mul"); + case 2: return Files.GetFilePath("anim2.mul"); + case 3: return Files.GetFilePath("anim3.mul"); + case 4: return Files.GetFilePath("anim4.mul"); + case 5: return Files.GetFilePath("anim5.mul"); + default: return null; + } + } + + /// + /// Builds our own index over one anim file, with the same constructor arguments + /// Animations uses — the entry lengths especially, since they decide how far + /// into the file an index runs. + /// + public static FileIndex OpenAnimIndex(int fileType) + { + if (AnimDataPath(fileType) == null) + return null; + + switch (fileType) + { + case 1: return new FileIndex("Anim.idx", "Anim.mul", 0x40000, 6); + case 2: return new FileIndex("Anim2.idx", "Anim2.mul", 0x10000, -1); + case 3: return new FileIndex("Anim3.idx", "Anim3.mul", 0x20000, -1); + case 4: return new FileIndex("Anim4.idx", "Anim4.mul", 0x20000, -1); + case 5: return new FileIndex("Anim5.idx", "Anim5.mul", 0x20000, -1); + default: return null; + } + } + + /// + /// Where a body's animation actually lives: which anim file, and which index in it. + /// + /// **This is the never-sweep-file-types rule, written as code** (§4.3). It asks + /// BodyConverter.Convert once, takes its answer, and if that answer leads + /// nowhere it reports nowhere. There is deliberately no loop here and no fallback, + /// because asking the *other* anim files for an index they do not own does not fail — + /// it returns 175 decodable action/direction combinations of **a giant spider** for + /// gargoyle 666, and misaligned colour fragments for the other two. Every one of those + /// reads reports success, and nothing downstream can tell them from art. + /// + /// A false return with set is the ordinary, expected answer + /// for a body this client has no art for — the caller reports absent, not an error. + /// + public static bool ResolveAnimation( + int body, int action, int direction, out int fileType, out int index, out string reason) + { + reason = null; + fileType = 0; + index = -1; + + if (body <= 0 || action < 0) + { + reason = "body " + body + " action " + action + " is not addressable"; + return false; + } + + // Directions 5-7 are the client mirroring 1-3, and `Frame` decodes them through its + // flip branch — different pointer arithmetic, which nothing below has checked. + // §5.1 fixed this protocol at direction 0 or 1, so refusing the rest costs nothing + // and keeps the validator honest about what it has actually verified. + if (direction < 0 || direction > 4) + { + reason = "direction " + direction + " is mirrored; this protocol reads 0-4 only"; + return false; + } + + int translated = body; + int hue = 0; + + try + { + // Exactly what GetAnimation(..., preserveHue: false, ...) does first. + Animations.Translate(ref translated, ref hue); + fileType = BodyConverter.Convert(ref translated); + } + catch (Exception e) + { + reason = "body.def/bodyconv.def lookup failed: " + e.GetType().Name; + return false; + } + + if (AnimDataPath(fileType) == null) + { + // Gargoyle 666 lands here: Bodyconv.def maps it to anim5, and this client has no + // anim5. Absent is the correct answer and the ONLY safe one. + reason = "bodyconv sends body " + body + " to file type " + fileType + + ", which this client does not have"; + return false; + } + + index = AnimIndexOf(translated, fileType) + (action * 5) + direction; + + return true; + } + + /// + /// Animations.GetFileIndex's own arithmetic, which is private. The banding is + /// per file type and the boundaries differ between them, so this is transcribed rather + /// than generalised — an index that disagrees with the library's by one is a picture + /// of the wrong creature, validated. + /// + private static int AnimIndexOf(int body, int fileType) + { + switch (fileType) + { + case 2: + return body < 200 ? body * 110 : 22000 + ((body - 200) * 65); + + case 3: + if (body < 300) + return body * 65; + + return body < 400 ? 33000 + ((body - 300) * 110) : 35000 + ((body - 400) * 175); + + case 5: + // "looks strange, though it works" — the library's own comment. Body 34 is + // excluded from the first band here and nowhere else. + if (body < 200 && body != 34) + return body * 110; + + return body < 400 ? 22000 + ((body - 200) * 65) : 35000 + ((body - 400) * 175); + + default: // 1 and 4 share their banding + if (body < 200) + return body * 110; + + return body < 400 ? 22000 + ((body - 200) * 65) : 35000 + ((body - 400) * 175); + } + } + + /// + /// Walks an animation record the way GetAnimation and Frame will, and + /// refuses it if that walk would read outside the record or write outside the bitmap. + /// + /// Two things make this stricter than the static walk, and both come from the library: + /// + /// GetAnimation decodes through new MemoryStream(m_StreamBuffer, false) — + /// the whole shared buffer, not the length bytes it just read into it. So a + /// truncated record does not hit end-of-stream and throw; the reader sails on into the + /// **previous** animation's bytes and returns a plausible frame. Bounding against + /// rather than against the buffer is the entire point. + /// + /// And Frame's run loop is a *write* through a LockBits pointer whose + /// origin comes from two signed shorts in the file (xCenter, yCenter), + /// with no bound of any kind. LoadStatic at least guards its writes; this does + /// not, so the destination of every run is checked against the bitmap it locked. + /// + /// is how many frames the caller will actually decode — + /// 1 for the catalogue's thumbnail (FirstFrame: true), 0 for all of them. + /// Checking frames nobody decodes would invent refusals, which §4.5 costs more than + /// it saves. + /// + public static bool AnimationRecordSane(byte[] record, int length, int maxFrames, out string reason) + { + reason = null; + + if (length < AnimPaletteBytes + 4) + { + reason = "record is " + length + " bytes; an animation needs " + + (AnimPaletteBytes + 4) + " for its palette and frame count"; + return false; + } + + int start = AnimPaletteBytes; + int frameCount = ReadInt32(record, start); + + if (frameCount <= 0) + { + reason = "declares " + frameCount + " frames"; + return false; + } + + if (frameCount > MaxAnimFrames) + { + reason = "declares " + frameCount + " frames, past the " + MaxAnimFrames + " ceiling"; + return false; + } + + // The lookup table is read in full whatever FirstFrame says, so it is bounded in full. + long tableEnd = (long)start + 4 + ((long)frameCount * 4); + + if (tableEnd > length) + { + reason = "frame table (" + frameCount + " entries) does not fit in a " + + length + "-byte record"; + return false; + } + + int check = maxFrames > 0 && maxFrames < frameCount ? maxFrames : frameCount; + + for (int i = 0; i < check; i++) + { + int at = start + ReadInt32(record, start + 4 + (i * 4)); + + if (!FrameSane(record, length, at, i, out reason)) + return false; + } + + return true; + } + + private static bool FrameSane(byte[] record, int length, int at, int frame, out string reason) + { + reason = null; + + if (at < 0 || at + 8 > length) + { + reason = "frame " + frame + " starts at " + at + ", outside the " + + length + "-byte record"; + return false; + } + + int xCenter = ReadInt16(record, at); + int yCenter = ReadInt16(record, at + 2); + int width = ReadUInt16(record, at + 4); + int height = ReadUInt16(record, at + 6); + + // Frame's constructor returns before locking anything for these, so they are empty + // rather than dangerous — and an empty frame is a real thing in this format. + if (width == 0 || height == 0) + return true; + + if (width > MaxArtDimension || height > MaxArtDimension) + { + reason = "frame " + frame + " declares " + width + "x" + height + ", past the " + + MaxArtDimension + "px ceiling"; + return false; + } + + // Settings.PixelFormat is 16bpp and GDI+ pads each scanline to four bytes, so a row + // is `delta` ushorts wide and the locked region is height*delta of them. This is the + // same `bd.Stride >> 1` Frame computes. + int delta = (((width * 2) + 3) & ~3) >> 1; + long pixels = (long)height * delta; + + long origin = (xCenter - 0x200) + ((long)((yCenter + height) - 0x200) * delta); + int cursor = at + 8; + + while (true) + { + if (cursor + 4 > length) + { + reason = "frame " + frame + + " runs off the end of the record looking for its terminator"; + return false; + } + + int header = ReadInt32(record, cursor); + cursor += 4; + + if (header == 0x7FFF7FFF) + break; + + header ^= DoubleXor; + + long dy = (header >> 12) & 0x3FF; + long dx = (header >> 22) & 0x3FF; + int run = header & 0xFFF; + + long first = origin + (dy * delta) + dx; + + if (first < 0 || first + run > pixels) + { + reason = "frame " + frame + " writes pixels " + first + ".." + (first + run) + + " outside its own " + pixels + "-pixel bitmap"; + return false; + } + + // One palette byte per pixel, read straight out of the record. + if (cursor + run > length) + { + reason = "frame " + frame + " declares a " + run + + "-pixel run running past the record"; + return false; + } + + cursor += run; + } + + return true; + } + + private static int ReadUInt16(byte[] b, int at) + { + return b[at] | (b[at + 1] << 8); + } + + private static int ReadInt16(byte[] b, int at) + { + return (short)(b[at] | (b[at + 1] << 8)); + } + + private static int ReadInt32(byte[] b, int at) + { + return b[at] | (b[at + 1] << 8) | (b[at + 2] << 16) | (b[at + 3] << 24); + } + + /// + /// Reads a record's actual bytes so or + /// can walk it. + /// + /// Holds its own handles rather than borrowing the library's, because FileIndex + /// hands out the stream it decodes from and moving that stream's position underneath + /// the decoder would be its own bug. Opened FileShare.ReadWrite to match how + /// FileIndex opens the same files. + /// + /// One reader serves one data file, so an animation sweep wants one per file type, + /// built from . + /// + public sealed class RecordReader : IDisposable + { + private readonly FileStream _mul; + private readonly FileStream _verdata; + private byte[] _scratch = new byte[64 * 1024]; + + public RecordReader(string mulPath, string verdataPath) + { + _mul = Open(mulPath); + _verdata = Open(verdataPath); + } + + private static FileStream Open(string path) + { + if (path == null || !File.Exists(path)) + return null; + + try + { + return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + catch + { + return null; + } + } + + /// + /// True when the record at is safe to hand to + /// Art.GetStatic. A record that cannot be read at all is reported sane — + /// has already judged the entry, and this must not + /// invent a second reason to refuse. + /// + public bool StaticSane(FileIndex index, int at, out string reason) + { + int length = ReadRecord(index, at, out reason); + + if (length < 0) + return true; + + if (length == 0) + return false; + + return StaticRecordSane(_scratch, length, out reason); + } + + /// + /// True when the record at is safe to hand to + /// Animations.GetAnimation. is how many frames + /// the caller will decode — 1 for a FirstFrame call, 0 for all of them. + /// + public bool AnimationSane(FileIndex index, int at, int maxFrames, out string reason) + { + int length = ReadRecord(index, at, out reason); + + if (length < 0) + return true; + + if (length == 0) + return false; + + return AnimationRecordSane(_scratch, length, maxFrames, out reason); + } + + /// + /// Reads one record into . Returns its length, 0 for a + /// failure (with set), or -1 when there is nothing to + /// read at all — has already judged the entry, and this + /// must not invent a second reason to refuse. + /// + private int ReadRecord(FileIndex index, int at, out string reason) + { + reason = null; + + if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) + return -1; + + Entry3D e = index.Index[at]; + bool patched = (e.length & (1 << 31)) != 0; + int length = e.length & 0x7FFFFFFF; + + var stream = patched ? _verdata : _mul; + + if (stream == null || length <= 0 || e.lookup < 0) + return -1; + + if (_scratch.Length < length) + _scratch = new byte[length]; + + int read; + + try + { + stream.Seek(e.lookup, SeekOrigin.Begin); + read = stream.Read(_scratch, 0, length); + } + catch (Exception ex) + { + reason = "cannot read the record: " + ex.GetType().Name; + return 0; + } + + // The short read the decoders discard. Refusing here is the whole point: the + // library would decode whatever the shared buffer happened to hold. + if (read < length) + { + reason = "short read — " + read + " of " + length + " bytes available"; + return 0; + } + + return length; + } + + public void Dispose() + { + if (_mul != null) + _mul.Dispose(); + + if (_verdata != null) + _verdata.Dispose(); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs new file mode 100644 index 0000000..a3e4714 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs @@ -0,0 +1,747 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The Asset Bridge's transport** (docs/link/v8.md §3, §6, §7 — protocol 8, phase 1). + /// + /// Everything else on this link answers on the Core thread, reads live world state, and + /// replies in microseconds. The asset plane cannot: it reads hundreds of megabytes of the + /// operator's client files and decodes pictures out of them, and doing either on the Core + /// thread would stop the world for every player on the shard. So this class is the one + /// genuinely new shape in protocol 8 — a handler that accepts on the Core thread, hands + /// the work to a **dedicated asset worker**, and returns immediately. + /// + /// Three rules hold it together, and each of them is answering a specific way this could + /// go wrong. + /// + /// **1. Replies, never events.** Every asset frame carries the caller's `reqId`, so + /// `rpc.rs`'s `try_route` consumes it before `app.rs` can persist it to SQLite and + /// broadcast it to every WebSocket subscriber. An asset stream on the event path would + /// grow the sidecar's store without bound and fan megabytes at every connected client, + /// forever. Nothing here emits an unsolicited frame — if a request has no `reqId` it is + /// refused rather than answered. + /// + /// **2. One request outstanding, always.** 's queue is bounded + /// drop-oldest in **lines, not bytes** — a design that is right for live events and + /// dangerous for bulk transfer, because 10,000 queued 200 KB replies is 2 GB of shard + /// memory. The bound that actually holds is flow control, not a bigger queue: this plane + /// has **one slot**, and a second asset request arriving while one is in flight is + /// answered `bridge.busy` (which the sidecar already maps to 425) rather than queued. + /// Queue depth therefore stays at approximately one by construction. A dropped or lost + /// reply just times out and is re-requested, which is safe because reading a client file + /// is idempotent and touches no world state. + /// + /// Note what that costs, deliberately: a status poll shares the slot with a batch, so + /// polling during a long import gets 425 until the batch lands. That is honest — this + /// plane really is doing one thing at a time — and the admin surface (phase 8) is where a + /// separate status lane would have to argue for itself. + /// + /// **3. Byte budgets, not counts.** Batches are cut by encoded size + /// (, 512 KB), not by item count, because the + /// ceilings this has to live inside are byte ceilings: the sidecar refuses an inbound line + /// over 1 MiB, and base64 costs 33% on top of whatever the payload measures. + /// is that budget, and every asset family shares it so the + /// envelope cannot drift apart between them. + /// + /// **Emitting from off the Core thread is safe here, and it is worth saying why.** + /// BridgeLink.Emit enqueues onto a ConcurrentQueue and never touches the + /// socket, so the enqueue itself is fine. The subtle part is + /// BridgeIdempotency.Observe, which Emit calls while a keyed command is in + /// flight: it captures a line only when that line's correlation field **exactly equals** + /// the open command's correlation value, and correlation values come from one monotonic + /// counter in the sidecar. An asset reply therefore cannot be mistaken for a keyed + /// command's reply, whatever the interleaving. + /// + public static class BridgeAssets + { + /// + /// What version of *our derivation* produced these bytes (§7). + /// + /// The source gate hashes the operator's client files, which answers "did the inputs + /// change". It cannot answer "did the way we read them change" — and that is the case + /// that bites, because a corrected frame offset or a fixed hue application changes + /// every derived byte while every source file stays identical. So this is folded into + /// stage 1 alongside the hashes, and bumping it makes the whole working set drift, + /// which is the intended and correct blast radius. + /// + /// Bump it whenever extraction changes what it produces from unchanged input. It is + /// the same rule spawnAtlasSource.js's `PARSER_VERSION` follows, and it applies + /// here more rather than less: this pipeline derives far more from far less. + /// + public const int EXTRACTOR_VERSION = 1; + + // ── the one slot (§3.2) ────────────────────────────────────────────────────────────── + + private static readonly object _sync = new object(); + private static Thread _worker; + private static readonly AutoResetEvent _wake = new AutoResetEvent(false); + private static Action _job; + private static string _inFlight; + private static DateTime _inFlightSince; + private static bool _running; + + private static long _served, _busied, _failed; + + // ── the hash cache (§6) ────────────────────────────────────────────────────────────── + + private static readonly Dictionary _hashes = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + private static Thread _hasher; + private static volatile bool _hashing; + + private sealed class CachedHash + { + public long Size; + public long MTime; + public string Sha256; + } + + // ── imaging (§4.4) ─────────────────────────────────────────────────────────────────── + + private static bool _imagingChecked; + private static bool _imagingOk; + private static string _imagingReason; + + public static void Initialize() + { + if (!BridgeConfig.Enabled) + return; + + BridgeBoot.RegisterHandler("assets.sources", OnSources); + } + + public static string Status() + { + int cached; + + lock (_hashes) + { + cached = _hashes.Count; + } + + lock (_sync) + { + return String.Format( + "assets(served={0} busied={1} failed={2} inFlight={3} hashing={4} cached={5})", + _served, _busied, _failed, _inFlight ?? "-", _hashing, cached); + } + } + + // ── the request plane ──────────────────────────────────────────────────────────────── + + /// + /// Stage 1 of §6: what the shard's client files currently are. No pixels, no assets — + /// just the gate that lets the website decide whether anything needs importing at all, + /// because the normal case is a restart that changed nothing and it must cost nothing. + /// + private static void OnSources(Dictionary o) + { + var reqId = BridgeJson.GetString(o, "reqId"); + + if (reqId == null) + { + // Rule 1. Without a correlation id this would land on the event path, be persisted + // to the sidecar's store and broadcast to every subscriber. Refuse instead. + Fail(null, "assets.sources requires a reqId"); + return; + } + + if (!BridgeConfig.AssetsEnabled) + { + Fail(reqId, "asset extraction is disabled on this shard"); + return; + } + + Accept(reqId, "assets.sources", () => ReplySources(reqId)); + } + + /// + /// Claims the single slot and hands the work to the worker, or answers `bridge.busy`. + /// Runs on the Core thread and does nothing expensive; runs on + /// the worker and must touch no world state. + /// + private static void Accept(string reqId, string kind, Action job) + { + lock (_sync) + { + if (_inFlight != null) + { + _busied++; + Busy(reqId, kind); + return; + } + + _inFlight = kind; + _inFlightSince = DateTime.UtcNow; + _job = job; + + try + { + EnsureWorker(); + } + catch (Exception e) + { + // The slot is claimed and there is now nothing that will ever free it. Give it + // back here or this plane answers `bridge.busy` for the life of the process. + _inFlight = null; + _job = null; + + Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message); + Fail(reqId, "the shard could not start its asset worker"); + return; + } + } + + _wake.Set(); + } + + private static void Busy(string reqId, string kind) + { + var held = (DateTime.UtcNow - _inFlightSince).TotalSeconds; + + var sb = BridgeJson.Begin("bridge.busy"); + + sb.Str("reqId", reqId) + // `busyKind`, never a second `kind` — `Begin` has already written this frame's own, + // and a JSON object carrying two makes every parser take the last. Protocol 6 shipped + // that bug once and it made the sidecar answer 200 for a refusal. + .Str("busyKind", kind) + .Num("heldForSec", (long)held) + .Str("reason", "the asset plane serves one request at a time"); + + BridgeLink.Emit(sb.End()); + } + + private static void Fail(string reqId, string reason) + { + var sb = BridgeJson.Begin("assets.error"); + + if (reqId != null) + sb.Str("reqId", reqId); + + sb.Str("reason", reason); + BridgeLink.Emit(sb.End()); + } + + // ── the worker ─────────────────────────────────────────────────────────────────────── + + /// + /// Started on first use rather than at boot, so a shard that never imports an asset + /// never carries the thread. Caller must hold . + /// + private static void EnsureWorker() + { + if (_worker != null) + return; + + _running = true; + + _worker = new Thread(WorkLoop) + { + Name = "BridgeAssets", + IsBackground = true + }; + + _worker.Start(); + } + + private static void WorkLoop() + { + while (_running) + { + _wake.WaitOne(1000); + + Action job; + + lock (_sync) + { + job = _job; + _job = null; + } + + if (job == null) + continue; + + try + { + job(); + Interlocked.Increment(ref _served); + } + catch (Exception e) + { + // A handler that throws must still free the slot, or this plane is wedged for + // the life of the process and every later request answers `bridge.busy`. + Interlocked.Increment(ref _failed); + Console.WriteLine("[Bridge] asset worker: {0}: {1}", e.GetType().Name, e.Message); + } + finally + { + lock (_sync) + { + _inFlight = null; + } + } + } + } + + // ── assets.sources ─────────────────────────────────────────────────────────────────── + + /// + /// The client files whose bytes decide everything downstream. + /// + /// Resolved through Ultima.Files rather than by joining a configured directory, + /// because that is what the decoders themselves do — a file this reports is a file + /// they would actually open. + /// + private static readonly string[] SourceFiles = + { + "cliloc.enu", + "artlegacymul.uop", "art.mul", "artidx.mul", + "anim.idx", "anim.mul", + "anim2.idx", "anim2.mul", + "anim3.idx", "anim3.mul", + "anim4.idx", "anim4.mul", + "anim5.idx", "anim5.mul", + "body.def", "bodyconv.def", + "hues.mul", + "verdata.mul" + }; + + private static void ReplySources(string reqId) + { + var sb = BridgeJson.Begin("assets.sources.ok"); + + sb.Str("reqId", reqId) + .Num("extractorVersion", EXTRACTOR_VERSION); + + WriteImaging(sb); + + // §4.6: whichever of art.mul / artLegacyMUL.uop `FileIndex` would actually open. An + // operator who added custom graphics to art.mul while the UOP is present is getting + // nothing, silently, and this is the only place that can tell them so. + string artData = BridgeAssetValidator.ArtDataPath(); + + sb.Str("artDataFile", artData == null ? null : Path.GetFileName(artData)); + + var page = new PageBuilder(sb, "files", BridgeConfig.AssetBatchBytes); + bool anyMissingHash = false; + + for (int i = 0; i < SourceFiles.Length; i++) + { + string name = SourceFiles[i]; + string path = ResolvePath(name); + + if (path == null) + continue; + + var item = new StringBuilder(256); + + item.Append("{\"name\":"); + BridgeJson.Text(item, name); + item.Append(",\"path\":"); + BridgeJson.Text(item, path); + + long size = 0, mtime = 0; + + try + { + var info = new FileInfo(path); + size = info.Length; + mtime = ToUnixMs(info.LastWriteTimeUtc); + } + catch (Exception e) + { + item.Append(",\"unreadable\":"); + BridgeJson.Text(item, e.GetType().Name); + } + + item.Append(",\"size\":").Append(size.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"mtime\":").Append(mtime.ToString(CultureInfo.InvariantCulture)); + + string hash = CachedHashFor(path, size, mtime); + + if (hash == null) + anyMissingHash = true; + + item.Append(",\"sha256\":"); + BridgeJson.Text(item, hash); + + // The one diagnostic §4.6 asks for: art.mul is present, and unread. + if (artData != null + && (name == "art.mul" || name == "artidx.mul") + && !artData.EndsWith(".mul", StringComparison.OrdinalIgnoreCase)) + { + item.Append(",\"shadowedBy\":"); + BridgeJson.Text(item, Path.GetFileName(artData)); + } + + item.Append('}'); + + if (!page.TryAdd(item.ToString(), name)) + break; + } + + page.Close(); + + // §6's gate is (size, mtime) first and a content hash only when those differ, because + // anim.mul and art.mul are 195 MB and 148 MB and a full hash on every status poll + // would make the admin panel feel broken. It would also blow the sidecar's 10 s reply + // timeout outright on the first call. So a hash that is not cached is reported `null` + // and computed in the background: this reply is always fast, and the next poll — after + // `hashing` goes false — carries the answer. + if (anyMissingHash) + StartHashing(); + + sb.Bool("hashing", _hashing); + sb.Bool("complete", !anyMissingHash); + + BridgeLink.Emit(sb.End()); + } + + private static string ResolvePath(string name) + { + try + { + return Files.GetFilePath(name); + } + catch + { + return null; + } + } + + private static long ToUnixMs(DateTime utc) + { + return (long)(utc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; + } + + // ── imaging (§4.4) ─────────────────────────────────────────────────────────────────── + + /// + /// Whether this host can turn a record into a picture at all. + /// + /// ServUO targets net48, so a Linux shard runs it under Mono, where + /// System.Drawing is a thin layer over **libgdiplus** — and §4.2 put + /// System.Drawing in the *decode* path, not merely the encode: Frame + /// writes ARGB1555 through a LockBits pointer. Without that library a Linux + /// shard cannot read a sprite at all, while clilocs and the atlas are unaffected + /// because neither touches pixels. + /// + /// It must never present as a stack trace or a 500. It is a named, actionable outcome + /// in the same family as the cliloc reader's `COMPRESSED`, and it is reported here — + /// on the *source gate*, the first call any import makes — so an operator learns it + /// while setting the shard up rather than from an empty bestiary weeks later. + /// + private static void WriteImaging(StringBuilder sb) + { + CheckImaging(); + + sb.Append(",\"imaging\":{\"ok\":").Append(_imagingOk ? "true" : "false"); + + if (!_imagingOk) + { + sb.Append(",\"code\":\"NO_IMAGING\",\"reason\":"); + BridgeJson.Text(sb, + "This shard host cannot render images — Mono's System.Drawing needs libgdiplus. " + + "Install it (apt-get install libgdiplus) and re-run the import. Cliloc and atlas " + + "import are unaffected. (" + _imagingReason + ")"); + } + + sb.Append('}'); + } + + private static void CheckImaging() + { + if (_imagingChecked) + return; + + _imagingChecked = true; + + try + { + TouchImaging(); + _imagingOk = true; + } + catch (Exception e) + { + // On a host with no libgdiplus this is a TypeInitializationException wrapping a + // DllNotFoundException, and it can surface as the method failing to JIT rather + // than as a throw from inside it — which is why the construction lives in its own + // method, so the failure is contained here instead of taking this class's + // static initialisation with it. + _imagingOk = false; + _imagingReason = e.GetType().Name + ": " + e.Message; + } + } + + private static void TouchImaging() + { + using (var bmp = new System.Drawing.Bitmap(1, 1)) + { + bmp.SetPixel(0, 0, System.Drawing.Color.Black); + } + } + + // ── the hash cache (§6) ────────────────────────────────────────────────────────────── + + private static string CachedHashFor(string path, long size, long mtime) + { + lock (_hashes) + { + CachedHash cached; + + if (_hashes.TryGetValue(path, out cached) + && cached.Size == size + && cached.MTime == mtime) + { + return cached.Sha256; + } + } + + return null; + } + + /// + /// Rehashes whatever the cache is missing, on its own thread. + /// + /// Deliberately **not** a job on the asset worker: hashing 343 MB takes seconds to + /// tens of seconds, and holding the single slot for that long would answer every + /// status poll `bridge.busy` for the whole pass — which is exactly the moment an + /// operator is watching the panel. It emits nothing and correlates with nothing; it + /// only fills the cache that the next `assets.sources` reads. + /// + private static void StartHashing() + { + lock (_sync) + { + if (_hashing) + return; + + _hashing = true; + + _hasher = new Thread(HashLoop) + { + Name = "BridgeAssetHash", + IsBackground = true + }; + + _hasher.Start(); + } + } + + private static void HashLoop() + { + try + { + for (int i = 0; i < SourceFiles.Length; i++) + { + string path = ResolvePath(SourceFiles[i]); + + if (path == null) + continue; + + long size, mtime; + + try + { + var info = new FileInfo(path); + size = info.Length; + mtime = ToUnixMs(info.LastWriteTimeUtc); + } + catch + { + continue; + } + + if (CachedHashFor(path, size, mtime) != null) + continue; + + string hash = HashFile(path); + + if (hash == null) + continue; + + lock (_hashes) + { + _hashes[path] = new CachedHash { Size = size, MTime = mtime, Sha256 = hash }; + } + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] asset hash pass: {0}: {1}", e.GetType().Name, e.Message); + } + finally + { + // Under _sync, matching StartHashing: cleared outside it, two passes could both + // pass the guard and hash the same 343 MB twice. + lock (_sync) + { + _hashing = false; + } + } + } + + private static string HashFile(string path) + { + try + { + using (var sha = SHA256.Create()) + using (var stream = new FileStream( + path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1 << 20)) + { + var buffer = new byte[1 << 20]; + int read; + + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + sha.TransformBlock(buffer, 0, read, null, 0); + + sha.TransformFinalBlock(buffer, 0, 0); + + return Hex(sha.Hash); + } + } + catch (Exception e) + { + Console.WriteLine("[Bridge] cannot hash {0}: {1}", path, e.Message); + return null; + } + } + + private static string Hex(byte[] bytes) + { + var sb = new StringBuilder(bytes.Length * 2); + + for (int i = 0; i < bytes.Length; i++) + sb.Append(bytes[i].ToString("x2", CultureInfo.InvariantCulture)); + + return sb.ToString(); + } + + // ── the paging envelope (§3.3) ─────────────────────────────────────────────────────── + + /// + /// **One envelope for every asset family**, defined here in phase 1 so that clilocs + /// (phase 2), the body catalogue (3), statics and land (5), deep animation keys (6) + /// and the ServUO tree files (7) all page the same way. They are otherwise five + /// chances to invent five slightly different shapes, and the website would have to + /// learn each one. + /// + /// The envelope a reply closes with: + /// + /// + /// "items": [ … ], + /// "more": true, // ask again with this cursor + /// "cursor": "s:4104", // opaque to everyone but the shard; absent when more:false + /// "cut": "budget" // budget | end | limit — WHY this page stopped + /// + /// + /// **The budget is bytes, and it is UTF-8 bytes.** Not item count, because the ceiling + /// this lives inside is the sidecar's inbound line cap; and not chars, because a + /// cliloc row is real text and a `StringBuilder`'s Length would undercount every + /// non-ASCII character in it. + /// + /// `cut` exists because "the page is short" has three different meanings and the + /// website must not have to guess which: the source ran out (`end`), the byte budget + /// was spent (`budget`), or the family stopped at its own limit (`limit`). Only the + /// first means the import is finished. + /// + /// **The first item is always admitted**, even if it alone exceeds the budget. + /// Otherwise an oversized item would make its family unable to make any progress at + /// all — it would be skipped for the budget on every page, forever. That is safe + /// precisely because the budget is set to half the sidecar's line cap + /// (), so one such item still fits the wire. + /// + public sealed class PageBuilder + { + private readonly StringBuilder _sb; + private readonly int _budget; + private int _bytes; + private int _count; + private string _cursor; + private string _cut = "end"; + + /// + /// Room kept back for the fields the envelope must still be able to write after + /// the last item — `more`, `cursor`, `cut` and the closing brace. Without it a + /// page could fill the budget exactly and then overrun it closing itself. + /// + private const int Reserve = 256; + + public PageBuilder(StringBuilder sb, string arrayName, int budget) + { + _sb = sb; + _budget = budget; + + sb.Append(",\"").Append(arrayName).Append("\":["); + + // The prefix is already written, and it counts: the cap the sidecar enforces is + // on the whole line, not on the array. + _bytes = Encoding.UTF8.GetByteCount(sb.ToString()); + } + + public int Count { get { return _count; } } + + /// + /// Adds one already-serialised item. is where the + /// family should resume if this turns out to be the last item on the page. + /// Returns false when the budget is spent — the caller stops, and `more` is true. + /// + public bool TryAdd(string item, string cursorAfter) + { + if (item == null) + return true; + + int cost = Encoding.UTF8.GetByteCount(item) + (_count > 0 ? 1 : 0); + + if (_count > 0 && _bytes + cost + Reserve > _budget) + { + _cut = "budget"; + return false; + } + + if (_count > 0) + _sb.Append(','); + + _sb.Append(item); + + _bytes += cost; + _count++; + _cursor = cursorAfter; + + return true; + } + + /// + /// Stops the page for a reason of the family's own — a per-request limit, say — + /// rather than because the budget ran out. + /// + public void Cut(string why) + { + _cut = why; + } + + public void Close() + { + bool more = _cut != "end"; + + _sb.Append(']'); + _sb.Bool("more", more); + _sb.Str("cut", _cut); + + if (more && _cursor != null) + _sb.Str("cursor", _cursor); + } + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs index 3d6c131..d09dc43 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeBoot.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeBoot.cs @@ -260,6 +260,7 @@ namespace Server.Custom.Bridge e.Mobile.SendMessage("Bridge: {0}", BridgeParticipation.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeWorld.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeOneShots.Status()); + e.Mobile.SendMessage("Bridge: {0}", BridgeAssets.Status()); break; } } diff --git a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs index dedf4bc..0f140f4 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeConfig.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeConfig.cs @@ -87,6 +87,16 @@ namespace Server.Custom.Bridge // morning. Those are different consents, and one switch cannot express both. public static bool EventsEnabled { get; private set; } + // ---- the asset plane (docs/link/v8.md §3, protocol 8) ---- + // + // Its own gate again, and for the same reason the event plane got one: enabling this is + // an operator consenting to the WEBSITE READING THEIR CLIENT FILES -- art, animations and + // the string table, off the host's disk, over the link. That is a different consent from + // publishing world state, and one switch cannot express both. Reads only: nothing on this + // plane writes anything, anywhere. + public static bool AssetsEnabled { get; private set; } + public static int AssetBatchBytes { get; private set; } + public static int LeaseMaxDurationSec { get; private set; } public static int LeaseGraceSec { get; private set; } @@ -144,6 +154,20 @@ namespace Server.Custom.Bridge Port = Config.Get("Bridge.Port", 7788); QueueCap = Config.Get("Bridge.QueueCap", 10000); + AssetsEnabled = Config.Get("Bridge.AssetsEnabled", true); + + // The largest reply this plane will build, in ENCODED bytes -- not items, because the + // ceiling it has to live inside is a byte ceiling. Clamped to half the sidecar's 1 MiB + // inbound line cap, and the halving is load-bearing rather than cautious: a page + // always admits its first item even when that item alone exceeds the budget (the + // alternative is an oversized item being skipped forever and its family never making + // progress), so the wire must still have room for one such overshoot. + AssetBatchBytes = Config.Get("Bridge.AssetBatchBytes", 512 * 1024); + if (AssetBatchBytes < 64 * 1024) + AssetBatchBytes = 64 * 1024; + if (AssetBatchBytes > 512 * 1024) + AssetBatchBytes = 512 * 1024; + StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30); DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60); EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300); diff --git a/tools/scaffolding/BridgeAssetProbe.cs b/tools/scaffolding/BridgeAssetProbe.cs index 6f0bfb9..7d42604 100644 --- a/tools/scaffolding/BridgeAssetProbe.cs +++ b/tools/scaffolding/BridgeAssetProbe.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading; using Server.Commands; +using Server.Custom.Bridge; using Ultima; @@ -475,76 +476,109 @@ namespace Server.Custom /// nothing downstream can tell. A "0 rows" outcome is the correct answer. /// /// So the sweep records the file type each body resolved to and whether that file - /// answered — and never a second opinion. + /// answered — and never a second opinion. ResolveAnimation is that rule as + /// code, and this sweep is now one of its callers rather than its own transcription. + /// + /// **Phase 1 added the validator to this sweep**, which phase 0 ran without one: + /// it reported "1,144 decoded, 0 faults" while the patched client's verdata entry for + /// body 34 pointed past verdata.mul's own end and the wolf rendered something else, + /// counted among those 1,144. REFUSED-BUT-DECODED is the cell that says so, and it is + /// the same cell the art sweeps have had since phase 0. /// private static void SectionBodies() { Head("bodies — Animations.GetAnimation, one direction, first frame"); int direction = Config.Get("Bridge.AssetProbeCreatureDirection", 1); - int decoded = 0, empty = 0, faulted = 0; + var tally = new Tally(); var byFileType = new int[8]; - var faults = new List(); + int unresolved = 0; - for (int body = 0; body < 2048; body++) + var indexes = new Dictionary(); + var readers = new Dictionary(); + var lengths = new Dictionary(); + + string verdataPath = Files.GetFilePath("verdata.mul"); + long verdataLength = BridgeAssetValidator.MulLength(verdataPath); + + try { - Checkpoint("bodies", body); - - int translated = body; - int fileType; - - try + for (int body = 0; body < 2048; body++) { - fileType = BodyConverter.Convert(ref translated); - } - catch (Exception e) - { - faulted++; - faults.Add("body " + body + " BodyConverter.Convert: " + e.GetType().Name + ": " + e.Message); - continue; - } + Checkpoint("bodies", body); - if (fileType >= 0 && fileType < byFileType.Length) - byFileType[fileType]++; + int fileType, at; + string reason; - try - { - int hue = 0; - var frames = Animations.GetAnimation(body, 0, direction, ref hue, false, true); - - if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null) + if (!BridgeAssetValidator.ResolveAnimation(body, 0, direction, out fileType, out at, out reason)) { - decoded++; - frames[0].Bitmap.Dispose(); + // The never-sweep-file-types rule's own outcome (§4.3): bodyconv sent this + // body to a file this client does not have, so we report nothing and ask + // no other file. Asking anim2 for gargoyle 666 returns a giant spider. + unresolved++; + continue; } - else - { - empty++; - } - } - catch (Exception e) - { - faulted++; - if (faults.Count < 40) - faults.Add("body " + body + " (fileType " + fileType + "): " + e.GetType().Name + ": " + e.Message); + if (fileType >= 0 && fileType < byFileType.Length) + byFileType[fileType]++; + + if (!indexes.ContainsKey(fileType)) + { + string dataPath = BridgeAssetValidator.AnimDataPath(fileType); + + indexes[fileType] = BridgeAssetValidator.OpenAnimIndex(fileType); + lengths[fileType] = BridgeAssetValidator.MulLength(dataPath); + readers[fileType] = new BridgeAssetValidator.RecordReader(dataPath, verdataPath); + } + + var index = indexes[fileType]; + var verdict = BridgeAssetValidator.CheckEntry( + index, at, lengths[fileType], verdataLength, out reason); + + // Only the entry has been judged so far. The record behind it is where the + // frame table and the unbounded run headers live. + if (verdict == BridgeAssetValidator.Verdict.Ok + && !readers[fileType].AnimationSane(index, at, 1, out reason)) + { + verdict = BridgeAssetValidator.Verdict.Refused; + } + + bool decoded = false; + string thrown = null; + + try + { + int hue = 0; + var frames = Animations.GetAnimation(body, 0, direction, ref hue, false, true); + + if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null) + { + decoded = true; + frames[0].Bitmap.Dispose(); + } + } + catch (Exception e) + { + thrown = e.GetType().Name + ": " + e.Message; + } + + Record(tally, verdict, reason, decoded, thrown, "body/" + body + "/a0"); + } + } + finally + { + foreach (var reader in readers.Values) + { + if (reader != null) + reader.Dispose(); } } Say("direction " + direction + " (creature default — §5.1)"); - Say(string.Format(" decoded {0} empty {1} FAULTED {2}", decoded, empty, faulted)); + Say(string.Format(" bodyconv resolves nowhere (correct — never swept): {0:N0}", unresolved)); Say(" by file type: " + string.Join(", ", FileTypeCounts(byFileType))); - - if (faults.Count > 0) - { - Say(""); - Say(" faults:"); - - foreach (var f in faults) - Say(" " + f); - } - Say(""); + tally.Report("bodies 0..2047, action 0, first frame"); } private static string[] FileTypeCounts(int[] byFileType) @@ -566,10 +600,16 @@ namespace Server.Custom /// The twelve (on stock 57.4) player-character bodies, each at direction 0 — head-on, /// because a character is a portrait and should look at you (§5.1). /// - /// Six of them are expected to report nothing on the legacy path: both human ghosts and - /// every gargoyle body are UOP-only. **That is the measurement, not a failure** — it is - /// what phase 4's UOP reader exists for, and a probe that flagged it red would teach an + /// Most of them are expected to have no art on the legacy path — the ghosts and every + /// gargoyle body are UOP-only. **That is the measurement, not a failure**: it is what + /// phase 4's UOP reader exists for, and a probe that flagged it red would teach an /// operator to ignore the panel. + /// + /// **What is a failure is the middle column.** Phase 0 ran this without the validator + /// and read the library's answer as the truth, which made it report six of twelve + /// decoding. Two of those six — the elf ghosts — have an index entry of `length 0` and + /// were returning whatever body was decoded immediately before them, at that body's + /// exact dimensions. Four of twelve have art on a stock client, not six. /// private static void SectionPlayers() { @@ -582,14 +622,42 @@ namespace Server.Custom } int direction = Config.Get("Bridge.AssetProbePlayerDirection", 0); - int decoded = 0, absent = 0; + int real = 0, absent = 0, wrong = 0; + + string verdataPath = Files.GetFilePath("verdata.mul"); + long verdataLength = BridgeAssetValidator.MulLength(verdataPath); foreach (var pb in _playerBodies) { Checkpoint("players", pb.Body); - int translated = pb.Body; - int fileType = BodyConverter.Convert(ref translated); + int fileType, at; + string reason; + bool resolved = BridgeAssetValidator.ResolveAnimation( + pb.Body, 0, direction, out fileType, out at, out reason); + + // What the validator says BEFORE the library is asked. This is the whole point of + // the section: phase 0 reported "6 of 12 decode" from the library's answer alone, + // and two of those six were the previous body's picture. + var verdict = BridgeAssetValidator.Verdict.Absent; + + if (resolved) + { + string dataPath = BridgeAssetValidator.AnimDataPath(fileType); + var index = BridgeAssetValidator.OpenAnimIndex(fileType); + long length = BridgeAssetValidator.MulLength(dataPath); + + using (var reader = new BridgeAssetValidator.RecordReader(dataPath, verdataPath)) + { + verdict = BridgeAssetValidator.CheckEntry(index, at, length, verdataLength, out reason); + + if (verdict == BridgeAssetValidator.Verdict.Ok + && !reader.AnimationSane(index, at, 1, out reason)) + { + verdict = BridgeAssetValidator.Verdict.Refused; + } + } + } string outcome; @@ -597,17 +665,32 @@ namespace Server.Custom { int hue = 0; var frames = Animations.GetAnimation(pb.Body, 0, direction, ref hue, false, true); + bool gotBitmap = frames != null && frames.Length > 0 + && frames[0] != null && frames[0].Bitmap != null; + string size = null; - if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null) + if (gotBitmap) { var bmp = frames[0].Bitmap; - outcome = "decoded " + bmp.Width + "x" + bmp.Height; + size = bmp.Width + "x" + bmp.Height; bmp.Dispose(); - decoded++; + } + + if (verdict == BridgeAssetValidator.Verdict.Ok && gotBitmap) + { + outcome = "art, " + size; + real++; + } + else if (gotBitmap) + { + // The elf ghosts land here on a stock client: index entry `length 0`, and + // a bitmap the exact size of whatever was decoded last. + outcome = "WRONG PICTURE " + size + " — " + reason; + wrong++; } else { - outcome = "no art on the legacy path (UOP-only — phase 4)"; + outcome = "no art on the legacy path (UOP-only — phase 4): " + reason; absent++; } } @@ -617,11 +700,12 @@ namespace Server.Custom } Say(string.Format(" {0,-10} {1,-14} body {2,-5} fileType {3,-3} {4}", - pb.Race, pb.Slot, pb.Body, fileType, outcome)); + pb.Race, pb.Slot, pb.Body, resolved ? fileType.ToString() : "-", outcome)); } Say(""); - Say(string.Format(" {0} decoded, {1} absent, of {2}", decoded, absent, _playerBodies.Count)); + Say(string.Format(" {0} with art, {1} absent, {2} WRONG PICTURES, of {3}", + real, absent, wrong, _playerBodies.Count)); Say(""); } @@ -978,369 +1062,4 @@ namespace Server.Custom to.SendMessage(text); } } - - /// - /// **Validate before calling** — the response the org lead chose for §4.2's residual risk, - /// prototyped here so phase 1 adopts it with measurements rather than on faith. - /// - /// The principle: `Ultima`'s decoders take their bounds from the file they are reading, so - /// the extractor must decide whether a record is worth handing over *before* handing it - /// over. Every check below is against the index entry and the record header — cheap, and - /// enough to turn an uncatchable corrupted-state exception into a skipped asset. - /// - /// It cannot be complete and does not claim to be. It closes the shapes that reading the - /// source showed are reachable; the probe's REFUSED-BUT-DECODED count is what says whether - /// the boundary is drawn in the right place. - /// - /// Promoted into the overlay in phase 1. - /// - public static class BridgeAssetValidator - { - public enum Verdict - { - /// Nothing at this id, and the index says so honestly. - Absent, - - /// The entry is self-consistent and inside its file. - Ok, - - /// The entry claims something the file cannot support. Do not decode it. - Refused - } - - /// Land tiles decode a fixed 44×44 diamond: 2 × (2+4+…+44) ushorts. - public const int LandRecordBytes = 2024; - - /// - /// A ceiling on decoded art dimensions. `LoadStatic` allocates - /// new Bitmap(width, height) straight from two ushorts in the record, so a - /// corrupt header asks for up to 65535×65535 — an 8 GB allocation, from a file. Real - /// art is a couple of hundred pixels at most. - /// - public const int MaxArtDimension = 1024; - - /// - /// Builds our own index over the same files, with the same constructor arguments - /// Art uses — including hasExtra: false, which is the whole reason the - /// art path is safe where the gump path is not (§4.1). - /// - public static FileIndex OpenArtIndex() - { - if (ArtDataPath() == null) - return null; - - return new FileIndex("Artidx.mul", "Art.mul", "artLegacyMUL.uop", 0x10000, 4, ".tga", 0x13FDC, false); - } - - /// - /// The file an art index entry's lookup is an offset **into** — which is not - /// art.mul on any current client. - /// - /// This cost a whole probe run to learn and it is the single most important thing - /// phase 1 must not get wrong. FileIndex's UOP constructor ends with a bare - /// MulPath = uopPath: **when artLegacyMUL.uop exists it wins outright**, - /// and art.mul / artidx.mul are never opened at all. A validator that - /// bounds offsets against art.mul while the index holds UOP offsets is not - /// merely approximate, it is nonsense — the first run of this probe refused 34,299 - /// perfectly good statics for "declaring 10533x2085" because it was reading UOP - /// offsets into the wrong file. - /// - /// So the resolution order here mirrors FileIndex's exactly, and anything that - /// needs the bytes behind an entry must ask this rather than assume. - /// - public static string ArtDataPath() - { - var uop = Files.GetFilePath("artlegacymul.uop"); - - if (uop != null) - return uop; - - return Files.GetFilePath("art.mul"); - } - - public static long MulLength(string path) - { - if (path == null) - return 0; - - try - { - return new FileInfo(path).Length; - } - catch - { - return 0; - } - } - - /// - /// Judges one index entry. - /// - /// The check FileIndex.Seek is missing is the last one: it tests - /// Stream.Length < e.lookup — that the record *starts* inside the file — and - /// never that it *ends* inside it. A record that begins two bytes before EOF and - /// declares a length of 4,000 passes, and stream.Read then returns a short count - /// that the decoders discard, leaving the previous asset's bytes in the shared buffer. - /// - public static Verdict CheckEntry(FileIndex index, int at, long mulLength, long verdataLength, out string reason) - { - reason = null; - - if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) - { - reason = "index " + at + " out of range"; - return Verdict.Absent; - } - - Entry3D e = index.Index[at]; - - if (e.lookup < 0) - { - reason = "lookup " + e.lookup; - return Verdict.Absent; - } - - bool patched = (e.length & (1 << 31)) != 0; - int length = e.length & 0x7FFFFFFF; - - if (!patched && e.length < 0) - { - reason = "length " + e.length; - return Verdict.Absent; - } - - if (length == 0) - { - reason = "lookup " + e.lookup + ", length 0"; - return Verdict.Absent; - } - - long ceiling = patched ? verdataLength : mulLength; - - if (ceiling <= 0) - { - reason = (patched ? "verdata.mul" : "the art data file") + " has no length"; - return Verdict.Refused; - } - - if (e.lookup >= ceiling) - { - reason = "lookup " + e.lookup + " past the end of " - + (patched ? "verdata.mul" : "the mul") + " (" + ceiling + ")"; - return Verdict.Refused; - } - - // The missing check. A short read is silent, and its consequence is the PREVIOUS - // asset's picture served under this id. - if (e.lookup + (long)length > ceiling) - { - reason = "record runs " + (e.lookup + (long)length - ceiling) + " bytes past the end of " - + (patched ? "verdata.mul" : "the mul"); - return Verdict.Refused; - } - - return Verdict.Ok; - } - - /// - /// `LoadLand` reads 2,024 bytes regardless of the declared length, so a shorter record - /// reads past the end of a buffer sized from that length. - /// - public static bool LandLengthSane(FileIndex index, int at, out string reason) - { - reason = null; - - if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) - return true; - - int length = index.Index[at].length & 0x7FFFFFFF; - - if (length > 0 && length < LandRecordBytes) - { - reason = "land record is " + length + " bytes; LoadLand always reads " + LandRecordBytes; - return false; - } - - return true; - } - - /// - /// Walks a static record's own row table the way LoadStatic will, and refuses - /// it if that walk would read outside the record. - /// - /// This is the check with teeth. LoadStatic's inner loop guards the write into - /// the bitmap (xOffset > delta, xOffset + xRun > delta) and does - /// nothing at all about the read cursor, which advances until it happens to find a - /// zero pair — potentially far outside a pinned array. Simulating the same walk with - /// a bound is the cheapest way to know whether handing the id over is safe. - /// - public static bool StaticRecordSane(byte[] record, int length, out string reason) - { - reason = null; - - if (length < 8) - { - reason = "record is " + length + " bytes; a static header needs 8"; - return false; - } - - int words = length / 2; - int width = ReadUInt16(record, 4); - int height = ReadUInt16(record, 6); - - // LoadStatic returns null for these rather than misbehaving, so it is not a refusal. - if (width <= 0 || height <= 0) - return true; - - if (width > MaxArtDimension || height > MaxArtDimension) - { - reason = "declares " + width + "x" + height + ", past the " + MaxArtDimension + "px ceiling"; - return false; - } - - // The row-lookup table: height ushorts starting at word 4. - if (4 + height > words) - { - reason = "row table (" + height + " entries) does not fit in a " + length + "-byte record"; - return false; - } - - int start = height + 4; - - for (int y = 0; y < height; y++) - { - int cursor = start + ReadUInt16(record, (4 + y) * 2); - - while (true) - { - // Two ushorts for the run header, and they must both be inside the record. - if (cursor < 0 || cursor + 1 >= words) - { - reason = "row " + y + " reads at word " + cursor + ", past the record's " + words; - return false; - } - - int xOffset = ReadUInt16(record, cursor * 2); - int xRun = ReadUInt16(record, (cursor + 1) * 2); - cursor += 2; - - if (xOffset + xRun == 0) - break; - - // LoadStatic stops the row here, so the read cursor stops with it. - if (xOffset > width || xOffset + xRun > width) - break; - - if (cursor + xRun > words) - { - reason = "row " + y + " declares a " + xRun + "-pixel run running past the record"; - return false; - } - - cursor += xRun; - } - } - - return true; - } - - private static int ReadUInt16(byte[] b, int at) - { - return b[at] | (b[at + 1] << 8); - } - - /// - /// Reads a record's actual bytes so can walk it. - /// - /// Holds its own handles rather than borrowing the library's, because FileIndex - /// hands out the stream it decodes from and moving that stream's position underneath - /// the decoder would be its own bug. Opened FileShare.ReadWrite to match how - /// FileIndex opens the same files. - /// - public sealed class RecordReader : IDisposable - { - private readonly FileStream _mul; - private readonly FileStream _verdata; - private byte[] _scratch = new byte[64 * 1024]; - - public RecordReader(string mulPath, string verdataPath) - { - _mul = Open(mulPath); - _verdata = Open(verdataPath); - } - - private static FileStream Open(string path) - { - if (path == null || !File.Exists(path)) - return null; - - try - { - return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - } - catch - { - return null; - } - } - - /// - /// True when the record at is safe to hand to - /// Art.GetStatic. A record that cannot be read at all is reported sane — - /// has already judged the entry, and this must not - /// invent a second reason to refuse. - /// - public bool StaticSane(FileIndex index, int at, out string reason) - { - reason = null; - - if (index == null || index.Index == null || at < 0 || at >= index.Index.Length) - return true; - - Entry3D e = index.Index[at]; - bool patched = (e.length & (1 << 31)) != 0; - int length = e.length & 0x7FFFFFFF; - - var stream = patched ? _verdata : _mul; - - if (stream == null || length <= 0 || e.lookup < 0) - return true; - - if (_scratch.Length < length) - _scratch = new byte[length]; - - int read; - - try - { - stream.Seek(e.lookup, SeekOrigin.Begin); - read = stream.Read(_scratch, 0, length); - } - catch (Exception ex) - { - reason = "cannot read the record: " + ex.GetType().Name; - return false; - } - - // The short read the decoders discard. Refusing here is the whole point: the - // library would decode whatever the shared buffer happened to hold. - if (read < length) - { - reason = "short read — " + read + " of " + length + " bytes available"; - return false; - } - - return StaticRecordSane(_scratch, length, out reason); - } - - public void Dispose() - { - if (_mul != null) - _mul.Dispose(); - - if (_verdata != null) - _verdata.Dispose(); - } - } - } }