From efbd45685ce6526c2b4a51ffc1e43e4af78d208b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 11 Sep 2026 04:55:11 -0500 Subject: [PATCH] feat(asset-bridge): the UOP animation reader, and 235 bodies the legacy path cannot see (Phase 4) ServUO's vendored `Ultima.Animations` reads legacy `anim*.mul` only -- it builds its five FileIndexes with the constructor that passes `uopFile: null` -- so everything a modern client moved into `AnimationFrame*.uop` is invisible to it. This adds the one reader docs/link/v8.md 4.3 reserved for phase 4, and wires it in as a fallback beneath the legacy path. What it actually recovers is not what the plan expected, and the difference was measured before any of this was written: - Of the EIGHT player-character bodies 4.8 assigned to this phase, only TWO are in the client at all: gargoyles 666 and 667, in AnimationFrame3.uop. The six ghost bodies (human 402/403, elf 607/608, gargoyle 694/695) are in no package. The five packages hold 10,724 entries between them and the `build/animationlegacyframe/%06d/%02d.bin` name scheme claims every one, so there is no other naming they could be hiding under. - The same fallback reaches 233 further bodies the catalogue had nothing for, so the working set goes from 787 to 1,022 (57 Monster, 26 Animal, 97 Equipment, 50 unlisted, 3 Human, 2 Sea). The catalogue was already 366 Equipment bodies before this, so its character does not change. Decided with the org lead before building: the fallback applies to every body rather than to player bodies alone; ghost ids leave the player-body set entirely (no client has art for any of them, and listing them only advertised keys that cannot exist); the UOP path gets its own PNG encoder rather than Bitmap.Save; and a host without libgdiplus keeps the flat NO_IMAGING refusal rather than serving a partial catalogue. Three things about the reader: - It is not the never-sweep rule being broken. That rule exists because a legacy index is addressed by POSITION, so asking the wrong file returns a giant spider that decodes cleanly. A UOP entry is addressed by the hash of a name containing the body id, and the payload declares that id again for `Group.TryOpen` to check, so a hit is proof of identity. Measured: no hash appears in two packages. - Validate-as-we-go replaces validate-before-calling, because here we ARE the library: the block chain is bounded against the file, the record against the file, the inflated length against the declared one, the frame table against the payload, and every run header against both the record's remaining bytes and the bitmap it writes into. Measured the way 4.5 was -- across every UOP body on a stock client it refuses nothing that carries art. The one body it refuses (286) declares a 0x0 frame, which the vendored decoder treats as absent too. - No System.Drawing anywhere in it, which is what 4.4 promised: the decode fills a ushort[] of ARGB1555 and BridgePng encodes that directly (zlib around net48's raw-deflate-only DeflateStream, CRC32, one IDAT, filter 0). EXTRACTOR_VERSION 1 -> 2: every client file is byte-identical and the answer is different, which is exactly what that number exists to say. The UOP packages join `assets.sources` and the catalogue id, so patching one is drift; `Ultima.Files` cannot resolve them (its table predates UOP animations) so BridgeUop.FindClientFile does it, case-insensitively by enumeration for Linux hosts. Manifest and fetch rows carry a new `source` field (`legacy` / `uop`). Protocol stays 8 -- no message shape changed, only fields added. Measured on the live rig (real sidecar, real ServUO, this machine's client): 1,022 rows in ONE page, 1,409 ms cold; six player bodies, all six with art for the first time (400/401/605/606 legacy, 666/667 uop), all at direction 0; 1,016 at direction 1; the six ghost bodies absent; 45 duplicate-hash groups of which exactly one is new, bodies 1531/1532, two distinct records whose first frames match -- legitimate, and provable only because each payload declares its own body id. The gargoyles were rendered and looked at, because 4.3's whole point is that this failure mode produces confident, wrong pictures. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016wDDVXWMDz82WqE1i969r4 --- overlay/Scripts/Custom/Bridge/BridgeAssets.cs | 24 +- .../Scripts/Custom/Bridge/BridgeCatalog.cs | 231 ++++- overlay/Scripts/Custom/Bridge/BridgePng.cs | 231 +++++ overlay/Scripts/Custom/Bridge/BridgeUop.cs | 823 ++++++++++++++++++ 4 files changed, 1281 insertions(+), 28 deletions(-) create mode 100644 overlay/Scripts/Custom/Bridge/BridgePng.cs create mode 100644 overlay/Scripts/Custom/Bridge/BridgeUop.cs diff --git a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs index d809aa6..9bde8cb 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeAssets.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeAssets.cs @@ -76,8 +76,14 @@ namespace Server.Custom.Bridge /// Bump it whenever extraction changes what it produces from unchanged input. It is /// the same rule spawnAtlasSource.js's `PARSER_VERSION` follows, and it applies /// here more rather than less: this pipeline derives far more from far less. + /// + /// **2** — phase 4 (§4.3, §4.9). The catalogue now falls back to + /// AnimationFrame*.uop for bodies the legacy path has nothing for, which on a + /// stock client is 235 new sprites and two of them player-character bodies; and the + /// player-body set no longer carries ghost ids. Every client file is byte-identical + /// and the answer is different, which is precisely what this number exists to say. /// - public const int EXTRACTOR_VERSION = 1; + public const int EXTRACTOR_VERSION = 2; // ── the one slot (§3.2) ────────────────────────────────────────────────────────────── @@ -325,6 +331,8 @@ namespace Server.Custom.Bridge "anim3.idx", "anim3.mul", "anim4.idx", "anim4.mul", "anim5.idx", "anim5.mul", + "animationframe1.uop", "animationframe2.uop", "animationframe3.uop", + "animationframe4.uop", "animationframe6.uop", "body.def", "bodyconv.def", "hues.mul", "verdata.mul" @@ -425,12 +433,22 @@ namespace Server.Custom.Bridge { try { - return Files.GetFilePath(name); + string path = Files.GetFilePath(name); + + if (path != null) + return path; } catch { - return null; + // Ultima's lookup reads the registry on Windows; a host where that throws still + // has the directories ServUO itself booted from, which is what the fallback uses. } + + // `Ultima.Files` has a fixed table of file names that predates UOP animations, so it + // answers null for every `AnimationFrame*.uop` however present they are (§4.3). Phase + // 4 added those to this list, so the fallback is what makes their size, mtime and hash + // reachable at all. + return BridgeUop.FindClientFile(name); } private static long ToUnixMs(DateTime utc) diff --git a/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs index 00b3ad0..20f19fd 100644 --- a/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs +++ b/overlay/Scripts/Custom/Bridge/BridgeCatalog.cs @@ -19,7 +19,9 @@ namespace Server.Custom.Bridge /// listing and a character sheet render. Everything deeper — every action, every frame — /// is the same addressing scheme at a deeper key, fetched on demand in a later phase; this /// is the set that is worth importing before anything asks for it, because on this - /// machine's client it is **787 sprites at about a kilobyte each**. + /// machine's client it is **1,022 sprites at about a kilobyte each** — 787 out of the + /// legacy `anim*.mul` files and, since phase 4, 235 more out of `AnimationFrame*.uop`, + /// which ServUO's vendored decoder never opens (§4.3, §4.9). /// /// Two request kinds, which are §6's two stages for assets rather than for sources: /// @@ -70,6 +72,17 @@ namespace Server.Custom.Bridge /// from the library's reused stream buffer. That is 357 of the 1,144 bodies the library /// claims on a stock client, and importing them would have written 357 duplicate /// portraits whose subject depended on the order this walk happened to run in. + /// + /// ── **The UOP fallback, and why it cannot reintroduce that** ── + /// + /// Phase 4 added beneath the legacy reader: a body the vendored + /// path has nothing for is looked for in the UOP packages before it is reported absent. + /// That is where two of the six player-character bodies live — `Bodyconv.def` sends + /// gargoyles 666 and 667 to `anim5`, at an index past the end of `anim5.idx` — and 233 + /// other bodies besides. It cannot produce a wrong picture the way a legacy sweep would, + /// because a UOP entry is addressed by the hash of a name that contains the body id and + /// the payload then declares that id again, which + /// checks. /// public static class BridgeCatalog { @@ -103,6 +116,15 @@ namespace Server.Custom.Bridge public byte[] Png; public int Width; public int Height; + + /// + /// Which reader produced it: `legacy` for ServUO's vendored Animations over + /// anim*.mul, `uop` for phase 4's own reader over + /// AnimationFrame*.uop (§4.3, §4.9). On the wire so that an operator + /// looking at a wrong picture can tell which half of the extractor to doubt, and + /// so the acceptance walk can prove the fallback fired at all. + /// + public string Source; } private sealed class Catalog @@ -243,6 +265,7 @@ namespace Server.Custom.Bridge item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"source\":\"").Append(sprite.Source).Append('"'); item.Append('}'); if (!page.TryAdd(item.ToString(), "b:" + body.ToString(CultureInfo.InvariantCulture))) @@ -446,6 +469,7 @@ namespace Server.Custom.Bridge item.Append(",\"height\":").Append(sprite.Height.ToString(CultureInfo.InvariantCulture)); item.Append(",\"body\":").Append(sprite.Body.ToString(CultureInfo.InvariantCulture)); item.Append(",\"direction\":").Append(sprite.Direction.ToString(CultureInfo.InvariantCulture)); + item.Append(",\"source\":\"").Append(sprite.Source).Append('"'); item.Append(",\"png\":\"").Append(Convert.ToBase64String(sprite.Png)).Append("\"}"); return item.ToString(); @@ -474,6 +498,34 @@ namespace Server.Custom.Bridge ? BridgeConfig.AssetPlayerDirection : BridgeConfig.AssetCreatureDirection; + // Legacy first, always. The vendored decoder is what 787 of this client's bodies come + // out of, it is what phase 3 measured, and the UOP packages hold a different and + // mostly disjoint set (measured: of the 244 bodies they carry, 8 also have legacy + // art). So this is a fallback rather than a choice, and no body changes reader while + // a client sits still. + Sprite sprite = ResolveLegacy(key, readers, body, direction) + ?? ResolveUop(key, readers, body, direction); + + if (sprite == null) + return null; + + lock (_sync) + { + if (!catalog.ByKey.ContainsKey(key)) + { + catalog.ByKey[key] = sprite; + catalog.Order.Add(sprite); + } + + return catalog.ByKey[key]; + } + } + + /// + /// ServUO's vendored Animations over anim*.mul, behind §4.5's validator. + /// + private static Sprite ResolveLegacy(string key, Readers readers, int body, int direction) + { int fileType, at; string reason; @@ -505,11 +557,9 @@ namespace Server.Custom.Bridge if (!reader.AnimationSane(index, at, 1, out reason)) return null; - Sprite sprite; - try { - sprite = Decode(key, body, direction, fileType); + return Decode(key, body, direction, fileType); } catch (Exception e) { @@ -517,20 +567,93 @@ namespace Server.Custom.Bridge body, e.GetType().Name, e.Message); return null; } + } - if (sprite == null) - return null; + /// + /// Phase 4's own reader over AnimationFrame*.uop (§4.3, §4.9), for the bodies + /// the legacy path has nothing for. + /// + /// On this machine's client that is **235 bodies** the catalogue could not reach + /// before, including the two gargoyle player bodies — `Bodyconv.def` sends 666 and 667 + /// to `anim5`, at an index past the end of `anim5.idx`, and the art has been in + /// `AnimationFrame3.uop` all along. + /// + /// Nothing here can produce §4.8's failure. A legacy index is addressed by position, + /// so a wrong lookup is another creature's picture; a UOP entry is addressed by the + /// hash of a name carrying the body id, and the payload repeats that id in its own + /// header for to check. A miss is a miss. + /// + private static Sprite ResolveUop(string key, Readers readers, int body, int direction) + { + ulong hash = BridgeUop.HashOf(body, CatalogAction); - lock (_sync) + byte[] payload = null; + string reason = null; + + foreach (int n in BridgeUop.Packages) { - if (!catalog.ByKey.ContainsKey(key)) + BridgeUop.Package package = readers.Package(n); + + if (package == null || !package.Has(hash)) + continue; + + if (!package.TryRead(hash, out payload, out reason)) { - catalog.ByKey[key] = sprite; - catalog.Order.Add(sprite); + Console.WriteLine("[Bridge] catalogue: body {0} in {1}: {2}", + body, BridgeUop.PackageName(n), reason); + return null; } - return catalog.ByKey[key]; + break; } + + if (payload == null) + return null; + + BridgeUop.Group group; + + if (!BridgeUop.Group.TryOpen(payload, body, out group, out reason)) + { + Console.WriteLine("[Bridge] catalogue: body {0} uop: {1}", body, reason); + return null; + } + + int frame = group.DirectionAt(direction); + + if (frame < 0) + return null; + + BridgeUop.Pixels pixels; + bool empty; + + if (!group.TryDecode(frame, out pixels, out empty, out reason)) + { + // A 0x0 frame is no art rather than damage — the vendored decoder returns early on + // exactly the same condition — so it is absent, silently. Anything else is a + // record this reader refused, and that is worth a line. + if (!empty) + Console.WriteLine("[Bridge] catalogue: body {0} uop: {1}", body, reason); + + return null; + } + + byte[] png = BridgePng.FromArgb1555(pixels.Argb1555, pixels.Width, pixels.Height); + + if (png == null) + return null; + + return new Sprite + { + Key = key, + Body = body, + Direction = direction, + FileType = 0, + Png = png, + Width = pixels.Width, + Height = pixels.Height, + Sha256 = Hash(png), + Source = "uop" + }; } private static Sprite Decode(string key, int body, int direction, int fileType) @@ -564,7 +687,8 @@ namespace Server.Custom.Bridge Png = png, Width = bitmap.Width, Height = bitmap.Height, - Sha256 = Hash(png) + Sha256 = Hash(png), + Source = "legacy" }; } @@ -674,21 +798,32 @@ namespace Server.Custom.Bridge /// /// Asked of the shard, never hardcoded. /// - /// Every registered race carries four body ids, and a shard that calls `RegisterRace` - /// adds ids no table of ours could contain. Even on stock ServUO a hardcoded list - /// would already be wrong in a way that is easy to miss: `RaceDefinitions.cs` passes - /// the gargoyle's ghost bodies in the OPPOSITE order to the other two races. + /// Every registered race carries its living male and female body ids, and a shard that + /// calls `RegisterRace` adds ids no table of ours could contain — which is the whole + /// argument against a hardcoded list, and it was never hypothetical: stock ServUO's + /// own `RaceDefinitions.cs` passes the gargoyle's ghost bodies in the OPPOSITE order + /// to the other two races. /// /// This set is the whole of what §5.1 gives direction 0 — head-on, facing the viewer, /// because a character is a portrait and should look at you. Everything else takes /// direction 1, the front three-quarter, because head-on is the least legible view of /// a four-legged creature: a wolf seen from the front is a dark blob. + /// + /// **Ghost bodies are deliberately not in it** (§5.2, decided 2026-09-10 in phase 4). + /// A race declares four ids and two of them are its ghosts, and no UO client has art + /// for any of them: 402/403 and 694/695 read `lookup -1` in `anim.idx`, 607/608 read + /// `length 0` — the §4.8 shape, where the library hands back the previously-decoded + /// body's picture — and none of the six is in any `AnimationFrame*.uop`, which phase 4 + /// established by claiming all 10,724 entries of the five packages with the one name + /// scheme. Listing them only advertised keys that cannot exist. A shard whose client + /// does ship ghost art still gets it: the body is catalogued like any other, at + /// direction 1 rather than 0. /// /// /// Cached for the life of the process: `RegisterRace` runs at Configure time, before /// anything on this plane can be asked a question, so the set cannot change under a /// running shard. Rebuilding it per body would enumerate every race 2,047 times per - /// scan to answer a question whose answer is twelve integers. + /// scan to answer a question whose answer is six integers. /// private static HashSet _playerBodies; @@ -710,8 +845,6 @@ namespace Server.Custom.Bridge set.Add(race.MaleBody); set.Add(race.FemaleBody); - set.Add(race.MaleGhostBody); - set.Add(race.FemaleGhostBody); } } catch (Exception e) @@ -829,12 +962,22 @@ namespace Server.Custom.Bridge .Append(':').Append(BridgeConfig.AssetPlayerDirection) .Append(':').Append(BridgeConfig.AssetCreatureDirection); + var paths = new List(); + for (int fileType = 1; fileType <= 5; fileType++) + paths.Add(BridgeAssetValidator.AnimDataPath(fileType)); + + // Since phase 4 the catalogue's bytes depend on the UOP packages too — 235 of its + // bodies come out of them — so patching one has to change the catalogue id, exactly as + // patching an anim*.mul does. Leaving them out would let an operator replace a + // gargoyle and have an Update find nothing to do. + foreach (int n in BridgeUop.Packages) + paths.Add(BridgeUop.PackagePath(n)); + + foreach (string path in paths) { sb.Append('|'); - string path = BridgeAssetValidator.AnimDataPath(fileType); - if (path == null) continue; @@ -883,10 +1026,11 @@ namespace Server.Custom.Bridge } /// - /// The five anim files' index and record readers, opened for one reply and closed with - /// it. Holding them across replies would keep handles on the operator's client files - /// for as long as the cache lives, for no gain: opening five is microseconds and a - /// page decodes hundreds of sprites through them. + /// The five anim files' index and record readers — and, since phase 4, the five UOP + /// packages beside them — opened for one reply and closed with it. Holding them across + /// replies would keep handles on the operator's client files for as long as the cache + /// lives, for no gain: opening them is microseconds and a page decodes hundreds of + /// sprites through them. /// private sealed class Readers : IDisposable { @@ -896,6 +1040,9 @@ namespace Server.Custom.Bridge private readonly long[] _length = new long[6]; private readonly bool[] _open = new bool[6]; + private readonly Dictionary _packages = + new Dictionary(); + public readonly long VerdataLength; public Readers() @@ -958,8 +1105,42 @@ namespace Server.Custom.Bridge return fileType >= 1 && fileType <= 5 ? _reader[fileType] : null; } + /// + /// One AnimationFrame*.uop, opened on first use. A package this client does + /// not ship is a null that is cached as one: the miss must not be re-resolved and + /// re-opened once per body across a 2,047-body walk. + /// + public BridgeUop.Package Package(int n) + { + BridgeUop.Package package; + + if (_packages.TryGetValue(n, out package)) + return package; + + package = BridgeUop.Package.Open(BridgeUop.PackagePath(n)); + + _packages[n] = package; + + return package; + } + public void Dispose() { + foreach (var package in _packages.Values) + { + if (package == null) + continue; + + try + { + package.Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + for (int i = 1; i <= 5; i++) { if (_reader[i] == null) diff --git a/overlay/Scripts/Custom/Bridge/BridgePng.cs b/overlay/Scripts/Custom/Bridge/BridgePng.cs new file mode 100644 index 0000000..47b89e1 --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgePng.cs @@ -0,0 +1,231 @@ +using System; +using System.IO; +using System.IO.Compression; + +namespace Server.Custom.Bridge +{ + /// + /// **A PNG encoder that does not go through GDI+** (docs/link/v8.md §4.4, §4.9 — phase 4). + /// + /// decodes into a ushort[] of ARGB1555 rather than into a + /// Bitmap, which is the whole point of §4.4's note that the UOP reader is written + /// without System.Drawing: libgdiplus was archived in March 2025, and every line of + /// extraction that does not depend on it is a line that survives its absence. That leaves + /// the encode, and Bitmap.Save(…, ImageFormat.Png) is GDI+ too — so this is the + /// other half. + /// + /// It is deliberately the smallest thing that produces a correct file: 8-bit RGBA, one + /// IDAT, filter type 0 on every row. No interlacing, no palette, no colour-type choice, no + /// filter heuristics. A sprite is a few hundred pixels across and the bytes go straight + /// into a base64 field; the compression difference between this and a tuned encoder is a + /// rounding error against the wire, and every knob not turned is a way this cannot be + /// subtly wrong. + /// + /// Phase 3's BridgeCatalog.ToPng is left exactly as it is. It is measured, shipped, + /// and its input really is a Bitmap from the vendored decoder — a path that needs + /// GDI+ to produce the pixels in the first place, so encoding them without it buys nothing. + /// + public static class BridgePng + { + private static readonly byte[] Signature = + { + 0x89, (byte)'P', (byte)'N', (byte)'G', 0x0D, 0x0A, 0x1A, 0x0A + }; + + private static readonly uint[] CrcTable = BuildCrcTable(); + + private static readonly byte[] Empty = new byte[0]; + + /// + /// ARGB1555 to an RGBA8 PNG with a transparent background. + /// + /// The expansion is the same one BridgeCatalog.ToPng documents and for the same + /// reason: alpha bit clear is fully transparent, and each 5-bit channel is widened by + /// repeating its high bits — (c << 3) | (c >> 2), not a plain shift, + /// which would cap white at 248 and tint every sprite. + /// + public static byte[] FromArgb1555(ushort[] pixels, int width, int height) + { + if (pixels == null || width <= 0 || height <= 0) + return null; + + if ((long)width * height > pixels.Length) + return null; + + // One filter byte per row, then RGBA per pixel. This is the PNG "raw" stream, the + // thing that gets deflated. Bounded by the caller's dimension ceiling + // (BridgeAssetValidator.MaxArtDimension), so the arithmetic cannot overflow an int — + // the check is here anyway, because that ceiling lives in another file. + long size = (((long)width * 4) + 1) * height; + + if (size > Int32.MaxValue / 2) + return null; + + var raw = new byte[size]; + + int at = 0; + + for (int y = 0; y < height; y++) + { + raw[at++] = 0; // filter: None + + int row = y * width; + + for (int x = 0; x < width; x++) + { + int p = pixels[row + x]; + + if ((p & 0x8000) == 0) + { + at += 4; // already zero: transparent black + continue; + } + + int r = (p >> 10) & 0x1F; + int g = (p >> 5) & 0x1F; + int b = p & 0x1F; + + raw[at++] = (byte)((r << 3) | (r >> 2)); + raw[at++] = (byte)((g << 3) | (g >> 2)); + raw[at++] = (byte)((b << 3) | (b >> 2)); + raw[at++] = 0xFF; + } + } + + using (var ms = new MemoryStream(raw.Length / 2)) + { + ms.Write(Signature, 0, Signature.Length); + + var header = new byte[13]; + + WriteBigEndian(header, 0, (uint)width); + WriteBigEndian(header, 4, (uint)height); + + header[8] = 8; // bit depth + header[9] = 6; // colour type: truecolour with alpha + header[10] = 0; // compression: deflate + header[11] = 0; // filter method 0 + header[12] = 0; // no interlace + + WriteChunk(ms, "IHDR", header, 0, header.Length); + + byte[] deflated = Zlib(raw); + + WriteChunk(ms, "IDAT", deflated, 0, deflated.Length); + WriteChunk(ms, "IEND", Empty, 0, 0); + + return ms.ToArray(); + } + } + + /// + /// A zlib stream around .NET Framework's raw-deflate-only DeflateStream: the + /// two-byte header PNG requires, the deflate data, and the adler32 trailer computed + /// here because nothing in the framework will do it. Written by hand for exactly the + /// same reason reads one by hand — net48 exposes deflate and + /// calls it zlib, and the two are not the same format. + /// + private static byte[] Zlib(byte[] data) + { + using (var ms = new MemoryStream(data.Length / 2)) + { + // CMF 0x78 (deflate, 32K window) and FLG 0x9C (default level, no dictionary): + // 0x789C is the pair whose value is divisible by 31, which is the check a decoder + // applies. + ms.WriteByte(0x78); + ms.WriteByte(0x9C); + + using (var deflate = new DeflateStream(ms, CompressionMode.Compress, true)) + deflate.Write(data, 0, data.Length); + + uint adler = Adler32(data); + + ms.WriteByte((byte)(adler >> 24)); + ms.WriteByte((byte)(adler >> 16)); + ms.WriteByte((byte)(adler >> 8)); + ms.WriteByte((byte)adler); + + return ms.ToArray(); + } + } + + private static void WriteChunk(Stream to, string type, byte[] data, int offset, int length) + { + var head = new byte[8]; + + WriteBigEndian(head, 0, (uint)length); + + head[4] = (byte)type[0]; + head[5] = (byte)type[1]; + head[6] = (byte)type[2]; + head[7] = (byte)type[3]; + + to.Write(head, 0, head.Length); + + if (length > 0) + to.Write(data, offset, length); + + // The CRC covers the type and the data, and not the length. + uint crc = Crc32(head, 4, 4, 0xFFFFFFFF); + + if (length > 0) + crc = Crc32(data, offset, length, crc); + + crc ^= 0xFFFFFFFF; + + var tail = new byte[4]; + + WriteBigEndian(tail, 0, crc); + + to.Write(tail, 0, tail.Length); + } + + private static void WriteBigEndian(byte[] into, int at, uint value) + { + into[at] = (byte)(value >> 24); + into[at + 1] = (byte)(value >> 16); + into[at + 2] = (byte)(value >> 8); + into[at + 3] = (byte)value; + } + + private static uint[] BuildCrcTable() + { + var table = new uint[256]; + + for (uint n = 0; n < 256; n++) + { + uint c = n; + + for (int k = 0; k < 8; k++) + c = (c & 1) != 0 ? 0xEDB88320 ^ (c >> 1) : c >> 1; + + table[n] = c; + } + + return table; + } + + private static uint Crc32(byte[] data, int offset, int length, uint crc) + { + for (int i = 0; i < length; i++) + crc = CrcTable[(crc ^ data[offset + i]) & 0xFF] ^ (crc >> 8); + + return crc; + } + + private static uint Adler32(byte[] data) + { + const uint Mod = 65521; + + uint a = 1, b = 0; + + for (int i = 0; i < data.Length; i++) + { + a = (a + data[i]) % Mod; + b = (b + a) % Mod; + } + + return (b << 16) | a; + } + } +} diff --git a/overlay/Scripts/Custom/Bridge/BridgeUop.cs b/overlay/Scripts/Custom/Bridge/BridgeUop.cs new file mode 100644 index 0000000..da7c41c --- /dev/null +++ b/overlay/Scripts/Custom/Bridge/BridgeUop.cs @@ -0,0 +1,823 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Text; + +using Ultima; + +namespace Server.Custom.Bridge +{ + /// + /// **The UOP animation reader** (docs/link/v8.md §4.3, §4.9 — protocol 8, phase 4): the + /// second and last decoder this protocol writes rather than calls. + /// + /// ServUO's vendored Ultima.Animations reads legacy anim*.mul only — it + /// constructs its five FileIndexes with the four-argument constructor, which passes + /// uopFile: null, so AnimationFrame*.uop is never opened. Everything a + /// modern client added there is invisible to it. This class opens those packages directly. + /// + /// ── **Why this is not the never-sweep rule being broken** ── + /// + /// §4.3's rule is that a body's file type comes from BodyConverter.Convert and is + /// never guessed, because asking another anim*.mul for an index it does not own + /// returns a decodable picture of something else — a giant spider on the gargoyle page. + /// That rule exists because a legacy index is addressed **by position**: nothing in the + /// file says which body a record belongs to. + /// + /// A UOP package is addressed by the **hash of a name that contains the body id** + /// (build/animationlegacyframe/000666/00.bin). Looking in all five packages for one + /// hash is therefore not a sweep — a hit is proof of identity, not a coincidence of + /// position, and the payload repeats the body id in its own header for us to check against. + /// Measured on this machine's client: 10,724 entries across the five packages, every one + /// of them claimed by that name scheme, and **no hash appears in more than one package**. + /// + /// ── **Validate as we go, because here we are the library** ── + /// + /// §4.5's rule is "validate before calling", and it exists because Ultima's decoders + /// take their bounds from the file they are reading. Nothing about this code can be + /// validated from outside — it *is* the decode — so the same discipline appears as a bound + /// on every read: the block chain against the file length, an entry's record against the + /// file, the inflated length against the declared one, the frame table against the + /// payload, and every run header against **both** the record's remaining bytes and the + /// bitmap it is writing into. A record that fails any of them is reported absent and no + /// pixel of it is kept. + /// + /// Measured the same way §4.5 was, which is the only measurement that says the boundary is + /// in the right place: across every UOP body on a stock client the walk refused **nothing** + /// that carries art, and the one body it does refuse (286) declares a 0×0 frame, which the + /// legacy decoder treats as absent too. + /// + /// ── **No System.Drawing, deliberately** ── + /// + /// §4.4 states it: libgdiplus was archived in March 2025, and the long-term argument for + /// moving extraction off System.Drawing is that a Linux shard depends on an + /// unmaintained library to see a sprite. This decoder writes ARGB1555 into a + /// ushort[] of its own and encodes that directly, so the + /// door stays open. (Phase 4 does not walk through it: the catalogue still refuses the + /// whole family when imaging is unavailable, because most of it genuinely needs GDI+.) + /// + public static class BridgeUop + { + /// 'MYP\0' — the Mythic package magic, as FileIndex reads it. + private const int PackageMagic = 0x50594D; + + /// 'AMOU' — the animation payload's own magic, little-endian. + private const int PayloadMagic = 0x554F4D41; + + /// Each frame record opens with its own palette: 0x100 ARGB1555 entries. + private const int PaletteBytes = 0x100 * 2; + + /// The frame table's row width: group, frame id, two unknowns, pixel offset. + private const int FrameRowBytes = 16; + + /// One block-chain record: offset, three lengths, hash, adler32, flag. + private const int BlockEntryBytes = 34; + + /// The xor Frame applies to every run header, and so must this. + private const int DoubleXor = (0x200 << 22) | (0x200 << 12); + + /// + /// A ceiling on a declared decompressed payload. One group file is a whole action for + /// one body across every direction; the largest on this machine's client is body + /// 1248's at 4.3 MB, so this is two orders of magnitude of headroom over real data and + /// still small enough that a corrupt length cannot ask for the host's memory. + /// + public const int MaxPayloadBytes = 64 * 1024 * 1024; + + /// + /// A ceiling on the block chain. Five packages hold 10,724 entries between them; this + /// bounds a cyclic or corrupt chain into a refusal rather than a hang. + /// + private const int MaxEntries = 1 << 20; + + /// The five packages this client ships. There is no AnimationFrame5.uop. + private static readonly int[] PackageNumbers = { 1, 2, 3, 4, 6 }; + + public static IEnumerable Packages + { + get { return PackageNumbers; } + } + + public static string PackageName(int n) + { + return "AnimationFrame" + n.ToString(CultureInfo.InvariantCulture) + ".uop"; + } + + /// + /// Where a UOP animation package lives. + /// + /// Ultima.Files.GetFilePath cannot answer this: its table of known client files + /// predates UOP animations and contains no AnimationFrame*.uop entry, so it + /// returns null for every one of them. So the lookup is done here, against the same + /// directories ServUO itself resolved at boot — Files.RootDir first, then + /// Core.DataDirectories, which §1 is built on. + /// + /// The comparison is case-insensitive **by enumeration** rather than by trying one + /// spelling. On Windows either would work; on a Linux shard host the client directory + /// is case-sensitive and the file may be shipped as `AnimationFrame1.uop`, + /// `animationframe1.uop` or anything between, which is exactly the shape of bug that + /// presents as "the gargoyles import on my machine and not on the server". + /// + /// is the general form, and `assets.sources` uses it for + /// the same reason: a file Ultima's table predates has to be found some other way. + /// + public static string PackagePath(int n) + { + return FindClientFile(PackageName(n)); + } + + private static readonly object _pathSync = new object(); + + private static readonly Dictionary _paths = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// + /// Finds a client file Ultima.Files has never heard of. + /// + /// Only successful answers are cached: a file an operator copies in while the shard is + /// up should be found by the next import, and nothing here is hot enough for a + /// negative cache to be worth that. + /// + public static string FindClientFile(string name) + { + if (String.IsNullOrEmpty(name)) + return null; + + lock (_pathSync) + { + string cached; + + if (_paths.TryGetValue(name, out cached)) + return cached; + } + + foreach (string dir in Directories()) + { + if (String.IsNullOrEmpty(dir)) + continue; + + try + { + if (!Directory.Exists(dir)) + continue; + + string direct = Path.Combine(dir, name); + string hit = File.Exists(direct) ? direct : null; + + if (hit == null) + { + foreach (string found in Directory.GetFiles(dir)) + { + if (String.Equals(Path.GetFileName(found), name, + StringComparison.OrdinalIgnoreCase)) + { + hit = found; + break; + } + } + } + + if (hit == null) + continue; + + lock (_pathSync) + _paths[name] = hit; + + return hit; + } + catch (Exception e) + { + Console.WriteLine("[Bridge] uop: cannot look in {0}: {1}", dir, e.Message); + } + } + + return null; + } + + private static IEnumerable Directories() + { + string root = null; + + try + { + root = Files.RootDir; + } + catch + { + // Ultima's static initialiser reads the registry on Windows. A host where that + // throws still has Core.DataDirectories, which is the path ServUO actually booted + // from. + } + + if (!String.IsNullOrEmpty(root)) + yield return root; + + List dirs = null; + + try + { + dirs = Core.DataDirectories; + } + catch + { + // Same reasoning; an empty list is a real answer and the caller reports absent. + } + + if (dirs == null) + yield break; + + foreach (string dir in dirs) + yield return dir; + } + + /// + /// The name a body's action file is stored under, hashed the way the container indexes + /// it. Ultima.FileIndex.HashFileName is pure arithmetic over a string — no file + /// is touched and no decoder is entered — so this is the one place phase 4 leans on + /// the vendored code, and it leans on it precisely so that our lookup cannot disagree + /// with the container's own. + /// + public static ulong HashOf(int body, int action) + { + string name = String.Format(CultureInfo.InvariantCulture, + "build/animationlegacyframe/{0:D6}/{1:D2}.bin", body, action); + + return FileIndex.HashFileName(name); + } + + // ── the container ──────────────────────────────────────────────────────────────────── + + private struct Entry + { + public long At; + public int CompressedLength; + public int DecompressedLength; + public short Flag; + } + + /// + /// One opened AnimationFrame*.uop: its entry table in memory, its bytes on + /// demand. Opening one is a single pass over the block chain — 10,724 entries across + /// all five on this client — and the handle is held for the life of a reply, exactly + /// like the legacy readers next to it. + /// + public sealed class Package : IDisposable + { + private readonly Dictionary _entries; + private readonly FileStream _stream; + + public readonly string Path; + + private Package(string path, FileStream stream, Dictionary entries) + { + Path = path; + _stream = stream; + _entries = entries; + } + + public int Count + { + get { return _entries.Count; } + } + + /// + /// Reads the block chain, refusing anything that does not fit inside the file. + /// Returns null — never throws — because a client that ships a truncated package + /// is an ordinary thing to survive, not an error to raise. + /// + public static Package Open(string path) + { + if (String.IsNullOrEmpty(path)) + return null; + + FileStream stream = null; + + try + { + stream = new FileStream(path, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + + long length = stream.Length; + + var entries = new Dictionary(); + + using (var br = new BinaryReader(stream, Encoding.UTF8, true)) + { + if (length < 28 || br.ReadInt32() != PackageMagic) + { + Console.WriteLine("[Bridge] uop: {0} is not a Mythic package", path); + stream.Dispose(); + return null; + } + + br.ReadInt32(); // version + br.ReadUInt32(); // signature + + long nextBlock = br.ReadInt64(); + + br.ReadInt32(); // block capacity + br.ReadInt32(); // declared file count + + while (nextBlock > 0) + { + // A block header is 12 bytes. Anything that does not leave room for + // one is a corrupt or cyclic chain, and this is where it stops. + if (nextBlock + 12 > length) + break; + + stream.Seek(nextBlock, SeekOrigin.Begin); + + int filesCount = br.ReadInt32(); + long following = br.ReadInt64(); + + if (filesCount < 0 + || nextBlock + 12 + ((long)filesCount * BlockEntryBytes) > length) + { + break; + } + + for (int i = 0; i < filesCount; i++) + { + long offset = br.ReadInt64(); + int headerLength = br.ReadInt32(); + int compressedLength = br.ReadInt32(); + int decompressedLength = br.ReadInt32(); + ulong hash = br.ReadUInt64(); + + br.ReadUInt32(); // adler32 + + short flag = br.ReadInt16(); + + if (offset <= 0 || headerLength < 0 || compressedLength <= 0) + continue; + + if (decompressedLength <= 0 || decompressedLength > MaxPayloadBytes) + continue; + + long at = offset + headerLength; + + // The check FileIndex.Seek is missing, in the place it matters + // here too: that the record ENDS inside the file, not merely that + // it starts inside it (§4.5). + if (at < 0 || at + compressedLength > length) + continue; + + if (entries.Count >= MaxEntries) + break; + + // First writer wins. Nothing on this client produces a collision + // — measured: no hash appears in two packages, and none twice in + // one — and if a patched client ever did, taking the first is the + // answer that does not depend on chain order. + if (!entries.ContainsKey(hash)) + entries[hash] = new Entry + { + At = at, + CompressedLength = compressedLength, + DecompressedLength = decompressedLength, + Flag = flag + }; + } + + if (following <= nextBlock) + break; // a chain that does not move forward is a loop + + nextBlock = following; + } + } + + return new Package(path, stream, entries); + } + catch (Exception e) + { + Console.WriteLine("[Bridge] uop: cannot open {0}: {1}: {2}", + path, e.GetType().Name, e.Message); + + if (stream != null) + { + try + { + stream.Dispose(); + } + catch + { + // Closing a read-only handle. + } + } + + return null; + } + } + + public bool Has(ulong hash) + { + return _entries.ContainsKey(hash); + } + + /// + /// The bytes behind one entry, decompressed. False with a reason is the ordinary + /// answer for "this package does not hold it". + /// + public bool TryRead(ulong hash, out byte[] payload, out string reason) + { + payload = null; + reason = null; + + Entry entry; + + if (!_entries.TryGetValue(hash, out entry)) + { + reason = "not in " + System.IO.Path.GetFileName(Path); + return false; + } + + byte[] raw; + + try + { + _stream.Seek(entry.At, SeekOrigin.Begin); + + raw = new byte[entry.CompressedLength]; + + if (!Fill(_stream, raw, raw.Length)) + { + // The §4.5 failure, in our own code this time: a short read that nobody + // checked is how the library ends up decoding the previous asset. + reason = "record is shorter than the index claims"; + return false; + } + } + catch (Exception e) + { + reason = "read failed: " + e.GetType().Name; + return false; + } + + if (entry.Flag != 1) + { + if (raw.Length != entry.DecompressedLength) + { + reason = "stored record is " + raw.Length + " bytes, not the declared " + + entry.DecompressedLength; + return false; + } + + payload = raw; + return true; + } + + return TryInflate(raw, entry.DecompressedLength, out payload, out reason); + } + + public void Dispose() + { + try + { + _stream.Dispose(); + } + catch + { + // Closing a read-only handle. Nothing useful is left to do. + } + } + } + + private static bool Fill(Stream stream, byte[] into, int count) + { + int read = 0; + + while (read < count) + { + int n = stream.Read(into, read, count - read); + + if (n <= 0) + return false; + + read += n; + } + + return true; + } + + /// + /// zlib, which .NET Framework 4.8 does not expose — only raw deflate. The two-byte + /// zlib header is checked and skipped rather than assumed, because handing a + /// DeflateStream a stream that is not deflate produces garbage as readily as an + /// exception, and the trailing adler32 is left to the length check below: a stream + /// that inflates to exactly the declared number of bytes did not silently truncate. + /// + private static bool TryInflate(byte[] raw, int declared, out byte[] payload, out string reason) + { + payload = null; + reason = null; + + if (raw.Length < 3) + { + reason = "compressed record is too short to be zlib"; + return false; + } + + int cmf = raw[0]; + int flg = raw[1]; + + if ((cmf & 0x0F) != 8 || (((cmf << 8) + flg) % 31) != 0 || (flg & 0x20) != 0) + { + reason = "compressed record is not a zlib stream"; + return false; + } + + try + { + var output = new byte[declared]; + + using (var source = new MemoryStream(raw, 2, raw.Length - 2, false)) + using (var inflate = new DeflateStream(source, CompressionMode.Decompress)) + { + int read = 0; + + while (read < declared) + { + int n = inflate.Read(output, read, declared - read); + + if (n <= 0) + break; + + read += n; + } + + if (read != declared) + { + reason = "inflated " + read + " bytes, not the declared " + declared; + return false; + } + + // One more byte would mean the record is longer than its own header says, + // which is a different file from the one we were promised. + if (inflate.ReadByte() != -1) + { + reason = "inflated past the declared " + declared + " bytes"; + return false; + } + } + + payload = output; + return true; + } + catch (Exception e) + { + reason = "inflate failed: " + e.GetType().Name; + return false; + } + } + + // ── the payload ────────────────────────────────────────────────────────────────────── + + /// One decoded frame: ARGB1555 in our own array, no Bitmap anywhere. + public sealed class Pixels + { + public int Width; + public int Height; + public int CenterX; + public int CenterY; + public ushort[] Argb1555; + } + + /// + /// One action of one body — every direction of it, concatenated. + /// + /// The legacy files address a frame as index + action * 5 + direction; a UOP + /// group file holds the whole action in one record and the directions are equal-length + /// runs inside its frame table. So is where "direction 1" is + /// turned into a frame number, and it is integer division exactly as the reference + /// implementations do it — see the note there for the nine bodies where that matters. + /// + public sealed class Group + { + private readonly byte[] _buf; + private readonly int _dataStart; + + public readonly int FrameCount; + public readonly int Body; + + private Group(byte[] buf, int body, int frameCount, int dataStart) + { + _buf = buf; + Body = body; + FrameCount = frameCount; + _dataStart = dataStart; + } + + public static bool TryOpen(byte[] buf, int expectedBody, out Group group, out string reason) + { + group = null; + reason = null; + + if (buf == null || buf.Length < 40) + { + reason = "payload is too short to carry a header"; + return false; + } + + if (BitConverter.ToInt32(buf, 0) != PayloadMagic) + { + reason = "payload is not an AMOU animation record"; + return false; + } + + int body = BitConverter.ToInt32(buf, 12); + + // The container said which body this is, by the name it was stored under; the + // payload says it again. They agree on every record of this client, and the day + // they do not is the day something is being read that was not asked for. + if (body != expectedBody) + { + reason = "payload declares body " + body + ", not " + expectedBody; + return false; + } + + int frameCount = BitConverter.ToInt32(buf, 32); + int dataStart = BitConverter.ToInt32(buf, 36); + + if (frameCount <= 0 || frameCount > BridgeAssetValidator.MaxAnimFrames) + { + reason = "payload declares " + frameCount + " frames"; + return false; + } + + if (dataStart < 40 || dataStart > buf.Length) + { + reason = "frame table starts at " + dataStart + " of " + buf.Length; + return false; + } + + if ((long)dataStart + ((long)frameCount * FrameRowBytes) > buf.Length) + { + reason = "frame table of " + frameCount + " rows runs past the record"; + return false; + } + + group = new Group(buf, body, frameCount, dataStart); + return true; + } + + /// + /// Which frame of this action faces a given direction. + /// + /// Five directions share the action's frames equally, so direction *d* starts at + /// d * (FrameCount / 5). On nine of this client's 244 UOP bodies the frame + /// count is **not** a multiple of five (41, 42, 46…), and integer division then + /// lands a direction or so early in the run. That is what ClassicUO does, it is + /// the right trade, and the reason is §4.8's: the failure being guarded against is + /// a picture of the **wrong creature**, and this cannot produce one — the worst + /// case is the right creature at a slightly different angle, on nine bodies, where + /// refusing them instead would lose nine creatures outright. + /// + public int DirectionAt(int direction) + { + int perDirection = FrameCount / 5; + + if (perDirection <= 0) + return direction == 0 ? 0 : -1; + + if (direction < 0 || direction > 4) + return -1; + + int at = direction * perDirection; + + return at < FrameCount ? at : -1; + } + + /// + /// Decodes one frame, bounding every read against the record and every write + /// against the bitmap. + /// + /// The run loop is Ultima.Frame's, with the two bounds it does not have. + /// Frame writes through a LockBits pointer whose origin comes from + /// two signed shorts in the file and never checks where a run lands; here a run + /// that would leave the bitmap, or read past the record, refuses the frame. Across + /// every UOP body on a stock client that refuses nothing that carries art. + /// + /// A 0×0 frame returns false with set: the legacy decoder + /// treats that as no art rather than as damage, and so must this, or body 286 + /// would be logged as a defect on every scan. + /// + public bool TryDecode(int index, out Pixels pixels, out bool empty, out string reason) + { + pixels = null; + empty = false; + reason = null; + + if (index < 0 || index >= FrameCount) + { + reason = "frame " + index + " of " + FrameCount; + return false; + } + + int row = _dataStart + (index * FrameRowBytes); + + long at = (long)row + (uint)BitConverter.ToInt32(_buf, row + 12); + + if (at < 0 || at + PaletteBytes + 8 > _buf.Length) + { + reason = "frame " + index + " points outside the record"; + return false; + } + + int pixelAt = (int)at; + + int centerX = BitConverter.ToInt16(_buf, pixelAt + PaletteBytes); + int centerY = BitConverter.ToInt16(_buf, pixelAt + PaletteBytes + 2); + int width = BitConverter.ToUInt16(_buf, pixelAt + PaletteBytes + 4); + int height = BitConverter.ToUInt16(_buf, pixelAt + PaletteBytes + 6); + + if (width <= 0 || height <= 0) + { + empty = true; + reason = "frame " + index + " is " + width + "x" + height; + return false; + } + + if (width > BridgeAssetValidator.MaxArtDimension + || height > BridgeAssetValidator.MaxArtDimension) + { + reason = "frame " + index + " declares " + width + "x" + height; + return false; + } + + var palette = new ushort[0x100]; + + for (int i = 0; i < palette.Length; i++) + { + // The library's own xor: the stored entry has its alpha bit clear and every + // palette colour is opaque. A pixel no run covers stays zero, which is how a + // sprite keeps its transparent background. + palette[i] = (ushort)(BitConverter.ToUInt16(_buf, pixelAt + (i * 2)) ^ 0x8000); + } + + var canvas = new ushort[width * height]; + + int p = pixelAt + PaletteBytes + 8; + + int xBase = centerX - 0x200; + int yBase = (centerY + height) - 0x200; + + while (true) + { + if (p + 4 > _buf.Length) + { + reason = "frame " + index + " has no terminator inside the record"; + return false; + } + + int header = BitConverter.ToInt32(_buf, p); + p += 4; + + if (header == 0x7FFF7FFF) + break; + + header ^= DoubleXor; + + int x = ((header >> 22) & 0x3FF) + xBase; + int y = ((header >> 12) & 0x3FF) + yBase; + int run = header & 0xFFF; + + if (run == 0) + continue; + + if (p + run > _buf.Length) + { + reason = "frame " + index + " has a run past the end of the record"; + return false; + } + + if (y < 0 || y >= height || x < 0 || x + run > width) + { + reason = "frame " + index + " has a run at " + x + "," + y + " of " + + run + " outside " + width + "x" + height; + return false; + } + + int cursor = (y * width) + x; + + for (int i = 0; i < run; i++) + canvas[cursor + i] = palette[_buf[p + i]]; + + p += run; + } + + pixels = new Pixels + { + Width = width, + Height = height, + CenterX = centerX, + CenterY = centerY, + Argb1555 = canvas + }; + + return true; + } + } + } +}