servuo-plugins/tools/scaffolding/BridgeAssetProbe.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Globalization;
using System.IO;
using System.Text;
using System.Threading;

using Server.Commands;

using Ultima;

namespace Server.Custom
{
    /// <summary>
    ///     **Asset Bridge phase 0 — the spike** (docs/link/v8.md §16).
    ///
    ///     §4 decided to call ServUO's own vendored <c>Ultima</c> decoders rather than reimplement
    ///     them. That decision rests on a probe run from **PowerShell** against a **stock** client,
    ///     and neither of those is the environment the extractor will actually live in. This runs
    ///     the same decoders from **inside a running ServUO**, against a **deliberately patched**
    ///     client, and its whole job is to find a fault on a path we call before eight phases are
    ///     built on top of one.
    ///
    ///     What "a fault" means here is wider than a crash, and the wider half is the dangerous
    ///     half. `Ultima`'s decoders take their bounds from the files they are reading, so a
    ///     malformed record does not usually throw — it produces **a confident, wrong picture**.
    ///     Four such shapes are known from reading the source and each has its own counter below:
    ///
    ///       * <c>LoadStatic</c> walks <c>bindata[count++]</c> with no bound on <c>count</c>. The
    ///         two guards in that loop bound the *write* into the bitmap and not the *read* out of
    ///         the record, so a record whose row table points outside itself reads adjacent heap.
    ///       * <c>stream.Read(m_StreamBuffer, 0, length)</c> ignores its return value, and
    ///         <c>m_StreamBuffer</c> is reused and only ever grown. A short read therefore decodes
    ///         **the previous asset's bytes** under this asset's id.
    ///       * <c>LoadLand</c> reads a fixed 2,024 bytes whatever <c>length</c> says.
    ///       * <c>Art.GetLegalItemID</c> returns **0** for an out-of-range id, so <c>GetStatic</c>
    ///         of a nonexistent id can hand back item 0's picture instead of nothing.
    ///
    ///     So every sweep here compares two answers: what <see cref="BridgeAssetValidator"/> says
    ///     about the index entry *before* the call, and what <c>Ultima</c> does *with* it. The
    ///     interesting cell is not the error count. It is **REFUSED-BUT-DECODED** — a record the
    ///     validator rejects and the library cheerfully returns a bitmap for. Those are the wrong
    ///     pictures, and they are invisible to any success count.
    ///
    ///     **Nothing here calls <c>Ultima.Gumps</c>**, which is a safety rule and not a preference
    ///     (v8.md §4.1) — with one deliberate exception, the opt-in <c>gump</c> section, whose
    ///     entire purpose is to reproduce the process-killing access violation from inside ServUO
    ///     so the rule has evidence behind it. It is off by default and it **takes the shard down**.
    ///
    ///     Test scaffolding. Never deployed; <c>deploy.ps1</c> copies only <c>overlay/</c>.
    ///     In game / from <c>BridgeRigDriver</c>: <c>[assetprobe [section] [stock|patched]</c>.
    ///     Flag: <c>AssetProbeOnStart</c>. Build the patched client with
    ///     <c>tools/patch_client.ps1</c>.
    /// </summary>
    public static class BridgeAssetProbe
    {
        // Where the run writes. The checkpoint is the point of the whole arrangement: some of
        // these faults are corrupted-state exceptions that no catch block sees, so the last id
        // written to disk is the only evidence of where the process died.
        private static readonly string OutputDir = Path.Combine(Core.BaseDirectory, "Logs", "AssetProbe");

        private static string _checkpointPath;
        private static StreamWriter _report;
        private static readonly object _sync = new object();
        private static bool _running;

        // Snapshot of the player-character body ids, taken on the Core thread (§5.2). Not
        // hardcoded: RaceDefinitions.cs passes the gargoyle's ghost bodies in the opposite order
        // to the other races, and a shard that calls RegisterRace adds ids no table of ours holds.
        private static List<PlayerBody> _playerBodies;

        private struct PlayerBody
        {
            public string Race;
            public string Slot;
            public int Body;

            public PlayerBody(string race, string slot, int body)
            {
                Race = race;
                Slot = slot;
                Body = body;
            }
        }

        public static void Initialize()
        {
            CommandSystem.Register("assetprobe", AccessLevel.Administrator, Probe_OnCommand);

            if (Config.Get("Bridge.AssetProbeOnStart", false))
                EventSink.ServerStarted += () => Timer.DelayCall(TimeSpan.FromSeconds(10.0), () => Begin(null, "all", null));
        }

        [Usage("assetprobe [all|paths|statics|land|bodies|players|cliloc|gump] [stock|patched]")]
        [Description("Drives ServUO's vendored Ultima decoders against a client and reports every fault and every wrong picture.")]
        private static void Probe_OnCommand(CommandEventArgs e)
        {
            var section = e.Arguments.Length > 0 ? e.Arguments[0].ToLowerInvariant() : "all";
            var which = e.Arguments.Length > 1 ? e.Arguments[1].ToLowerInvariant() : null;

            Begin(e.Mobile, section, which);
        }

        // ── Entry ────────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Reads what only the Core thread may read, then hands the sweep to a background
        ///     thread. That split is not tidiness — it is the §8 threading shape this protocol
        ///     introduces, rehearsed here: the decode must run OFF the Core thread (a 66,000-id
        ///     sweep would freeze the shard), and the world reads it depends on must run ON it.
        /// </summary>
        public static void Begin(Mobile from, string section, string which)
        {
            lock (_sync)
            {
                if (_running)
                {
                    Tell(from, "already running — one sweep at a time, so the checkpoint means something");
                    return;
                }

                _running = true;
            }

            // Core-thread reads first.
            _playerBodies = ReadPlayerBodies();

            var thread = new Thread(() => Run(from, section, which));
            thread.IsBackground = true;
            thread.Name = "BridgeAssetProbe";
            thread.Priority = ThreadPriority.BelowNormal;
            thread.Start();

            Tell(from, "started on a background thread — output in Logs/AssetProbe");
        }

        /// <summary>§5.2: ask the shard which bodies are player characters; never hardcode them.</summary>
        private static List<PlayerBody> ReadPlayerBodies()
        {
            var list = new List<PlayerBody>();

            foreach (var race in Race.AllRaces)
            {
                if (race == null)
                    continue;

                list.Add(new PlayerBody(race.Name, "male", race.MaleBody));
                list.Add(new PlayerBody(race.Name, "female", race.FemaleBody));
                list.Add(new PlayerBody(race.Name, "male ghost", race.MaleGhostBody));
                list.Add(new PlayerBody(race.Name, "female ghost", race.FemaleGhostBody));
            }

            return list;
        }

        private static void Run(Mobile from, string section, string which)
        {
            try
            {
                Directory.CreateDirectory(OutputDir);

                var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
                _checkpointPath = Path.Combine(OutputDir, "checkpoint.txt");
                _report = new StreamWriter(Path.Combine(OutputDir, "report-" + stamp + ".txt"), false, new UTF8Encoding(false));
                _report.AutoFlush = true;

                Say("Asset Bridge phase 0 probe — " + DateTime.UtcNow.ToString("u", CultureInfo.InvariantCulture));
                Say("section=" + section + " client=" + (which ?? "(config default)"));
                Say("");

                // Caching off, or a full sweep holds ~49,000 live Bitmaps. That is not only memory:
                // every Bitmap is a GDI object and Windows caps a process at 10,000 of them, so a
                // cached sweep fails partway through for a reason that has nothing to do with the
                // files being read. Each bitmap below is disposed as soon as it is measured.
                Files.CacheData = false;

                if (!PointAtClient(which))
                    return;

                bool all = section == "all";

                if (all || section == "paths") SectionPaths();
                if (all || section == "statics") SectionStatics();
                if (all || section == "land") SectionLand();
                if (all || section == "bodies") SectionBodies();
                if (all || section == "players") SectionPlayers();
                if (all || section == "cliloc") SectionCliloc();

                // Never part of "all". This one is expected to kill the process.
                if (section == "gump") SectionGump();

                Checkpoint("done", 0);
                Say("");
                Say("complete.");
            }
            catch (Exception e)
            {
                Say("PROBE FAILED: " + e);
            }
            finally
            {
                if (_report != null)
                {
                    _report.Dispose();
                    _report = null;
                }

                lock (_sync)
                    _running = false;
            }
        }

        // ── Pointing Ultima at a client ──────────────────────────────────────────────────────

        /// <summary>
        ///     Re-points <c>Ultima.Files</c> at the stock or the patched client and reloads every
        ///     index built from it.
        ///
        ///     **Not <c>Files.SetMulPath(string)</c>.** That overload keeps any entry already
        ///     holding an absolute path — its own comment reads <c>// absolut dir ignore</c> — and
        ///     it *writes* absolute paths. So it works exactly once: the second call, and every
        ///     call that would switch back, is a silent no-op, and the probe would report a run
        ///     against the patched client while reading the stock one. The two-argument overload
        ///     writes the key directly and is the only one that can be called twice.
        /// </summary>
        private static bool PointAtClient(string which)
        {
            var patched = Config.Get("Bridge.AssetProbeClient", (string)null);

            if (which == null)
                which = string.IsNullOrEmpty(patched) ? "stock" : "patched";

            string root;

            if (which == "patched")
            {
                if (string.IsNullOrEmpty(patched))
                {
                    Say("no Bridge.AssetProbeClient configured — build one with tools/patch_client.ps1");
                    return false;
                }

                root = patched;
            }
            else
            {
                // What the shard itself resolved at boot: the §1 premise, read rather than assumed.
                root = Core.DataDirectories.Count > 0 ? Core.DataDirectories[0] : Files.Directory;
            }

            if (string.IsNullOrEmpty(root) || !Directory.Exists(root))
            {
                Say("client directory does not exist: " + (root ?? "(null)"));
                return false;
            }

            foreach (var file in InterestingFiles)
            {
                var full = Path.Combine(root, file);
                // An absent file must resolve to nothing, NOT fall through to the stock client —
                // otherwise a patched tree missing a file quietly borrows the real one and the
                // run proves nothing.
                Files.SetMulPath(File.Exists(full) ? full : string.Empty, file);
            }

            Verdata.Initialize();
            Art.Reload();
            Animations.Reload();
            BodyConverter.Initialize();
            Hues.Initialize();

            Say("pointed at the " + which + " client: " + root);
            Say("verdata patches loaded: " + (Verdata.Patches == null ? 0 : Verdata.Patches.Length));
            Say("");

            return true;
        }

        /// <summary>
        ///     Every key this protocol reads, lowercase because that is how <c>Files.MulPath</c> is
        ///     keyed. Gump keys are absent on purpose (§4.1).
        /// </summary>
        private static readonly string[] InterestingFiles =
        {
            "anim.idx", "anim.mul", "anim2.idx", "anim2.mul", "anim3.idx", "anim3.mul",
            "anim4.idx", "anim4.mul", "anim5.idx", "anim5.mul",
            "art.mul", "artidx.mul", "artlegacymul.uop",
            "body.def", "bodyconv.def", "hues.mul", "verdata.mul", "cliloc.enu"
        };

        // ── paths ────────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     §1's premise, verified from inside the shard rather than argued: the client files
        ///     are already here, and the shard already knows where.
        /// </summary>
        private static void SectionPaths()
        {
            Head("paths");

            Say("Core.DataDirectories (" + Core.DataDirectories.Count + "):");

            foreach (var dir in Core.DataDirectories)
                Say("  " + dir);

            Say("Ultima.Files.Directory: " + (Files.Directory ?? "(null)"));
            Say("");

            foreach (var file in InterestingFiles)
            {
                var path = Files.GetFilePath(file);

                if (path == null)
                {
                    Say(string.Format("  {0,-22} ABSENT", file));
                    continue;
                }

                var info = new FileInfo(path);
                Say(string.Format(
                    "  {0,-22} {1,14:N0} bytes  {2}", file, info.Length,
                    info.LastWriteTimeUtc.ToString("u", CultureInfo.InvariantCulture)));
            }

            Say("");
        }

        // ── statics ──────────────────────────────────────────────────────────────────────────

        private static void SectionStatics()
        {
            Head("statics — Art.GetStatic");

            var index = BridgeAssetValidator.OpenArtIndex();

            if (index == null)
            {
                Say("no art index — artidx.mul/art.mul did not resolve");
                return;
            }

            var dataPath = BridgeAssetValidator.ArtDataPath();
            long mulLength = BridgeAssetValidator.MulLength(dataPath);
            long verdataLength = BridgeAssetValidator.MulLength(Files.GetFilePath("verdata.mul"));

            Say("  index offsets are into: " + dataPath);

            var tally = new Tally();
            int max = Config.Get("Bridge.AssetProbeMaxStatic", 0xFFFF);

            using (var reader = new BridgeAssetValidator.RecordReader(
                dataPath, Files.GetFilePath("verdata.mul")))
            {
                for (int id = 0; id <= max; id++)
                {
                    Checkpoint("statics", id);

                    // The validator's verdict, taken from the index entry BEFORE the library is
                    // asked. 0x4000 is the static offset Art applies internally.
                    string reason;
                    var verdict = BridgeAssetValidator.CheckEntry(index, id + 0x4000, mulLength, verdataLength, out reason);

                    // The entry can be well-formed and the record inside it still hostile, so a
                    // passing entry gets its row table walked before the library sees the id.
                    // This is the check with teeth — LoadStatic's read cursor is unbounded.
                    if (verdict == BridgeAssetValidator.Verdict.Ok)
                    {
                        string deepReason;

                        if (!reader.StaticSane(index, id + 0x4000, out deepReason))
                        {
                            verdict = BridgeAssetValidator.Verdict.Refused;
                            reason = deepReason;
                        }
                    }

                    // checkmaxid:false deliberately. With it true, GetLegalItemID maps an
                    // out-of-range id to 0 and the call returns ITEM 0's picture — the
                    // out-of-range answer would be a real bitmap of the wrong thing, which is
                    // precisely the confusion being counted.
                    Bitmap bmp = null;
                    string thrown = null;

                    try
                    {
                        bmp = Art.GetStatic(id, false);
                    }
                    catch (Exception e)
                    {
                        thrown = e.GetType().Name + ": " + e.Message;
                    }

                    Record(tally, verdict, reason, bmp != null, thrown, "static/" + id);

                    if (bmp != null)
                        bmp.Dispose();
                }
            }

            tally.Report("statics 0.." + max);
        }

        // ── land ─────────────────────────────────────────────────────────────────────────────

        private static void SectionLand()
        {
            Head("land — Art.GetLand");

            var index = BridgeAssetValidator.OpenArtIndex();

            if (index == null)
            {
                Say("no art index");
                return;
            }

            long mulLength = BridgeAssetValidator.MulLength(BridgeAssetValidator.ArtDataPath());
            long verdataLength = BridgeAssetValidator.MulLength(Files.GetFilePath("verdata.mul"));

            var tally = new Tally();

            for (int id = 0; id < 0x4000; id++)
            {
                Checkpoint("land", id);

                string reason;
                var verdict = BridgeAssetValidator.CheckEntry(index, id, mulLength, verdataLength, out reason);

                // LoadLand reads a fixed 2,024 bytes whatever the record says, so a short record
                // reads past the buffer. The validator's land rule is the only thing standing
                // between that and an out-of-bounds read.
                if (verdict == BridgeAssetValidator.Verdict.Ok)
                {
                    string landReason;

                    if (!BridgeAssetValidator.LandLengthSane(index, id, out landReason))
                    {
                        verdict = BridgeAssetValidator.Verdict.Refused;
                        reason = landReason;
                    }
                }

                Bitmap bmp = null;
                string thrown = null;

                try
                {
                    bmp = Art.GetLand(id);
                }
                catch (Exception e)
                {
                    thrown = e.GetType().Name + ": " + e.Message;
                }

                Record(tally, verdict, reason, bmp != null, thrown, "land/" + id);

                if (bmp != null)
                    bmp.Dispose();
            }

            tally.Report("land 0..16383");
        }

        // ── bodies ───────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Sweeps every body id, taking <c>BodyConverter.Convert</c>'s answer and stopping
        ///     there.
        ///
        ///     **It must never ask the other anim files when that answer yields nothing** (v8.md
        ///     §4.3). Doing so does not find missing art: gargoyle 666 maps to <c>anim5</c>, where
        ///     this client has nothing, and asking <c>anim2</c> for index 666 returns 175
        ///     decodable frames of a giant spider. Every one of those reads reports success, and
        ///     nothing downstream can tell. A "0 rows" outcome is the correct answer.
        ///
        ///     So the sweep records the file type each body resolved to and whether that file
        ///     answered — and never a second opinion.
        /// </summary>
        private static void SectionBodies()
        {
            Head("bodies — Animations.GetAnimation, one direction, first frame");

            int direction = Config.Get("Bridge.AssetProbeCreatureDirection", 1);
            int decoded = 0, empty = 0, faulted = 0;
            var byFileType = new int[8];
            var faults = new List<string>();

            for (int body = 0; body < 2048; body++)
            {
                Checkpoint("bodies", body);

                int translated = body;
                int fileType;

                try
                {
                    fileType = BodyConverter.Convert(ref translated);
                }
                catch (Exception e)
                {
                    faulted++;
                    faults.Add("body " + body + " BodyConverter.Convert: " + e.GetType().Name + ": " + e.Message);
                    continue;
                }

                if (fileType >= 0 && fileType < byFileType.Length)
                    byFileType[fileType]++;

                try
                {
                    int hue = 0;
                    var frames = Animations.GetAnimation(body, 0, direction, ref hue, false, true);

                    if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null)
                    {
                        decoded++;
                        frames[0].Bitmap.Dispose();
                    }
                    else
                    {
                        empty++;
                    }
                }
                catch (Exception e)
                {
                    faulted++;

                    if (faults.Count < 40)
                        faults.Add("body " + body + " (fileType " + fileType + "): " + e.GetType().Name + ": " + e.Message);
                }
            }

            Say("direction " + direction + " (creature default — §5.1)");
            Say(string.Format("  decoded {0}   empty {1}   FAULTED {2}", decoded, empty, faulted));
            Say("  by file type: " + string.Join(", ", FileTypeCounts(byFileType)));

            if (faults.Count > 0)
            {
                Say("");
                Say("  faults:");

                foreach (var f in faults)
                    Say("    " + f);
            }

            Say("");
        }

        private static string[] FileTypeCounts(int[] byFileType)
        {
            var parts = new List<string>();

            for (int i = 0; i < byFileType.Length; i++)
            {
                if (byFileType[i] > 0)
                    parts.Add(i + "=" + byFileType[i]);
            }

            return parts.ToArray();
        }

        // ── players ──────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     The twelve (on stock 57.4) player-character bodies, each at direction 0 — head-on,
        ///     because a character is a portrait and should look at you (§5.1).
        ///
        ///     Six of them are expected to report nothing on the legacy path: both human ghosts and
        ///     every gargoyle body are UOP-only. **That is the measurement, not a failure** — it is
        ///     what phase 4's UOP reader exists for, and a probe that flagged it red would teach an
        ///     operator to ignore the panel.
        /// </summary>
        private static void SectionPlayers()
        {
            Head("player bodies — Race.AllRaces, direction 0");

            if (_playerBodies == null || _playerBodies.Count == 0)
            {
                Say("no races registered (was the Core-thread snapshot taken?)");
                return;
            }

            int direction = Config.Get("Bridge.AssetProbePlayerDirection", 0);
            int decoded = 0, absent = 0;

            foreach (var pb in _playerBodies)
            {
                Checkpoint("players", pb.Body);

                int translated = pb.Body;
                int fileType = BodyConverter.Convert(ref translated);

                string outcome;

                try
                {
                    int hue = 0;
                    var frames = Animations.GetAnimation(pb.Body, 0, direction, ref hue, false, true);

                    if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null)
                    {
                        var bmp = frames[0].Bitmap;
                        outcome = "decoded " + bmp.Width + "x" + bmp.Height;
                        bmp.Dispose();
                        decoded++;
                    }
                    else
                    {
                        outcome = "no art on the legacy path (UOP-only — phase 4)";
                        absent++;
                    }
                }
                catch (Exception e)
                {
                    outcome = "FAULTED " + e.GetType().Name + ": " + e.Message;
                }

                Say(string.Format("  {0,-10} {1,-14} body {2,-5} fileType {3,-3} {4}",
                    pb.Race, pb.Slot, pb.Body, fileType, outcome));
            }

            Say("");
            Say(string.Format("  {0} decoded, {1} absent, of {2}", decoded, absent, _playerBodies.Count));
            Say("");
        }

        // ── cliloc ───────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Runs <see cref="BridgeMythicCliloc"/> against the client's own Cliloc.enu and, when
        ///     a reference is configured, diffs it against UOFiddler's output entry by entry.
        ///
        ///     The reference is what makes this a test rather than a demonstration. A decompressor
        ///     that is subtly wrong still produces a plausible table — mostly-right strings with a
        ///     few mangled ones is the expected shape of a bug in an inverse-BWT coder, and a row
        ///     count alone would pass it. Produce the reference with
        ///     <c>website/server/tools/cliloc-export --tsv</c>.
        /// </summary>
        private static void SectionCliloc()
        {
            Head("cliloc — the ported Mythic reader (§9)");

            var path = Files.GetFilePath("cliloc.enu");

            if (path == null)
            {
                Say("cliloc.enu did not resolve");
                return;
            }

            Checkpoint("cliloc", 0);

            var started = DateTime.UtcNow;

            List<BridgeMythicCliloc.Entry> entries;
            string warning, error;

            if (!BridgeMythicCliloc.TryLoadFile(path, out entries, out warning, out error))
            {
                Say("FAILED: " + error);
                return;
            }

            var elapsed = DateTime.UtcNow - started;

            int blank = 0;

            foreach (var entry in entries)
            {
                if (string.IsNullOrEmpty(entry.Text))
                    blank++;
            }

            Say(string.Format("  {0:N0} entries in {1:N0} ms ({2:N0} blank, {3:N0} would be stored)",
                entries.Count, elapsed.TotalMilliseconds, blank, entries.Count - blank));

            if (warning != null)
                Say("  WARNING: " + warning);

            var reference = Config.Get("Bridge.AssetProbeClilocRef", (string)null);

            if (string.IsNullOrEmpty(reference))
            {
                Say("  no Bridge.AssetProbeClilocRef set — row count only, which proves nothing about the strings");
                Say("");
                return;
            }

            CompareToReference(entries, reference);
        }

        /// <summary>
        ///     Diffs against a UOFiddler-produced tab-separated table. The comparison is
        ///     deliberately two-sided: an id we produced and it did not is as much a defect as a
        ///     mismatched string, and only checking the ids we happen to hold would hide a table
        ///     that stopped early.
        /// </summary>
        private static void CompareToReference(List<BridgeMythicCliloc.Entry> entries, string reference)
        {
            if (!File.Exists(reference))
            {
                Say("  reference not found: " + reference);
                return;
            }

            var theirs = new Dictionary<int, string>();

            foreach (var line in File.ReadAllLines(reference))
            {
                var tab = line.IndexOf('\t');

                if (tab <= 0)
                    continue;

                int number;

                if (!int.TryParse(line.Substring(0, tab), NumberStyles.Integer, CultureInfo.InvariantCulture, out number))
                    continue;

                theirs[number] = line.Substring(tab + 1);
            }

            var ours = new Dictionary<int, string>();

            foreach (var entry in entries)
                ours[entry.Number] = entry.Text;

            int matched = 0, differed = 0, onlyOurs = 0, onlyTheirs = 0;
            var examples = new List<string>();

            foreach (var pair in ours)
            {
                string theirText;

                if (!theirs.TryGetValue(pair.Key, out theirText))
                {
                    onlyOurs++;
                    continue;
                }

                // The reference is written by a tool that collapses tabs and newlines to spaces,
                // so compare on the same footing rather than reporting whitespace as a defect.
                if (Flatten(pair.Value) == theirText)
                {
                    matched++;
                }
                else
                {
                    differed++;

                    if (examples.Count < 10)
                    {
                        examples.Add("    #" + pair.Key
                                     + "\n      ours:  " + Truncate(Flatten(pair.Value))
                                     + "\n      theirs: " + Truncate(theirText));
                    }
                }
            }

            foreach (var key in theirs.Keys)
            {
                if (!ours.ContainsKey(key))
                    onlyTheirs++;
            }

            Say(string.Format("  vs UOFiddler: {0:N0} identical, {1:N0} differ, {2:N0} only ours, {3:N0} only theirs",
                matched, differed, onlyOurs, onlyTheirs));

            if (differed == 0 && onlyOurs == 0 && onlyTheirs == 0)
                Say("  IDENTICAL — the port reproduces UOFiddler's table exactly");

            foreach (var example in examples)
                Say(example);

            Say("");
        }

        private static string Flatten(string s)
        {
            return s.Replace('\t', ' ').Replace('\r', ' ').Replace('\n', ' ');
        }

        private static string Truncate(string s)
        {
            return s.Length <= 90 ? s : s.Substring(0, 90) + "…";
        }

        // ── gump: the deliberate crash ───────────────────────────────────────────────────────

        /// <summary>
        ///     Reproduces §4.1's access violation **from inside a running ServUO**, which is the
        ///     only place the claim actually matters. `Gumps` is the one decoder that builds its
        ///     `FileIndex` with <c>hasExtra: true</c>, and <c>FileIndex.cs</c>'s own comment says
        ///     that branch exists for <c>gumpartlegacy.uop</c>.
        ///
        ///     <c>AccessViolationException</c> is a corrupted-state exception and .NET Framework
        ///     4.8 does not deliver it to an ordinary catch, so **this takes the shard down** and
        ///     there is no in-process defence. That is the finding, and the reason "nothing calls
        ///     <c>Ultima.Gumps</c>" is a safety rule rather than a scoping preference. Never part
        ///     of "all"; never run on anything but a rig.
        /// </summary>
        private static void SectionGump()
        {
            Head("gump — DELIBERATE CRASH (§4.1)");
            Say("  This is expected to kill the process. Nothing in Protocol 8 calls Ultima.Gumps.");
            Say("  If the shard survives this section, that is itself the finding — record it.");

            Checkpoint("gump", 2);

            try
            {
                var bmp = Ultima.Gumps.GetGump(2);
                Say("  SURVIVED: GetGump(2) returned " + (bmp == null ? "null" : bmp.Width + "x" + bmp.Height));

                if (bmp != null)
                    bmp.Dispose();
            }
            catch (Exception e)
            {
                Say("  caught (so it was not a corrupted-state exception): " + e.GetType().Name + ": " + e.Message);
            }

            Say("");
        }

        // ── Tally ────────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     The four counts that matter, and one of them is the point of the whole probe.
        ///
        ///     <c>RefusedButDecoded</c> is a record the validator rejects and the library returned
        ///     a picture for anyway. On a stock client that number should be zero. On a patched
        ///     one it is the population of wrong pictures — the failure this protocol most needs
        ///     to avoid, because it raises no error anywhere and no success count can see it.
        /// </summary>
        private sealed class Tally
        {
            public int Ok;                 // validator passed, decoded
            public int Absent;             // validator says nothing there, library agreed
            public int AbsentButDecoded;   // NOTHING is there, and the library returned a picture
            public int Refused;            // validator refused, library also returned nothing
            public int RefusedButDecoded;  // validator refused, library returned a picture anyway
            public int OkButNothing;       // validator passed, library returned nothing
            public int Threw;              // the library threw
            public readonly List<string> Examples = new List<string>();

            public void Report(string label)
            {
                Say(string.Format("  {0}:", label));
                Say(string.Format("    ok {0:N0}   absent {1:N0}   refused {2:N0}   threw {3:N0}", Ok, Absent, Refused, Threw));
                Say(string.Format("    validator passed but nothing decoded: {0:N0}", OkButNothing));
                Say(string.Format("    WRONG PICTURES, empty record:  {0:N0}", AbsentButDecoded));
                Say(string.Format("    WRONG PICTURES, bad record:    {0:N0}", RefusedButDecoded));

                if (Examples.Count > 0)
                {
                    Say("    examples:");

                    foreach (var example in Examples)
                        Say("      " + example);
                }

                Say("");
            }
        }

        private static void Record(Tally tally, BridgeAssetValidator.Verdict verdict, string reason, bool decoded, string thrown, string key)
        {
            if (thrown != null)
            {
                tally.Threw++;

                if (tally.Examples.Count < 20)
                    tally.Examples.Add(key + " THREW " + thrown + (reason == null ? "" : " [validator: " + reason + "]"));

                return;
            }

            switch (verdict)
            {
                case BridgeAssetValidator.Verdict.Ok:
                    if (decoded)
                        tally.Ok++;
                    else
                        tally.OkButNothing++;

                    break;

                case BridgeAssetValidator.Verdict.Absent:
                    // An empty record that still yields a bitmap is not a disagreement about
                    // strictness. It is the shared-buffer defect: LoadStatic reuses
                    // m_StreamBuffer, only ever grows it, and discards stream.Read's return, so
                    // a zero-length record decodes whatever the PREVIOUS asset left behind.
                    if (decoded)
                    {
                        tally.AbsentButDecoded++;

                        if (tally.Examples.Count < 20)
                            tally.Examples.Add(key + " has no record (" + reason + ") — the library returned a picture");
                    }
                    else
                    {
                        tally.Absent++;
                    }

                    break;

                case BridgeAssetValidator.Verdict.Refused:
                    if (decoded)
                    {
                        tally.RefusedButDecoded++;

                        if (tally.Examples.Count < 20)
                            tally.Examples.Add(key + " REFUSED (" + reason + ") — library returned a picture anyway");
                    }
                    else
                    {
                        tally.Refused++;
                    }

                    break;
            }
        }

        // ── Output ───────────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Writes the id the probe is **about to** touch, then flushes.
        ///
        ///     Learned the expensive way from the PowerShell probes and it matters more here: an
        ///     access violation is not catchable and does not unwind, so nothing in this file runs
        ///     after one. The last line in this file is the only evidence of which id killed the
        ///     shard.
        /// </summary>
        private static void Checkpoint(string section, int id)
        {
            try
            {
                File.WriteAllText(_checkpointPath, section + " " + id + " @ "
                                                   + DateTime.UtcNow.ToString("u", CultureInfo.InvariantCulture) + Environment.NewLine);
            }
            catch
            {
                // A checkpoint that cannot be written must not stop the sweep.
            }
        }

        private static void Head(string title)
        {
            Say("── " + title + " " + new string('─', Math.Max(0, 70 - title.Length)));
        }

        private static void Say(string text)
        {
            Console.WriteLine("[assetprobe] {0}", text);

            var report = _report;

            if (report != null)
            {
                try
                {
                    report.WriteLine(text);
                }
                catch
                {
                    // Reporting must never be the thing that fails the run.
                }
            }
        }

        private static void Tell(Mobile to, string text)
        {
            Console.WriteLine("[assetprobe] {0}", text);

            if (to != null)
                to.SendMessage(text);
        }
    }

    /// <summary>
    ///     **Validate before calling** — the response the org lead chose for §4.2's residual risk,
    ///     prototyped here so phase 1 adopts it with measurements rather than on faith.
    ///
    ///     The principle: `Ultima`'s decoders take their bounds from the file they are reading, so
    ///     the extractor must decide whether a record is worth handing over *before* handing it
    ///     over. Every check below is against the index entry and the record header — cheap, and
    ///     enough to turn an uncatchable corrupted-state exception into a skipped asset.
    ///
    ///     It cannot be complete and does not claim to be. It closes the shapes that reading the
    ///     source showed are reachable; the probe's REFUSED-BUT-DECODED count is what says whether
    ///     the boundary is drawn in the right place.
    ///
    ///     Promoted into the overlay in phase 1.
    /// </summary>
    public static class BridgeAssetValidator
    {
        public enum Verdict
        {
            /// <summary>Nothing at this id, and the index says so honestly.</summary>
            Absent,

            /// <summary>The entry is self-consistent and inside its file.</summary>
            Ok,

            /// <summary>The entry claims something the file cannot support. Do not decode it.</summary>
            Refused
        }

        /// <summary>Land tiles decode a fixed 44×44 diamond: 2 × (2+4+…+44) ushorts.</summary>
        public const int LandRecordBytes = 2024;

        /// <summary>
        ///     A ceiling on decoded art dimensions. `LoadStatic` allocates
        ///     <c>new Bitmap(width, height)</c> straight from two ushorts in the record, so a
        ///     corrupt header asks for up to 65535×65535 — an 8 GB allocation, from a file. Real
        ///     art is a couple of hundred pixels at most.
        /// </summary>
        public const int MaxArtDimension = 1024;

        /// <summary>
        ///     Builds our own index over the same files, with the same constructor arguments
        ///     <c>Art</c> uses — including <c>hasExtra: false</c>, which is the whole reason the
        ///     art path is safe where the gump path is not (§4.1).
        /// </summary>
        public static FileIndex OpenArtIndex()
        {
            if (ArtDataPath() == null)
                return null;

            return new FileIndex("Artidx.mul", "Art.mul", "artLegacyMUL.uop", 0x10000, 4, ".tga", 0x13FDC, false);
        }

        /// <summary>
        ///     The file an art index entry's <c>lookup</c> is an offset **into** — which is not
        ///     <c>art.mul</c> on any current client.
        ///
        ///     This cost a whole probe run to learn and it is the single most important thing
        ///     phase 1 must not get wrong. <c>FileIndex</c>'s UOP constructor ends with a bare
        ///     <c>MulPath = uopPath</c>: **when <c>artLegacyMUL.uop</c> exists it wins outright**,
        ///     and <c>art.mul</c> / <c>artidx.mul</c> are never opened at all. A validator that
        ///     bounds offsets against <c>art.mul</c> while the index holds UOP offsets is not
        ///     merely approximate, it is nonsense — the first run of this probe refused 34,299
        ///     perfectly good statics for "declaring 10533x2085" because it was reading UOP
        ///     offsets into the wrong file.
        ///
        ///     So the resolution order here mirrors <c>FileIndex</c>'s exactly, and anything that
        ///     needs the bytes behind an entry must ask this rather than assume.
        /// </summary>
        public static string ArtDataPath()
        {
            var uop = Files.GetFilePath("artlegacymul.uop");

            if (uop != null)
                return uop;

            return Files.GetFilePath("art.mul");
        }

        public static long MulLength(string path)
        {
            if (path == null)
                return 0;

            try
            {
                return new FileInfo(path).Length;
            }
            catch
            {
                return 0;
            }
        }

        /// <summary>
        ///     Judges one index entry.
        ///
        ///     The check <c>FileIndex.Seek</c> is missing is the last one: it tests
        ///     <c>Stream.Length &lt; e.lookup</c> — that the record *starts* inside the file — and
        ///     never that it *ends* inside it. A record that begins two bytes before EOF and
        ///     declares a length of 4,000 passes, and <c>stream.Read</c> then returns a short count
        ///     that the decoders discard, leaving the previous asset's bytes in the shared buffer.
        /// </summary>
        public static Verdict CheckEntry(FileIndex index, int at, long mulLength, long verdataLength, out string reason)
        {
            reason = null;

            if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
            {
                reason = "index " + at + " out of range";
                return Verdict.Absent;
            }

            Entry3D e = index.Index[at];

            if (e.lookup < 0)
            {
                reason = "lookup " + e.lookup;
                return Verdict.Absent;
            }

            bool patched = (e.length & (1 << 31)) != 0;
            int length = e.length & 0x7FFFFFFF;

            if (!patched && e.length < 0)
            {
                reason = "length " + e.length;
                return Verdict.Absent;
            }

            if (length == 0)
            {
                reason = "lookup " + e.lookup + ", length 0";
                return Verdict.Absent;
            }

            long ceiling = patched ? verdataLength : mulLength;

            if (ceiling <= 0)
            {
                reason = (patched ? "verdata.mul" : "the art data file") + " has no length";
                return Verdict.Refused;
            }

            if (e.lookup >= ceiling)
            {
                reason = "lookup " + e.lookup + " past the end of "
                         + (patched ? "verdata.mul" : "the mul") + " (" + ceiling + ")";
                return Verdict.Refused;
            }

            // The missing check. A short read is silent, and its consequence is the PREVIOUS
            // asset's picture served under this id.
            if (e.lookup + (long)length > ceiling)
            {
                reason = "record runs " + (e.lookup + (long)length - ceiling) + " bytes past the end of "
                         + (patched ? "verdata.mul" : "the mul");
                return Verdict.Refused;
            }

            return Verdict.Ok;
        }

        /// <summary>
        ///     `LoadLand` reads 2,024 bytes regardless of the declared length, so a shorter record
        ///     reads past the end of a buffer sized from that length.
        /// </summary>
        public static bool LandLengthSane(FileIndex index, int at, out string reason)
        {
            reason = null;

            if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
                return true;

            int length = index.Index[at].length & 0x7FFFFFFF;

            if (length > 0 && length < LandRecordBytes)
            {
                reason = "land record is " + length + " bytes; LoadLand always reads " + LandRecordBytes;
                return false;
            }

            return true;
        }

        /// <summary>
        ///     Walks a static record's own row table the way <c>LoadStatic</c> will, and refuses
        ///     it if that walk would read outside the record.
        ///
        ///     This is the check with teeth. <c>LoadStatic</c>'s inner loop guards the write into
        ///     the bitmap (<c>xOffset &gt; delta</c>, <c>xOffset + xRun &gt; delta</c>) and does
        ///     nothing at all about the read cursor, which advances until it happens to find a
        ///     zero pair — potentially far outside a pinned array. Simulating the same walk with
        ///     a bound is the cheapest way to know whether handing the id over is safe.
        /// </summary>
        public static bool StaticRecordSane(byte[] record, int length, out string reason)
        {
            reason = null;

            if (length < 8)
            {
                reason = "record is " + length + " bytes; a static header needs 8";
                return false;
            }

            int words = length / 2;
            int width = ReadUInt16(record, 4);
            int height = ReadUInt16(record, 6);

            // LoadStatic returns null for these rather than misbehaving, so it is not a refusal.
            if (width <= 0 || height <= 0)
                return true;

            if (width > MaxArtDimension || height > MaxArtDimension)
            {
                reason = "declares " + width + "x" + height + ", past the " + MaxArtDimension + "px ceiling";
                return false;
            }

            // The row-lookup table: height ushorts starting at word 4.
            if (4 + height > words)
            {
                reason = "row table (" + height + " entries) does not fit in a " + length + "-byte record";
                return false;
            }

            int start = height + 4;

            for (int y = 0; y < height; y++)
            {
                int cursor = start + ReadUInt16(record, (4 + y) * 2);

                while (true)
                {
                    // Two ushorts for the run header, and they must both be inside the record.
                    if (cursor < 0 || cursor + 1 >= words)
                    {
                        reason = "row " + y + " reads at word " + cursor + ", past the record's " + words;
                        return false;
                    }

                    int xOffset = ReadUInt16(record, cursor * 2);
                    int xRun = ReadUInt16(record, (cursor + 1) * 2);
                    cursor += 2;

                    if (xOffset + xRun == 0)
                        break;

                    // LoadStatic stops the row here, so the read cursor stops with it.
                    if (xOffset > width || xOffset + xRun > width)
                        break;

                    if (cursor + xRun > words)
                    {
                        reason = "row " + y + " declares a " + xRun + "-pixel run running past the record";
                        return false;
                    }

                    cursor += xRun;
                }
            }

            return true;
        }

        private static int ReadUInt16(byte[] b, int at)
        {
            return b[at] | (b[at + 1] << 8);
        }

        /// <summary>
        ///     Reads a record's actual bytes so <see cref="StaticRecordSane"/> can walk it.
        ///
        ///     Holds its own handles rather than borrowing the library's, because <c>FileIndex</c>
        ///     hands out the stream it decodes from and moving that stream's position underneath
        ///     the decoder would be its own bug. Opened <c>FileShare.ReadWrite</c> to match how
        ///     <c>FileIndex</c> opens the same files.
        /// </summary>
        public sealed class RecordReader : IDisposable
        {
            private readonly FileStream _mul;
            private readonly FileStream _verdata;
            private byte[] _scratch = new byte[64 * 1024];

            public RecordReader(string mulPath, string verdataPath)
            {
                _mul = Open(mulPath);
                _verdata = Open(verdataPath);
            }

            private static FileStream Open(string path)
            {
                if (path == null || !File.Exists(path))
                    return null;

                try
                {
                    return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                }
                catch
                {
                    return null;
                }
            }

            /// <summary>
            ///     True when the record at <paramref name="at"/> is safe to hand to
            ///     <c>Art.GetStatic</c>. A record that cannot be read at all is reported sane —
            ///     <see cref="CheckEntry"/> has already judged the entry, and this must not
            ///     invent a second reason to refuse.
            /// </summary>
            public bool StaticSane(FileIndex index, int at, out string reason)
            {
                reason = null;

                if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
                    return true;

                Entry3D e = index.Index[at];
                bool patched = (e.length & (1 << 31)) != 0;
                int length = e.length & 0x7FFFFFFF;

                var stream = patched ? _verdata : _mul;

                if (stream == null || length <= 0 || e.lookup < 0)
                    return true;

                if (_scratch.Length < length)
                    _scratch = new byte[length];

                int read;

                try
                {
                    stream.Seek(e.lookup, SeekOrigin.Begin);
                    read = stream.Read(_scratch, 0, length);
                }
                catch (Exception ex)
                {
                    reason = "cannot read the record: " + ex.GetType().Name;
                    return false;
                }

                // The short read the decoders discard. Refusing here is the whole point: the
                // library would decode whatever the shared buffer happened to hold.
                if (read < length)
                {
                    reason = "short read — " + read + " of " + length + " bytes available";
                    return false;
                }

                return StaticRecordSane(_scratch, length, out reason);
            }

            public void Dispose()
            {
                if (_mul != null)
                    _mul.Dispose();

                if (_verdata != null)
                    _verdata.Dispose();
            }
        }
    }
}