servuo-plugins/overlay/Scripts/Custom/Bridge/BridgeAssets.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading;

using Ultima;

namespace Server.Custom.Bridge
{
    /// <summary>
    ///     **The Asset Bridge's transport** (docs/link/v8.md §3, §6, §7 — protocol 8, phase 1).
    ///
    ///     Everything else on this link answers on the Core thread, reads live world state, and
    ///     replies in microseconds. The asset plane cannot: it reads hundreds of megabytes of the
    ///     operator's client files and decodes pictures out of them, and doing either on the Core
    ///     thread would stop the world for every player on the shard. So this class is the one
    ///     genuinely new shape in protocol 8 — a handler that accepts on the Core thread, hands
    ///     the work to a **dedicated asset worker**, and returns immediately.
    ///
    ///     Three rules hold it together, and each of them is answering a specific way this could
    ///     go wrong.
    ///
    ///     **1. Replies, never events.** Every asset frame carries the caller's `reqId`, so
    ///     `rpc.rs`'s `try_route` consumes it before `app.rs` can persist it to SQLite and
    ///     broadcast it to every WebSocket subscriber. An asset stream on the event path would
    ///     grow the sidecar's store without bound and fan megabytes at every connected client,
    ///     forever. Nothing here emits an unsolicited frame — if a request has no `reqId` it is
    ///     refused rather than answered.
    ///
    ///     **2. One request outstanding, always.** <see cref="BridgeLink.Emit"/>'s queue is bounded
    ///     drop-oldest in **lines, not bytes** — a design that is right for live events and
    ///     dangerous for bulk transfer, because 10,000 queued 200 KB replies is 2 GB of shard
    ///     memory. The bound that actually holds is flow control, not a bigger queue: this plane
    ///     has **one slot**, and a second asset request arriving while one is in flight is
    ///     answered `bridge.busy` (which the sidecar already maps to 425) rather than queued.
    ///     Queue depth therefore stays at approximately one by construction. A dropped or lost
    ///     reply just times out and is re-requested, which is safe because reading a client file
    ///     is idempotent and touches no world state.
    ///
    ///     Note what that costs, deliberately: a status poll shares the slot with a batch, so
    ///     polling during a long import gets 425 until the batch lands. That is honest — this
    ///     plane really is doing one thing at a time — and the admin surface (phase 8) is where a
    ///     separate status lane would have to argue for itself.
    ///
    ///     **3. Byte budgets, not counts.** Batches are cut by encoded size
    ///     (<see cref="BridgeConfig.AssetBatchBytes"/>, 512 KB), not by item count, because the
    ///     ceilings this has to live inside are byte ceilings: the sidecar refuses an inbound line
    ///     over 1 MiB, and base64 costs 33% on top of whatever the payload measures.
    ///     <see cref="PageBuilder"/> is that budget, and every asset family shares it so the
    ///     envelope cannot drift apart between them.
    ///
    ///     **Emitting from off the Core thread is safe here, and it is worth saying why.**
    ///     <c>BridgeLink.Emit</c> enqueues onto a <c>ConcurrentQueue</c> and never touches the
    ///     socket, so the enqueue itself is fine. The subtle part is
    ///     <c>BridgeIdempotency.Observe</c>, which <c>Emit</c> calls while a keyed command is in
    ///     flight: it captures a line only when that line's correlation field **exactly equals**
    ///     the open command's correlation value, and correlation values come from one monotonic
    ///     counter in the sidecar. An asset reply therefore cannot be mistaken for a keyed
    ///     command's reply, whatever the interleaving.
    /// </summary>
    public static class BridgeAssets
    {
        /// <summary>
        ///     What version of *our derivation* produced these bytes (§7).
        ///
        ///     The source gate hashes the operator's client files, which answers "did the inputs
        ///     change". It cannot answer "did the way we read them change" — and that is the case
        ///     that bites, because a corrected frame offset or a fixed hue application changes
        ///     every derived byte while every source file stays identical. So this is folded into
        ///     stage 1 alongside the hashes, and bumping it makes the whole working set drift,
        ///     which is the intended and correct blast radius.
        ///
        ///     Bump it whenever extraction changes what it produces from unchanged input. It is
        ///     the same rule <c>spawnAtlasSource.js</c>'s `PARSER_VERSION` follows, and it applies
        ///     here more rather than less: this pipeline derives far more from far less.
        /// </summary>
        public const int EXTRACTOR_VERSION = 1;

        // ── the one slot (§3.2) ──────────────────────────────────────────────────────────────

        private static readonly object _sync = new object();
        private static Thread _worker;
        private static readonly AutoResetEvent _wake = new AutoResetEvent(false);
        private static Action _job;
        private static string _inFlight;
        private static DateTime _inFlightSince;
        private static bool _running;

        private static long _served, _busied, _failed;

        // ── the hash cache (§6) ──────────────────────────────────────────────────────────────

        private static readonly Dictionary<string, CachedHash> _hashes =
            new Dictionary<string, CachedHash>(StringComparer.OrdinalIgnoreCase);

        private static Thread _hasher;
        private static volatile bool _hashing;

        private sealed class CachedHash
        {
            public long Size;
            public long MTime;
            public string Sha256;
        }

        // ── imaging (§4.4) ───────────────────────────────────────────────────────────────────

        private static bool _imagingChecked;
        private static bool _imagingOk;
        private static string _imagingReason;

        public static void Initialize()
        {
            if (!BridgeConfig.Enabled)
                return;

            BridgeBoot.RegisterHandler("assets.sources", OnSources);
        }

        public static string Status()
        {
            int cached;

            lock (_hashes)
            {
                cached = _hashes.Count;
            }

            lock (_sync)
            {
                return String.Format(
                    "assets(served={0} busied={1} failed={2} inFlight={3} hashing={4} cached={5})",
                    _served, _busied, _failed, _inFlight ?? "-", _hashing, cached);
            }
        }

        // ── the request plane ────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Stage 1 of §6: what the shard's client files currently are. No pixels, no assets —
        ///     just the gate that lets the website decide whether anything needs importing at all,
        ///     because the normal case is a restart that changed nothing and it must cost nothing.
        /// </summary>
        private static void OnSources(Dictionary<string, object> o)
        {
            var reqId = BridgeJson.GetString(o, "reqId");

            if (reqId == null)
            {
                // Rule 1. Without a correlation id this would land on the event path, be persisted
                // to the sidecar's store and broadcast to every subscriber. Refuse instead.
                Fail(null, "assets.sources requires a reqId");
                return;
            }

            if (!BridgeConfig.AssetsEnabled)
            {
                Fail(reqId, "asset extraction is disabled on this shard");
                return;
            }

            Accept(reqId, "assets.sources", () => ReplySources(reqId));
        }

        /// <summary>
        ///     Claims the single slot and hands the work to the worker, or answers `bridge.busy`.
        ///     Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on
        ///     the worker and must touch no world state.
        /// </summary>
        private static void Accept(string reqId, string kind, Action job)
        {
            lock (_sync)
            {
                if (_inFlight != null)
                {
                    _busied++;
                    Busy(reqId, kind);
                    return;
                }

                _inFlight = kind;
                _inFlightSince = DateTime.UtcNow;
                _job = job;

                try
                {
                    EnsureWorker();
                }
                catch (Exception e)
                {
                    // The slot is claimed and there is now nothing that will ever free it. Give it
                    // back here or this plane answers `bridge.busy` for the life of the process.
                    _inFlight = null;
                    _job = null;

                    Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message);
                    Fail(reqId, "the shard could not start its asset worker");
                    return;
                }
            }

            _wake.Set();
        }

        private static void Busy(string reqId, string kind)
        {
            var held = (DateTime.UtcNow - _inFlightSince).TotalSeconds;

            var sb = BridgeJson.Begin("bridge.busy");

            sb.Str("reqId", reqId)
              // `busyKind`, never a second `kind` — `Begin` has already written this frame's own,
              // and a JSON object carrying two makes every parser take the last. Protocol 6 shipped
              // that bug once and it made the sidecar answer 200 for a refusal.
              .Str("busyKind", kind)
              .Num("heldForSec", (long)held)
              .Str("reason", "the asset plane serves one request at a time");

            BridgeLink.Emit(sb.End());
        }

        private static void Fail(string reqId, string reason)
        {
            var sb = BridgeJson.Begin("assets.error");

            if (reqId != null)
                sb.Str("reqId", reqId);

            sb.Str("reason", reason);
            BridgeLink.Emit(sb.End());
        }

        // ── the worker ───────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Started on first use rather than at boot, so a shard that never imports an asset
        ///     never carries the thread. Caller must hold <see cref="_sync"/>.
        /// </summary>
        private static void EnsureWorker()
        {
            if (_worker != null)
                return;

            _running = true;

            _worker = new Thread(WorkLoop)
            {
                Name = "BridgeAssets",
                IsBackground = true
            };

            _worker.Start();
        }

        private static void WorkLoop()
        {
            while (_running)
            {
                _wake.WaitOne(1000);

                Action job;

                lock (_sync)
                {
                    job = _job;
                    _job = null;
                }

                if (job == null)
                    continue;

                try
                {
                    job();
                    Interlocked.Increment(ref _served);
                }
                catch (Exception e)
                {
                    // A handler that throws must still free the slot, or this plane is wedged for
                    // the life of the process and every later request answers `bridge.busy`.
                    Interlocked.Increment(ref _failed);
                    Console.WriteLine("[Bridge] asset worker: {0}: {1}", e.GetType().Name, e.Message);
                }
                finally
                {
                    lock (_sync)
                    {
                        _inFlight = null;
                    }
                }
            }
        }

        // ── assets.sources ───────────────────────────────────────────────────────────────────

        /// <summary>
        ///     The client files whose bytes decide everything downstream.
        ///
        ///     Resolved through <c>Ultima.Files</c> rather than by joining a configured directory,
        ///     because that is what the decoders themselves do — a file this reports is a file
        ///     they would actually open.
        /// </summary>
        private static readonly string[] SourceFiles =
        {
            "cliloc.enu",
            "artlegacymul.uop", "art.mul", "artidx.mul",
            "anim.idx", "anim.mul",
            "anim2.idx", "anim2.mul",
            "anim3.idx", "anim3.mul",
            "anim4.idx", "anim4.mul",
            "anim5.idx", "anim5.mul",
            "body.def", "bodyconv.def",
            "hues.mul",
            "verdata.mul"
        };

        private static void ReplySources(string reqId)
        {
            var sb = BridgeJson.Begin("assets.sources.ok");

            sb.Str("reqId", reqId)
              .Num("extractorVersion", EXTRACTOR_VERSION);

            WriteImaging(sb);

            // §4.6: whichever of art.mul / artLegacyMUL.uop `FileIndex` would actually open. An
            // operator who added custom graphics to art.mul while the UOP is present is getting
            // nothing, silently, and this is the only place that can tell them so.
            string artData = BridgeAssetValidator.ArtDataPath();

            sb.Str("artDataFile", artData == null ? null : Path.GetFileName(artData));

            var page = new PageBuilder(sb, "files", BridgeConfig.AssetBatchBytes);
            bool anyMissingHash = false;

            for (int i = 0; i < SourceFiles.Length; i++)
            {
                string name = SourceFiles[i];
                string path = ResolvePath(name);

                if (path == null)
                    continue;

                var item = new StringBuilder(256);

                item.Append("{\"name\":");
                BridgeJson.Text(item, name);
                item.Append(",\"path\":");
                BridgeJson.Text(item, path);

                long size = 0, mtime = 0;

                try
                {
                    var info = new FileInfo(path);
                    size = info.Length;
                    mtime = ToUnixMs(info.LastWriteTimeUtc);
                }
                catch (Exception e)
                {
                    item.Append(",\"unreadable\":");
                    BridgeJson.Text(item, e.GetType().Name);
                }

                item.Append(",\"size\":").Append(size.ToString(CultureInfo.InvariantCulture));
                item.Append(",\"mtime\":").Append(mtime.ToString(CultureInfo.InvariantCulture));

                string hash = CachedHashFor(path, size, mtime);

                if (hash == null)
                    anyMissingHash = true;

                item.Append(",\"sha256\":");
                BridgeJson.Text(item, hash);

                // The one diagnostic §4.6 asks for: art.mul is present, and unread.
                if (artData != null
                    && (name == "art.mul" || name == "artidx.mul")
                    && !artData.EndsWith(".mul", StringComparison.OrdinalIgnoreCase))
                {
                    item.Append(",\"shadowedBy\":");
                    BridgeJson.Text(item, Path.GetFileName(artData));
                }

                item.Append('}');

                if (!page.TryAdd(item.ToString(), name))
                    break;
            }

            page.Close();

            // §6's gate is (size, mtime) first and a content hash only when those differ, because
            // anim.mul and art.mul are 195 MB and 148 MB and a full hash on every status poll
            // would make the admin panel feel broken. It would also blow the sidecar's 10 s reply
            // timeout outright on the first call. So a hash that is not cached is reported `null`
            // and computed in the background: this reply is always fast, and the next poll — after
            // `hashing` goes false — carries the answer.
            if (anyMissingHash)
                StartHashing();

            sb.Bool("hashing", _hashing);
            sb.Bool("complete", !anyMissingHash);

            BridgeLink.Emit(sb.End());
        }

        private static string ResolvePath(string name)
        {
            try
            {
                return Files.GetFilePath(name);
            }
            catch
            {
                return null;
            }
        }

        private static long ToUnixMs(DateTime utc)
        {
            return (long)(utc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;
        }

        // ── imaging (§4.4) ───────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Whether this host can turn a record into a picture at all.
        ///
        ///     ServUO targets net48, so a Linux shard runs it under Mono, where
        ///     <c>System.Drawing</c> is a thin layer over **libgdiplus** — and §4.2 put
        ///     <c>System.Drawing</c> in the *decode* path, not merely the encode: <c>Frame</c>
        ///     writes ARGB1555 through a <c>LockBits</c> pointer. Without that library a Linux
        ///     shard cannot read a sprite at all, while clilocs and the atlas are unaffected
        ///     because neither touches pixels.
        ///
        ///     It must never present as a stack trace or a 500. It is a named, actionable outcome
        ///     in the same family as the cliloc reader's `COMPRESSED`, and it is reported here —
        ///     on the *source gate*, the first call any import makes — so an operator learns it
        ///     while setting the shard up rather than from an empty bestiary weeks later.
        /// </summary>
        private static void WriteImaging(StringBuilder sb)
        {
            CheckImaging();

            sb.Append(",\"imaging\":{\"ok\":").Append(_imagingOk ? "true" : "false");

            if (!_imagingOk)
            {
                sb.Append(",\"code\":\"NO_IMAGING\",\"reason\":");
                BridgeJson.Text(sb,
                    "This shard host cannot render images — Mono's System.Drawing needs libgdiplus. "
                    + "Install it (apt-get install libgdiplus) and re-run the import. Cliloc and atlas "
                    + "import are unaffected. (" + _imagingReason + ")");
            }

            sb.Append('}');
        }

        private static void CheckImaging()
        {
            if (_imagingChecked)
                return;

            _imagingChecked = true;

            try
            {
                TouchImaging();
                _imagingOk = true;
            }
            catch (Exception e)
            {
                // On a host with no libgdiplus this is a TypeInitializationException wrapping a
                // DllNotFoundException, and it can surface as the method failing to JIT rather
                // than as a throw from inside it — which is why the construction lives in its own
                // method, so the failure is contained here instead of taking this class's
                // static initialisation with it.
                _imagingOk = false;
                _imagingReason = e.GetType().Name + ": " + e.Message;
            }
        }

        private static void TouchImaging()
        {
            using (var bmp = new System.Drawing.Bitmap(1, 1))
            {
                bmp.SetPixel(0, 0, System.Drawing.Color.Black);
            }
        }

        // ── the hash cache (§6) ──────────────────────────────────────────────────────────────

        private static string CachedHashFor(string path, long size, long mtime)
        {
            lock (_hashes)
            {
                CachedHash cached;

                if (_hashes.TryGetValue(path, out cached)
                    && cached.Size == size
                    && cached.MTime == mtime)
                {
                    return cached.Sha256;
                }
            }

            return null;
        }

        /// <summary>
        ///     Rehashes whatever the cache is missing, on its own thread.
        ///
        ///     Deliberately **not** a job on the asset worker: hashing 343 MB takes seconds to
        ///     tens of seconds, and holding the single slot for that long would answer every
        ///     status poll `bridge.busy` for the whole pass — which is exactly the moment an
        ///     operator is watching the panel. It emits nothing and correlates with nothing; it
        ///     only fills the cache that the next `assets.sources` reads.
        /// </summary>
        private static void StartHashing()
        {
            lock (_sync)
            {
                if (_hashing)
                    return;

                _hashing = true;

                _hasher = new Thread(HashLoop)
                {
                    Name = "BridgeAssetHash",
                    IsBackground = true
                };

                _hasher.Start();
            }
        }

        private static void HashLoop()
        {
            try
            {
                for (int i = 0; i < SourceFiles.Length; i++)
                {
                    string path = ResolvePath(SourceFiles[i]);

                    if (path == null)
                        continue;

                    long size, mtime;

                    try
                    {
                        var info = new FileInfo(path);
                        size = info.Length;
                        mtime = ToUnixMs(info.LastWriteTimeUtc);
                    }
                    catch
                    {
                        continue;
                    }

                    if (CachedHashFor(path, size, mtime) != null)
                        continue;

                    string hash = HashFile(path);

                    if (hash == null)
                        continue;

                    lock (_hashes)
                    {
                        _hashes[path] = new CachedHash { Size = size, MTime = mtime, Sha256 = hash };
                    }
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("[Bridge] asset hash pass: {0}: {1}", e.GetType().Name, e.Message);
            }
            finally
            {
                // Under _sync, matching StartHashing: cleared outside it, two passes could both
                // pass the guard and hash the same 343 MB twice.
                lock (_sync)
                {
                    _hashing = false;
                }
            }
        }

        private static string HashFile(string path)
        {
            try
            {
                using (var sha = SHA256.Create())
                using (var stream = new FileStream(
                    path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1 << 20))
                {
                    var buffer = new byte[1 << 20];
                    int read;

                    while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
                        sha.TransformBlock(buffer, 0, read, null, 0);

                    sha.TransformFinalBlock(buffer, 0, 0);

                    return Hex(sha.Hash);
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("[Bridge] cannot hash {0}: {1}", path, e.Message);
                return null;
            }
        }

        private static string Hex(byte[] bytes)
        {
            var sb = new StringBuilder(bytes.Length * 2);

            for (int i = 0; i < bytes.Length; i++)
                sb.Append(bytes[i].ToString("x2", CultureInfo.InvariantCulture));

            return sb.ToString();
        }

        // ── the paging envelope (§3.3) ───────────────────────────────────────────────────────

        /// <summary>
        ///     **One envelope for every asset family**, defined here in phase 1 so that clilocs
        ///     (phase 2), the body catalogue (3), statics and land (5), deep animation keys (6)
        ///     and the ServUO tree files (7) all page the same way. They are otherwise five
        ///     chances to invent five slightly different shapes, and the website would have to
        ///     learn each one.
        ///
        ///     The envelope a reply closes with:
        ///
        ///     <code>
        ///     "items": [ … ],
        ///     "more":   true,        // ask again with this cursor
        ///     "cursor": "s:4104",    // opaque to everyone but the shard; absent when more:false
        ///     "cut":    "budget"     // budget | end | limit — WHY this page stopped
        ///     </code>
        ///
        ///     **The budget is bytes, and it is UTF-8 bytes.** Not item count, because the ceiling
        ///     this lives inside is the sidecar's inbound line cap; and not chars, because a
        ///     cliloc row is real text and a `StringBuilder`'s Length would undercount every
        ///     non-ASCII character in it.
        ///
        ///     `cut` exists because "the page is short" has three different meanings and the
        ///     website must not have to guess which: the source ran out (`end`), the byte budget
        ///     was spent (`budget`), or the family stopped at its own limit (`limit`). Only the
        ///     first means the import is finished.
        ///
        ///     **The first item is always admitted**, even if it alone exceeds the budget.
        ///     Otherwise an oversized item would make its family unable to make any progress at
        ///     all — it would be skipped for the budget on every page, forever. That is safe
        ///     precisely because the budget is set to half the sidecar's line cap
        ///     (<see cref="BridgeConfig.AssetBatchBytes"/>), so one such item still fits the wire.
        /// </summary>
        public sealed class PageBuilder
        {
            private readonly StringBuilder _sb;
            private readonly int _budget;
            private int _bytes;
            private int _count;
            private string _cursor;
            private string _cut = "end";

            /// <summary>
            ///     Room kept back for the fields the envelope must still be able to write after
            ///     the last item — `more`, `cursor`, `cut` and the closing brace. Without it a
            ///     page could fill the budget exactly and then overrun it closing itself.
            /// </summary>
            private const int Reserve = 256;

            public PageBuilder(StringBuilder sb, string arrayName, int budget)
            {
                _sb = sb;
                _budget = budget;

                sb.Append(",\"").Append(arrayName).Append("\":[");

                // The prefix is already written, and it counts: the cap the sidecar enforces is
                // on the whole line, not on the array.
                _bytes = Encoding.UTF8.GetByteCount(sb.ToString());
            }

            public int Count { get { return _count; } }

            /// <summary>
            ///     Adds one already-serialised item. <paramref name="cursorAfter"/> is where the
            ///     family should resume if this turns out to be the last item on the page.
            ///     Returns false when the budget is spent — the caller stops, and `more` is true.
            /// </summary>
            public bool TryAdd(string item, string cursorAfter)
            {
                if (item == null)
                    return true;

                int cost = Encoding.UTF8.GetByteCount(item) + (_count > 0 ? 1 : 0);

                if (_count > 0 && _bytes + cost + Reserve > _budget)
                {
                    _cut = "budget";
                    return false;
                }

                if (_count > 0)
                    _sb.Append(',');

                _sb.Append(item);

                _bytes += cost;
                _count++;
                _cursor = cursorAfter;

                return true;
            }

            /// <summary>
            ///     Stops the page for a reason of the family's own — a per-request limit, say —
            ///     rather than because the budget ran out.
            /// </summary>
            public void Cut(string why)
            {
                _cut = why;
            }

            public void Close()
            {
                bool more = _cut != "end";

                _sb.Append(']');
                _sb.Bool("more", more);
                _sb.Str("cut", _cut);

                if (more && _cursor != null)
                    _sb.Str("cursor", _cursor);
            }
        }
    }
}