servuo-plugins/tools/scaffolding/BridgeMythicCliloc.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;

namespace Server.Custom
{
    /// <summary>
    ///     A reader for the **Mythic compressed** cliloc container, in plain .NET Framework 4.8 C#.
    ///
    ///     This is the Asset Bridge's §9 decoder — the ONE decoder Protocol 8 writes rather than
    ///     calls (docs/link/v8.md §4, §9). ServUO's bundled <c>Ultima.StringList</c> implements only
    ///     the plain layout and throws <c>Non-negative number required</c> on every modern client's
    ///     file, which is also why the shard's own <c>VendorSearch.GetItemName</c> is already inert.
    ///
    ///     **Provenance.** Ported from UOFiddler's <c>Ultima/Helpers/MythicDecompress.cs</c>,
    ///     <c>MoveToFront.cs</c> and <c>StringList.TryParse</c> (polserver/UOFiddler). UOFiddler is
    ///     released under the **Beerware** licence, so carrying its algorithm into this
    ///     GPL-3.0-or-later tree is clean — see v8.md §9.
    ///
    ///     **What the port had to change**, and why the differences are not cosmetic:
    ///
    ///       * UOFiddler targets net10.0 and its implementation is written in <c>Span&lt;T&gt;</c>,
    ///         <c>stackalloc</c>, <c>ArrayPool</c> and <c>BinaryPrimitives</c>. ServUO compiles the
    ///         overlay against net48 with no package feed, so all of that becomes plain arrays.
    ///       * Every read of the compressed payload is **bounds-checked here and is not there**.
    ///         Upstream indexes <c>input[m + 1024]</c> and <c>input[firstVal + 1024]</c> with
    ///         offsets derived from the file's own frequency header, inside a
    ///         <c>try { } catch (Exception) { return false; }</c>. That is adequate for a desktop
    ///         tool and is not adequate for us: this runs inside a live shard, and a corrupt or
    ///         hostile Cliloc.enu must produce a refusal, not an exception unwinding through the
    ///         bridge. Every such index is tested before use and returns <c>false</c> instead.
    ///
    ///     Phase 0 uses this from <see cref="BridgeAssetProbe"/> to prove the port reproduces
    ///     UOFiddler's own output exactly. **Phase 2 promotes this file into
    ///     <c>overlay/Scripts/Custom/Bridge/</c>** — it lives in scaffolding only for as long as it
    ///     is a spike.
    /// </summary>
    public static class BridgeMythicCliloc
    {
        /// <summary>The first DWORD of a compressed file is the decompressed length, XORed with this.</summary>
        private const uint HeaderXorKey = 0x8E2C9A3D;

        /// <summary>256 little-endian int32 symbol frequencies precede the coded payload.</summary>
        private const int FrequencyHeaderSize = 1024;

        /// <summary>One decoded cliloc row. Mirrors <c>Ultima.StringEntry</c>'s three fields.</summary>
        public struct Entry
        {
            public int Number;
            public byte Flag;
            public string Text;

            public Entry(int number, byte flag, string text)
            {
                Number = number;
                Flag = flag;
                Text = text;
            }
        }

        // ── Container detection ──────────────────────────────────────────────────────────────

        /// <summary>
        ///     True when the file looks like the Mythic container. The marker is the high byte of
        ///     the first DWORD being <c>0x8E</c> — which is not a magic number in the file so much
        ///     as a consequence of <see cref="HeaderXorKey"/>: a plausible decompressed length is
        ///     small enough that its top byte is zero, so the XOR leaves 0x8E showing.
        /// </summary>
        public static bool LooksCompressed(byte[] buffer)
        {
            return buffer != null && buffer.Length >= 4 && buffer[3] == 0x8E;
        }

        // ── The public entry point ───────────────────────────────────────────────────────────

        /// <summary>
        ///     Reads a cliloc file, compressed or plain, and returns its entries.
        ///
        ///     Tries the layout the header suggests first and the other one second — the same
        ///     fallback UOFiddler performs, and the reason an already-converted file passes
        ///     straight through. <paramref name="warning"/> is non-null when a layout parsed
        ///     *partially*: that is the case a caller must surface rather than swallow, because a
        ///     quietly short table is the failure mode the website's importer refuses.
        /// </summary>
        public static bool TryLoadFile(string path, out List<Entry> entries, out string warning, out string error)
        {
            entries = null;
            warning = null;
            error = null;

            byte[] buffer;

            try
            {
                buffer = File.ReadAllBytes(path);
            }
            catch (Exception e)
            {
                error = "cannot read " + path + ": " + e.Message;
                return false;
            }

            return TryLoad(buffer, out entries, out warning, out error);
        }

        /// <summary>Reads an in-memory cliloc file. See <see cref="TryLoadFile"/>.</summary>
        public static bool TryLoad(byte[] buffer, out List<Entry> entries, out string warning, out string error)
        {
            entries = null;
            warning = null;
            error = null;

            bool compressedFirst = LooksCompressed(buffer);

            List<Entry> primary;
            string primaryError;
            bool primaryComplete;

            if (TryParse(buffer, compressedFirst, out primary, out primaryComplete, out primaryError) && primaryComplete)
            {
                entries = primary;
                return true;
            }

            List<Entry> fallback;
            string fallbackError;
            bool fallbackComplete;

            if (TryParse(buffer, !compressedFirst, out fallback, out fallbackComplete, out fallbackError) && fallbackComplete)
            {
                entries = fallback;
                return true;
            }

            // Neither layout parsed to the end. Take whichever salvaged more rows and say so.
            int primaryCount = primary == null ? 0 : primary.Count;
            int fallbackCount = fallback == null ? 0 : fallback.Count;

            if (primaryCount == 0 && fallbackCount == 0)
            {
                error = "as " + Label(compressedFirst) + ": " + primaryError
                        + "; as " + Label(!compressedFirst) + ": " + fallbackError;
                return false;
            }

            if (primaryCount >= fallbackCount)
            {
                entries = primary;
                warning = "parsed partially as " + Label(compressedFirst) + ": " + primaryError
                          + " (" + primaryCount + " entries salvaged)";
            }
            else
            {
                entries = fallback;
                warning = "parsed partially as " + Label(!compressedFirst) + ": " + fallbackError
                          + " (" + fallbackCount + " entries salvaged)";
            }

            return true;
        }

        private static string Label(bool compressed)
        {
            return compressed ? "compressed" : "uncompressed";
        }

        // ── Record layout ────────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Walks the plain record layout: a 4-byte and a 2-byte header, then repeating
        ///     [int32 number][byte flag][uint16 length][length bytes of UTF-8].
        ///
        ///     <paramref name="complete"/> distinguishes "parsed to the end of the file" from
        ///     "stopped early but salvaged rows", which is the distinction the caller needs and
        ///     an exception would destroy.
        /// </summary>
        private static bool TryParse(byte[] buffer, bool decompress, out List<Entry> entries, out bool complete, out string error)
        {
            entries = new List<Entry>();
            complete = false;
            error = null;

            byte[] data;

            if (decompress)
            {
                if (!TryDecompress(buffer, out data, out error))
                    return false;
            }
            else
            {
                data = buffer;
            }

            if (data.Length < 6)
            {
                error = "file is " + data.Length + " bytes, smaller than the 6-byte header";
                return false;
            }

            int cursor = 6; // int32 version marker + int16 language marker
            int lastNumber = -1;

            while (cursor < data.Length)
            {
                int entryStart = cursor;
                int remaining = data.Length - cursor;

                if (remaining < 7)
                {
                    error = "unexpected " + remaining + " trailing byte(s) at 0x" + entryStart.ToString("X")
                            + " after entry #" + lastNumber + "; an entry header needs 7";
                    return true;
                }

                int number = ReadInt32(data, cursor);
                byte flag = data[cursor + 4];
                // Deliberately UNSIGNED. Read as Int16, a string of 32768 bytes or more comes back
                // negative and corrupts every record after it.
                int length = data[cursor + 5] | (data[cursor + 6] << 8);
                cursor += 7;

                if (length > data.Length - cursor)
                {
                    error = "entry #" + number + " at 0x" + entryStart.ToString("X") + " declares length "
                            + length + " but only " + (data.Length - cursor) + " byte(s) remain (parsed "
                            + entries.Count + " so far)";
                    return true;
                }

                string text;

                try
                {
                    text = Encoding.UTF8.GetString(data, cursor, length);
                }
                catch (Exception e)
                {
                    error = "entry #" + number + " at 0x" + entryStart.ToString("X") + " has " + length
                            + " body bytes that are not valid UTF-8: " + e.Message;
                    return true;
                }

                cursor += length;

                entries.Add(new Entry(number, flag, text));
                lastNumber = number;
            }

            complete = true;
            return true;
        }

        // ── Mythic stage 1: the XOR header and the move-to-front code ────────────────────────

        /// <summary>
        ///     Reads the obfuscated decompressed length from the first DWORD. Public so a caller
        ///     can size a buffer before committing to the decode.
        /// </summary>
        public static uint PeekDecompressedLength(byte[] source)
        {
            if (source == null || source.Length < 4)
                return 0;

            return ReadUInt32(source, 0) ^ HeaderXorKey;
        }

        /// <summary>
        ///     Decompresses the Mythic container: strip the 4-byte length header, undo the
        ///     move-to-front coding, then run stage 2.
        /// </summary>
        public static bool TryDecompress(byte[] source, out byte[] output, out string error)
        {
            output = null;
            error = null;

            if (source == null || source.Length < 4)
            {
                error = "payload shorter than the 4-byte length header";
                return false;
            }

            uint dataLength = ReadUInt32(source, 0) ^ HeaderXorKey;

            // A wrong guess about the container makes this astronomically large, which is the
            // cheapest possible rejection and must happen before any allocation.
            if (dataLength == 0 || dataLength > int.MaxValue)
            {
                error = "implausible decompressed length " + dataLength + " — not the compressed layout";
                return false;
            }

            var mtf = new byte[source.Length - 4];
            MoveToFrontDecode(source, 4, mtf);

            var destination = new byte[(int)dataLength];
            int written;

            if (!TryInternalDecompress(mtf, destination, out written, out error))
                return false;

            if (written != (int)dataLength)
            {
                error = "decompressed " + written + " bytes, header declared " + dataLength;
                return false;
            }

            output = destination;
            return true;
        }

        /// <summary>
        ///     Move-to-front decode. Each input byte is an index into a 256-symbol table; the
        ///     symbol found there is emitted and moved to the front.
        /// </summary>
        private static void MoveToFrontDecode(byte[] input, int offset, byte[] output)
        {
            var symbols = new byte[256];

            for (int i = 0; i < 256; i++)
                symbols[i] = (byte)i;

            for (int i = 0; i < output.Length; i++)
            {
                int index = input[offset + i];
                byte symbol = symbols[index];
                output[i] = symbol;

                for (int j = index; j > 0; j--)
                    symbols[j] = symbols[j - 1];

                symbols[0] = symbol;
            }
        }

        // ── Mythic stage 2 ───────────────────────────────────────────────────────────────────

        /// <summary>
        ///     Turns the MTF-decoded payload back into the original bytes.
        ///
        ///     The payload is a 1024-byte frequency header (256 little-endian int32 symbol counts)
        ///     followed by the coded stream. The counts partition the stream into one run per
        ///     symbol; <c>cursor[]</c> holds each run's read position and <c>limit[]</c> its end,
        ///     and the walk emits a symbol, advances that symbol's run, and re-orders the symbol
        ///     table by the index it reads.
        ///
        ///     Every index derived from file content is checked. Upstream's equivalent is wrapped
        ///     in a blanket catch; here a malformed file is a <c>false</c> with a reason.
        /// </summary>
        private static bool TryInternalDecompress(byte[] input, byte[] destination, out int written, out string error)
        {
            written = 0;
            error = null;

            if (input.Length < FrequencyHeaderSize)
            {
                error = "payload (" + input.Length + " bytes) is smaller than the 1024-byte frequency header";
                return false;
            }

            var counts = new int[256]; // symbol → number of occurrences
            var cursor = new int[256]; // symbol → next unread position in its run
            var limit = new int[256];  // symbol → one past the end of its run

            int sum = 0;

            for (int i = 0; i < 256; i++)
            {
                counts[i] = ReadInt32(input, i * 4);

                if (counts[i] < 0)
                {
                    error = "frequency header declares a negative count for symbol " + i;
                    return false;
                }

                sum += counts[i];

                if (sum < 0)
                {
                    error = "frequency header sums past int range at symbol " + i;
                    return false;
                }
            }

            if (sum == 0)
            {
                written = 0;
                return true;
            }

            if (destination.Length < sum)
            {
                error = "destination holds " + destination.Length + " bytes, payload needs " + sum;
                return false;
            }

            int nonZeroCount = 0;

            for (int i = 0; i < 256; i++)
            {
                if (counts[i] != 0)
                    nonZeroCount++;
            }

            // The coded stream must be long enough to hold one index per emitted byte.
            if (input.Length - FrequencyHeaderSize < sum)
            {
                error = "coded stream holds " + (input.Length - FrequencyHeaderSize) + " bytes, frequency header claims " + sum;
                return false;
            }

            var order = new byte[256];
            FrequencyOrder(counts, order);

            var symbolTable = new byte[256];

            for (int i = 0; i < 256; i++)
                symbolTable[i] = (byte)i;

            for (int i = 0, m = 0; i < nonZeroCount; ++i)
            {
                byte symbol = order[i];

                // m indexes the coded stream and comes from the file's own counts.
                if (m < 0 || m >= input.Length - FrequencyHeaderSize)
                {
                    error = "run table for symbol " + symbol + " starts at " + m + ", past the coded stream";
                    return false;
                }

                symbolTable[input[m + FrequencyHeaderSize]] = symbol;
                cursor[symbol] = m + 1;
                m += counts[symbol];
                limit[symbol] = m;
            }

            byte val = symbolTable[0];
            int count = 0;
            int liveSymbols = nonZeroCount;

            do
            {
                destination[count] = val;

                if (cursor[val] < limit[val])
                {
                    int at = cursor[val] + FrequencyHeaderSize;

                    if (at < FrequencyHeaderSize || at >= input.Length)
                    {
                        error = "run for symbol " + val + " reads at " + at + ", past the " + input.Length + "-byte payload";
                        return false;
                    }

                    byte index = input[at];
                    cursor[val]++;

                    if (index != 0)
                    {
                        ShiftLeft(symbolTable, index);
                        symbolTable[index] = val;
                        val = symbolTable[0];
                    }
                }
                else if (liveSymbols-- > 0)
                {
                    ShiftLeft(symbolTable, liveSymbols);
                    val = symbolTable[0];
                }

                count++;
            }
            while (count < sum);

            written = sum;
            return true;
        }

        /// <summary>
        ///     Orders symbols by descending frequency: repeatedly take the largest remaining count
        ///     and record its symbol. Ties go to the lower symbol, because the scan keeps the first
        ///     strictly-greater value — matching upstream, and the tie-break is load-bearing.
        /// </summary>
        private static void FrequencyOrder(int[] counts, byte[] output)
        {
            var tmp = new int[256];
            Array.Copy(counts, tmp, 256);

            for (int i = 0; i < 256; i++)
            {
                int best = 0;
                byte index = 0;

                for (int j = 0; j < 256; j++)
                {
                    if (tmp[j] > best)
                    {
                        index = (byte)j;
                        best = tmp[j];
                    }
                }

                if (best == 0)
                    break;

                output[i] = index;
                tmp[index] = 0;
            }
        }

        /// <summary>Shifts <c>[1..element]</c> down one slot, dropping element 0.</summary>
        private static void ShiftLeft(byte[] input, int element)
        {
            for (int i = 0; i < element; ++i)
                input[i] = input[i + 1];
        }

        // ── Little-endian readers (BinaryPrimitives is not available on net48) ───────────────

        private static int ReadInt32(byte[] b, int at)
        {
            return b[at] | (b[at + 1] << 8) | (b[at + 2] << 16) | (b[at + 3] << 24);
        }

        private static uint ReadUInt32(byte[] b, int at)
        {
            return (uint)(b[at] | (b[at + 1] << 8) | (b[at + 2] << 16) | (b[at + 3] << 24));
        }
    }
}