Merge pull request 'feat(events): the Event System — core (Phase 16b cutover, 2 of 6)' (#199) from edge into main

Reviewed-on: #199
Reviewed-by: Colby Whitlock <whitlocktech@gmail.com>

server/tools/cliloc-export/Program.cs

@@ -0,0 +1,140 @@
+// Cliloc export — converts a modern client's COMPRESSED Cliloc.enu into the
+// plain format the website can read (docs/website/CLILOCS.md).
+//
+// Why this exists at all: every current UO client ships its cliloc files in the
+// compressed "Mythic" format — the first DWORD's high byte is 0x8E — and the
+// plain layout the website parses is what those files looked like before that
+// change. Decompressing is a bit-level inverse-BWT coder that the site has no
+// business carrying at runtime, and ServUO's own bundled `Ultima.StringList`
+// cannot read it either (which is why `VendorSearch.GetItemName` is already
+// inert on such a shard, and why the shard cannot supply names instead).
+//
+// So the conversion happens ONCE, here, against a decompressor that already
+// exists and is maintained: UOFiddler's `Ultima.dll`.
+//
+// ── Why reflection rather than a project reference ────────────────────────
+//
+// UOFiddler ships as net10.0. Referencing it from a project built by an older
+// SDK fails at COMPILE time with CS1705 ("uses System.Runtime 10.0 which has a
+// higher version than referenced assembly"). Loading it reflectively moves that
+// question to run time, where `RollForward: LatestMajor` answers it — so this
+// builds on whatever SDK an operator happens to have and runs on the newest
+// runtime installed.
+//
+// ── Why not StringList.SaveStringList ────────────────────────────────────
+//
+// It looks like exactly the right method and it is not: it RE-COMPRESSES on
+// save, because its purpose is round-tripping a file back into the client. The
+// output is byte-identical to the compressed input. The plain records below are
+// written by hand for that reason.
+//
+// Usage:
+//   dotnet run -- <Ultima.dll> <Cliloc.enu> <output> [--tsv]
+//
+// Nothing produced by this tool is committed. See docs/website/CLILOCS.md.
+
+using System;
+using System.Collections;
+using System.IO;
+using System.Reflection;
+using System.Text;
+
+internal static class Program
+{
+    private static int Main(string[] args)
+    {
+        if (args.Length < 3)
+        {
+            Console.Error.WriteLine("usage: clilocexport <path-to-Ultima.dll> <cliloc-file> <output-file> [--tsv]");
+            Console.Error.WriteLine("  Ultima.dll ships with UOFiddler (https://github.com/polserver/UOFiddler).");
+            return 2;
+        }
+
+        var (ultimaDll, input, output) = (args[0], args[1], args[2]);
+        var asTsv = Array.IndexOf(args, "--tsv") >= 0;
+
+        // The language code only names the file when StringList resolves the path
+        // itself; here the path is explicit, so it is cosmetic.
+        var language = Path.GetExtension(input).TrimStart('.');
+        if (string.IsNullOrWhiteSpace(language)) language = "enu";
+
+        var assembly = Assembly.LoadFrom(Path.GetFullPath(ultimaDll));
+        var stringListType = assembly.GetType("Ultima.StringList")
+            ?? throw new InvalidOperationException("Ultima.StringList not found — is that really UOFiddler's Ultima.dll?");
+
+        // (language, path, decompress). `decompress: true` is the whole point;
+        // the loader falls back to a plain read on its own if the file turns out
+        // not to be compressed, so an already-converted file passes through.
+        var ctor = stringListType.GetConstructor(new[] { typeof(string), typeof(string), typeof(bool) })
+            ?? throw new InvalidOperationException("Unexpected Ultima.StringList API — this tool targets UOFiddler 4.21+.");
+
+        var stringList = ctor.Invoke(new object[] { language, Path.GetFullPath(input), true });
+
+        // A partial parse is reported rather than thrown. Surfacing it matters:
+        // the output would otherwise be a quietly short table, which is exactly
+        // the failure mode the website's parser refuses to import.
+        var warning = stringListType.GetProperty("LoadWarning")?.GetValue(stringList) as string;
+        if (!string.IsNullOrWhiteSpace(warning)) Console.Error.WriteLine("warning: " + warning);
+
+        var entries = (IEnumerable)stringListType.GetProperty("Entries")!.GetValue(stringList)!;
+        var entryType = assembly.GetType("Ultima.StringEntry")!;
+        var numberProp = entryType.GetProperty("Number")!;
+        var textProp = entryType.GetProperty("Text")!;
+        var flagProp = entryType.GetProperty("Flag")!;
+
+        int written = 0, skipped = 0, maxBytes = 0;
+
+        if (asTsv)
+        {
+            using var writer = new StreamWriter(output, false, new UTF8Encoding(false));
+            foreach (var entry in entries)
+            {
+                var number = (int)numberProp.GetValue(entry)!;
+                var text = (string?)textProp.GetValue(entry) ?? "";
+                maxBytes = Math.Max(maxBytes, Encoding.UTF8.GetByteCount(text));
+                // A tab or newline inside a cliloc string would break the row.
+                // Neither occurs in real tables, but silently emitting a broken
+                // file is worse than collapsing the whitespace.
+                writer.WriteLine($"{number}\t{text.Replace('\t', ' ').Replace('\r', ' ').Replace('\n', ' ')}");
+                written++;
+            }
+        }
+        else
+        {
+            using var stream = new FileStream(output, FileMode.Create, FileAccess.Write);
+            using var binary = new BinaryWriter(stream);
+            binary.Write(2);        // int32 — the plain-format version marker
+            binary.Write((short)1); // int16 — language marker
+
+            foreach (var entry in entries)
+            {
+                var number = (int)numberProp.GetValue(entry)!;
+                var text = (string?)textProp.GetValue(entry) ?? "";
+                var flag = Convert.ToByte(Convert.ToInt32(flagProp.GetValue(entry)));
+
+                var utf8 = Encoding.UTF8.GetBytes(text);
+                maxBytes = Math.Max(maxBytes, utf8.Length);
+
+                // The length field is 16 bits. Real tables peak around 12 KB, so
+                // this has never fired — but writing a truncated length would
+                // corrupt every record after it, so an oversize entry is dropped
+                // and counted instead.
+                if (utf8.Length > ushort.MaxValue) { skipped++; continue; }
+
+                binary.Write(number);
+                binary.Write(flag);
+                binary.Write((ushort)utf8.Length);
+                binary.Write(utf8);
+                written++;
+            }
+        }
+
+        Console.WriteLine($"wrote {written} entries to {output} (maxTextBytes={maxBytes}, skippedOversize={skipped})");
+        if (written == 0)
+        {
+            Console.Error.WriteLine("no entries were written — is that a cliloc file?");
+            return 1;
+        }
+        return 0;
+    }
+}

server/tools/cliloc-export/README.md

@@ -0,0 +1,64 @@
+# cliloc-export
+
+Converts a UO client's **compressed** `Cliloc.enu` into the plain format the
+website can read.
+
+This is a one-off operator utility, not part of the website build. Nothing in the
+Node application references it and CI never touches it. Full background —
+including why the conversion is necessary at all — is in
+[`docs/website/CLILOCS.md`](https://gitea.whitlocktech.com/RunicGateway/docs/src/branch/edge/website/CLILOCS.md).
+
+## The short version
+
+Every current UO client ships its cliloc files in the compressed "Mythic"
+container (the first DWORD's high byte is `0x8E`). The website parses the plain
+layout those files used before that change. Decompressing is an inverse-BWT coder
+that the site has no business carrying at runtime — and ServUO's own bundled
+`Ultima.StringList` cannot read it either, so the shard cannot supply item names
+on our behalf.
+
+So: convert once, here, using a decompressor that already exists and is already
+maintained — [UOFiddler](https://github.com/polserver/UOFiddler)'s `Ultima.dll`.
+
+## Usage
+
+```bash
+dotnet build -c Release
+
+# plain binary (recommended — exact)
+dotnet run -- "<UOFiddler>/Ultima.dll" "<UO client>/Cliloc.enu" /srv/uo-data/clilocs.plain
+
+# tab-delimited text (convenient; does not preserve leading/trailing whitespace)
+dotnet run -- "<UOFiddler>/Ultima.dll" "<UO client>/Cliloc.enu" /srv/uo-data/clilocs.tsv --tsv
+```
+
+Then point the site at the output: **Admin → Shard → cliloc path**, or the
+`UO_CLIENT_PATH` environment variable. The setting wins over the environment.
+
+Expected output for a stock English client:
+
+```
+wrote 123490 entries to /srv/uo-data/clilocs.plain (maxTextBytes=12150, skippedOversize=0)
+```
+
+The site stores ~67,500 of those — roughly half a cliloc table is empty strings
+for ids the client reserves and never uses.
+
+## Two implementation notes worth keeping
+
+**`Ultima.dll` is loaded reflectively, not referenced.** UOFiddler ships as
+net10.0; a project reference from an older SDK fails at *compile* time with
+CS1705. Reflection moves that to run time, where `RollForward: LatestMajor`
+answers it — so this builds on whatever SDK you have and runs on the newest
+runtime installed.
+
+**`StringList.SaveStringList` is not the export path**, despite looking exactly
+like it. It *re-compresses* on save, because its purpose is round-tripping a file
+back into the client — its output is byte-identical to its input. The plain
+records are written by hand for that reason.
+
+## Output is never committed
+
+UO's strings are EA's. `.gitignore` covers this project's build output and the
+conventional in-repo output location, but the supported arrangement is a path
+**outside** the repository entirely.

server/tools/cliloc-export/clilocexport.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <!--
+    A one-off operator utility, not part of the website build. Nothing in the
+    Node application references it and CI never touches it; it exists so an
+    operator can convert their client's compressed cliloc file without clicking
+    through a GUI. See README.md and docs/website/CLILOCS.md.
+
+    TargetFramework is deliberately net8.0 — the OLDEST runtime this needs — so
+    it builds on whatever SDK an operator already has. UOFiddler's Ultima.dll is
+    net10.0 and is loaded reflectively at run time rather than referenced, which
+    is what keeps that version difference from being a compile error; the
+    RollForward below is what lets the resulting binary run on it.
+  -->
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <AssemblyName>clilocexport</AssemblyName>
+    <RootNamespace>ClilocExport</RootNamespace>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>disable</ImplicitUsings>
+    <RollForward>LatestMajor</RollForward>
+    <InvariantGlobalization>true</InvariantGlobalization>
+  </PropertyGroup>
+
+</Project>