feat(asset-bridge): the cliloc table, decompressed on the shard (Phase 2) #29

Merged
whitlocktech merged 2 commits from feat/asset-bridge-p2 into edge 2026-09-10 16:19:50 +00:00
2 changed files with 791 additions and 6 deletions

View File

@@ -152,13 +152,13 @@ namespace Server.Custom.Bridge
{ {
// Rule 1. Without a correlation id this would land on the event path, be persisted // Rule 1. Without a correlation id this would land on the event path, be persisted
// to the sidecar's store and broadcast to every subscriber. Refuse instead. // to the sidecar's store and broadcast to every subscriber. Refuse instead.
Fail(null, "assets.sources requires a reqId"); Fail(null, "BAD_REQUEST", "assets.sources requires a reqId");
return; return;
} }
if (!BridgeConfig.AssetsEnabled) if (!BridgeConfig.AssetsEnabled)
{ {
Fail(reqId, "asset extraction is disabled on this shard"); Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
return; return;
} }
@@ -170,7 +170,7 @@ namespace Server.Custom.Bridge
/// Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on /// Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on
/// the worker and must touch no world state. /// the worker and must touch no world state.
/// </summary> /// </summary>
private static void Accept(string reqId, string kind, Action job) internal static void Accept(string reqId, string kind, Action job)
{ {
lock (_sync) lock (_sync)
{ {
@@ -197,7 +197,7 @@ namespace Server.Custom.Bridge
_job = null; _job = null;
Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message); Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message);
Fail(reqId, "the shard could not start its asset worker"); Fail(reqId, "UNAVAILABLE", "the shard could not start its asset worker");
return; return;
} }
} }
@@ -222,14 +222,27 @@ namespace Server.Custom.Bridge
BridgeLink.Emit(sb.End()); BridgeLink.Emit(sb.End());
} }
private static void Fail(string reqId, string reason) /// <summary>
/// The asset plane's one refusal frame, shared by every family on it.
///
/// <paramref name="code"/> is what the sidecar maps to a status, and it exists because
/// the alternative it replaced — matching on the words in <paramref name="reason"/> —
/// makes an operator-facing sentence load-bearing. Rewording "disabled" would silently
/// turn a 403 into a 400. The codes are `DISABLED` (the operator switched this plane
/// off), `NOT_FOUND` (the shard has no such file), `UNREADABLE` (it has it and cannot
/// decode it), `UNAVAILABLE` (the shard cannot do this right now) and `BAD_REQUEST`
/// (the default, and the caller's fault).
/// </summary>
internal static void Fail(string reqId, string code, string reason)
{ {
var sb = BridgeJson.Begin("assets.error"); var sb = BridgeJson.Begin("assets.error");
if (reqId != null) if (reqId != null)
sb.Str("reqId", reqId); sb.Str("reqId", reqId);
sb.Str("reason", reason); sb.Str("code", code)
.Str("reason", reason);
BridgeLink.Emit(sb.End()); BridgeLink.Emit(sb.End());
} }

View File

@@ -0,0 +1,772 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
using Ultima;
namespace Server.Custom.Bridge
{
/// <summary>
/// **The cliloc table, over the bridge** (docs/link/v8.md §9 — protocol 8, phase 2).
///
/// A "cliloc" is UO's localization table: an integer id mapped to a display string. Items
/// on the wire carry a `LabelNumber`, never a name, so without this table the website can
/// only render `id 1023721` where the game renders "quarter staff". The number was never
/// the missing piece; the table was.
///
/// Until this phase the operator supplied it by hand: install UOFiddler, build a converter
/// against its `Ultima.dll`, run it over their own `Cliloc.enu`, copy a 5 MB file to the
/// web host and point a setting at it. That whole pipeline existed for one reason — the
/// file is compressed and **nothing in this stack could read it**. ServUO's own bundled
/// `Ultima.StringList` implements the plain layout only and throws on a modern client's
/// file, which is also why the shard's `VendorSearch.GetItemName` has always been inert.
///
/// So this class is the one decoder protocol 8 **writes** rather than calls (§4): a port
/// of UOFiddler's Mythic decompressor into the overlay, after which the shard can read its
/// own client's table and hand it to the website over the same request/reply path as
/// everything else. The operator installs nothing.
///
/// **Attribution.** The decompression below is a port of `Ultima/Helpers/MythicDecompress`
/// and `MoveToFront` from UOFiddler (https://github.com/polserver/UOFiddler), which is
/// released under the **Beerware** licence — compatible with this tree's GPL-3.0-or-later.
/// It is rewritten for .NET Framework 4.8: the original is written against `Span&lt;T&gt;`,
/// `ArrayPool&lt;T&gt;` and `BinaryPrimitives`, none of which ServUO's `net48` target has.
///
/// **What is NOT here, deliberately.** Shard-added items carry cliloc ids no client table
/// contains, and ServUO has no server-side notion of a custom cliloc — there is nothing in
/// the tree to read. That gap is in the *game*, not in this pipeline, so the website keeps
/// its `custom/` overlay directory and merges it over whatever arrives here. This class
/// answers exactly one question: what does the client's own table say.
/// </summary>
public static class BridgeCliloc
{
/// <summary>
/// Languages this can serve.
///
/// Not an arbitrary code: <c>Ultima.Files</c> resolves only the names in its own file
/// table, and cliloc files are represented there by these four. Asking for anything
/// else cannot resolve to a path however the client is laid out, so it is refused by
/// name rather than answered with an empty table.
///
/// `custom1` / `custom2` are the *client-side* custom cliloc files a shard ships to
/// its players. Nothing on the website imports them today — its `custom/` overlay
/// directory is the supported answer — but they are the shard's files and they are
/// readable, so they are not artificially excluded.
/// </summary>
private static readonly string[] Languages = { "enu", "deu", "custom1", "custom2" };
private const string DefaultLanguage = "enu";
/// <summary>
/// How long a decoded table is kept in memory after its last page.
///
/// A stock `Cliloc.enu` decodes to ~67,000 live strings; holding that forever on a
/// shard that imports once a month is rude, and decoding it again costs about a
/// second. So it is cached only for as long as an import is plausibly still running:
/// freed when the last page is served, and expired on the next request if one never
/// comes (an import abandoned halfway leaves nothing behind).
/// </summary>
private static readonly TimeSpan CacheIdle = TimeSpan.FromMinutes(5);
private static readonly object _sync = new object();
private static Table _cached;
public static void Initialize()
{
if (!BridgeConfig.Enabled)
return;
BridgeBoot.RegisterHandler("cliloc.table", OnTable);
}
// ── the request plane ────────────────────────────────────────────────────────────────
/// <summary>
/// Core thread. Validates, then hands the decode to the asset worker — reading and
/// decompressing five megabytes is emphatically not something to do while the world
/// is waiting, and <see cref="BridgeAssets"/>'s single slot is what keeps the shard's
/// outbound queue at a depth of about one while it happens.
/// </summary>
private static void OnTable(Dictionary<string, object> o)
{
var reqId = BridgeJson.GetString(o, "reqId");
if (reqId == null)
{
// Without a correlation id this reply would land on the event path, be persisted
// to the sidecar's store and broadcast to every subscriber — a megabyte of
// strings to every connected client, forever. Refuse instead (§3.1).
BridgeAssets.Fail(null, "BAD_REQUEST", "cliloc.table requires a reqId");
return;
}
if (!BridgeConfig.AssetsEnabled)
{
BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
return;
}
var lang = BridgeJson.GetString(o, "lang");
if (String.IsNullOrEmpty(lang))
lang = DefaultLanguage;
lang = lang.ToLowerInvariant();
if (Array.IndexOf(Languages, lang) < 0)
{
BridgeAssets.Fail(reqId, "NOT_FOUND",
"no cliloc file for language '" + lang + "' (this shard can serve: "
+ String.Join(", ", Languages) + ")");
return;
}
// The cursor is this family's own resume point and it is a cliloc NUMBER, not an
// offset into anything. That matters: the cache behind it can be dropped and rebuilt
// between two pages of the same import (idle expiry, a second import, a restart), and
// an index into a list would silently mean something different afterwards. "Resume
// after id N" survives all of it, because the table is served in id order.
int after = -1;
var cursor = BridgeJson.GetString(o, "cursor");
if (!String.IsNullOrEmpty(cursor))
{
if (!TryParseCursor(cursor, out after))
{
BridgeAssets.Fail(reqId, "BAD_REQUEST", "malformed cursor: " + cursor);
return;
}
}
string language = lang;
int resumeAfter = after;
BridgeAssets.Accept(reqId, "cliloc.table", () => ReplyTable(reqId, language, resumeAfter));
}
private static bool TryParseCursor(string cursor, out int after)
{
after = -1;
if (!cursor.StartsWith("n:", StringComparison.Ordinal))
return false;
return Int32.TryParse(
cursor.Substring(2), NumberStyles.Integer, CultureInfo.InvariantCulture, out after);
}
/// <summary>
/// Asset worker. Decodes (or reuses) the table and writes one page of it.
/// </summary>
private static void ReplyTable(string reqId, string lang, int after)
{
string path = ResolvePath(lang);
if (path == null)
{
BridgeAssets.Fail(reqId, "NOT_FOUND",
"this shard's client has no cliloc." + lang + " (looked where ServUO's own "
+ "data path points)");
return;
}
Table table;
string code, reason;
if (!TryLoad(lang, path, out table, out code, out reason))
{
BridgeAssets.Fail(reqId, code, reason);
return;
}
var sb = BridgeJson.Begin("cliloc.table.ok");
sb.Str("reqId", reqId)
.Str("lang", lang)
.Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION)
.Str("file", Path.GetFileName(path))
// The website pages this table over several round trips and must be able to tell
// that the file changed underneath it — an operator patching their client mid-import
// would otherwise produce one table stitched from two, with no error anywhere. It
// compares these two fields across pages and starts over if they move.
.Num("size", table.Size)
.Num("mtime", table.MTime)
.Num("total", table.Count)
.Bool("compressed", table.Compressed);
// Before the page opens, not after it closes: PageBuilder reserves room for the
// envelope it still has to write, and a field appended past Close() is spent outside
// that reserve. It fits today by a wide margin, and it is the kind of thing the next
// family copies.
int start = table.IndexAfter(after);
sb.Num("from", start);
var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes);
int i = start;
for (; i < table.Count; i++)
{
var item = new StringBuilder(96);
item.Append("{\"n\":").Append(table.Numbers[i].ToString(CultureInfo.InvariantCulture));
item.Append(",\"f\":").Append(table.Flags[i].ToString(CultureInfo.InvariantCulture));
item.Append(",\"t\":");
BridgeJson.Text(item, table.Texts[i]);
item.Append('}');
if (!page.TryAdd(item.ToString(), "n:" + table.Numbers[i].ToString(CultureInfo.InvariantCulture)))
break;
}
page.Close();
bool finished = i >= table.Count;
BridgeLink.Emit(sb.End());
// The last page is also the end of the import, so let the strings go. A retry of that
// page re-decodes, which costs a second and happens approximately never; holding ~67k
// strings against that is the wrong trade.
if (finished)
Release(lang);
}
private static string ResolvePath(string lang)
{
try
{
// ServUO's own `Scripts/Misc/DataPath.cs` calls `Files.SetMulPath` for every
// configured data directory at Configure time, so this resolves against the
// client the SHARD is running on — including on Linux, where `Ultima.Files`'s
// registry lookup finds nothing on its own.
return Files.GetFilePath("cliloc." + lang);
}
catch
{
return null;
}
}
// ── the decoded table ────────────────────────────────────────────────────────────────
private sealed class Table
{
public string Lang;
public long Size;
public long MTime;
public bool Compressed;
public int[] Numbers;
public byte[] Flags;
public string[] Texts;
public DateTime LastUsed;
public int Count { get { return Numbers.Length; } }
/// <summary>
/// Index of the first row with a number greater than <paramref name="after"/>.
/// Binary search, because the rows are in id order by construction and a page
/// deep into the table would otherwise walk everything before it.
/// </summary>
public int IndexAfter(int after)
{
if (after < 0)
return 0;
int lo = 0, hi = Numbers.Length;
while (lo < hi)
{
int mid = lo + ((hi - lo) >> 1);
if (Numbers[mid] <= after)
lo = mid + 1;
else
hi = mid;
}
return lo;
}
}
private static bool TryLoad(string lang, string path, out Table table, out string code, out string reason)
{
code = null;
reason = null;
long size, mtime;
try
{
var info = new FileInfo(path);
size = info.Length;
mtime = (long)(info.LastWriteTimeUtc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))
.TotalMilliseconds;
}
catch (Exception e)
{
table = null;
code = "UNREADABLE";
reason = "cannot stat " + Path.GetFileName(path) + ": " + e.Message;
return false;
}
lock (_sync)
{
if (_cached != null)
{
bool stale = _cached.Lang != lang
|| _cached.Size != size
|| _cached.MTime != mtime
|| DateTime.UtcNow - _cached.LastUsed > CacheIdle;
if (stale)
_cached = null;
}
if (_cached != null)
{
_cached.LastUsed = DateTime.UtcNow;
table = _cached;
return true;
}
}
byte[] raw;
try
{
raw = File.ReadAllBytes(path);
}
catch (Exception e)
{
table = null;
code = "UNREADABLE";
reason = "cannot read " + Path.GetFileName(path) + ": " + e.Message;
return false;
}
bool compressed = IsCompressed(raw);
byte[] plain;
if (compressed)
{
try
{
plain = Mythic.Decompress(raw);
}
catch (Exception e)
{
table = null;
code = "UNREADABLE";
reason = "cannot decompress " + Path.GetFileName(path) + ": " + e.Message;
return false;
}
}
else
{
plain = raw;
}
var built = new Table
{
Lang = lang,
Size = size,
MTime = mtime,
Compressed = compressed,
LastUsed = DateTime.UtcNow
};
if (!TryParseRecords(plain, built, out reason))
{
table = null;
code = "UNREADABLE";
return false;
}
lock (_sync)
{
_cached = built;
}
table = built;
return true;
}
private static void Release(string lang)
{
lock (_sync)
{
if (_cached != null && _cached.Lang == lang)
_cached = null;
}
}
/// <summary>
/// Every compressed cliloc begins with a DWORD whose high byte is <c>0x8E</c> — the
/// top byte of UOFiddler's `HeaderXorKey`, showing through because the value it hides
/// (a length) is far smaller than the key. That single byte is what tells a modern
/// client's file from the pre-2010 plain layout, and both are accepted here: a shard
/// running an old or hand-built client is not a broken shard.
/// </summary>
private static bool IsCompressed(byte[] buffer)
{
return buffer.Length >= 4 && buffer[3] == 0x8E;
}
// ── the plain layout ─────────────────────────────────────────────────────────────────
private const int HeaderBytes = 6; // int32 version + int16 language marker
private const int RecordHeaderBytes = 7; // int32 number + byte flag + uint16 length
/// <summary>
/// Parses the plain layout into the sorted, blank-free arrays the wire wants.
///
/// **Strict about truncation**, and that strictness is the point: a half-decoded table
/// is indistinguishable from a complete one downstream — you would simply see some
/// items named and some not, which is exactly what "no table at all" looks like. So a
/// record running past the end of the buffer is an error naming its offset, never a
/// short table.
///
/// **Blanks are dropped here rather than on the website.** Roughly 56,000 of a stock
/// table's 123,490 entries are empty strings the client reserves and never uses, the
/// website discards them at import already, and a row that resolves to no name is
/// indistinguishable from no row at all to every caller. Dropping them halves what
/// crosses the wire for data that would be thrown away on arrival.
///
/// **A repeated id is resolved last-wins**, matching the client's own loader (its
/// dictionary assignment overwrites). The plain format permits it, so a file the game
/// itself would load happily must not fail here.
/// </summary>
private static bool TryParseRecords(byte[] data, Table into, out string reason)
{
reason = null;
if (data.Length < HeaderBytes)
{
reason = "cliloc file is shorter than its 6-byte header";
return false;
}
var byNumber = new Dictionary<int, Entry>(140000);
int offset = HeaderBytes;
int read = 0;
while (offset < data.Length)
{
if (offset + RecordHeaderBytes > data.Length)
{
reason = "truncated record header at byte " + offset + " (" + read + " entries read)";
return false;
}
int number = ReadInt32(data, offset);
byte flag = data[offset + 4];
// Unsigned: reading this signed (as ServUO's own SDK does) turns any string over
// 32 KB into a negative length. Real tables top out around 12 KB, so it changes
// nothing today and costs nothing to get right.
int length = data[offset + 5] | (data[offset + 6] << 8);
offset += RecordHeaderBytes;
if (offset + length > data.Length)
{
reason = "truncated record body at byte " + offset + " (" + read + " entries read)";
return false;
}
string text;
try
{
text = Encoding.UTF8.GetString(data, offset, length);
}
catch (Exception e)
{
reason = "entry " + number + " at byte " + offset + " is not valid UTF-8: " + e.Message;
return false;
}
offset += length;
read++;
byNumber[number] = new Entry { Flag = flag, Text = text };
}
var numbers = new List<int>(byNumber.Count);
foreach (var pair in byNumber)
{
if (IsBlank(pair.Value.Text))
continue;
numbers.Add(pair.Key);
}
numbers.Sort();
into.Numbers = numbers.ToArray();
into.Flags = new byte[numbers.Count];
into.Texts = new string[numbers.Count];
for (int i = 0; i < numbers.Count; i++)
{
var entry = byNumber[numbers[i]];
into.Flags[i] = entry.Flag;
into.Texts[i] = entry.Text;
}
return true;
}
private struct Entry
{
public byte Flag;
public string Text;
}
private static bool IsBlank(string text)
{
if (String.IsNullOrEmpty(text))
return true;
for (int i = 0; i < text.Length; i++)
{
if (!Char.IsWhiteSpace(text[i]))
return false;
}
return true;
}
private static int ReadInt32(byte[] data, int at)
{
return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24);
}
// ── the Mythic container ─────────────────────────────────────────────────────────────
/// <summary>
/// The decompressor, ported from UOFiddler (Beerware; see this class's summary).
///
/// The container is two stages over the plain cliloc bytes, undone in reverse:
///
/// 1. A 4-byte header holding the decompressed length, XORed with `0x8E2C9A3D` —
/// which is where the `0x8E` sniff byte comes from.
/// 2. A **move-to-front** coding of…
/// 3. …a Burrows-Wheeler-style transform whose 1 KB frequency header (256 little-endian
/// counts, one per byte value) is both the table sizes and the total output length.
///
/// Rewritten against plain arrays: the upstream is `Span&lt;T&gt;`/`ArrayPool&lt;T&gt;`
/// code and ServUO targets `net48`, which has neither without a package this tree does
/// not vendor. The algorithm is unchanged, including the parts that read oddly — the
/// three-region `partial` table (counts, cursors, ends) and the symbol-table shifts are
/// the original's, deliberately, because this is a format decoder and a tidier
/// rewrite is a chance to be subtly wrong about someone else's bytes.
/// </summary>
private static class Mythic
{
private const uint HeaderXorKey = 0x8E2C9A3D;
private const int FrequencyHeaderSize = 1024; // 256 little-endian ints
public static byte[] Decompress(byte[] source)
{
if (source.Length < 4)
throw new InvalidDataException("compressed cliloc is shorter than its header");
uint declared = (uint)ReadInt32(source, 0) ^ HeaderXorKey;
if (declared == 0 || declared > Int32.MaxValue)
throw new InvalidDataException("compressed cliloc declares an impossible length");
var mtf = new byte[source.Length - 4];
MoveToFrontDecode(source, 4, mtf);
var output = new byte[(int)declared];
int written = InverseTransform(mtf, output);
if (written != (int)declared)
{
throw new InvalidDataException(
"decompressed length " + written + " does not match the declared " + declared);
}
return output;
}
private static void MoveToFrontDecode(byte[] input, int from, byte[] output)
{
var symbols = new byte[256];
for (int i = 0; i < 256; i++)
symbols[i] = (byte)i;
for (int i = 0; i < output.Length; i++)
{
int index = input[from + i];
byte symbol = symbols[index];
output[i] = symbol;
for (int j = index; j > 0; j--)
symbols[j] = symbols[j - 1];
symbols[0] = symbol;
}
}
private static int InverseTransform(byte[] input, byte[] destination)
{
if (input.Length < FrequencyHeaderSize)
throw new InvalidDataException("compressed cliloc is smaller than its frequency header");
// Three regions of 256: [0..255] the counts read from the header, [256..511] a
// moving cursor per symbol, [512..767] where that symbol's run ends.
var partial = new int[256 * 3];
for (int i = 0; i < 256; i++)
partial[i] = ReadInt32(input, i * 4);
int sum = 0;
for (int i = 0; i < 256; i++)
{
if (partial[i] < 0)
throw new InvalidDataException("compressed cliloc has a negative symbol count");
sum += partial[i];
}
if (sum == 0)
return 0;
if (destination.Length < sum)
throw new InvalidDataException("compressed cliloc's frequency header outruns its declared length");
int nonZero = 0;
for (int i = 0; i < 256; i++)
{
if (partial[i] != 0)
nonZero++;
}
var frequency = new byte[256];
Frequency(partial, frequency);
var symbols = new byte[256];
for (int i = 0; i < 256; i++)
symbols[i] = (byte)i;
for (int i = 0, m = 0; i < nonZero; ++i)
{
byte freq = frequency[i];
Need(input, m + FrequencyHeaderSize);
symbols[input[m + FrequencyHeaderSize]] = freq;
partial[freq + 256] = m + 1;
m += partial[freq];
partial[freq + 512] = m;
}
byte val = symbols[0];
int count = 0;
do
{
destination[count] = val;
if (partial[val + 256] < partial[val + 512])
{
Need(input, partial[val + 256] + FrequencyHeaderSize);
byte idx = input[partial[val + 256] + FrequencyHeaderSize];
partial[val + 256]++;
if (idx != 0)
{
ShiftLeft(symbols, idx);
symbols[idx] = val;
val = symbols[0];
}
}
else if (nonZero-- > 0)
{
ShiftLeft(symbols, nonZero);
val = symbols[0];
}
count++;
}
while (count < sum);
return sum;
}
/// <summary>
/// The upstream indexes the payload without bounds-checking it, which is safe for
/// a file the client wrote and is not safe for a file this shard was handed. A
/// truncated or hand-edited container would otherwise read whatever follows the
/// buffer in memory — or, on .NET, throw an `IndexOutOfRangeException` from inside
/// a decoder, which says nothing useful to an operator. This turns both into one
/// named, reportable failure.
/// </summary>
private static void Need(byte[] input, int at)
{
if (at < 0 || at >= input.Length)
throw new InvalidDataException("compressed cliloc ends mid-stream (wanted byte " + at + ")");
}
/// <summary>
/// Symbol values ordered by descending count — the order the coder assigned its
/// runs in. Repeated max-finding rather than a sort, as upstream: 256 passes over
/// 256 entries is nothing, and it reproduces the original's tie-breaking (the
/// lowest index wins), which a comparison sort would not.
/// </summary>
private static void Frequency(int[] counts, byte[] output)
{
var tmp = new int[256];
Array.Copy(counts, tmp, 256);
for (int i = 0; i < 256; i++)
{
int value = 0;
byte index = 0;
for (int j = 0; j < 256; j++)
{
if (tmp[j] > value)
{
index = (byte)j;
value = tmp[j];
}
}
if (value == 0)
break;
output[i] = index;
tmp[index] = 0;
}
}
private static void ShiftLeft(byte[] symbols, int upTo)
{
for (int i = 0; i < upTo; ++i)
symbols[i] = symbols[i + 1];
}
private static int ReadInt32(byte[] data, int at)
{
return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24);
}
}
}
}