feat(asset-bridge): the cliloc table, decompressed on the shard (Phase 2) #29
@@ -152,13 +152,13 @@ namespace Server.Custom.Bridge
|
|||||||
{
|
{
|
||||||
// Rule 1. Without a correlation id this would land on the event path, be persisted
|
// Rule 1. Without a correlation id this would land on the event path, be persisted
|
||||||
// to the sidecar's store and broadcast to every subscriber. Refuse instead.
|
// to the sidecar's store and broadcast to every subscriber. Refuse instead.
|
||||||
Fail(null, "assets.sources requires a reqId");
|
Fail(null, "BAD_REQUEST", "assets.sources requires a reqId");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!BridgeConfig.AssetsEnabled)
|
if (!BridgeConfig.AssetsEnabled)
|
||||||
{
|
{
|
||||||
Fail(reqId, "asset extraction is disabled on this shard");
|
Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,7 +170,7 @@ namespace Server.Custom.Bridge
|
|||||||
/// Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on
|
/// Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on
|
||||||
/// the worker and must touch no world state.
|
/// the worker and must touch no world state.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static void Accept(string reqId, string kind, Action job)
|
internal static void Accept(string reqId, string kind, Action job)
|
||||||
{
|
{
|
||||||
lock (_sync)
|
lock (_sync)
|
||||||
{
|
{
|
||||||
@@ -197,7 +197,7 @@ namespace Server.Custom.Bridge
|
|||||||
_job = null;
|
_job = null;
|
||||||
|
|
||||||
Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message);
|
Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message);
|
||||||
Fail(reqId, "the shard could not start its asset worker");
|
Fail(reqId, "UNAVAILABLE", "the shard could not start its asset worker");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -222,14 +222,27 @@ namespace Server.Custom.Bridge
|
|||||||
BridgeLink.Emit(sb.End());
|
BridgeLink.Emit(sb.End());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void Fail(string reqId, string reason)
|
/// <summary>
|
||||||
|
/// The asset plane's one refusal frame, shared by every family on it.
|
||||||
|
///
|
||||||
|
/// <paramref name="code"/> is what the sidecar maps to a status, and it exists because
|
||||||
|
/// the alternative it replaced — matching on the words in <paramref name="reason"/> —
|
||||||
|
/// makes an operator-facing sentence load-bearing. Rewording "disabled" would silently
|
||||||
|
/// turn a 403 into a 400. The codes are `DISABLED` (the operator switched this plane
|
||||||
|
/// off), `NOT_FOUND` (the shard has no such file), `UNREADABLE` (it has it and cannot
|
||||||
|
/// decode it), `UNAVAILABLE` (the shard cannot do this right now) and `BAD_REQUEST`
|
||||||
|
/// (the default, and the caller's fault).
|
||||||
|
/// </summary>
|
||||||
|
internal static void Fail(string reqId, string code, string reason)
|
||||||
{
|
{
|
||||||
var sb = BridgeJson.Begin("assets.error");
|
var sb = BridgeJson.Begin("assets.error");
|
||||||
|
|
||||||
if (reqId != null)
|
if (reqId != null)
|
||||||
sb.Str("reqId", reqId);
|
sb.Str("reqId", reqId);
|
||||||
|
|
||||||
sb.Str("reason", reason);
|
sb.Str("code", code)
|
||||||
|
.Str("reason", reason);
|
||||||
|
|
||||||
BridgeLink.Emit(sb.End());
|
BridgeLink.Emit(sb.End());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
767
overlay/Scripts/Custom/Bridge/BridgeCliloc.cs
Normal file
767
overlay/Scripts/Custom/Bridge/BridgeCliloc.cs
Normal file
@@ -0,0 +1,767 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.IO;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
using Ultima;
|
||||||
|
|
||||||
|
namespace Server.Custom.Bridge
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// **The cliloc table, over the bridge** (docs/link/v8.md §9 — protocol 8, phase 2).
|
||||||
|
///
|
||||||
|
/// A "cliloc" is UO's localization table: an integer id mapped to a display string. Items
|
||||||
|
/// on the wire carry a `LabelNumber`, never a name, so without this table the website can
|
||||||
|
/// only render `id 1023721` where the game renders "quarter staff". The number was never
|
||||||
|
/// the missing piece; the table was.
|
||||||
|
///
|
||||||
|
/// Until this phase the operator supplied it by hand: install UOFiddler, build a converter
|
||||||
|
/// against its `Ultima.dll`, run it over their own `Cliloc.enu`, copy a 5 MB file to the
|
||||||
|
/// web host and point a setting at it. That whole pipeline existed for one reason — the
|
||||||
|
/// file is compressed and **nothing in this stack could read it**. ServUO's own bundled
|
||||||
|
/// `Ultima.StringList` implements the plain layout only and throws on a modern client's
|
||||||
|
/// file, which is also why the shard's `VendorSearch.GetItemName` has always been inert.
|
||||||
|
///
|
||||||
|
/// So this class is the one decoder protocol 8 **writes** rather than calls (§4): a port
|
||||||
|
/// of UOFiddler's Mythic decompressor into the overlay, after which the shard can read its
|
||||||
|
/// own client's table and hand it to the website over the same request/reply path as
|
||||||
|
/// everything else. The operator installs nothing.
|
||||||
|
///
|
||||||
|
/// **Attribution.** The decompression below is a port of `Ultima/Helpers/MythicDecompress`
|
||||||
|
/// and `MoveToFront` from UOFiddler (https://github.com/polserver/UOFiddler), which is
|
||||||
|
/// released under the **Beerware** licence — compatible with this tree's GPL-3.0-or-later.
|
||||||
|
/// It is rewritten for .NET Framework 4.8: the original is written against `Span<T>`,
|
||||||
|
/// `ArrayPool<T>` and `BinaryPrimitives`, none of which ServUO's `net48` target has.
|
||||||
|
///
|
||||||
|
/// **What is NOT here, deliberately.** Shard-added items carry cliloc ids no client table
|
||||||
|
/// contains, and ServUO has no server-side notion of a custom cliloc — there is nothing in
|
||||||
|
/// the tree to read. That gap is in the *game*, not in this pipeline, so the website keeps
|
||||||
|
/// its `custom/` overlay directory and merges it over whatever arrives here. This class
|
||||||
|
/// answers exactly one question: what does the client's own table say.
|
||||||
|
/// </summary>
|
||||||
|
public static class BridgeCliloc
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Languages this can serve.
|
||||||
|
///
|
||||||
|
/// Not an arbitrary code: <c>Ultima.Files</c> resolves only the names in its own file
|
||||||
|
/// table, and cliloc files are represented there by these four. Asking for anything
|
||||||
|
/// else cannot resolve to a path however the client is laid out, so it is refused by
|
||||||
|
/// name rather than answered with an empty table.
|
||||||
|
///
|
||||||
|
/// `custom1` / `custom2` are the *client-side* custom cliloc files a shard ships to
|
||||||
|
/// its players. Nothing on the website imports them today — its `custom/` overlay
|
||||||
|
/// directory is the supported answer — but they are the shard's files and they are
|
||||||
|
/// readable, so they are not artificially excluded.
|
||||||
|
/// </summary>
|
||||||
|
private static readonly string[] Languages = { "enu", "deu", "custom1", "custom2" };
|
||||||
|
|
||||||
|
private const string DefaultLanguage = "enu";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How long a decoded table is kept in memory after its last page.
|
||||||
|
///
|
||||||
|
/// A stock `Cliloc.enu` decodes to ~67,000 live strings; holding that forever on a
|
||||||
|
/// shard that imports once a month is rude, and decoding it again costs about a
|
||||||
|
/// second. So it is cached only for as long as an import is plausibly still running:
|
||||||
|
/// freed when the last page is served, and expired on the next request if one never
|
||||||
|
/// comes (an import abandoned halfway leaves nothing behind).
|
||||||
|
/// </summary>
|
||||||
|
private static readonly TimeSpan CacheIdle = TimeSpan.FromMinutes(5);
|
||||||
|
|
||||||
|
private static readonly object _sync = new object();
|
||||||
|
private static Table _cached;
|
||||||
|
|
||||||
|
public static void Initialize()
|
||||||
|
{
|
||||||
|
if (!BridgeConfig.Enabled)
|
||||||
|
return;
|
||||||
|
|
||||||
|
BridgeBoot.RegisterHandler("cliloc.table", OnTable);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── the request plane ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Core thread. Validates, then hands the decode to the asset worker — reading and
|
||||||
|
/// decompressing five megabytes is emphatically not something to do while the world
|
||||||
|
/// is waiting, and <see cref="BridgeAssets"/>'s single slot is what keeps the shard's
|
||||||
|
/// outbound queue at a depth of about one while it happens.
|
||||||
|
/// </summary>
|
||||||
|
private static void OnTable(Dictionary<string, object> o)
|
||||||
|
{
|
||||||
|
var reqId = BridgeJson.GetString(o, "reqId");
|
||||||
|
|
||||||
|
if (reqId == null)
|
||||||
|
{
|
||||||
|
// Without a correlation id this reply would land on the event path, be persisted
|
||||||
|
// to the sidecar's store and broadcast to every subscriber — a megabyte of
|
||||||
|
// strings to every connected client, forever. Refuse instead (§3.1).
|
||||||
|
BridgeAssets.Fail(null, "BAD_REQUEST", "cliloc.table requires a reqId");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!BridgeConfig.AssetsEnabled)
|
||||||
|
{
|
||||||
|
BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var lang = BridgeJson.GetString(o, "lang");
|
||||||
|
|
||||||
|
if (String.IsNullOrEmpty(lang))
|
||||||
|
lang = DefaultLanguage;
|
||||||
|
|
||||||
|
lang = lang.ToLowerInvariant();
|
||||||
|
|
||||||
|
if (Array.IndexOf(Languages, lang) < 0)
|
||||||
|
{
|
||||||
|
BridgeAssets.Fail(reqId, "NOT_FOUND",
|
||||||
|
"no cliloc file for language '" + lang + "' (this shard can serve: "
|
||||||
|
+ String.Join(", ", Languages) + ")");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The cursor is this family's own resume point and it is a cliloc NUMBER, not an
|
||||||
|
// offset into anything. That matters: the cache behind it can be dropped and rebuilt
|
||||||
|
// between two pages of the same import (idle expiry, a second import, a restart), and
|
||||||
|
// an index into a list would silently mean something different afterwards. "Resume
|
||||||
|
// after id N" survives all of it, because the table is served in id order.
|
||||||
|
int after = -1;
|
||||||
|
var cursor = BridgeJson.GetString(o, "cursor");
|
||||||
|
|
||||||
|
if (!String.IsNullOrEmpty(cursor))
|
||||||
|
{
|
||||||
|
if (!TryParseCursor(cursor, out after))
|
||||||
|
{
|
||||||
|
BridgeAssets.Fail(reqId, "BAD_REQUEST", "malformed cursor: " + cursor);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
string language = lang;
|
||||||
|
int resumeAfter = after;
|
||||||
|
|
||||||
|
BridgeAssets.Accept(reqId, "cliloc.table", () => ReplyTable(reqId, language, resumeAfter));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryParseCursor(string cursor, out int after)
|
||||||
|
{
|
||||||
|
after = -1;
|
||||||
|
|
||||||
|
if (!cursor.StartsWith("n:", StringComparison.Ordinal))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
return Int32.TryParse(
|
||||||
|
cursor.Substring(2), NumberStyles.Integer, CultureInfo.InvariantCulture, out after);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Asset worker. Decodes (or reuses) the table and writes one page of it.
|
||||||
|
/// </summary>
|
||||||
|
private static void ReplyTable(string reqId, string lang, int after)
|
||||||
|
{
|
||||||
|
string path = ResolvePath(lang);
|
||||||
|
|
||||||
|
if (path == null)
|
||||||
|
{
|
||||||
|
BridgeAssets.Fail(reqId, "NOT_FOUND",
|
||||||
|
"this shard's client has no cliloc." + lang + " (looked where ServUO's own "
|
||||||
|
+ "data path points)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Table table;
|
||||||
|
string code, reason;
|
||||||
|
|
||||||
|
if (!TryLoad(lang, path, out table, out code, out reason))
|
||||||
|
{
|
||||||
|
BridgeAssets.Fail(reqId, code, reason);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sb = BridgeJson.Begin("cliloc.table.ok");
|
||||||
|
|
||||||
|
sb.Str("reqId", reqId)
|
||||||
|
.Str("lang", lang)
|
||||||
|
.Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION)
|
||||||
|
.Str("file", Path.GetFileName(path))
|
||||||
|
// The website pages this table over several round trips and must be able to tell
|
||||||
|
// that the file changed underneath it — an operator patching their client mid-import
|
||||||
|
// would otherwise produce one table stitched from two, with no error anywhere. It
|
||||||
|
// compares these two fields across pages and starts over if they move.
|
||||||
|
.Num("size", table.Size)
|
||||||
|
.Num("mtime", table.MTime)
|
||||||
|
.Num("total", table.Count)
|
||||||
|
.Bool("compressed", table.Compressed);
|
||||||
|
|
||||||
|
var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes);
|
||||||
|
|
||||||
|
int start = table.IndexAfter(after);
|
||||||
|
int i = start;
|
||||||
|
|
||||||
|
for (; i < table.Count; i++)
|
||||||
|
{
|
||||||
|
var item = new StringBuilder(96);
|
||||||
|
|
||||||
|
item.Append("{\"n\":").Append(table.Numbers[i].ToString(CultureInfo.InvariantCulture));
|
||||||
|
item.Append(",\"f\":").Append(table.Flags[i].ToString(CultureInfo.InvariantCulture));
|
||||||
|
item.Append(",\"t\":");
|
||||||
|
BridgeJson.Text(item, table.Texts[i]);
|
||||||
|
item.Append('}');
|
||||||
|
|
||||||
|
if (!page.TryAdd(item.ToString(), "n:" + table.Numbers[i].ToString(CultureInfo.InvariantCulture)))
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
page.Close();
|
||||||
|
|
||||||
|
bool finished = i >= table.Count;
|
||||||
|
|
||||||
|
sb.Num("from", start);
|
||||||
|
BridgeLink.Emit(sb.End());
|
||||||
|
|
||||||
|
// The last page is also the end of the import, so let the strings go. A retry of that
|
||||||
|
// page re-decodes, which costs a second and happens approximately never; holding ~67k
|
||||||
|
// strings against that is the wrong trade.
|
||||||
|
if (finished)
|
||||||
|
Release(lang);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolvePath(string lang)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// ServUO's own `Scripts/Misc/DataPath.cs` calls `Files.SetMulPath` for every
|
||||||
|
// configured data directory at Configure time, so this resolves against the
|
||||||
|
// client the SHARD is running on — including on Linux, where `Ultima.Files`'s
|
||||||
|
// registry lookup finds nothing on its own.
|
||||||
|
return Files.GetFilePath("cliloc." + lang);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── the decoded table ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
private sealed class Table
|
||||||
|
{
|
||||||
|
public string Lang;
|
||||||
|
public long Size;
|
||||||
|
public long MTime;
|
||||||
|
public bool Compressed;
|
||||||
|
public int[] Numbers;
|
||||||
|
public byte[] Flags;
|
||||||
|
public string[] Texts;
|
||||||
|
public DateTime LastUsed;
|
||||||
|
|
||||||
|
public int Count { get { return Numbers.Length; } }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Index of the first row with a number greater than <paramref name="after"/>.
|
||||||
|
/// Binary search, because the rows are in id order by construction and a page
|
||||||
|
/// deep into the table would otherwise walk everything before it.
|
||||||
|
/// </summary>
|
||||||
|
public int IndexAfter(int after)
|
||||||
|
{
|
||||||
|
if (after < 0)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
int lo = 0, hi = Numbers.Length;
|
||||||
|
|
||||||
|
while (lo < hi)
|
||||||
|
{
|
||||||
|
int mid = lo + ((hi - lo) >> 1);
|
||||||
|
|
||||||
|
if (Numbers[mid] <= after)
|
||||||
|
lo = mid + 1;
|
||||||
|
else
|
||||||
|
hi = mid;
|
||||||
|
}
|
||||||
|
|
||||||
|
return lo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryLoad(string lang, string path, out Table table, out string code, out string reason)
|
||||||
|
{
|
||||||
|
code = null;
|
||||||
|
reason = null;
|
||||||
|
|
||||||
|
long size, mtime;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var info = new FileInfo(path);
|
||||||
|
size = info.Length;
|
||||||
|
mtime = (long)(info.LastWriteTimeUtc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))
|
||||||
|
.TotalMilliseconds;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
table = null;
|
||||||
|
code = "UNREADABLE";
|
||||||
|
reason = "cannot stat " + Path.GetFileName(path) + ": " + e.Message;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
lock (_sync)
|
||||||
|
{
|
||||||
|
if (_cached != null)
|
||||||
|
{
|
||||||
|
bool stale = _cached.Lang != lang
|
||||||
|
|| _cached.Size != size
|
||||||
|
|| _cached.MTime != mtime
|
||||||
|
|| DateTime.UtcNow - _cached.LastUsed > CacheIdle;
|
||||||
|
|
||||||
|
if (stale)
|
||||||
|
_cached = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_cached != null)
|
||||||
|
{
|
||||||
|
_cached.LastUsed = DateTime.UtcNow;
|
||||||
|
table = _cached;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[] raw;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
raw = File.ReadAllBytes(path);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
table = null;
|
||||||
|
code = "UNREADABLE";
|
||||||
|
reason = "cannot read " + Path.GetFileName(path) + ": " + e.Message;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compressed = IsCompressed(raw);
|
||||||
|
byte[] plain;
|
||||||
|
|
||||||
|
if (compressed)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
plain = Mythic.Decompress(raw);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
table = null;
|
||||||
|
code = "UNREADABLE";
|
||||||
|
reason = "cannot decompress " + Path.GetFileName(path) + ": " + e.Message;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
plain = raw;
|
||||||
|
}
|
||||||
|
|
||||||
|
var built = new Table
|
||||||
|
{
|
||||||
|
Lang = lang,
|
||||||
|
Size = size,
|
||||||
|
MTime = mtime,
|
||||||
|
Compressed = compressed,
|
||||||
|
LastUsed = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!TryParseRecords(plain, built, out reason))
|
||||||
|
{
|
||||||
|
table = null;
|
||||||
|
code = "UNREADABLE";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
lock (_sync)
|
||||||
|
{
|
||||||
|
_cached = built;
|
||||||
|
}
|
||||||
|
|
||||||
|
table = built;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void Release(string lang)
|
||||||
|
{
|
||||||
|
lock (_sync)
|
||||||
|
{
|
||||||
|
if (_cached != null && _cached.Lang == lang)
|
||||||
|
_cached = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Every compressed cliloc begins with a DWORD whose high byte is <c>0x8E</c> — the
|
||||||
|
/// top byte of UOFiddler's `HeaderXorKey`, showing through because the value it hides
|
||||||
|
/// (a length) is far smaller than the key. That single byte is what tells a modern
|
||||||
|
/// client's file from the pre-2010 plain layout, and both are accepted here: a shard
|
||||||
|
/// running an old or hand-built client is not a broken shard.
|
||||||
|
/// </summary>
|
||||||
|
private static bool IsCompressed(byte[] buffer)
|
||||||
|
{
|
||||||
|
return buffer.Length >= 4 && buffer[3] == 0x8E;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── the plain layout ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
private const int HeaderBytes = 6; // int32 version + int16 language marker
|
||||||
|
private const int RecordHeaderBytes = 7; // int32 number + byte flag + uint16 length
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parses the plain layout into the sorted, blank-free arrays the wire wants.
|
||||||
|
///
|
||||||
|
/// **Strict about truncation**, and that strictness is the point: a half-decoded table
|
||||||
|
/// is indistinguishable from a complete one downstream — you would simply see some
|
||||||
|
/// items named and some not, which is exactly what "no table at all" looks like. So a
|
||||||
|
/// record running past the end of the buffer is an error naming its offset, never a
|
||||||
|
/// short table.
|
||||||
|
///
|
||||||
|
/// **Blanks are dropped here rather than on the website.** Roughly 56,000 of a stock
|
||||||
|
/// table's 123,490 entries are empty strings the client reserves and never uses, the
|
||||||
|
/// website discards them at import already, and a row that resolves to no name is
|
||||||
|
/// indistinguishable from no row at all to every caller. Dropping them halves what
|
||||||
|
/// crosses the wire for data that would be thrown away on arrival.
|
||||||
|
///
|
||||||
|
/// **A repeated id is resolved last-wins**, matching the client's own loader (its
|
||||||
|
/// dictionary assignment overwrites). The plain format permits it, so a file the game
|
||||||
|
/// itself would load happily must not fail here.
|
||||||
|
/// </summary>
|
||||||
|
private static bool TryParseRecords(byte[] data, Table into, out string reason)
|
||||||
|
{
|
||||||
|
reason = null;
|
||||||
|
|
||||||
|
if (data.Length < HeaderBytes)
|
||||||
|
{
|
||||||
|
reason = "cliloc file is shorter than its 6-byte header";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var byNumber = new Dictionary<int, Entry>(140000);
|
||||||
|
int offset = HeaderBytes;
|
||||||
|
int read = 0;
|
||||||
|
|
||||||
|
while (offset < data.Length)
|
||||||
|
{
|
||||||
|
if (offset + RecordHeaderBytes > data.Length)
|
||||||
|
{
|
||||||
|
reason = "truncated record header at byte " + offset + " (" + read + " entries read)";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
int number = ReadInt32(data, offset);
|
||||||
|
byte flag = data[offset + 4];
|
||||||
|
// Unsigned: reading this signed (as ServUO's own SDK does) turns any string over
|
||||||
|
// 32 KB into a negative length. Real tables top out around 12 KB, so it changes
|
||||||
|
// nothing today and costs nothing to get right.
|
||||||
|
int length = data[offset + 5] | (data[offset + 6] << 8);
|
||||||
|
|
||||||
|
offset += RecordHeaderBytes;
|
||||||
|
|
||||||
|
if (offset + length > data.Length)
|
||||||
|
{
|
||||||
|
reason = "truncated record body at byte " + offset + " (" + read + " entries read)";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
string text;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
text = Encoding.UTF8.GetString(data, offset, length);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
reason = "entry " + number + " at byte " + offset + " is not valid UTF-8: " + e.Message;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += length;
|
||||||
|
read++;
|
||||||
|
|
||||||
|
byNumber[number] = new Entry { Flag = flag, Text = text };
|
||||||
|
}
|
||||||
|
|
||||||
|
var numbers = new List<int>(byNumber.Count);
|
||||||
|
|
||||||
|
foreach (var pair in byNumber)
|
||||||
|
{
|
||||||
|
if (IsBlank(pair.Value.Text))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
numbers.Add(pair.Key);
|
||||||
|
}
|
||||||
|
|
||||||
|
numbers.Sort();
|
||||||
|
|
||||||
|
into.Numbers = numbers.ToArray();
|
||||||
|
into.Flags = new byte[numbers.Count];
|
||||||
|
into.Texts = new string[numbers.Count];
|
||||||
|
|
||||||
|
for (int i = 0; i < numbers.Count; i++)
|
||||||
|
{
|
||||||
|
var entry = byNumber[numbers[i]];
|
||||||
|
|
||||||
|
into.Flags[i] = entry.Flag;
|
||||||
|
into.Texts[i] = entry.Text;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct Entry
|
||||||
|
{
|
||||||
|
public byte Flag;
|
||||||
|
public string Text;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsBlank(string text)
|
||||||
|
{
|
||||||
|
if (String.IsNullOrEmpty(text))
|
||||||
|
return true;
|
||||||
|
|
||||||
|
for (int i = 0; i < text.Length; i++)
|
||||||
|
{
|
||||||
|
if (!Char.IsWhiteSpace(text[i]))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int ReadInt32(byte[] data, int at)
|
||||||
|
{
|
||||||
|
return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── the Mythic container ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The decompressor, ported from UOFiddler (Beerware; see this class's summary).
|
||||||
|
///
|
||||||
|
/// The container is two stages over the plain cliloc bytes, undone in reverse:
|
||||||
|
///
|
||||||
|
/// 1. A 4-byte header holding the decompressed length, XORed with `0x8E2C9A3D` —
|
||||||
|
/// which is where the `0x8E` sniff byte comes from.
|
||||||
|
/// 2. A **move-to-front** coding of…
|
||||||
|
/// 3. …a Burrows-Wheeler-style transform whose 1 KB frequency header (256 little-endian
|
||||||
|
/// counts, one per byte value) is both the table sizes and the total output length.
|
||||||
|
///
|
||||||
|
/// Rewritten against plain arrays: the upstream is `Span<T>`/`ArrayPool<T>`
|
||||||
|
/// code and ServUO targets `net48`, which has neither without a package this tree does
|
||||||
|
/// not vendor. The algorithm is unchanged, including the parts that read oddly — the
|
||||||
|
/// three-region `partial` table (counts, cursors, ends) and the symbol-table shifts are
|
||||||
|
/// the original's, deliberately, because this is a format decoder and a tidier
|
||||||
|
/// rewrite is a chance to be subtly wrong about someone else's bytes.
|
||||||
|
/// </summary>
|
||||||
|
private static class Mythic
|
||||||
|
{
|
||||||
|
private const uint HeaderXorKey = 0x8E2C9A3D;
|
||||||
|
private const int FrequencyHeaderSize = 1024; // 256 little-endian ints
|
||||||
|
|
||||||
|
public static byte[] Decompress(byte[] source)
|
||||||
|
{
|
||||||
|
if (source.Length < 4)
|
||||||
|
throw new InvalidDataException("compressed cliloc is shorter than its header");
|
||||||
|
|
||||||
|
uint declared = (uint)ReadInt32(source, 0) ^ HeaderXorKey;
|
||||||
|
|
||||||
|
if (declared == 0 || declared > Int32.MaxValue)
|
||||||
|
throw new InvalidDataException("compressed cliloc declares an impossible length");
|
||||||
|
|
||||||
|
var mtf = new byte[source.Length - 4];
|
||||||
|
MoveToFrontDecode(source, 4, mtf);
|
||||||
|
|
||||||
|
var output = new byte[(int)declared];
|
||||||
|
int written = InverseTransform(mtf, output);
|
||||||
|
|
||||||
|
if (written != (int)declared)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException(
|
||||||
|
"decompressed length " + written + " does not match the declared " + declared);
|
||||||
|
}
|
||||||
|
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void MoveToFrontDecode(byte[] input, int from, byte[] output)
|
||||||
|
{
|
||||||
|
var symbols = new byte[256];
|
||||||
|
|
||||||
|
for (int i = 0; i < 256; i++)
|
||||||
|
symbols[i] = (byte)i;
|
||||||
|
|
||||||
|
for (int i = 0; i < output.Length; i++)
|
||||||
|
{
|
||||||
|
int index = input[from + i];
|
||||||
|
byte symbol = symbols[index];
|
||||||
|
|
||||||
|
output[i] = symbol;
|
||||||
|
|
||||||
|
for (int j = index; j > 0; j--)
|
||||||
|
symbols[j] = symbols[j - 1];
|
||||||
|
|
||||||
|
symbols[0] = symbol;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int InverseTransform(byte[] input, byte[] destination)
|
||||||
|
{
|
||||||
|
if (input.Length < FrequencyHeaderSize)
|
||||||
|
throw new InvalidDataException("compressed cliloc is smaller than its frequency header");
|
||||||
|
|
||||||
|
// Three regions of 256: [0..255] the counts read from the header, [256..511] a
|
||||||
|
// moving cursor per symbol, [512..767] where that symbol's run ends.
|
||||||
|
var partial = new int[256 * 3];
|
||||||
|
|
||||||
|
for (int i = 0; i < 256; i++)
|
||||||
|
partial[i] = ReadInt32(input, i * 4);
|
||||||
|
|
||||||
|
int sum = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < 256; i++)
|
||||||
|
{
|
||||||
|
if (partial[i] < 0)
|
||||||
|
throw new InvalidDataException("compressed cliloc has a negative symbol count");
|
||||||
|
|
||||||
|
sum += partial[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sum == 0)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
if (destination.Length < sum)
|
||||||
|
throw new InvalidDataException("compressed cliloc's frequency header outruns its declared length");
|
||||||
|
|
||||||
|
int nonZero = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < 256; i++)
|
||||||
|
{
|
||||||
|
if (partial[i] != 0)
|
||||||
|
nonZero++;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frequency = new byte[256];
|
||||||
|
Frequency(partial, frequency);
|
||||||
|
|
||||||
|
var symbols = new byte[256];
|
||||||
|
|
||||||
|
for (int i = 0; i < 256; i++)
|
||||||
|
symbols[i] = (byte)i;
|
||||||
|
|
||||||
|
for (int i = 0, m = 0; i < nonZero; ++i)
|
||||||
|
{
|
||||||
|
byte freq = frequency[i];
|
||||||
|
|
||||||
|
Need(input, m + FrequencyHeaderSize);
|
||||||
|
|
||||||
|
symbols[input[m + FrequencyHeaderSize]] = freq;
|
||||||
|
partial[freq + 256] = m + 1;
|
||||||
|
m += partial[freq];
|
||||||
|
partial[freq + 512] = m;
|
||||||
|
}
|
||||||
|
|
||||||
|
byte val = symbols[0];
|
||||||
|
int count = 0;
|
||||||
|
|
||||||
|
do
|
||||||
|
{
|
||||||
|
destination[count] = val;
|
||||||
|
|
||||||
|
if (partial[val + 256] < partial[val + 512])
|
||||||
|
{
|
||||||
|
Need(input, partial[val + 256] + FrequencyHeaderSize);
|
||||||
|
|
||||||
|
byte idx = input[partial[val + 256] + FrequencyHeaderSize];
|
||||||
|
partial[val + 256]++;
|
||||||
|
|
||||||
|
if (idx != 0)
|
||||||
|
{
|
||||||
|
ShiftLeft(symbols, idx);
|
||||||
|
|
||||||
|
symbols[idx] = val;
|
||||||
|
val = symbols[0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (nonZero-- > 0)
|
||||||
|
{
|
||||||
|
ShiftLeft(symbols, nonZero);
|
||||||
|
|
||||||
|
val = symbols[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
while (count < sum);
|
||||||
|
|
||||||
|
return sum;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The upstream indexes the payload without bounds-checking it, which is safe for
|
||||||
|
/// a file the client wrote and is not safe for a file this shard was handed. A
|
||||||
|
/// truncated or hand-edited container would otherwise read whatever follows the
|
||||||
|
/// buffer in memory — or, on .NET, throw an `IndexOutOfRangeException` from inside
|
||||||
|
/// a decoder, which says nothing useful to an operator. This turns both into one
|
||||||
|
/// named, reportable failure.
|
||||||
|
/// </summary>
|
||||||
|
private static void Need(byte[] input, int at)
|
||||||
|
{
|
||||||
|
if (at < 0 || at >= input.Length)
|
||||||
|
throw new InvalidDataException("compressed cliloc ends mid-stream (wanted byte " + at + ")");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Symbol values ordered by descending count — the order the coder assigned its
|
||||||
|
/// runs in. Repeated max-finding rather than a sort, as upstream: 256 passes over
|
||||||
|
/// 256 entries is nothing, and it reproduces the original's tie-breaking (the
|
||||||
|
/// lowest index wins), which a comparison sort would not.
|
||||||
|
/// </summary>
|
||||||
|
private static void Frequency(int[] counts, byte[] output)
|
||||||
|
{
|
||||||
|
var tmp = new int[256];
|
||||||
|
Array.Copy(counts, tmp, 256);
|
||||||
|
|
||||||
|
for (int i = 0; i < 256; i++)
|
||||||
|
{
|
||||||
|
int value = 0;
|
||||||
|
byte index = 0;
|
||||||
|
|
||||||
|
for (int j = 0; j < 256; j++)
|
||||||
|
{
|
||||||
|
if (tmp[j] > value)
|
||||||
|
{
|
||||||
|
index = (byte)j;
|
||||||
|
value = tmp[j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value == 0)
|
||||||
|
break;
|
||||||
|
|
||||||
|
output[i] = index;
|
||||||
|
tmp[index] = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ShiftLeft(byte[] symbols, int upTo)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < upTo; ++i)
|
||||||
|
symbols[i] = symbols[i + 1];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int ReadInt32(byte[] data, int at)
|
||||||
|
{
|
||||||
|
return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user