Merge pull request 'feat(asset-bridge): the cliloc table, decompressed on the shard (Phase 2)' (#29) from feat/asset-bridge-p2 into edge
Reviewed-on: #29
This commit is contained in:
@@ -152,13 +152,13 @@ namespace Server.Custom.Bridge
|
||||
{
|
||||
// Rule 1. Without a correlation id this would land on the event path, be persisted
|
||||
// to the sidecar's store and broadcast to every subscriber. Refuse instead.
|
||||
Fail(null, "assets.sources requires a reqId");
|
||||
Fail(null, "BAD_REQUEST", "assets.sources requires a reqId");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!BridgeConfig.AssetsEnabled)
|
||||
{
|
||||
Fail(reqId, "asset extraction is disabled on this shard");
|
||||
Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ namespace Server.Custom.Bridge
|
||||
/// Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on
|
||||
/// the worker and must touch no world state.
|
||||
/// </summary>
|
||||
private static void Accept(string reqId, string kind, Action job)
|
||||
internal static void Accept(string reqId, string kind, Action job)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
@@ -197,7 +197,7 @@ namespace Server.Custom.Bridge
|
||||
_job = null;
|
||||
|
||||
Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message);
|
||||
Fail(reqId, "the shard could not start its asset worker");
|
||||
Fail(reqId, "UNAVAILABLE", "the shard could not start its asset worker");
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -222,14 +222,27 @@ namespace Server.Custom.Bridge
|
||||
BridgeLink.Emit(sb.End());
|
||||
}
|
||||
|
||||
private static void Fail(string reqId, string reason)
|
||||
/// <summary>
|
||||
/// The asset plane's one refusal frame, shared by every family on it.
|
||||
///
|
||||
/// <paramref name="code"/> is what the sidecar maps to a status, and it exists because
|
||||
/// the alternative it replaced — matching on the words in <paramref name="reason"/> —
|
||||
/// makes an operator-facing sentence load-bearing. Rewording "disabled" would silently
|
||||
/// turn a 403 into a 400. The codes are `DISABLED` (the operator switched this plane
|
||||
/// off), `NOT_FOUND` (the shard has no such file), `UNREADABLE` (it has it and cannot
|
||||
/// decode it), `UNAVAILABLE` (the shard cannot do this right now) and `BAD_REQUEST`
|
||||
/// (the default, and the caller's fault).
|
||||
/// </summary>
|
||||
internal static void Fail(string reqId, string code, string reason)
|
||||
{
|
||||
var sb = BridgeJson.Begin("assets.error");
|
||||
|
||||
if (reqId != null)
|
||||
sb.Str("reqId", reqId);
|
||||
|
||||
sb.Str("reason", reason);
|
||||
sb.Str("code", code)
|
||||
.Str("reason", reason);
|
||||
|
||||
BridgeLink.Emit(sb.End());
|
||||
}
|
||||
|
||||
|
||||
772
overlay/Scripts/Custom/Bridge/BridgeCliloc.cs
Normal file
772
overlay/Scripts/Custom/Bridge/BridgeCliloc.cs
Normal file
@@ -0,0 +1,772 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
using Ultima;
|
||||
|
||||
namespace Server.Custom.Bridge
|
||||
{
|
||||
/// <summary>
|
||||
/// **The cliloc table, over the bridge** (docs/link/v8.md §9 — protocol 8, phase 2).
|
||||
///
|
||||
/// A "cliloc" is UO's localization table: an integer id mapped to a display string. Items
|
||||
/// on the wire carry a `LabelNumber`, never a name, so without this table the website can
|
||||
/// only render `id 1023721` where the game renders "quarter staff". The number was never
|
||||
/// the missing piece; the table was.
|
||||
///
|
||||
/// Until this phase the operator supplied it by hand: install UOFiddler, build a converter
|
||||
/// against its `Ultima.dll`, run it over their own `Cliloc.enu`, copy a 5 MB file to the
|
||||
/// web host and point a setting at it. That whole pipeline existed for one reason — the
|
||||
/// file is compressed and **nothing in this stack could read it**. ServUO's own bundled
|
||||
/// `Ultima.StringList` implements the plain layout only and throws on a modern client's
|
||||
/// file, which is also why the shard's `VendorSearch.GetItemName` has always been inert.
|
||||
///
|
||||
/// So this class is the one decoder protocol 8 **writes** rather than calls (§4): a port
|
||||
/// of UOFiddler's Mythic decompressor into the overlay, after which the shard can read its
|
||||
/// own client's table and hand it to the website over the same request/reply path as
|
||||
/// everything else. The operator installs nothing.
|
||||
///
|
||||
/// **Attribution.** The decompression below is a port of `Ultima/Helpers/MythicDecompress`
|
||||
/// and `MoveToFront` from UOFiddler (https://github.com/polserver/UOFiddler), which is
|
||||
/// released under the **Beerware** licence — compatible with this tree's GPL-3.0-or-later.
|
||||
/// It is rewritten for .NET Framework 4.8: the original is written against `Span<T>`,
|
||||
/// `ArrayPool<T>` and `BinaryPrimitives`, none of which ServUO's `net48` target has.
|
||||
///
|
||||
/// **What is NOT here, deliberately.** Shard-added items carry cliloc ids no client table
|
||||
/// contains, and ServUO has no server-side notion of a custom cliloc — there is nothing in
|
||||
/// the tree to read. That gap is in the *game*, not in this pipeline, so the website keeps
|
||||
/// its `custom/` overlay directory and merges it over whatever arrives here. This class
|
||||
/// answers exactly one question: what does the client's own table say.
|
||||
/// </summary>
|
||||
public static class BridgeCliloc
|
||||
{
|
||||
/// <summary>
|
||||
/// Languages this can serve.
|
||||
///
|
||||
/// Not an arbitrary code: <c>Ultima.Files</c> resolves only the names in its own file
|
||||
/// table, and cliloc files are represented there by these four. Asking for anything
|
||||
/// else cannot resolve to a path however the client is laid out, so it is refused by
|
||||
/// name rather than answered with an empty table.
|
||||
///
|
||||
/// `custom1` / `custom2` are the *client-side* custom cliloc files a shard ships to
|
||||
/// its players. Nothing on the website imports them today — its `custom/` overlay
|
||||
/// directory is the supported answer — but they are the shard's files and they are
|
||||
/// readable, so they are not artificially excluded.
|
||||
/// </summary>
|
||||
private static readonly string[] Languages = { "enu", "deu", "custom1", "custom2" };
|
||||
|
||||
private const string DefaultLanguage = "enu";
|
||||
|
||||
/// <summary>
|
||||
/// How long a decoded table is kept in memory after its last page.
|
||||
///
|
||||
/// A stock `Cliloc.enu` decodes to ~67,000 live strings; holding that forever on a
|
||||
/// shard that imports once a month is rude, and decoding it again costs about a
|
||||
/// second. So it is cached only for as long as an import is plausibly still running:
|
||||
/// freed when the last page is served, and expired on the next request if one never
|
||||
/// comes (an import abandoned halfway leaves nothing behind).
|
||||
/// </summary>
|
||||
private static readonly TimeSpan CacheIdle = TimeSpan.FromMinutes(5);
|
||||
|
||||
private static readonly object _sync = new object();
|
||||
private static Table _cached;
|
||||
|
||||
public static void Initialize()
|
||||
{
|
||||
if (!BridgeConfig.Enabled)
|
||||
return;
|
||||
|
||||
BridgeBoot.RegisterHandler("cliloc.table", OnTable);
|
||||
}
|
||||
|
||||
// ── the request plane ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Core thread. Validates, then hands the decode to the asset worker — reading and
|
||||
/// decompressing five megabytes is emphatically not something to do while the world
|
||||
/// is waiting, and <see cref="BridgeAssets"/>'s single slot is what keeps the shard's
|
||||
/// outbound queue at a depth of about one while it happens.
|
||||
/// </summary>
|
||||
private static void OnTable(Dictionary<string, object> o)
|
||||
{
|
||||
var reqId = BridgeJson.GetString(o, "reqId");
|
||||
|
||||
if (reqId == null)
|
||||
{
|
||||
// Without a correlation id this reply would land on the event path, be persisted
|
||||
// to the sidecar's store and broadcast to every subscriber — a megabyte of
|
||||
// strings to every connected client, forever. Refuse instead (§3.1).
|
||||
BridgeAssets.Fail(null, "BAD_REQUEST", "cliloc.table requires a reqId");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!BridgeConfig.AssetsEnabled)
|
||||
{
|
||||
BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
|
||||
return;
|
||||
}
|
||||
|
||||
var lang = BridgeJson.GetString(o, "lang");
|
||||
|
||||
if (String.IsNullOrEmpty(lang))
|
||||
lang = DefaultLanguage;
|
||||
|
||||
lang = lang.ToLowerInvariant();
|
||||
|
||||
if (Array.IndexOf(Languages, lang) < 0)
|
||||
{
|
||||
BridgeAssets.Fail(reqId, "NOT_FOUND",
|
||||
"no cliloc file for language '" + lang + "' (this shard can serve: "
|
||||
+ String.Join(", ", Languages) + ")");
|
||||
return;
|
||||
}
|
||||
|
||||
// The cursor is this family's own resume point and it is a cliloc NUMBER, not an
|
||||
// offset into anything. That matters: the cache behind it can be dropped and rebuilt
|
||||
// between two pages of the same import (idle expiry, a second import, a restart), and
|
||||
// an index into a list would silently mean something different afterwards. "Resume
|
||||
// after id N" survives all of it, because the table is served in id order.
|
||||
int after = -1;
|
||||
var cursor = BridgeJson.GetString(o, "cursor");
|
||||
|
||||
if (!String.IsNullOrEmpty(cursor))
|
||||
{
|
||||
if (!TryParseCursor(cursor, out after))
|
||||
{
|
||||
BridgeAssets.Fail(reqId, "BAD_REQUEST", "malformed cursor: " + cursor);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
string language = lang;
|
||||
int resumeAfter = after;
|
||||
|
||||
BridgeAssets.Accept(reqId, "cliloc.table", () => ReplyTable(reqId, language, resumeAfter));
|
||||
}
|
||||
|
||||
private static bool TryParseCursor(string cursor, out int after)
|
||||
{
|
||||
after = -1;
|
||||
|
||||
if (!cursor.StartsWith("n:", StringComparison.Ordinal))
|
||||
return false;
|
||||
|
||||
return Int32.TryParse(
|
||||
cursor.Substring(2), NumberStyles.Integer, CultureInfo.InvariantCulture, out after);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asset worker. Decodes (or reuses) the table and writes one page of it.
|
||||
/// </summary>
|
||||
private static void ReplyTable(string reqId, string lang, int after)
|
||||
{
|
||||
string path = ResolvePath(lang);
|
||||
|
||||
if (path == null)
|
||||
{
|
||||
BridgeAssets.Fail(reqId, "NOT_FOUND",
|
||||
"this shard's client has no cliloc." + lang + " (looked where ServUO's own "
|
||||
+ "data path points)");
|
||||
return;
|
||||
}
|
||||
|
||||
Table table;
|
||||
string code, reason;
|
||||
|
||||
if (!TryLoad(lang, path, out table, out code, out reason))
|
||||
{
|
||||
BridgeAssets.Fail(reqId, code, reason);
|
||||
return;
|
||||
}
|
||||
|
||||
var sb = BridgeJson.Begin("cliloc.table.ok");
|
||||
|
||||
sb.Str("reqId", reqId)
|
||||
.Str("lang", lang)
|
||||
.Num("extractorVersion", BridgeAssets.EXTRACTOR_VERSION)
|
||||
.Str("file", Path.GetFileName(path))
|
||||
// The website pages this table over several round trips and must be able to tell
|
||||
// that the file changed underneath it — an operator patching their client mid-import
|
||||
// would otherwise produce one table stitched from two, with no error anywhere. It
|
||||
// compares these two fields across pages and starts over if they move.
|
||||
.Num("size", table.Size)
|
||||
.Num("mtime", table.MTime)
|
||||
.Num("total", table.Count)
|
||||
.Bool("compressed", table.Compressed);
|
||||
|
||||
// Before the page opens, not after it closes: PageBuilder reserves room for the
|
||||
// envelope it still has to write, and a field appended past Close() is spent outside
|
||||
// that reserve. It fits today by a wide margin, and it is the kind of thing the next
|
||||
// family copies.
|
||||
int start = table.IndexAfter(after);
|
||||
sb.Num("from", start);
|
||||
|
||||
var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes);
|
||||
|
||||
int i = start;
|
||||
|
||||
for (; i < table.Count; i++)
|
||||
{
|
||||
var item = new StringBuilder(96);
|
||||
|
||||
item.Append("{\"n\":").Append(table.Numbers[i].ToString(CultureInfo.InvariantCulture));
|
||||
item.Append(",\"f\":").Append(table.Flags[i].ToString(CultureInfo.InvariantCulture));
|
||||
item.Append(",\"t\":");
|
||||
BridgeJson.Text(item, table.Texts[i]);
|
||||
item.Append('}');
|
||||
|
||||
if (!page.TryAdd(item.ToString(), "n:" + table.Numbers[i].ToString(CultureInfo.InvariantCulture)))
|
||||
break;
|
||||
}
|
||||
|
||||
page.Close();
|
||||
|
||||
bool finished = i >= table.Count;
|
||||
|
||||
BridgeLink.Emit(sb.End());
|
||||
|
||||
// The last page is also the end of the import, so let the strings go. A retry of that
|
||||
// page re-decodes, which costs a second and happens approximately never; holding ~67k
|
||||
// strings against that is the wrong trade.
|
||||
if (finished)
|
||||
Release(lang);
|
||||
}
|
||||
|
||||
private static string ResolvePath(string lang)
|
||||
{
|
||||
try
|
||||
{
|
||||
// ServUO's own `Scripts/Misc/DataPath.cs` calls `Files.SetMulPath` for every
|
||||
// configured data directory at Configure time, so this resolves against the
|
||||
// client the SHARD is running on — including on Linux, where `Ultima.Files`'s
|
||||
// registry lookup finds nothing on its own.
|
||||
return Files.GetFilePath("cliloc." + lang);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── the decoded table ────────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Table
|
||||
{
|
||||
public string Lang;
|
||||
public long Size;
|
||||
public long MTime;
|
||||
public bool Compressed;
|
||||
public int[] Numbers;
|
||||
public byte[] Flags;
|
||||
public string[] Texts;
|
||||
public DateTime LastUsed;
|
||||
|
||||
public int Count { get { return Numbers.Length; } }
|
||||
|
||||
/// <summary>
|
||||
/// Index of the first row with a number greater than <paramref name="after"/>.
|
||||
/// Binary search, because the rows are in id order by construction and a page
|
||||
/// deep into the table would otherwise walk everything before it.
|
||||
/// </summary>
|
||||
public int IndexAfter(int after)
|
||||
{
|
||||
if (after < 0)
|
||||
return 0;
|
||||
|
||||
int lo = 0, hi = Numbers.Length;
|
||||
|
||||
while (lo < hi)
|
||||
{
|
||||
int mid = lo + ((hi - lo) >> 1);
|
||||
|
||||
if (Numbers[mid] <= after)
|
||||
lo = mid + 1;
|
||||
else
|
||||
hi = mid;
|
||||
}
|
||||
|
||||
return lo;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryLoad(string lang, string path, out Table table, out string code, out string reason)
|
||||
{
|
||||
code = null;
|
||||
reason = null;
|
||||
|
||||
long size, mtime;
|
||||
|
||||
try
|
||||
{
|
||||
var info = new FileInfo(path);
|
||||
size = info.Length;
|
||||
mtime = (long)(info.LastWriteTimeUtc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))
|
||||
.TotalMilliseconds;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
table = null;
|
||||
code = "UNREADABLE";
|
||||
reason = "cannot stat " + Path.GetFileName(path) + ": " + e.Message;
|
||||
return false;
|
||||
}
|
||||
|
||||
lock (_sync)
|
||||
{
|
||||
if (_cached != null)
|
||||
{
|
||||
bool stale = _cached.Lang != lang
|
||||
|| _cached.Size != size
|
||||
|| _cached.MTime != mtime
|
||||
|| DateTime.UtcNow - _cached.LastUsed > CacheIdle;
|
||||
|
||||
if (stale)
|
||||
_cached = null;
|
||||
}
|
||||
|
||||
if (_cached != null)
|
||||
{
|
||||
_cached.LastUsed = DateTime.UtcNow;
|
||||
table = _cached;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
byte[] raw;
|
||||
|
||||
try
|
||||
{
|
||||
raw = File.ReadAllBytes(path);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
table = null;
|
||||
code = "UNREADABLE";
|
||||
reason = "cannot read " + Path.GetFileName(path) + ": " + e.Message;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool compressed = IsCompressed(raw);
|
||||
byte[] plain;
|
||||
|
||||
if (compressed)
|
||||
{
|
||||
try
|
||||
{
|
||||
plain = Mythic.Decompress(raw);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
table = null;
|
||||
code = "UNREADABLE";
|
||||
reason = "cannot decompress " + Path.GetFileName(path) + ": " + e.Message;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
plain = raw;
|
||||
}
|
||||
|
||||
var built = new Table
|
||||
{
|
||||
Lang = lang,
|
||||
Size = size,
|
||||
MTime = mtime,
|
||||
Compressed = compressed,
|
||||
LastUsed = DateTime.UtcNow
|
||||
};
|
||||
|
||||
if (!TryParseRecords(plain, built, out reason))
|
||||
{
|
||||
table = null;
|
||||
code = "UNREADABLE";
|
||||
return false;
|
||||
}
|
||||
|
||||
lock (_sync)
|
||||
{
|
||||
_cached = built;
|
||||
}
|
||||
|
||||
table = built;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void Release(string lang)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
if (_cached != null && _cached.Lang == lang)
|
||||
_cached = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every compressed cliloc begins with a DWORD whose high byte is <c>0x8E</c> — the
|
||||
/// top byte of UOFiddler's `HeaderXorKey`, showing through because the value it hides
|
||||
/// (a length) is far smaller than the key. That single byte is what tells a modern
|
||||
/// client's file from the pre-2010 plain layout, and both are accepted here: a shard
|
||||
/// running an old or hand-built client is not a broken shard.
|
||||
/// </summary>
|
||||
private static bool IsCompressed(byte[] buffer)
|
||||
{
|
||||
return buffer.Length >= 4 && buffer[3] == 0x8E;
|
||||
}
|
||||
|
||||
// ── the plain layout ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private const int HeaderBytes = 6; // int32 version + int16 language marker
|
||||
private const int RecordHeaderBytes = 7; // int32 number + byte flag + uint16 length
|
||||
|
||||
/// <summary>
|
||||
/// Parses the plain layout into the sorted, blank-free arrays the wire wants.
|
||||
///
|
||||
/// **Strict about truncation**, and that strictness is the point: a half-decoded table
|
||||
/// is indistinguishable from a complete one downstream — you would simply see some
|
||||
/// items named and some not, which is exactly what "no table at all" looks like. So a
|
||||
/// record running past the end of the buffer is an error naming its offset, never a
|
||||
/// short table.
|
||||
///
|
||||
/// **Blanks are dropped here rather than on the website.** Roughly 56,000 of a stock
|
||||
/// table's 123,490 entries are empty strings the client reserves and never uses, the
|
||||
/// website discards them at import already, and a row that resolves to no name is
|
||||
/// indistinguishable from no row at all to every caller. Dropping them halves what
|
||||
/// crosses the wire for data that would be thrown away on arrival.
|
||||
///
|
||||
/// **A repeated id is resolved last-wins**, matching the client's own loader (its
|
||||
/// dictionary assignment overwrites). The plain format permits it, so a file the game
|
||||
/// itself would load happily must not fail here.
|
||||
/// </summary>
|
||||
private static bool TryParseRecords(byte[] data, Table into, out string reason)
|
||||
{
|
||||
reason = null;
|
||||
|
||||
if (data.Length < HeaderBytes)
|
||||
{
|
||||
reason = "cliloc file is shorter than its 6-byte header";
|
||||
return false;
|
||||
}
|
||||
|
||||
var byNumber = new Dictionary<int, Entry>(140000);
|
||||
int offset = HeaderBytes;
|
||||
int read = 0;
|
||||
|
||||
while (offset < data.Length)
|
||||
{
|
||||
if (offset + RecordHeaderBytes > data.Length)
|
||||
{
|
||||
reason = "truncated record header at byte " + offset + " (" + read + " entries read)";
|
||||
return false;
|
||||
}
|
||||
|
||||
int number = ReadInt32(data, offset);
|
||||
byte flag = data[offset + 4];
|
||||
// Unsigned: reading this signed (as ServUO's own SDK does) turns any string over
|
||||
// 32 KB into a negative length. Real tables top out around 12 KB, so it changes
|
||||
// nothing today and costs nothing to get right.
|
||||
int length = data[offset + 5] | (data[offset + 6] << 8);
|
||||
|
||||
offset += RecordHeaderBytes;
|
||||
|
||||
if (offset + length > data.Length)
|
||||
{
|
||||
reason = "truncated record body at byte " + offset + " (" + read + " entries read)";
|
||||
return false;
|
||||
}
|
||||
|
||||
string text;
|
||||
|
||||
try
|
||||
{
|
||||
text = Encoding.UTF8.GetString(data, offset, length);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
reason = "entry " + number + " at byte " + offset + " is not valid UTF-8: " + e.Message;
|
||||
return false;
|
||||
}
|
||||
|
||||
offset += length;
|
||||
read++;
|
||||
|
||||
byNumber[number] = new Entry { Flag = flag, Text = text };
|
||||
}
|
||||
|
||||
var numbers = new List<int>(byNumber.Count);
|
||||
|
||||
foreach (var pair in byNumber)
|
||||
{
|
||||
if (IsBlank(pair.Value.Text))
|
||||
continue;
|
||||
|
||||
numbers.Add(pair.Key);
|
||||
}
|
||||
|
||||
numbers.Sort();
|
||||
|
||||
into.Numbers = numbers.ToArray();
|
||||
into.Flags = new byte[numbers.Count];
|
||||
into.Texts = new string[numbers.Count];
|
||||
|
||||
for (int i = 0; i < numbers.Count; i++)
|
||||
{
|
||||
var entry = byNumber[numbers[i]];
|
||||
|
||||
into.Flags[i] = entry.Flag;
|
||||
into.Texts[i] = entry.Text;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private struct Entry
|
||||
{
|
||||
public byte Flag;
|
||||
public string Text;
|
||||
}
|
||||
|
||||
private static bool IsBlank(string text)
|
||||
{
|
||||
if (String.IsNullOrEmpty(text))
|
||||
return true;
|
||||
|
||||
for (int i = 0; i < text.Length; i++)
|
||||
{
|
||||
if (!Char.IsWhiteSpace(text[i]))
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static int ReadInt32(byte[] data, int at)
|
||||
{
|
||||
return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24);
|
||||
}
|
||||
|
||||
// ── the Mythic container ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// The decompressor, ported from UOFiddler (Beerware; see this class's summary).
|
||||
///
|
||||
/// The container is two stages over the plain cliloc bytes, undone in reverse:
|
||||
///
|
||||
/// 1. A 4-byte header holding the decompressed length, XORed with `0x8E2C9A3D` —
|
||||
/// which is where the `0x8E` sniff byte comes from.
|
||||
/// 2. A **move-to-front** coding of…
|
||||
/// 3. …a Burrows-Wheeler-style transform whose 1 KB frequency header (256 little-endian
|
||||
/// counts, one per byte value) is both the table sizes and the total output length.
|
||||
///
|
||||
/// Rewritten against plain arrays: the upstream is `Span<T>`/`ArrayPool<T>`
|
||||
/// code and ServUO targets `net48`, which has neither without a package this tree does
|
||||
/// not vendor. The algorithm is unchanged, including the parts that read oddly — the
|
||||
/// three-region `partial` table (counts, cursors, ends) and the symbol-table shifts are
|
||||
/// the original's, deliberately, because this is a format decoder and a tidier
|
||||
/// rewrite is a chance to be subtly wrong about someone else's bytes.
|
||||
/// </summary>
|
||||
private static class Mythic
|
||||
{
|
||||
private const uint HeaderXorKey = 0x8E2C9A3D;
|
||||
private const int FrequencyHeaderSize = 1024; // 256 little-endian ints
|
||||
|
||||
public static byte[] Decompress(byte[] source)
|
||||
{
|
||||
if (source.Length < 4)
|
||||
throw new InvalidDataException("compressed cliloc is shorter than its header");
|
||||
|
||||
uint declared = (uint)ReadInt32(source, 0) ^ HeaderXorKey;
|
||||
|
||||
if (declared == 0 || declared > Int32.MaxValue)
|
||||
throw new InvalidDataException("compressed cliloc declares an impossible length");
|
||||
|
||||
var mtf = new byte[source.Length - 4];
|
||||
MoveToFrontDecode(source, 4, mtf);
|
||||
|
||||
var output = new byte[(int)declared];
|
||||
int written = InverseTransform(mtf, output);
|
||||
|
||||
if (written != (int)declared)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
"decompressed length " + written + " does not match the declared " + declared);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
private static void MoveToFrontDecode(byte[] input, int from, byte[] output)
|
||||
{
|
||||
var symbols = new byte[256];
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
symbols[i] = (byte)i;
|
||||
|
||||
for (int i = 0; i < output.Length; i++)
|
||||
{
|
||||
int index = input[from + i];
|
||||
byte symbol = symbols[index];
|
||||
|
||||
output[i] = symbol;
|
||||
|
||||
for (int j = index; j > 0; j--)
|
||||
symbols[j] = symbols[j - 1];
|
||||
|
||||
symbols[0] = symbol;
|
||||
}
|
||||
}
|
||||
|
||||
private static int InverseTransform(byte[] input, byte[] destination)
|
||||
{
|
||||
if (input.Length < FrequencyHeaderSize)
|
||||
throw new InvalidDataException("compressed cliloc is smaller than its frequency header");
|
||||
|
||||
// Three regions of 256: [0..255] the counts read from the header, [256..511] a
|
||||
// moving cursor per symbol, [512..767] where that symbol's run ends.
|
||||
var partial = new int[256 * 3];
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
partial[i] = ReadInt32(input, i * 4);
|
||||
|
||||
int sum = 0;
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
{
|
||||
if (partial[i] < 0)
|
||||
throw new InvalidDataException("compressed cliloc has a negative symbol count");
|
||||
|
||||
sum += partial[i];
|
||||
}
|
||||
|
||||
if (sum == 0)
|
||||
return 0;
|
||||
|
||||
if (destination.Length < sum)
|
||||
throw new InvalidDataException("compressed cliloc's frequency header outruns its declared length");
|
||||
|
||||
int nonZero = 0;
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
{
|
||||
if (partial[i] != 0)
|
||||
nonZero++;
|
||||
}
|
||||
|
||||
var frequency = new byte[256];
|
||||
Frequency(partial, frequency);
|
||||
|
||||
var symbols = new byte[256];
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
symbols[i] = (byte)i;
|
||||
|
||||
for (int i = 0, m = 0; i < nonZero; ++i)
|
||||
{
|
||||
byte freq = frequency[i];
|
||||
|
||||
Need(input, m + FrequencyHeaderSize);
|
||||
|
||||
symbols[input[m + FrequencyHeaderSize]] = freq;
|
||||
partial[freq + 256] = m + 1;
|
||||
m += partial[freq];
|
||||
partial[freq + 512] = m;
|
||||
}
|
||||
|
||||
byte val = symbols[0];
|
||||
int count = 0;
|
||||
|
||||
do
|
||||
{
|
||||
destination[count] = val;
|
||||
|
||||
if (partial[val + 256] < partial[val + 512])
|
||||
{
|
||||
Need(input, partial[val + 256] + FrequencyHeaderSize);
|
||||
|
||||
byte idx = input[partial[val + 256] + FrequencyHeaderSize];
|
||||
partial[val + 256]++;
|
||||
|
||||
if (idx != 0)
|
||||
{
|
||||
ShiftLeft(symbols, idx);
|
||||
|
||||
symbols[idx] = val;
|
||||
val = symbols[0];
|
||||
}
|
||||
}
|
||||
else if (nonZero-- > 0)
|
||||
{
|
||||
ShiftLeft(symbols, nonZero);
|
||||
|
||||
val = symbols[0];
|
||||
}
|
||||
|
||||
count++;
|
||||
}
|
||||
while (count < sum);
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The upstream indexes the payload without bounds-checking it, which is safe for
|
||||
/// a file the client wrote and is not safe for a file this shard was handed. A
|
||||
/// truncated or hand-edited container would otherwise read whatever follows the
|
||||
/// buffer in memory — or, on .NET, throw an `IndexOutOfRangeException` from inside
|
||||
/// a decoder, which says nothing useful to an operator. This turns both into one
|
||||
/// named, reportable failure.
|
||||
/// </summary>
|
||||
private static void Need(byte[] input, int at)
|
||||
{
|
||||
if (at < 0 || at >= input.Length)
|
||||
throw new InvalidDataException("compressed cliloc ends mid-stream (wanted byte " + at + ")");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Symbol values ordered by descending count — the order the coder assigned its
|
||||
/// runs in. Repeated max-finding rather than a sort, as upstream: 256 passes over
|
||||
/// 256 entries is nothing, and it reproduces the original's tie-breaking (the
|
||||
/// lowest index wins), which a comparison sort would not.
|
||||
/// </summary>
|
||||
private static void Frequency(int[] counts, byte[] output)
|
||||
{
|
||||
var tmp = new int[256];
|
||||
Array.Copy(counts, tmp, 256);
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
{
|
||||
int value = 0;
|
||||
byte index = 0;
|
||||
|
||||
for (int j = 0; j < 256; j++)
|
||||
{
|
||||
if (tmp[j] > value)
|
||||
{
|
||||
index = (byte)j;
|
||||
value = tmp[j];
|
||||
}
|
||||
}
|
||||
|
||||
if (value == 0)
|
||||
break;
|
||||
|
||||
output[i] = index;
|
||||
tmp[index] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private static void ShiftLeft(byte[] symbols, int upTo)
|
||||
{
|
||||
for (int i = 0; i < upTo; ++i)
|
||||
symbols[i] = symbols[i + 1];
|
||||
}
|
||||
|
||||
private static int ReadInt32(byte[] data, int at)
|
||||
{
|
||||
return data[at] | (data[at + 1] << 8) | (data[at + 2] << 16) | (data[at + 3] << 24);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user