RunicGateway/servuo-plugins
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Merge pull request 'feat(asset-bridge): the transport, and the 357 wrong pictures it found' (#28) from feat/asset-bridge-p1 into edge

Browse Source 
Reviewed-on: #28
This commit is contained in:
Colby Whitlock
2026-09-10 15:02:53 +00:00
parent 1b7edebd31 c89e818dbf
commit e87c103406
7 changed files with 1683 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
overlay.toml
View File

@@ -23,8 +23,9 @@
# manual duty: when the protocol changes, bump it here in the same PR that # manual duty: when the protocol changes, bump it here in the same PR that
# changes the emitters, exactly as link bumps PROTOCOL_VERSION. # changes the emitters, exactly as link bumps PROTOCOL_VERSION.
# #
# Current: 6 — see docs/link/v6.md (idempotency keys on inbound commands, champ.boss.killed). # Current: 8 — see docs/link/v8.md (the Asset Bridge: client assets over the loopback link
protocol = 7 # instead of a converter on somebody's desktop).
protocol = 8
# ── ServUO compatibility ───────────────────────────────────────────────────── # ── ServUO compatibility ─────────────────────────────────────────────────────
# #

12
overlay/Config/Bridge.cfg
View File

@@ -295,6 +295,18 @@ EventsMaxGrantStack=1000
# would land at a moment nobody chose. Set to 0 to allow a save at any time. # would land at a moment nobody chose. Set to 0 to allow a save at any time.
EventsMinSaveIntervalSec=300 EventsMinSaveIntervalSec=300
# The asset plane (docs/link/v8.md, protocol 8). Its own switch, deliberately: turning
# this on is consenting to the website reading this host's UO CLIENT FILES -- art,
# animations, the string table -- over the link. Nothing on this plane writes anything.
AssetsEnabled=true
# The largest reply the asset plane will build, in encoded bytes. Not an item count:
# the ceiling it lives inside is the sidecar's 1 MiB inbound line cap, and base64 adds
# 33% to every payload. Clamped to [64 KiB, 512 KiB] -- half the wire cap, so that a
# single oversized item (always admitted, or its family could never make progress)
# still fits.
AssetBatchBytes=524288
# The test scaffolding in tools/scaffolding/ reads its own flags from this file # The test scaffolding in tools/scaffolding/ reads its own flags from this file
# (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose: # (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose:
# Config.Get returns the default of false when a key is missing, so a deployed # Config.Get returns the default of false when a key is missing, so a deployed

751
overlay/Scripts/Custom/Bridge/BridgeAssetValidator.cs Normal file
View File

@@ -0,0 +1,751 @@
using System;
using System.IO;
using Ultima;
namespace Server.Custom.Bridge
{
/// <summary>
/// **Validate before calling** (docs/link/v8.md §4.5) — the boundary between this protocol
/// and ServUO's vendored <c>Ultima</c> decoders. Phase 0 prototyped it in
/// <c>tools/scaffolding/BridgeAssetProbe.cs</c> and measured it both ways; phase 1 promoted
/// it here, into the overlay, and extended it to animations.
///
/// The principle: `Ultima`'s decoders take their bounds from the file they are reading, so
/// the extractor must decide whether a record is worth handing over *before* handing it
/// over. Every check below is against the index entry and the record header — cheap, and
/// enough to turn an uncatchable corrupted-state exception into a skipped asset.
///
/// **The failure this exists for is a wrong picture, not a crash.** `LoadStatic`,
/// `LoadLand` and `GetAnimation` all decode out of a shared <c>m_StreamBuffer</c> that is
/// reused, only ever grown, and filled by a <c>stream.Read</c> whose return value is
/// discarded. A record that is short, absent or out of bounds therefore renders **whatever
/// the previously-decoded asset left behind**, reports success, and is undetectable by
/// anything downstream. On the stock client on the machine phase 0 ran on that is 22,102
/// ids whose index entry reads <c>lookup 0, length 0</c>.
///
/// It cannot be complete and does not claim to be. It closes the shapes that reading the
/// source showed are reachable. What says the boundary is in the right place is the second
/// measurement rather than the first: against a client patched 21 ways it refused all eight
/// record-level defects, and against the **stock** client it refused **nothing** across
/// 49,151 statics and 16,384 land tiles. A checker that refuses real art would be worse
/// than no checker.
/// </summary>
public static class BridgeAssetValidator
{
public enum Verdict
{
/// <summary>Nothing at this id, and the index says so honestly.</summary>
Absent,
/// <summary>The entry is self-consistent and inside its file.</summary>
Ok,
/// <summary>The entry claims something the file cannot support. Do not decode it.</summary>
Refused
}
/// <summary>Land tiles decode a fixed 44×44 diamond: 2 × (2+4+…+44) ushorts.</summary>
public const int LandRecordBytes = 2024;
/// <summary>
/// A ceiling on decoded art dimensions. `LoadStatic` allocates
/// <c>new Bitmap(width, height)</c> straight from two ushorts in the record, so a
/// corrupt header asks for up to 65535×65535 — an 8 GB allocation, from a file. Real
/// art is a couple of hundred pixels at most.
/// </summary>
public const int MaxArtDimension = 1024;
/// <summary>
/// Builds our own index over the same files, with the same constructor arguments
/// <c>Art</c> uses — including <c>hasExtra: false</c>, which is the whole reason the
/// art path is safe where the gump path is not (§4.1).
/// </summary>
public static FileIndex OpenArtIndex()
{
if (ArtDataPath() == null)
return null;
return new FileIndex("Artidx.mul", "Art.mul", "artLegacyMUL.uop", 0x10000, 4, ".tga", 0x13FDC, false);
}
/// <summary>
/// The file an art index entry's <c>lookup</c> is an offset **into** — which is not
/// <c>art.mul</c> on any current client.
///
/// This cost a whole probe run to learn and it is the single most important thing
/// phase 1 must not get wrong. <c>FileIndex</c>'s UOP constructor ends with a bare
/// <c>MulPath = uopPath</c>: **when <c>artLegacyMUL.uop</c> exists it wins outright**,
/// and <c>art.mul</c> / <c>artidx.mul</c> are never opened at all. A validator that
/// bounds offsets against <c>art.mul</c> while the index holds UOP offsets is not
/// merely approximate, it is nonsense — the first run of this probe refused 34,299
/// perfectly good statics for "declaring 10533x2085" because it was reading UOP
/// offsets into the wrong file.
///
/// So the resolution order here mirrors <c>FileIndex</c>'s exactly, and anything that
/// needs the bytes behind an entry must ask this rather than assume.
/// </summary>
public static string ArtDataPath()
{
var uop = Files.GetFilePath("artlegacymul.uop");
if (uop != null)
return uop;
return Files.GetFilePath("art.mul");
}
public static long MulLength(string path)
{
if (path == null)
return 0;
try
{
return new FileInfo(path).Length;
}
catch
{
return 0;
}
}
/// <summary>
/// Judges one index entry.
///
/// The check <c>FileIndex.Seek</c> is missing is the last one: it tests
/// <c>Stream.Length &lt; e.lookup</c> — that the record *starts* inside the file — and
/// never that it *ends* inside it. A record that begins two bytes before EOF and
/// declares a length of 4,000 passes, and <c>stream.Read</c> then returns a short count
/// that the decoders discard, leaving the previous asset's bytes in the shared buffer.
/// </summary>
public static Verdict CheckEntry(FileIndex index, int at, long mulLength, long verdataLength, out string reason)
{
reason = null;
if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
{
reason = "index " + at + " out of range";
return Verdict.Absent;
}
Entry3D e = index.Index[at];
if (e.lookup < 0)
{
reason = "lookup " + e.lookup;
return Verdict.Absent;
}
bool patched = (e.length & (1 << 31)) != 0;
int length = e.length & 0x7FFFFFFF;
if (!patched && e.length < 0)
{
reason = "length " + e.length;
return Verdict.Absent;
}
if (length == 0)
{
reason = "lookup " + e.lookup + ", length 0";
return Verdict.Absent;
}
long ceiling = patched ? verdataLength : mulLength;
if (ceiling <= 0)
{
reason = (patched ? "verdata.mul" : "the art data file") + " has no length";
return Verdict.Refused;
}
if (e.lookup >= ceiling)
{
reason = "lookup " + e.lookup + " past the end of "
+ (patched ? "verdata.mul" : "the mul") + " (" + ceiling + ")";
return Verdict.Refused;
}
// The missing check. A short read is silent, and its consequence is the PREVIOUS
// asset's picture served under this id.
if (e.lookup + (long)length > ceiling)
{
reason = "record runs " + (e.lookup + (long)length - ceiling) + " bytes past the end of "
+ (patched ? "verdata.mul" : "the mul");
return Verdict.Refused;
}
return Verdict.Ok;
}
/// <summary>
/// `LoadLand` reads 2,024 bytes regardless of the declared length, so a shorter record
/// reads past the end of a buffer sized from that length.
/// </summary>
public static bool LandLengthSane(FileIndex index, int at, out string reason)
{
reason = null;
if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
return true;
int length = index.Index[at].length & 0x7FFFFFFF;
if (length > 0 && length < LandRecordBytes)
{
reason = "land record is " + length + " bytes; LoadLand always reads " + LandRecordBytes;
return false;
}
return true;
}
/// <summary>
/// Walks a static record's own row table the way <c>LoadStatic</c> will, and refuses
/// it if that walk would read outside the record.
///
/// This is the check with teeth. <c>LoadStatic</c>'s inner loop guards the write into
/// the bitmap (<c>xOffset &gt; delta</c>, <c>xOffset + xRun &gt; delta</c>) and does
/// nothing at all about the read cursor, which advances until it happens to find a
/// zero pair — potentially far outside a pinned array. Simulating the same walk with
/// a bound is the cheapest way to know whether handing the id over is safe.
/// </summary>
public static bool StaticRecordSane(byte[] record, int length, out string reason)
{
reason = null;
if (length < 8)
{
reason = "record is " + length + " bytes; a static header needs 8";
return false;
}
int words = length / 2;
int width = ReadUInt16(record, 4);
int height = ReadUInt16(record, 6);
// LoadStatic returns null for these rather than misbehaving, so it is not a refusal.
if (width <= 0 || height <= 0)
return true;
if (width > MaxArtDimension || height > MaxArtDimension)
{
reason = "declares " + width + "x" + height + ", past the " + MaxArtDimension + "px ceiling";
return false;
}
// The row-lookup table: height ushorts starting at word 4.
if (4 + height > words)
{
reason = "row table (" + height + " entries) does not fit in a " + length + "-byte record";
return false;
}
int start = height + 4;
for (int y = 0; y < height; y++)
{
int cursor = start + ReadUInt16(record, (4 + y) * 2);
while (true)
{
// Two ushorts for the run header, and they must both be inside the record.
if (cursor < 0 || cursor + 1 >= words)
{
reason = "row " + y + " reads at word " + cursor + ", past the record's " + words;
return false;
}
int xOffset = ReadUInt16(record, cursor * 2);
int xRun = ReadUInt16(record, (cursor + 1) * 2);
cursor += 2;
if (xOffset + xRun == 0)
break;
// LoadStatic stops the row here, so the read cursor stops with it.
if (xOffset > width || xOffset + xRun > width)
break;
if (cursor + xRun > words)
{
reason = "row " + y + " declares a " + xRun + "-pixel run running past the record";
return false;
}
cursor += xRun;
}
}
return true;
}
// ── animations (phase 1) ─────────────────────────────────────────────────────────────
//
// Phase 0 measured the art path and left this half unbuilt, and then proved it was
// needed: the patched client's verdata entry for body 34 points past verdata.mul's own
// end, and the wolf still "decoded" — counted among the 1,144 successes while rendering
// something else entirely. `GetAnimation` has every weakness `LoadStatic` has and one
// more, because the buffer it decodes from is longer than the record it read.
/// <summary>The palette every animation record opens with: 0x100 ushorts.</summary>
public const int AnimPaletteBytes = 0x100 * 2;
/// <summary>
/// A ceiling on an animation's declared frame count. <c>GetAnimation</c> does
/// <c>new int[frameCount]</c> straight from four bytes in the file, before it has
/// looked at anything else. Real actions are tens of frames.
/// </summary>
public const int MaxAnimFrames = 1024;
/// <summary>The xor <c>Frame</c> applies to every run header before decoding it.</summary>
private const int DoubleXor = (0x200 << 22) | (0x200 << 12);
/// <summary>
/// The <c>anim*.mul</c> an animation index entry's <c>lookup</c> is an offset into.
///
/// Unlike art (§4.6) there is no UOP precedence to get wrong here, and that is not
/// luck: <c>Animations</c> constructs its five <c>FileIndex</c>es with the four-argument
/// constructor, which passes <c>uopFile: null</c>. It never reads
/// <c>AnimationFrame*.uop</c> at all — which is the same fact that leaves six of the
/// twelve player-character bodies undecodable until §4.3's reader lands in phase 4.
/// </summary>
public static string AnimDataPath(int fileType)
{
switch (fileType)
{
case 1: return Files.GetFilePath("anim.mul");
case 2: return Files.GetFilePath("anim2.mul");
case 3: return Files.GetFilePath("anim3.mul");
case 4: return Files.GetFilePath("anim4.mul");
case 5: return Files.GetFilePath("anim5.mul");
default: return null;
}
}
/// <summary>
/// Builds our own index over one anim file, with the same constructor arguments
/// <c>Animations</c> uses — the entry lengths especially, since they decide how far
/// into the file an index runs.
/// </summary>
public static FileIndex OpenAnimIndex(int fileType)
{
if (AnimDataPath(fileType) == null)
return null;
switch (fileType)
{
case 1: return new FileIndex("Anim.idx", "Anim.mul", 0x40000, 6);
case 2: return new FileIndex("Anim2.idx", "Anim2.mul", 0x10000, -1);
case 3: return new FileIndex("Anim3.idx", "Anim3.mul", 0x20000, -1);
case 4: return new FileIndex("Anim4.idx", "Anim4.mul", 0x20000, -1);
case 5: return new FileIndex("Anim5.idx", "Anim5.mul", 0x20000, -1);
default: return null;
}
}
/// <summary>
/// Where a body's animation actually lives: which anim file, and which index in it.
///
/// **This is the never-sweep-file-types rule, written as code** (§4.3). It asks
/// <c>BodyConverter.Convert</c> once, takes its answer, and if that answer leads
/// nowhere it reports nowhere. There is deliberately no loop here and no fallback,
/// because asking the *other* anim files for an index they do not own does not fail —
/// it returns 175 decodable action/direction combinations of **a giant spider** for
/// gargoyle 666, and misaligned colour fragments for the other two. Every one of those
/// reads reports success, and nothing downstream can tell them from art.
///
/// A false return with <paramref name="reason"/> set is the ordinary, expected answer
/// for a body this client has no art for — the caller reports absent, not an error.
/// </summary>
public static bool ResolveAnimation(
int body, int action, int direction, out int fileType, out int index, out string reason)
{
reason = null;
fileType = 0;
index = -1;
if (body <= 0 || action < 0)
{
reason = "body " + body + " action " + action + " is not addressable";
return false;
}
// Directions 5-7 are the client mirroring 1-3, and `Frame` decodes them through its
// flip branch — different pointer arithmetic, which nothing below has checked.
// §5.1 fixed this protocol at direction 0 or 1, so refusing the rest costs nothing
// and keeps the validator honest about what it has actually verified.
if (direction < 0 || direction > 4)
{
reason = "direction " + direction + " is mirrored; this protocol reads 0-4 only";
return false;
}
int translated = body;
int hue = 0;
try
{
// Exactly what GetAnimation(..., preserveHue: false, ...) does first.
Animations.Translate(ref translated, ref hue);
fileType = BodyConverter.Convert(ref translated);
}
catch (Exception e)
{
reason = "body.def/bodyconv.def lookup failed: " + e.GetType().Name;
return false;
}
if (AnimDataPath(fileType) == null)
{
// Gargoyle 666 lands here: Bodyconv.def maps it to anim5, and this client has no
// anim5. Absent is the correct answer and the ONLY safe one.
reason = "bodyconv sends body " + body + " to file type " + fileType
+ ", which this client does not have";
return false;
}
index = AnimIndexOf(translated, fileType) + (action * 5) + direction;
return true;
}
/// <summary>
/// <c>Animations.GetFileIndex</c>'s own arithmetic, which is private. The banding is
/// per file type and the boundaries differ between them, so this is transcribed rather
/// than generalised — an index that disagrees with the library's by one is a picture
/// of the wrong creature, validated.
/// </summary>
private static int AnimIndexOf(int body, int fileType)
{
switch (fileType)
{
case 2:
return body < 200 ? body * 110 : 22000 + ((body - 200) * 65);
case 3:
if (body < 300)
return body * 65;
return body < 400 ? 33000 + ((body - 300) * 110) : 35000 + ((body - 400) * 175);
case 5:
// "looks strange, though it works" — the library's own comment. Body 34 is
// excluded from the first band here and nowhere else.
if (body < 200 && body != 34)
return body * 110;
return body < 400 ? 22000 + ((body - 200) * 65) : 35000 + ((body - 400) * 175);
default: // 1 and 4 share their banding
if (body < 200)
return body * 110;
return body < 400 ? 22000 + ((body - 200) * 65) : 35000 + ((body - 400) * 175);
}
}
/// <summary>
/// Walks an animation record the way <c>GetAnimation</c> and <c>Frame</c> will, and
/// refuses it if that walk would read outside the record or write outside the bitmap.
///
/// Two things make this stricter than the static walk, and both come from the library:
///
/// <c>GetAnimation</c> decodes through <c>new MemoryStream(m_StreamBuffer, false)</c> —
/// the whole shared buffer, not the <c>length</c> bytes it just read into it. So a
/// truncated record does not hit end-of-stream and throw; the reader sails on into the
/// **previous** animation's bytes and returns a plausible frame. Bounding against
/// <paramref name="length"/> rather than against the buffer is the entire point.
///
/// And <c>Frame</c>'s run loop is a *write* through a <c>LockBits</c> pointer whose
/// origin comes from two signed shorts in the file (<c>xCenter</c>, <c>yCenter</c>),
/// with no bound of any kind. <c>LoadStatic</c> at least guards its writes; this does
/// not, so the destination of every run is checked against the bitmap it locked.
///
/// <paramref name="maxFrames"/> is how many frames the caller will actually decode —
/// 1 for the catalogue's thumbnail (<c>FirstFrame: true</c>), 0 for all of them.
/// Checking frames nobody decodes would invent refusals, which §4.5 costs more than
/// it saves.
/// </summary>
public static bool AnimationRecordSane(byte[] record, int length, int maxFrames, out string reason)
{
reason = null;
if (length < AnimPaletteBytes + 4)
{
reason = "record is " + length + " bytes; an animation needs "
+ (AnimPaletteBytes + 4) + " for its palette and frame count";
return false;
}
int start = AnimPaletteBytes;
int frameCount = ReadInt32(record, start);
if (frameCount <= 0)
{
reason = "declares " + frameCount + " frames";
return false;
}
if (frameCount > MaxAnimFrames)
{
reason = "declares " + frameCount + " frames, past the " + MaxAnimFrames + " ceiling";
return false;
}
// The lookup table is read in full whatever FirstFrame says, so it is bounded in full.
long tableEnd = (long)start + 4 + ((long)frameCount * 4);
if (tableEnd > length)
{
reason = "frame table (" + frameCount + " entries) does not fit in a "
+ length + "-byte record";
return false;
}
int check = maxFrames > 0 && maxFrames < frameCount ? maxFrames : frameCount;
for (int i = 0; i < check; i++)
{
int at = start + ReadInt32(record, start + 4 + (i * 4));
if (!FrameSane(record, length, at, i, out reason))
return false;
}
return true;
}
private static bool FrameSane(byte[] record, int length, int at, int frame, out string reason)
{
reason = null;
if (at < 0 || at + 8 > length)
{
reason = "frame " + frame + " starts at " + at + ", outside the "
+ length + "-byte record";
return false;
}
int xCenter = ReadInt16(record, at);
int yCenter = ReadInt16(record, at + 2);
int width = ReadUInt16(record, at + 4);
int height = ReadUInt16(record, at + 6);
// Frame's constructor returns before locking anything for these, so they are empty
// rather than dangerous — and an empty frame is a real thing in this format.
if (width == 0 || height == 0)
return true;
if (width > MaxArtDimension || height > MaxArtDimension)
{
reason = "frame " + frame + " declares " + width + "x" + height + ", past the "
+ MaxArtDimension + "px ceiling";
return false;
}
// Settings.PixelFormat is 16bpp and GDI+ pads each scanline to four bytes, so a row
// is `delta` ushorts wide and the locked region is height*delta of them. This is the
// same `bd.Stride >> 1` Frame computes.
int delta = (((width * 2) + 3) & ~3) >> 1;
long pixels = (long)height * delta;
long origin = (xCenter - 0x200) + ((long)((yCenter + height) - 0x200) * delta);
int cursor = at + 8;
while (true)
{
if (cursor + 4 > length)
{
reason = "frame " + frame
+ " runs off the end of the record looking for its terminator";
return false;
}
int header = ReadInt32(record, cursor);
cursor += 4;
if (header == 0x7FFF7FFF)
break;
header ^= DoubleXor;
long dy = (header >> 12) & 0x3FF;
long dx = (header >> 22) & 0x3FF;
int run = header & 0xFFF;
long first = origin + (dy * delta) + dx;
if (first < 0 || first + run > pixels)
{
reason = "frame " + frame + " writes pixels " + first + ".." + (first + run)
+ " outside its own " + pixels + "-pixel bitmap";
return false;
}
// One palette byte per pixel, read straight out of the record.
if (cursor + run > length)
{
reason = "frame " + frame + " declares a " + run
+ "-pixel run running past the record";
return false;
}
cursor += run;
}
return true;
}
private static int ReadUInt16(byte[] b, int at)
{
return b[at] | (b[at + 1] << 8);
}
private static int ReadInt16(byte[] b, int at)
{
return (short)(b[at] | (b[at + 1] << 8));
}
private static int ReadInt32(byte[] b, int at)
{
return b[at] | (b[at + 1] << 8) | (b[at + 2] << 16) | (b[at + 3] << 24);
}
/// <summary>
/// Reads a record's actual bytes so <see cref="StaticRecordSane"/> or
/// <see cref="AnimationRecordSane"/> can walk it.
///
/// Holds its own handles rather than borrowing the library's, because <c>FileIndex</c>
/// hands out the stream it decodes from and moving that stream's position underneath
/// the decoder would be its own bug. Opened <c>FileShare.ReadWrite</c> to match how
/// <c>FileIndex</c> opens the same files.
///
/// One reader serves one data file, so an animation sweep wants one per file type,
/// built from <see cref="AnimDataPath"/>.
/// </summary>
public sealed class RecordReader : IDisposable
{
private readonly FileStream _mul;
private readonly FileStream _verdata;
private byte[] _scratch = new byte[64 * 1024];
public RecordReader(string mulPath, string verdataPath)
{
_mul = Open(mulPath);
_verdata = Open(verdataPath);
}
private static FileStream Open(string path)
{
if (path == null || !File.Exists(path))
return null;
try
{
return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
catch
{
return null;
}
}
/// <summary>
/// True when the record at <paramref name="at"/> is safe to hand to
/// <c>Art.GetStatic</c>. A record that cannot be read at all is reported sane —
/// <see cref="CheckEntry"/> has already judged the entry, and this must not
/// invent a second reason to refuse.
/// </summary>
public bool StaticSane(FileIndex index, int at, out string reason)
{
int length = ReadRecord(index, at, out reason);
if (length < 0)
return true;
if (length == 0)
return false;
return StaticRecordSane(_scratch, length, out reason);
}
/// <summary>
/// True when the record at <paramref name="at"/> is safe to hand to
/// <c>Animations.GetAnimation</c>. <paramref name="maxFrames"/> is how many frames
/// the caller will decode — 1 for a <c>FirstFrame</c> call, 0 for all of them.
/// </summary>
public bool AnimationSane(FileIndex index, int at, int maxFrames, out string reason)
{
int length = ReadRecord(index, at, out reason);
if (length < 0)
return true;
if (length == 0)
return false;
return AnimationRecordSane(_scratch, length, maxFrames, out reason);
}
/// <summary>
/// Reads one record into <see cref="_scratch"/>. Returns its length, 0 for a
/// failure (with <paramref name="reason"/> set), or -1 when there is nothing to
/// read at all — <see cref="CheckEntry"/> has already judged the entry, and this
/// must not invent a second reason to refuse.
/// </summary>
private int ReadRecord(FileIndex index, int at, out string reason)
{
reason = null;
if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
return -1;
Entry3D e = index.Index[at];
bool patched = (e.length & (1 << 31)) != 0;
int length = e.length & 0x7FFFFFFF;
var stream = patched ? _verdata : _mul;
if (stream == null || length <= 0 || e.lookup < 0)
return -1;
if (_scratch.Length < length)
_scratch = new byte[length];
int read;
try
{
stream.Seek(e.lookup, SeekOrigin.Begin);
read = stream.Read(_scratch, 0, length);
}
catch (Exception ex)
{
reason = "cannot read the record: " + ex.GetType().Name;
return 0;
}
// The short read the decoders discard. Refusing here is the whole point: the
// library would decode whatever the shared buffer happened to hold.
if (read < length)
{
reason = "short read — " + read + " of " + length + " bytes available";
return 0;
}
return length;
}
public void Dispose()
{
if (_mul != null)
_mul.Dispose();
if (_verdata != null)
_verdata.Dispose();
}
}
}
}

747
overlay/Scripts/Custom/Bridge/BridgeAssets.cs Normal file
View File

@@ -0,0 +1,747 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using Ultima;
namespace Server.Custom.Bridge
{
/// <summary>
/// **The Asset Bridge's transport** (docs/link/v8.md §3, §6, §7 — protocol 8, phase 1).
///
/// Everything else on this link answers on the Core thread, reads live world state, and
/// replies in microseconds. The asset plane cannot: it reads hundreds of megabytes of the
/// operator's client files and decodes pictures out of them, and doing either on the Core
/// thread would stop the world for every player on the shard. So this class is the one
/// genuinely new shape in protocol 8 — a handler that accepts on the Core thread, hands
/// the work to a **dedicated asset worker**, and returns immediately.
///
/// Three rules hold it together, and each of them is answering a specific way this could
/// go wrong.
///
/// **1. Replies, never events.** Every asset frame carries the caller's `reqId`, so
/// `rpc.rs`'s `try_route` consumes it before `app.rs` can persist it to SQLite and
/// broadcast it to every WebSocket subscriber. An asset stream on the event path would
/// grow the sidecar's store without bound and fan megabytes at every connected client,
/// forever. Nothing here emits an unsolicited frame — if a request has no `reqId` it is
/// refused rather than answered.
///
/// **2. One request outstanding, always.** <see cref="BridgeLink.Emit"/>'s queue is bounded
/// drop-oldest in **lines, not bytes** — a design that is right for live events and
/// dangerous for bulk transfer, because 10,000 queued 200 KB replies is 2 GB of shard
/// memory. The bound that actually holds is flow control, not a bigger queue: this plane
/// has **one slot**, and a second asset request arriving while one is in flight is
/// answered `bridge.busy` (which the sidecar already maps to 425) rather than queued.
/// Queue depth therefore stays at approximately one by construction. A dropped or lost
/// reply just times out and is re-requested, which is safe because reading a client file
/// is idempotent and touches no world state.
///
/// Note what that costs, deliberately: a status poll shares the slot with a batch, so
/// polling during a long import gets 425 until the batch lands. That is honest — this
/// plane really is doing one thing at a time — and the admin surface (phase 8) is where a
/// separate status lane would have to argue for itself.
///
/// **3. Byte budgets, not counts.** Batches are cut by encoded size
/// (<see cref="BridgeConfig.AssetBatchBytes"/>, 512 KB), not by item count, because the
/// ceilings this has to live inside are byte ceilings: the sidecar refuses an inbound line
/// over 1 MiB, and base64 costs 33% on top of whatever the payload measures.
/// <see cref="PageBuilder"/> is that budget, and every asset family shares it so the
/// envelope cannot drift apart between them.
///
/// **Emitting from off the Core thread is safe here, and it is worth saying why.**
/// <c>BridgeLink.Emit</c> enqueues onto a <c>ConcurrentQueue</c> and never touches the
/// socket, so the enqueue itself is fine. The subtle part is
/// <c>BridgeIdempotency.Observe</c>, which <c>Emit</c> calls while a keyed command is in
/// flight: it captures a line only when that line's correlation field **exactly equals**
/// the open command's correlation value, and correlation values come from one monotonic
/// counter in the sidecar. An asset reply therefore cannot be mistaken for a keyed
/// command's reply, whatever the interleaving.
/// </summary>
public static class BridgeAssets
{
/// <summary>
/// What version of *our derivation* produced these bytes (§7).
///
/// The source gate hashes the operator's client files, which answers "did the inputs
/// change". It cannot answer "did the way we read them change" — and that is the case
/// that bites, because a corrected frame offset or a fixed hue application changes
/// every derived byte while every source file stays identical. So this is folded into
/// stage 1 alongside the hashes, and bumping it makes the whole working set drift,
/// which is the intended and correct blast radius.
///
/// Bump it whenever extraction changes what it produces from unchanged input. It is
/// the same rule <c>spawnAtlasSource.js</c>'s `PARSER_VERSION` follows, and it applies
/// here more rather than less: this pipeline derives far more from far less.
/// </summary>
public const int EXTRACTOR_VERSION = 1;
// ── the one slot (§3.2) ──────────────────────────────────────────────────────────────
private static readonly object _sync = new object();
private static Thread _worker;
private static readonly AutoResetEvent _wake = new AutoResetEvent(false);
private static Action _job;
private static string _inFlight;
private static DateTime _inFlightSince;
private static bool _running;
private static long _served, _busied, _failed;
// ── the hash cache (§6) ──────────────────────────────────────────────────────────────
private static readonly Dictionary<string, CachedHash> _hashes =
new Dictionary<string, CachedHash>(StringComparer.OrdinalIgnoreCase);
private static Thread _hasher;
private static volatile bool _hashing;
private sealed class CachedHash
{
public long Size;
public long MTime;
public string Sha256;
}
// ── imaging (§4.4) ───────────────────────────────────────────────────────────────────
private static bool _imagingChecked;
private static bool _imagingOk;
private static string _imagingReason;
public static void Initialize()
{
if (!BridgeConfig.Enabled)
return;
BridgeBoot.RegisterHandler("assets.sources", OnSources);
}
public static string Status()
{
int cached;
lock (_hashes)
{
cached = _hashes.Count;
}
lock (_sync)
{
return String.Format(
"assets(served={0} busied={1} failed={2} inFlight={3} hashing={4} cached={5})",
_served, _busied, _failed, _inFlight ?? "-", _hashing, cached);
}
}
// ── the request plane ────────────────────────────────────────────────────────────────
/// <summary>
/// Stage 1 of §6: what the shard's client files currently are. No pixels, no assets —
/// just the gate that lets the website decide whether anything needs importing at all,
/// because the normal case is a restart that changed nothing and it must cost nothing.
/// </summary>
private static void OnSources(Dictionary<string, object> o)
{
var reqId = BridgeJson.GetString(o, "reqId");
if (reqId == null)
{
// Rule 1. Without a correlation id this would land on the event path, be persisted
// to the sidecar's store and broadcast to every subscriber. Refuse instead.
Fail(null, "assets.sources requires a reqId");
return;
}
if (!BridgeConfig.AssetsEnabled)
{
Fail(reqId, "asset extraction is disabled on this shard");
return;
}
Accept(reqId, "assets.sources", () => ReplySources(reqId));
}
/// <summary>
/// Claims the single slot and hands the work to the worker, or answers `bridge.busy`.
/// Runs on the Core thread and does nothing expensive; <paramref name="job"/> runs on
/// the worker and must touch no world state.
/// </summary>
private static void Accept(string reqId, string kind, Action job)
{
lock (_sync)
{
if (_inFlight != null)
{
_busied++;
Busy(reqId, kind);
return;
}
_inFlight = kind;
_inFlightSince = DateTime.UtcNow;
_job = job;
try
{
EnsureWorker();
}
catch (Exception e)
{
// The slot is claimed and there is now nothing that will ever free it. Give it
// back here or this plane answers `bridge.busy` for the life of the process.
_inFlight = null;
_job = null;
Console.WriteLine("[Bridge] cannot start the asset worker: {0}", e.Message);
Fail(reqId, "the shard could not start its asset worker");
return;
}
}
_wake.Set();
}
private static void Busy(string reqId, string kind)
{
var held = (DateTime.UtcNow - _inFlightSince).TotalSeconds;
var sb = BridgeJson.Begin("bridge.busy");
sb.Str("reqId", reqId)
// `busyKind`, never a second `kind` — `Begin` has already written this frame's own,
// and a JSON object carrying two makes every parser take the last. Protocol 6 shipped
// that bug once and it made the sidecar answer 200 for a refusal.
.Str("busyKind", kind)
.Num("heldForSec", (long)held)
.Str("reason", "the asset plane serves one request at a time");
BridgeLink.Emit(sb.End());
}
private static void Fail(string reqId, string reason)
{
var sb = BridgeJson.Begin("assets.error");
if (reqId != null)
sb.Str("reqId", reqId);
sb.Str("reason", reason);
BridgeLink.Emit(sb.End());
}
// ── the worker ───────────────────────────────────────────────────────────────────────
/// <summary>
/// Started on first use rather than at boot, so a shard that never imports an asset
/// never carries the thread. Caller must hold <see cref="_sync"/>.
/// </summary>
private static void EnsureWorker()
{
if (_worker != null)
return;
_running = true;
_worker = new Thread(WorkLoop)
{
Name = "BridgeAssets",
IsBackground = true
};
_worker.Start();
}
private static void WorkLoop()
{
while (_running)
{
_wake.WaitOne(1000);
Action job;
lock (_sync)
{
job = _job;
_job = null;
}
if (job == null)
continue;
try
{
job();
Interlocked.Increment(ref _served);
}
catch (Exception e)
{
// A handler that throws must still free the slot, or this plane is wedged for
// the life of the process and every later request answers `bridge.busy`.
Interlocked.Increment(ref _failed);
Console.WriteLine("[Bridge] asset worker: {0}: {1}", e.GetType().Name, e.Message);
}
finally
{
lock (_sync)
{
_inFlight = null;
}
}
}
}
// ── assets.sources ───────────────────────────────────────────────────────────────────
/// <summary>
/// The client files whose bytes decide everything downstream.
///
/// Resolved through <c>Ultima.Files</c> rather than by joining a configured directory,
/// because that is what the decoders themselves do — a file this reports is a file
/// they would actually open.
/// </summary>
private static readonly string[] SourceFiles =
{
"cliloc.enu",
"artlegacymul.uop", "art.mul", "artidx.mul",
"anim.idx", "anim.mul",
"anim2.idx", "anim2.mul",
"anim3.idx", "anim3.mul",
"anim4.idx", "anim4.mul",
"anim5.idx", "anim5.mul",
"body.def", "bodyconv.def",
"hues.mul",
"verdata.mul"
};
private static void ReplySources(string reqId)
{
var sb = BridgeJson.Begin("assets.sources.ok");
sb.Str("reqId", reqId)
.Num("extractorVersion", EXTRACTOR_VERSION);
WriteImaging(sb);
// §4.6: whichever of art.mul / artLegacyMUL.uop `FileIndex` would actually open. An
// operator who added custom graphics to art.mul while the UOP is present is getting
// nothing, silently, and this is the only place that can tell them so.
string artData = BridgeAssetValidator.ArtDataPath();
sb.Str("artDataFile", artData == null ? null : Path.GetFileName(artData));
var page = new PageBuilder(sb, "files", BridgeConfig.AssetBatchBytes);
bool anyMissingHash = false;
for (int i = 0; i < SourceFiles.Length; i++)
{
string name = SourceFiles[i];
string path = ResolvePath(name);
if (path == null)
continue;
var item = new StringBuilder(256);
item.Append("{\"name\":");
BridgeJson.Text(item, name);
item.Append(",\"path\":");
BridgeJson.Text(item, path);
long size = 0, mtime = 0;
try
{
var info = new FileInfo(path);
size = info.Length;
mtime = ToUnixMs(info.LastWriteTimeUtc);
}
catch (Exception e)
{
item.Append(",\"unreadable\":");
BridgeJson.Text(item, e.GetType().Name);
}
item.Append(",\"size\":").Append(size.ToString(CultureInfo.InvariantCulture));
item.Append(",\"mtime\":").Append(mtime.ToString(CultureInfo.InvariantCulture));
string hash = CachedHashFor(path, size, mtime);
if (hash == null)
anyMissingHash = true;
item.Append(",\"sha256\":");
BridgeJson.Text(item, hash);
// The one diagnostic §4.6 asks for: art.mul is present, and unread.
if (artData != null
&& (name == "art.mul" || name == "artidx.mul")
&& !artData.EndsWith(".mul", StringComparison.OrdinalIgnoreCase))
{
item.Append(",\"shadowedBy\":");
BridgeJson.Text(item, Path.GetFileName(artData));
}
item.Append('}');
if (!page.TryAdd(item.ToString(), name))
break;
}
page.Close();
// §6's gate is (size, mtime) first and a content hash only when those differ, because
// anim.mul and art.mul are 195 MB and 148 MB and a full hash on every status poll
// would make the admin panel feel broken. It would also blow the sidecar's 10 s reply
// timeout outright on the first call. So a hash that is not cached is reported `null`
// and computed in the background: this reply is always fast, and the next poll — after
// `hashing` goes false — carries the answer.
if (anyMissingHash)
StartHashing();
sb.Bool("hashing", _hashing);
sb.Bool("complete", !anyMissingHash);
BridgeLink.Emit(sb.End());
}
private static string ResolvePath(string name)
{
try
{
return Files.GetFilePath(name);
}
catch
{
return null;
}
}
private static long ToUnixMs(DateTime utc)
{
return (long)(utc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;
}
// ── imaging (§4.4) ───────────────────────────────────────────────────────────────────
/// <summary>
/// Whether this host can turn a record into a picture at all.
///
/// ServUO targets net48, so a Linux shard runs it under Mono, where
/// <c>System.Drawing</c> is a thin layer over **libgdiplus** — and §4.2 put
/// <c>System.Drawing</c> in the *decode* path, not merely the encode: <c>Frame</c>
/// writes ARGB1555 through a <c>LockBits</c> pointer. Without that library a Linux
/// shard cannot read a sprite at all, while clilocs and the atlas are unaffected
/// because neither touches pixels.
///
/// It must never present as a stack trace or a 500. It is a named, actionable outcome
/// in the same family as the cliloc reader's `COMPRESSED`, and it is reported here —
/// on the *source gate*, the first call any import makes — so an operator learns it
/// while setting the shard up rather than from an empty bestiary weeks later.
/// </summary>
private static void WriteImaging(StringBuilder sb)
{
CheckImaging();
sb.Append(",\"imaging\":{\"ok\":").Append(_imagingOk ? "true" : "false");
if (!_imagingOk)
{
sb.Append(",\"code\":\"NO_IMAGING\",\"reason\":");
BridgeJson.Text(sb,
"This shard host cannot render images — Mono's System.Drawing needs libgdiplus. "
+ "Install it (apt-get install libgdiplus) and re-run the import. Cliloc and atlas "
+ "import are unaffected. (" + _imagingReason + ")");
}
sb.Append('}');
}
private static void CheckImaging()
{
if (_imagingChecked)
return;
_imagingChecked = true;
try
{
TouchImaging();
_imagingOk = true;
}
catch (Exception e)
{
// On a host with no libgdiplus this is a TypeInitializationException wrapping a
// DllNotFoundException, and it can surface as the method failing to JIT rather
// than as a throw from inside it — which is why the construction lives in its own
// method, so the failure is contained here instead of taking this class's
// static initialisation with it.
_imagingOk = false;
_imagingReason = e.GetType().Name + ": " + e.Message;
}
}
private static void TouchImaging()
{
using (var bmp = new System.Drawing.Bitmap(1, 1))
{
bmp.SetPixel(0, 0, System.Drawing.Color.Black);
}
}
// ── the hash cache (§6) ──────────────────────────────────────────────────────────────
private static string CachedHashFor(string path, long size, long mtime)
{
lock (_hashes)
{
CachedHash cached;
if (_hashes.TryGetValue(path, out cached)
&& cached.Size == size
&& cached.MTime == mtime)
{
return cached.Sha256;
}
}
return null;
}
/// <summary>
/// Rehashes whatever the cache is missing, on its own thread.
///
/// Deliberately **not** a job on the asset worker: hashing 343 MB takes seconds to
/// tens of seconds, and holding the single slot for that long would answer every
/// status poll `bridge.busy` for the whole pass — which is exactly the moment an
/// operator is watching the panel. It emits nothing and correlates with nothing; it
/// only fills the cache that the next `assets.sources` reads.
/// </summary>
private static void StartHashing()
{
lock (_sync)
{
if (_hashing)
return;
_hashing = true;
_hasher = new Thread(HashLoop)
{
Name = "BridgeAssetHash",
IsBackground = true
};
_hasher.Start();
}
}
private static void HashLoop()
{
try
{
for (int i = 0; i < SourceFiles.Length; i++)
{
string path = ResolvePath(SourceFiles[i]);
if (path == null)
continue;
long size, mtime;
try
{
var info = new FileInfo(path);
size = info.Length;
mtime = ToUnixMs(info.LastWriteTimeUtc);
}
catch
{
continue;
}
if (CachedHashFor(path, size, mtime) != null)
continue;
string hash = HashFile(path);
if (hash == null)
continue;
lock (_hashes)
{
_hashes[path] = new CachedHash { Size = size, MTime = mtime, Sha256 = hash };
}
}
}
catch (Exception e)
{
Console.WriteLine("[Bridge] asset hash pass: {0}: {1}", e.GetType().Name, e.Message);
}
finally
{
// Under _sync, matching StartHashing: cleared outside it, two passes could both
// pass the guard and hash the same 343 MB twice.
lock (_sync)
{
_hashing = false;
}
}
}
private static string HashFile(string path)
{
try
{
using (var sha = SHA256.Create())
using (var stream = new FileStream(
path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1 << 20))
{
var buffer = new byte[1 << 20];
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
sha.TransformBlock(buffer, 0, read, null, 0);
sha.TransformFinalBlock(buffer, 0, 0);
return Hex(sha.Hash);
}
}
catch (Exception e)
{
Console.WriteLine("[Bridge] cannot hash {0}: {1}", path, e.Message);
return null;
}
}
private static string Hex(byte[] bytes)
{
var sb = new StringBuilder(bytes.Length * 2);
for (int i = 0; i < bytes.Length; i++)
sb.Append(bytes[i].ToString("x2", CultureInfo.InvariantCulture));
return sb.ToString();
}
// ── the paging envelope (§3.3) ───────────────────────────────────────────────────────
/// <summary>
/// **One envelope for every asset family**, defined here in phase 1 so that clilocs
/// (phase 2), the body catalogue (3), statics and land (5), deep animation keys (6)
/// and the ServUO tree files (7) all page the same way. They are otherwise five
/// chances to invent five slightly different shapes, and the website would have to
/// learn each one.
///
/// The envelope a reply closes with:
///
/// <code>
/// "items": [ … ],
/// "more": true, // ask again with this cursor
/// "cursor": "s:4104", // opaque to everyone but the shard; absent when more:false
/// "cut": "budget" // budget | end | limit — WHY this page stopped
/// </code>
///
/// **The budget is bytes, and it is UTF-8 bytes.** Not item count, because the ceiling
/// this lives inside is the sidecar's inbound line cap; and not chars, because a
/// cliloc row is real text and a `StringBuilder`'s Length would undercount every
/// non-ASCII character in it.
///
/// `cut` exists because "the page is short" has three different meanings and the
/// website must not have to guess which: the source ran out (`end`), the byte budget
/// was spent (`budget`), or the family stopped at its own limit (`limit`). Only the
/// first means the import is finished.
///
/// **The first item is always admitted**, even if it alone exceeds the budget.
/// Otherwise an oversized item would make its family unable to make any progress at
/// all — it would be skipped for the budget on every page, forever. That is safe
/// precisely because the budget is set to half the sidecar's line cap
/// (<see cref="BridgeConfig.AssetBatchBytes"/>), so one such item still fits the wire.
/// </summary>
public sealed class PageBuilder
{
private readonly StringBuilder _sb;
private readonly int _budget;
private int _bytes;
private int _count;
private string _cursor;
private string _cut = "end";
/// <summary>
/// Room kept back for the fields the envelope must still be able to write after
/// the last item — `more`, `cursor`, `cut` and the closing brace. Without it a
/// page could fill the budget exactly and then overrun it closing itself.
/// </summary>
private const int Reserve = 256;
public PageBuilder(StringBuilder sb, string arrayName, int budget)
{
_sb = sb;
_budget = budget;
sb.Append(",\"").Append(arrayName).Append("\":[");
// The prefix is already written, and it counts: the cap the sidecar enforces is
// on the whole line, not on the array.
_bytes = Encoding.UTF8.GetByteCount(sb.ToString());
}
public int Count { get { return _count; } }
/// <summary>
/// Adds one already-serialised item. <paramref name="cursorAfter"/> is where the
/// family should resume if this turns out to be the last item on the page.
/// Returns false when the budget is spent — the caller stops, and `more` is true.
/// </summary>
public bool TryAdd(string item, string cursorAfter)
{
if (item == null)
return true;
int cost = Encoding.UTF8.GetByteCount(item) + (_count > 0 ? 1 : 0);
if (_count > 0 && _bytes + cost + Reserve > _budget)
{
_cut = "budget";
return false;
}
if (_count > 0)
_sb.Append(',');
_sb.Append(item);
_bytes += cost;
_count++;
_cursor = cursorAfter;
return true;
}
/// <summary>
/// Stops the page for a reason of the family's own — a per-request limit, say —
/// rather than because the budget ran out.
/// </summary>
public void Cut(string why)
{
_cut = why;
}
public void Close()
{
bool more = _cut != "end";
_sb.Append(']');
_sb.Bool("more", more);
_sb.Str("cut", _cut);
if (more && _cursor != null)
_sb.Str("cursor", _cursor);
}
}
}
}

1
overlay/Scripts/Custom/Bridge/BridgeBoot.cs
View File

@@ -260,6 +260,7 @@ namespace Server.Custom.Bridge
e.Mobile.SendMessage("Bridge: {0}", BridgeParticipation.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeParticipation.Status());
e.Mobile.SendMessage("Bridge: {0}", BridgeWorld.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeWorld.Status());
e.Mobile.SendMessage("Bridge: {0}", BridgeOneShots.Status()); e.Mobile.SendMessage("Bridge: {0}", BridgeOneShots.Status());
e.Mobile.SendMessage("Bridge: {0}", BridgeAssets.Status());
break; break;
} }
} }

24
overlay/Scripts/Custom/Bridge/BridgeConfig.cs
View File

@@ -87,6 +87,16 @@ namespace Server.Custom.Bridge
// morning. Those are different consents, and one switch cannot express both. // morning. Those are different consents, and one switch cannot express both.
public static bool EventsEnabled { get; private set; } public static bool EventsEnabled { get; private set; }
// ---- the asset plane (docs/link/v8.md §3, protocol 8) ----
//
// Its own gate again, and for the same reason the event plane got one: enabling this is
// an operator consenting to the WEBSITE READING THEIR CLIENT FILES -- art, animations and
// the string table, off the host's disk, over the link. That is a different consent from
// publishing world state, and one switch cannot express both. Reads only: nothing on this
// plane writes anything, anywhere.
public static bool AssetsEnabled { get; private set; }
public static int AssetBatchBytes { get; private set; }
public static int LeaseMaxDurationSec { get; private set; } public static int LeaseMaxDurationSec { get; private set; }
public static int LeaseGraceSec { get; private set; } public static int LeaseGraceSec { get; private set; }
@@ -144,6 +154,20 @@ namespace Server.Custom.Bridge
Port = Config.Get("Bridge.Port", 7788); Port = Config.Get("Bridge.Port", 7788);
QueueCap = Config.Get("Bridge.QueueCap", 10000); QueueCap = Config.Get("Bridge.QueueCap", 10000);
AssetsEnabled = Config.Get("Bridge.AssetsEnabled", true);
// The largest reply this plane will build, in ENCODED bytes -- not items, because the
// ceiling it has to live inside is a byte ceiling. Clamped to half the sidecar's 1 MiB
// inbound line cap, and the halving is load-bearing rather than cautious: a page
// always admits its first item even when that item alone exceeds the budget (the
// alternative is an oversized item being skipped forever and its family never making
// progress), so the wire must still have room for one such overshoot.
AssetBatchBytes = Config.Get("Bridge.AssetBatchBytes", 512 * 1024);
if (AssetBatchBytes < 64 * 1024)
AssetBatchBytes = 64 * 1024;
if (AssetBatchBytes > 512 * 1024)
AssetBatchBytes = 512 * 1024;
StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30); StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30);
DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60); DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60);
EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300); EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300);

535
tools/scaffolding/BridgeAssetProbe.cs
View File

@@ -7,6 +7,7 @@ using System.Text;
using System.Threading; using System.Threading;
using Server.Commands; using Server.Commands;
using Server.Custom.Bridge;
using Ultima; using Ultima;
@@ -475,38 +476,76 @@ namespace Server.Custom
/// nothing downstream can tell. A "0 rows" outcome is the correct answer. /// nothing downstream can tell. A "0 rows" outcome is the correct answer.
/// ///
/// So the sweep records the file type each body resolved to and whether that file /// So the sweep records the file type each body resolved to and whether that file
/// answered — and never a second opinion. /// answered — and never a second opinion. <c>ResolveAnimation</c> is that rule as
/// code, and this sweep is now one of its callers rather than its own transcription.
///
/// **Phase 1 added the validator to this sweep**, which phase 0 ran without one:
/// it reported "1,144 decoded, 0 faults" while the patched client's verdata entry for
/// body 34 pointed past verdata.mul's own end and the wolf rendered something else,
/// counted among those 1,144. REFUSED-BUT-DECODED is the cell that says so, and it is
/// the same cell the art sweeps have had since phase 0.
/// </summary> /// </summary>
private static void SectionBodies() private static void SectionBodies()
{ {
Head("bodies — Animations.GetAnimation, one direction, first frame"); Head("bodies — Animations.GetAnimation, one direction, first frame");
int direction = Config.Get("Bridge.AssetProbeCreatureDirection", 1); int direction = Config.Get("Bridge.AssetProbeCreatureDirection", 1);
int decoded = 0, empty = 0, faulted = 0; var tally = new Tally();
var byFileType = new int[8]; var byFileType = new int[8];
var faults = new List<string>(); int unresolved = 0;
var indexes = new Dictionary<int, FileIndex>();
var readers = new Dictionary<int, BridgeAssetValidator.RecordReader>();
var lengths = new Dictionary<int, long>();
string verdataPath = Files.GetFilePath("verdata.mul");
long verdataLength = BridgeAssetValidator.MulLength(verdataPath);
try
{
for (int body = 0; body < 2048; body++) for (int body = 0; body < 2048; body++)
{ {
Checkpoint("bodies", body); Checkpoint("bodies", body);
int translated = body; int fileType, at;
int fileType; string reason;
try if (!BridgeAssetValidator.ResolveAnimation(body, 0, direction, out fileType, out at, out reason))
{ {
fileType = BodyConverter.Convert(ref translated); // The never-sweep-file-types rule's own outcome (§4.3): bodyconv sent this
} // body to a file this client does not have, so we report nothing and ask
catch (Exception e) // no other file. Asking anim2 for gargoyle 666 returns a giant spider.
{ unresolved++;
faulted++;
faults.Add("body " + body + " BodyConverter.Convert: " + e.GetType().Name + ": " + e.Message);
continue; continue;
} }
if (fileType >= 0 && fileType < byFileType.Length) if (fileType >= 0 && fileType < byFileType.Length)
byFileType[fileType]++; byFileType[fileType]++;
if (!indexes.ContainsKey(fileType))
{
string dataPath = BridgeAssetValidator.AnimDataPath(fileType);
indexes[fileType] = BridgeAssetValidator.OpenAnimIndex(fileType);
lengths[fileType] = BridgeAssetValidator.MulLength(dataPath);
readers[fileType] = new BridgeAssetValidator.RecordReader(dataPath, verdataPath);
}
var index = indexes[fileType];
var verdict = BridgeAssetValidator.CheckEntry(
index, at, lengths[fileType], verdataLength, out reason);
// Only the entry has been judged so far. The record behind it is where the
// frame table and the unbounded run headers live.
if (verdict == BridgeAssetValidator.Verdict.Ok
&& !readers[fileType].AnimationSane(index, at, 1, out reason))
{
verdict = BridgeAssetValidator.Verdict.Refused;
}
bool decoded = false;
string thrown = null;
try try
{ {
int hue = 0; int hue = 0;
@@ -514,37 +553,32 @@ namespace Server.Custom
if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null) if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null)
{ {
decoded++; decoded = true;
frames[0].Bitmap.Dispose(); frames[0].Bitmap.Dispose();
} }
else
{
empty++;
}
} }
catch (Exception e) catch (Exception e)
{ {
faulted++; thrown = e.GetType().Name + ": " + e.Message;
}
if (faults.Count < 40) Record(tally, verdict, reason, decoded, thrown, "body/" + body + "/a0");
faults.Add("body " + body + " (fileType " + fileType + "): " + e.GetType().Name + ": " + e.Message); }
}
finally
{
foreach (var reader in readers.Values)
{
if (reader != null)
reader.Dispose();
} }
} }
Say("direction " + direction + " (creature default — §5.1)"); Say("direction " + direction + " (creature default — §5.1)");
Say(string.Format(" decoded {0} empty {1} FAULTED {2}", decoded, empty, faulted)); Say(string.Format(" bodyconv resolves nowhere (correct — never swept): {0:N0}", unresolved));
Say(" by file type: " + string.Join(", ", FileTypeCounts(byFileType))); Say(" by file type: " + string.Join(", ", FileTypeCounts(byFileType)));
if (faults.Count > 0)
{
Say("");
Say(" faults:");
foreach (var f in faults)
Say(" " + f);
}
Say(""); Say("");
tally.Report("bodies 0..2047, action 0, first frame");
} }
private static string[] FileTypeCounts(int[] byFileType) private static string[] FileTypeCounts(int[] byFileType)
@@ -566,10 +600,16 @@ namespace Server.Custom
/// The twelve (on stock 57.4) player-character bodies, each at direction 0 — head-on, /// The twelve (on stock 57.4) player-character bodies, each at direction 0 — head-on,
/// because a character is a portrait and should look at you (§5.1). /// because a character is a portrait and should look at you (§5.1).
/// ///
/// Six of them are expected to report nothing on the legacy path: both human ghosts and /// Most of them are expected to have no art on the legacy path — the ghosts and every
/// every gargoyle body are UOP-only. **That is the measurement, not a failure** — it is /// gargoyle body are UOP-only. **That is the measurement, not a failure**: it is what
/// what phase 4's UOP reader exists for, and a probe that flagged it red would teach an /// phase 4's UOP reader exists for, and a probe that flagged it red would teach an
/// operator to ignore the panel. /// operator to ignore the panel.
///
/// **What is a failure is the middle column.** Phase 0 ran this without the validator
/// and read the library's answer as the truth, which made it report six of twelve
/// decoding. Two of those six — the elf ghosts — have an index entry of `length 0` and
/// were returning whatever body was decoded immediately before them, at that body's
/// exact dimensions. Four of twelve have art on a stock client, not six.
/// </summary> /// </summary>
private static void SectionPlayers() private static void SectionPlayers()
{ {
@@ -582,14 +622,42 @@ namespace Server.Custom
} }
int direction = Config.Get("Bridge.AssetProbePlayerDirection", 0); int direction = Config.Get("Bridge.AssetProbePlayerDirection", 0);
int decoded = 0, absent = 0; int real = 0, absent = 0, wrong = 0;
string verdataPath = Files.GetFilePath("verdata.mul");
long verdataLength = BridgeAssetValidator.MulLength(verdataPath);
foreach (var pb in _playerBodies) foreach (var pb in _playerBodies)
{ {
Checkpoint("players", pb.Body); Checkpoint("players", pb.Body);
int translated = pb.Body; int fileType, at;
int fileType = BodyConverter.Convert(ref translated); string reason;
bool resolved = BridgeAssetValidator.ResolveAnimation(
pb.Body, 0, direction, out fileType, out at, out reason);
// What the validator says BEFORE the library is asked. This is the whole point of
// the section: phase 0 reported "6 of 12 decode" from the library's answer alone,
// and two of those six were the previous body's picture.
var verdict = BridgeAssetValidator.Verdict.Absent;
if (resolved)
{
string dataPath = BridgeAssetValidator.AnimDataPath(fileType);
var index = BridgeAssetValidator.OpenAnimIndex(fileType);
long length = BridgeAssetValidator.MulLength(dataPath);
using (var reader = new BridgeAssetValidator.RecordReader(dataPath, verdataPath))
{
verdict = BridgeAssetValidator.CheckEntry(index, at, length, verdataLength, out reason);
if (verdict == BridgeAssetValidator.Verdict.Ok
&& !reader.AnimationSane(index, at, 1, out reason))
{
verdict = BridgeAssetValidator.Verdict.Refused;
}
}
}
string outcome; string outcome;
@@ -597,17 +665,32 @@ namespace Server.Custom
{ {
int hue = 0; int hue = 0;
var frames = Animations.GetAnimation(pb.Body, 0, direction, ref hue, false, true); var frames = Animations.GetAnimation(pb.Body, 0, direction, ref hue, false, true);
bool gotBitmap = frames != null && frames.Length > 0
&& frames[0] != null && frames[0].Bitmap != null;
string size = null;
if (frames != null && frames.Length > 0 && frames[0] != null && frames[0].Bitmap != null) if (gotBitmap)
{ {
var bmp = frames[0].Bitmap; var bmp = frames[0].Bitmap;
outcome = "decoded " + bmp.Width + "x" + bmp.Height; size = bmp.Width + "x" + bmp.Height;
bmp.Dispose(); bmp.Dispose();
decoded++; }
if (verdict == BridgeAssetValidator.Verdict.Ok && gotBitmap)
{
outcome = "art, " + size;
real++;
}
else if (gotBitmap)
{
// The elf ghosts land here on a stock client: index entry `length 0`, and
// a bitmap the exact size of whatever was decoded last.
outcome = "WRONG PICTURE " + size + " — " + reason;
wrong++;
} }
else else
{ {
outcome = "no art on the legacy path (UOP-only — phase 4)"; outcome = "no art on the legacy path (UOP-only — phase 4): " + reason;
absent++; absent++;
} }
} }
@@ -617,11 +700,12 @@ namespace Server.Custom
} }
Say(string.Format(" {0,-10} {1,-14} body {2,-5} fileType {3,-3} {4}", Say(string.Format(" {0,-10} {1,-14} body {2,-5} fileType {3,-3} {4}",
pb.Race, pb.Slot, pb.Body, fileType, outcome)); pb.Race, pb.Slot, pb.Body, resolved ? fileType.ToString() : "-", outcome));
} }
Say(""); Say("");
Say(string.Format(" {0} decoded, {1} absent, of {2}", decoded, absent, _playerBodies.Count)); Say(string.Format(" {0} with art, {1} absent, {2} WRONG PICTURES, of {3}",
real, absent, wrong, _playerBodies.Count));
Say(""); Say("");
} }
@@ -978,369 +1062,4 @@ namespace Server.Custom
to.SendMessage(text); to.SendMessage(text);
} }
} }
/// <summary>
/// **Validate before calling** — the response the org lead chose for §4.2's residual risk,
/// prototyped here so phase 1 adopts it with measurements rather than on faith.
///
/// The principle: `Ultima`'s decoders take their bounds from the file they are reading, so
/// the extractor must decide whether a record is worth handing over *before* handing it
/// over. Every check below is against the index entry and the record header — cheap, and
/// enough to turn an uncatchable corrupted-state exception into a skipped asset.
///
/// It cannot be complete and does not claim to be. It closes the shapes that reading the
/// source showed are reachable; the probe's REFUSED-BUT-DECODED count is what says whether
/// the boundary is drawn in the right place.
///
/// Promoted into the overlay in phase 1.
/// </summary>
public static class BridgeAssetValidator
{
public enum Verdict
{
/// <summary>Nothing at this id, and the index says so honestly.</summary>
Absent,
/// <summary>The entry is self-consistent and inside its file.</summary>
Ok,
/// <summary>The entry claims something the file cannot support. Do not decode it.</summary>
Refused
}
/// <summary>Land tiles decode a fixed 44×44 diamond: 2 × (2+4+…+44) ushorts.</summary>
public const int LandRecordBytes = 2024;
/// <summary>
/// A ceiling on decoded art dimensions. `LoadStatic` allocates
/// <c>new Bitmap(width, height)</c> straight from two ushorts in the record, so a
/// corrupt header asks for up to 65535×65535 — an 8 GB allocation, from a file. Real
/// art is a couple of hundred pixels at most.
/// </summary>
public const int MaxArtDimension = 1024;
/// <summary>
/// Builds our own index over the same files, with the same constructor arguments
/// <c>Art</c> uses — including <c>hasExtra: false</c>, which is the whole reason the
/// art path is safe where the gump path is not (§4.1).
/// </summary>
public static FileIndex OpenArtIndex()
{
if (ArtDataPath() == null)
return null;
return new FileIndex("Artidx.mul", "Art.mul", "artLegacyMUL.uop", 0x10000, 4, ".tga", 0x13FDC, false);
}
/// <summary>
/// The file an art index entry's <c>lookup</c> is an offset **into** — which is not
/// <c>art.mul</c> on any current client.
///
/// This cost a whole probe run to learn and it is the single most important thing
/// phase 1 must not get wrong. <c>FileIndex</c>'s UOP constructor ends with a bare
/// <c>MulPath = uopPath</c>: **when <c>artLegacyMUL.uop</c> exists it wins outright**,
/// and <c>art.mul</c> / <c>artidx.mul</c> are never opened at all. A validator that
/// bounds offsets against <c>art.mul</c> while the index holds UOP offsets is not
/// merely approximate, it is nonsense — the first run of this probe refused 34,299
/// perfectly good statics for "declaring 10533x2085" because it was reading UOP
/// offsets into the wrong file.
///
/// So the resolution order here mirrors <c>FileIndex</c>'s exactly, and anything that
/// needs the bytes behind an entry must ask this rather than assume.
/// </summary>
public static string ArtDataPath()
{
var uop = Files.GetFilePath("artlegacymul.uop");
if (uop != null)
return uop;
return Files.GetFilePath("art.mul");
}
public static long MulLength(string path)
{
if (path == null)
return 0;
try
{
return new FileInfo(path).Length;
}
catch
{
return 0;
}
}
/// <summary>
/// Judges one index entry.
///
/// The check <c>FileIndex.Seek</c> is missing is the last one: it tests
/// <c>Stream.Length &lt; e.lookup</c> — that the record *starts* inside the file — and
/// never that it *ends* inside it. A record that begins two bytes before EOF and
/// declares a length of 4,000 passes, and <c>stream.Read</c> then returns a short count
/// that the decoders discard, leaving the previous asset's bytes in the shared buffer.
/// </summary>
public static Verdict CheckEntry(FileIndex index, int at, long mulLength, long verdataLength, out string reason)
{
reason = null;
if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
{
reason = "index " + at + " out of range";
return Verdict.Absent;
}
Entry3D e = index.Index[at];
if (e.lookup < 0)
{
reason = "lookup " + e.lookup;
return Verdict.Absent;
}
bool patched = (e.length & (1 << 31)) != 0;
int length = e.length & 0x7FFFFFFF;
if (!patched && e.length < 0)
{
reason = "length " + e.length;
return Verdict.Absent;
}
if (length == 0)
{
reason = "lookup " + e.lookup + ", length 0";
return Verdict.Absent;
}
long ceiling = patched ? verdataLength : mulLength;
if (ceiling <= 0)
{
reason = (patched ? "verdata.mul" : "the art data file") + " has no length";
return Verdict.Refused;
}
if (e.lookup >= ceiling)
{
reason = "lookup " + e.lookup + " past the end of "
+ (patched ? "verdata.mul" : "the mul") + " (" + ceiling + ")";
return Verdict.Refused;
}
// The missing check. A short read is silent, and its consequence is the PREVIOUS
// asset's picture served under this id.
if (e.lookup + (long)length > ceiling)
{
reason = "record runs " + (e.lookup + (long)length - ceiling) + " bytes past the end of "
+ (patched ? "verdata.mul" : "the mul");
return Verdict.Refused;
}
return Verdict.Ok;
}
/// <summary>
/// `LoadLand` reads 2,024 bytes regardless of the declared length, so a shorter record
/// reads past the end of a buffer sized from that length.
/// </summary>
public static bool LandLengthSane(FileIndex index, int at, out string reason)
{
reason = null;
if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
return true;
int length = index.Index[at].length & 0x7FFFFFFF;
if (length > 0 && length < LandRecordBytes)
{
reason = "land record is " + length + " bytes; LoadLand always reads " + LandRecordBytes;
return false;
}
return true;
}
/// <summary>
/// Walks a static record's own row table the way <c>LoadStatic</c> will, and refuses
/// it if that walk would read outside the record.
///
/// This is the check with teeth. <c>LoadStatic</c>'s inner loop guards the write into
/// the bitmap (<c>xOffset &gt; delta</c>, <c>xOffset + xRun &gt; delta</c>) and does
/// nothing at all about the read cursor, which advances until it happens to find a
/// zero pair — potentially far outside a pinned array. Simulating the same walk with
/// a bound is the cheapest way to know whether handing the id over is safe.
/// </summary>
public static bool StaticRecordSane(byte[] record, int length, out string reason)
{
reason = null;
if (length < 8)
{
reason = "record is " + length + " bytes; a static header needs 8";
return false;
}
int words = length / 2;
int width = ReadUInt16(record, 4);
int height = ReadUInt16(record, 6);
// LoadStatic returns null for these rather than misbehaving, so it is not a refusal.
if (width <= 0 || height <= 0)
return true;
if (width > MaxArtDimension || height > MaxArtDimension)
{
reason = "declares " + width + "x" + height + ", past the " + MaxArtDimension + "px ceiling";
return false;
}
// The row-lookup table: height ushorts starting at word 4.
if (4 + height > words)
{
reason = "row table (" + height + " entries) does not fit in a " + length + "-byte record";
return false;
}
int start = height + 4;
for (int y = 0; y < height; y++)
{
int cursor = start + ReadUInt16(record, (4 + y) * 2);
while (true)
{
// Two ushorts for the run header, and they must both be inside the record.
if (cursor < 0 || cursor + 1 >= words)
{
reason = "row " + y + " reads at word " + cursor + ", past the record's " + words;
return false;
}
int xOffset = ReadUInt16(record, cursor * 2);
int xRun = ReadUInt16(record, (cursor + 1) * 2);
cursor += 2;
if (xOffset + xRun == 0)
break;
// LoadStatic stops the row here, so the read cursor stops with it.
if (xOffset > width || xOffset + xRun > width)
break;
if (cursor + xRun > words)
{
reason = "row " + y + " declares a " + xRun + "-pixel run running past the record";
return false;
}
cursor += xRun;
}
}
return true;
}
private static int ReadUInt16(byte[] b, int at)
{
return b[at] | (b[at + 1] << 8);
}
/// <summary>
/// Reads a record's actual bytes so <see cref="StaticRecordSane"/> can walk it.
///
/// Holds its own handles rather than borrowing the library's, because <c>FileIndex</c>
/// hands out the stream it decodes from and moving that stream's position underneath
/// the decoder would be its own bug. Opened <c>FileShare.ReadWrite</c> to match how
/// <c>FileIndex</c> opens the same files.
/// </summary>
public sealed class RecordReader : IDisposable
{
private readonly FileStream _mul;
private readonly FileStream _verdata;
private byte[] _scratch = new byte[64 * 1024];
public RecordReader(string mulPath, string verdataPath)
{
_mul = Open(mulPath);
_verdata = Open(verdataPath);
}
private static FileStream Open(string path)
{
if (path == null || !File.Exists(path))
return null;
try
{
return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
catch
{
return null;
}
}
/// <summary>
/// True when the record at <paramref name="at"/> is safe to hand to
/// <c>Art.GetStatic</c>. A record that cannot be read at all is reported sane —
/// <see cref="CheckEntry"/> has already judged the entry, and this must not
/// invent a second reason to refuse.
/// </summary>
public bool StaticSane(FileIndex index, int at, out string reason)
{
reason = null;
if (index == null || index.Index == null || at < 0 || at >= index.Index.Length)
return true;
Entry3D e = index.Index[at];
bool patched = (e.length & (1 << 31)) != 0;
int length = e.length & 0x7FFFFFFF;
var stream = patched ? _verdata : _mul;
if (stream == null || length <= 0 || e.lookup < 0)
return true;
if (_scratch.Length < length)
_scratch = new byte[length];
int read;
try
{
stream.Seek(e.lookup, SeekOrigin.Begin);
read = stream.Read(_scratch, 0, length);
}
catch (Exception ex)
{
reason = "cannot read the record: " + ex.GetType().Name;
return false;
}
// The short read the decoders discard. Refusing here is the whole point: the
// library would decode whatever the shared buffer happened to hold.
if (read < length)
{
reason = "short read — " + read + " of " + length + " bytes available";
return false;
}
return StaticRecordSane(_scratch, length, out reason);
}
public void Dispose()
{
if (_mul != null)
_mul.Dispose();
if (_verdata != null)
_verdata.Dispose();
}
}
}
} }