RunicGateway/servuo-plugins
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Merge pull request 'feat(asset-bridge): the shard's own files stop needing a shared filesystem (Phase 7)' (#34) from feat/asset-bridge-p7 into edge

Browse Source 
Reviewed-on: #34
This commit is contained in:
Colby Whitlock
2026-09-14 07:36:54 +00:00
parent 577688b993 936a922487
commit 2539764cf7
7 changed files with 1067 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
overlay/Config/Bridge.cfg
View File

@@ -340,6 +340,30 @@ AssetScanMs=3000
AssetPlayerDirection=0
AssetCreatureDirection=1
# The tree plane (docs/link/v8.md §10, phase 7). A THIRD switch, for a third consent:
# the asset switch above is about this host's UO client, which came from EA. This one is
# about the shard's own configuration -- Spawns/*.xml, Data/Regions.xml,
# Data/Locations/*.xml, Config/ChampionSpawns.xml and Data/Decoration/**.cfg -- which is
# the operator's own work and is what the website's spawn atlas is built from. Before
# protocol 8 the website read those files off a shared filesystem; that was the one place
# the platform's own rule (only the sidecar bridges the shard) was broken, and broken by
# the component that faces the internet. Turning this off closes the bridge route and
# leaves that shared-filesystem path as the only way an atlas can be built.
#
# Reads only, and only those five groups. Nothing here joins a path the website sent: a
# request names a label this shard itself enumerated, or it is refused.
TreeEnabled=true
# How much of a tree file one chunk carries, BEFORE compression. Chunking is not an
# optimisation here, it is what makes a spawn file transferable: a stock trammel.xml is
# 4.03 MB, the sidecar discards any inbound line over 1 MiB, and the whole file as one
# base64 row would time out and be re-requested forever with no error anywhere. Each
# chunk is gzipped (a spawn file compresses ~18x, so a chunk is typically 40 KB on the
# wire), but the BOUND comes from the chunk rather than the compression, because nothing
# guarantees input compresses at all. Clamped to [64 KiB, 512 KiB]: at the ceiling a
# worst-case incompressible chunk is ~683 KiB of base64, which still fits the wire.
TreeChunkBytes=524288
# The test scaffolding in tools/scaffolding/ reads its own flags from this file
# (SeedOnStart, CensusOnStart, ProbeOnStart). They are absent here on purpose:
# Config.Get returns the default of false when a key is missing, so a deployed

2
overlay/Scripts/Custom/Bridge/BridgeArt.cs
View File

@@ -175,7 +175,7 @@ namespace Server.Custom.Bridge
string id = SourceId();
if (expected != null && expected != id)
if (BridgeAssets.CatalogMismatch(expected, id))
{
BridgeAssets.Fail(reqId, "UNREADABLE",
"the shard's client files changed since that catalogue was read (catalog "

239
overlay/Scripts/Custom/Bridge/BridgeAssets.cs
View File

@@ -135,6 +135,10 @@ namespace Server.Custom.Bridge
BridgeBoot.RegisterHandler("assets.sources", OnSources);
BridgeBoot.RegisterHandler("assets.fetch", OnFetch);
// Owned here since phase 7, for the same reason `assets.fetch` moved here in phase 5:
// it is the transport, and more than one family has something to enumerate.
BridgeBoot.RegisterHandler("assets.manifest", OnManifest);
}
/// <summary>
@@ -221,7 +225,11 @@ namespace Server.Custom.Bridge
return;
}
if (!BridgeConfig.AssetsEnabled)
// Stage 1 answers for the whole plane, not for the client files alone: since phase 7
// an operator can serve the shard's own configuration tree while declining to serve
// their UO client, and `families` is where a website discovers which. Refused only
// when there is nothing at all to report.
if (Families().Count == 0)
{
Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
return;
@@ -294,8 +302,33 @@ namespace Server.Custom.Bridge
/// </summary>
internal delegate void FamilyFetch(string reqId, List<string> keys, string catalog, string cursor);
private static readonly Dictionary<string, FamilyFetch> _families =
new Dictionary<string, FamilyFetch>(StringComparer.Ordinal);
/// <summary>
/// One family's answer to a manifest walk — everything it can serve, no payload.
/// Runs on the asset worker, never the Core thread. A family with nothing to
/// enumerate (statics and land are addressed, not listed) registers none.
/// </summary>
internal delegate void FamilyManifest(string reqId, string cursor);
/// <summary>
/// What one §5 key family registered: how to serve it, how to list it, and — since
/// phase 7 — which operator consent it answers to.
///
/// The gate is per family rather than per plane because the planes are not one
/// consent. `body`, `static` and `land` are the operator's UO CLIENT, licensed from
/// EA and read off their disk; `tree` is the shard's OWN configuration, which they
/// wrote. An operator can reasonably want the second published and not the first, and
/// before this the atlas would have been what silently disappeared when they said so.
/// </summary>
private sealed class FamilyReader
{
public FamilyFetch Fetch;
public FamilyManifest Manifest;
public Func<bool> Enabled;
public string DisabledReason;
}
private static readonly Dictionary<string, FamilyReader> _families =
new Dictionary<string, FamilyReader>(StringComparer.Ordinal);
/// <summary>
/// Claims one §5 key family for a reader.
@@ -312,33 +345,114 @@ namespace Server.Custom.Bridge
/// handler does not read until a request arrives.
/// </summary>
internal static void RegisterFamily(string name, FamilyFetch fetch)
{
RegisterFamily(name, fetch, null, null, null);
}
/// <summary>
/// The full registration: a fetch reader, an optional manifest reader, and the
/// consent this family answers to.
///
/// <paramref name="enabled"/> null means the asset plane's own gate
/// (<c>Bridge.AssetsEnabled</c>), which is what every client-file family wants.
/// A family that reads something else entirely passes its own.
/// </summary>
internal static void RegisterFamily(string name, FamilyFetch fetch, FamilyManifest manifest,
Func<bool> enabled, string disabledReason)
{
lock (_families)
{
_families[name] = fetch;
_families[name] = new FamilyReader
{
Fetch = fetch,
Manifest = manifest,
Enabled = enabled,
DisabledReason = disabledReason
};
}
}
/// <summary>The families this shard can serve, for §6's stage 1 and for diagnostics.</summary>
/// <summary>
/// The families this shard can serve **right now**, for §6's stage 1 and for
/// diagnostics.
///
/// Filtered by consent rather than by registration, because that is the question the
/// website is actually asking: a family it can see in this list is one it can fetch.
/// Listing a family the operator has switched off would turn one clear refusal at
/// import time into a per-key refusal on every pass, forever — which is exactly the
/// failure `families` was added in phase 5 to prevent.
/// </summary>
internal static List<string> Families()
{
var names = new List<string>();
lock (_families)
{
var names = new List<string>(_families.Keys);
names.Sort(StringComparer.Ordinal);
return names;
foreach (var pair in _families)
{
if (EnabledFor(pair.Value))
names.Add(pair.Key);
}
}
names.Sort(StringComparer.Ordinal);
return names;
}
private static bool EnabledFor(FamilyReader reader)
{
if (reader == null)
return false;
try
{
return reader.Enabled == null ? BridgeConfig.AssetsEnabled : reader.Enabled();
}
catch
{
// A gate that throws is a gate that has not consented.
return false;
}
}
private static FamilyFetch FamilyFor(string name)
private static FamilyReader FamilyFor(string name)
{
lock (_families)
{
FamilyFetch fetch;
return _families.TryGetValue(name, out fetch) ? fetch : null;
FamilyReader reader;
return _families.TryGetValue(name, out reader) ? reader : null;
}
}
/// <summary>
/// Resolves a named family and answers the request itself when it cannot.
///
/// Shared by <c>assets.fetch</c> and <c>assets.manifest</c> so the two cannot drift
/// apart about what "this shard does not serve that" means — and so the consent check
/// happens in exactly one place for both.
/// </summary>
private static bool Resolve(string reqId, string family, out FamilyReader reader)
{
reader = FamilyFor(family);
if (reader == null)
{
Fail(reqId, "BAD_REQUEST",
"this shard serves no '" + family + "' asset family (it serves "
+ String.Join(", ", Families().ToArray()) + ")");
return false;
}
if (!EnabledFor(reader))
{
Fail(reqId, "DISABLED", reader.DisabledReason
?? "asset extraction is disabled on this shard");
return false;
}
return true;
}
/// <summary>
/// The family segment of a §5 key: everything before the first `/`.
/// </summary>
@@ -373,12 +487,9 @@ namespace Server.Custom.Bridge
return;
}
if (!BridgeConfig.AssetsEnabled)
{
Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
return;
}
// The consent check is NOT here any more (phase 7). It cannot be: which consent this
// request needs is a property of the keys, and the keys have not been read yet. So the
// shape checks come first and the gate happens in `Resolve`, once the family is known.
var keys = BridgeJson.GetStringList(o, "keys");
if (keys.Count == 0)
@@ -408,22 +519,70 @@ namespace Server.Custom.Bridge
return;
}
FamilyFetch fetch = FamilyFor(family);
FamilyReader reader;
if (fetch == null)
if (!Resolve(reqId, family, out reader))
return;
if (reader.Fetch == null)
{
Fail(reqId, "BAD_REQUEST",
"this shard serves no '" + family + "' asset family (it serves "
+ String.Join(", ", Families().ToArray()) + ")");
"the '" + family + "' family cannot be fetched by key on this shard");
return;
}
var catalog = BridgeJson.GetString(o, "catalog");
var cursor = BridgeJson.GetString(o, "cursor");
FamilyFetch fetch = reader.Fetch;
Accept(reqId, "assets.fetch", () => fetch(reqId, keys, catalog, cursor));
}
/// <summary>
/// §14's `assets.manifest`, for every family that has one.
///
/// Phase 3 gave this command to the body catalogue outright and phase 5 learned, for
/// `assets.fetch`, that the command is the transport and the family is a property of
/// the key. Phase 7 is where the same lesson lands one level up: the tree family
/// enumerates its files exactly the way the catalogue enumerates its bodies, and
/// nothing about the envelope, the cursor or the consent differs between them.
///
/// **`family` still defaults to `body`.** A phase-3 website asks without naming one
/// and must keep getting the catalogue it asked for.
/// </summary>
private static void OnManifest(Dictionary<string, object> o)
{
var reqId = BridgeJson.GetString(o, "reqId");
if (reqId == null)
{
Fail(null, "BAD_REQUEST", "assets.manifest requires a reqId");
return;
}
var family = BridgeJson.GetString(o, "family") ?? "body";
FamilyReader reader;
if (!Resolve(reqId, family, out reader))
return;
if (reader.Manifest == null)
{
// Named rather than defaulted: statics and land are ADDRESSED (§11.1) rather than
// listed, and a website that asked for a list of 49,152 item graphics has made a
// mistake it needs told about rather than an empty page it will read as "none".
Fail(reqId, "BAD_REQUEST",
"the '" + family + "' family is fetched by key and has no manifest");
return;
}
var cursor = BridgeJson.GetString(o, "cursor");
FamilyManifest manifest = reader.Manifest;
Accept(reqId, "assets.manifest", () => manifest(reqId, cursor));
}
/// <summary>
/// ARGB1555 to a PNG with a transparent background.
///
@@ -511,6 +670,30 @@ namespace Server.Custom.Bridge
}
}
/// <summary>
/// Does a caller's asserted catalog id disagree with what this shard holds?
///
/// **An absent fingerprint and an empty one mean the same thing**, and that is the
/// whole reason this is a function rather than an inline `expected != null`. A caller
/// with nothing to assert sends the field absent or empty depending on how its own
/// client serialises a missing value, and treating `""` as a real id refuses **every**
/// fetch it makes — with a sentence naming no catalog at all ("catalog is now
/// 8159778b"), which reads as a shard fault rather than a caller one.
///
/// Phase 7 found this on the tree family, where a probe passed an empty string by
/// accident. It was inline in three places by then; it is one function now, because
/// three copies of a comparison are three chances for the next family to get it wrong
/// in a way only a differently-written client would ever reveal.
///
/// Note this is deliberately NOT the shape `BridgeLeases` uses for its own `expected`:
/// there the value is a world property, where an empty string is a legitimate thing to
/// expect and `!= null` is correct.
/// </summary>
internal static bool CatalogMismatch(string expected, string actual)
{
return !String.IsNullOrEmpty(expected) && !String.Equals(expected, actual, StringComparison.Ordinal);
}
/// <summary>
/// SHA-256, lowercase hex. Shared because the hash in a manifest row, the hash in a
/// fetch row and the hash the website stores must be one function.
@@ -644,7 +827,13 @@ namespace Server.Custom.Bridge
var sb = BridgeJson.Begin("assets.sources.ok");
sb.Str("reqId", reqId)
.Num("extractorVersion", EXTRACTOR_VERSION);
.Num("extractorVersion", EXTRACTOR_VERSION)
// Which of the two consents this shard has given (phase 7). Without it a website
// whose operator switched client-file extraction off would read an empty `files`
// array as "your client has no cliloc.enu" — a sentence that sends them looking at
// their client install for a setting that lives on their shard.
.Bool("assetsEnabled", BridgeConfig.AssetsEnabled)
.Bool("treeEnabled", BridgeConfig.TreeEnabled);
WriteImaging(sb);
@@ -676,7 +865,11 @@ namespace Server.Custom.Bridge
var page = new PageBuilder(sb, "files", BridgeConfig.AssetBatchBytes);
bool anyMissingHash = false;
for (int i = 0; i < SourceFiles.Length; i++)
// The client files are the asset plane's own subject, so they are listed under the
// asset plane's own consent. A tree-only shard answers this call — that is how its
// website learns the `tree` family exists — and reports no client files at all,
// which is the truthful answer to "what may I read here".
for (int i = 0; BridgeConfig.AssetsEnabled && i < SourceFiles.Length; i++)
{
string name = SourceFiles[i];
string path = ResolvePath(name);

1
overlay/Scripts/Custom/Bridge/BridgeBoot.cs
View File

@@ -263,6 +263,7 @@ namespace Server.Custom.Bridge
e.Mobile.SendMessage("Bridge: {0}", BridgeAssets.Status());
e.Mobile.SendMessage("Bridge: {0}", BridgeCatalog.Status());
e.Mobile.SendMessage("Bridge: {0}", BridgeArt.Status());
e.Mobile.SendMessage("Bridge: {0}", BridgeTree.Status());
break;
}
}

62
overlay/Scripts/Custom/Bridge/BridgeCatalog.cs
View File

@@ -120,11 +120,12 @@ namespace Server.Custom.Bridge
if (!BridgeConfig.Enabled)
return;
BridgeBoot.RegisterHandler("assets.manifest", OnManifest);
// `assets.fetch` is shared plumbing as of phase 5 (§5): BridgeAssets owns the command,
// decides which family a batch of keys belongs to, and calls the reader that owns it.
BridgeAssets.RegisterFamily(Family, ReplyFetch);
// Both commands are shared plumbing: `assets.fetch` since phase 5 and
// `assets.manifest` since phase 7 (§5, §10). BridgeAssets owns the correlation id, the
// operator's consent, the key ceiling and the family decision; what is registered here
// is only this family's two readers, and each is called on the asset worker with work
// it owns.
BridgeAssets.RegisterFamily(Family, ReplyFetch, ReplyManifest, null, null);
}
// ── the cache ────────────────────────────────────────────────────────────────────────
@@ -186,32 +187,6 @@ namespace Server.Custom.Bridge
// ── assets.manifest ──────────────────────────────────────────────────────────────────
private static void OnManifest(Dictionary<string, object> o)
{
string reqId;
if (!Admit(o, "assets.manifest", out reqId))
return;
var family = BridgeJson.GetString(o, "family") ?? Family;
if (!String.Equals(family, Family, StringComparison.Ordinal))
{
// Named rather than ignored: `family` exists so §5's statics and land can join
// this envelope in phase 5 without a second request kind, and a website that
// asked for one of those against a phase-3 overlay must be told it asked too
// early rather than handed a body catalogue it did not request.
BridgeAssets.Fail(reqId, "BAD_REQUEST",
"this shard serves the '" + Family + "' asset family only (asked for '"
+ family + "')");
return;
}
var cursor = BridgeJson.GetString(o, "cursor");
BridgeAssets.Accept(reqId, "assets.manifest", () => ReplyManifest(reqId, cursor));
}
/// <summary>
/// Worker thread. Scans forward from the cursor until the byte budget or the time
/// budget is spent, hashing what it decodes and keeping the bytes for the fetch.
@@ -382,7 +357,7 @@ namespace Server.Custom.Bridge
string id = SourceId();
if (expected != null && expected != id)
if (BridgeAssets.CatalogMismatch(expected, id))
{
// The client files moved between the manifest and this fetch. Refusing is the only
// honest answer: the keys were chosen against a catalogue that no longer describes
@@ -996,29 +971,6 @@ namespace Server.Custom.Bridge
// ── shared plumbing ──────────────────────────────────────────────────────────────────
/// <summary>
/// The two gates every request on this plane passes: a correlation id, and the
/// operator's consent. Both refuse rather than answer.
/// </summary>
private static bool Admit(Dictionary<string, object> o, string kind, out string reqId)
{
reqId = BridgeJson.GetString(o, "reqId");
if (reqId == null)
{
BridgeAssets.Fail(null, "BAD_REQUEST", kind + " requires a reqId");
return false;
}
if (!BridgeConfig.AssetsEnabled)
{
BridgeAssets.Fail(reqId, "DISABLED", "asset extraction is disabled on this shard");
return false;
}
return true;
}
/// <summary>
/// The five anim files' index and record readers — and, since phase 4, the five UOP
/// packages beside them — opened for one reply and closed with it. Holding them across

43
overlay/Scripts/Custom/Bridge/BridgeConfig.cs
View File

@@ -124,6 +124,36 @@ namespace Server.Custom.Bridge
public static int AssetPlayerDirection { get; private set; }
public static int AssetCreatureDirection { get; private set; }
// ---- the tree plane (docs/link/v8.md §10, phase 7) ----
//
// Its OWN gate, and the third one on this link for the third kind of consent. The asset
// gate above is the operator agreeing that the website may read THEIR UO CLIENT -- art
// and animations and a string table that came from EA. This one is the operator agreeing
// that it may read THE SHARD'S OWN CONFIGURATION: the spawn files, the region and
// location definitions, the champion table, the decoration lists. Those are the
// operator's own work rather than a licensed client, and they are what the spawn atlas is
// built out of -- so a shard that declines to serve client art must still be able to
// publish where its creatures live. One switch could not have expressed both, and the
// atlas would have been the thing that silently disappeared.
//
// Reads only, and only the five labelled groups SPAWN_ATLAS.md already names. Nothing
// here joins a path the website sent: a request names a label this shard enumerated, or
// it is refused.
public static bool TreeEnabled { get; private set; }
// How much of a tree file one chunk carries, BEFORE compression (§10). The chunk is the
// thing that makes this transferable at all: a stock Spawns/trammel.xml is 4.03 MB and
// the sidecar discards any inbound line over 1 MiB, so the file as a single base64 row
// could never arrive -- it would time out and be re-requested forever, which is a failure
// with no error in it anywhere.
//
// Compression is what makes it cheap (a spawn file gzips ~18x, so a chunk is typically
// 40 KB on the wire) and the chunk is what makes it BOUNDED: gzip cannot be relied on to
// shrink anything, so the ceiling has to hold for input that does not compress at all.
// At 512 KiB a worst-case incompressible chunk is ~683 KiB of base64, which still fits
// the wire under AssetBatchBytes' deliberate factor of two.
public static int TreeChunkBytes { get; private set; }
// How many bytes of rendered item and land art the shard holds between requests (§11,
// phase 5). This is a convenience, not a store: the website keeps every picture it fetches
// and does not ask twice, so what this actually buys is the second page of a batch, a
@@ -239,6 +269,19 @@ namespace Server.Custom.Bridge
if (AssetArtCacheBytes > 64 * 1024 * 1024)
AssetArtCacheBytes = 64 * 1024 * 1024;
TreeEnabled = Config.Get("Bridge.TreeEnabled", true);
// Floor and ceiling both matter. Below 64 KiB a stock tree is thousands of chunks and
// the per-row overhead starts to dominate the payload; above 512 KiB an incompressible
// chunk stops fitting inside the sidecar's inbound line cap, which is the one bound
// this number exists to respect. Kept equal to AssetBatchBytes' own ceiling so the two
// budgets cannot drift into disagreeing about the same wire.
TreeChunkBytes = Config.Get("Bridge.TreeChunkBytes", 512 * 1024);
if (TreeChunkBytes < 64 * 1024)
TreeChunkBytes = 64 * 1024;
if (TreeChunkBytes > 512 * 1024)
TreeChunkBytes = 512 * 1024;
StatSweepSeconds = Config.Get("Bridge.StatSweepSeconds", 30);
DecaySweepSeconds = Config.Get("Bridge.DecaySweepSeconds", 60);
EconomySweepSeconds = Config.Get("Bridge.EconomySweepSeconds", 300);

775
overlay/Scripts/Custom/Bridge/BridgeTree.cs Normal file
View File

@@ -0,0 +1,775 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Text;
namespace Server.Custom.Bridge
{
/// <summary>
/// **The shard's own configuration, over the bridge** (docs/link/v8.md §10 — protocol 8,
/// phase 7).
///
/// Everything else on the asset plane reads the operator's UO CLIENT. This family reads
/// the shard's own files: the spawn tables, the region and location definitions, the
/// champion list and the decoration lists. The website parses those into its spawn atlas —
/// where every creature lives, which regions exist, what this shard calls scenery — and
/// until protocol 8 it did so by **reading the ServUO tree off a shared filesystem**:
/// same host, a bind mount, or a shared volume.
///
/// That was the one place the platform's own rule was broken, and broken by the component
/// that faces the internet. This closes it. The parsers do not move — `spawnAtlasParse.js`
/// is pure, fs-free and covered by CI without a ServUO tree anywhere near it, and every
/// quirk it handles stays exactly where it is. The shard sends bytes; the website still
/// decides what they mean.
///
/// ── What phase 7 measured, and the shape it forced ────────────────────────────────
///
/// §10 said "the shard serves `tree/&lt;label&gt;` → bytes". Measured against a stock 57.4
/// tree, it cannot: `Spawns/trammel.xml` is **4.03 MB**, the sidecar discards any inbound
/// line over **1 MiB** (`shard.rs` `MAX_INBOUND_LINE_BYTES`), and that file as a single
/// base64 row is 5.4 MiB. It would never arrive — the reply would be discarded, the
/// request would time out, and the import would retry forever with no error anywhere in
/// it. Two files on a *stock* tree are in that state; a shard with hand-built spawn tables
/// has more.
///
/// So a file crosses as **chunks, each gzipped**:
///
/// <code>
/// tree/Spawns/trammel.xml the manifest row — size, hash, chunk count
/// tree/Spawns/trammel.xml/c0 the first 512 KiB of it, gzipped
/// tree/Spawns/trammel.xml/c1 the next
/// </code>
///
/// which is §5's depth scheme at work a second time, exactly as `body/400/a0/f0` is —
/// and, as there, nothing about it needed a protocol change.
///
/// **The chunk is the bound and the compression is the saving**, and it matters which is
/// which. Compression is what makes this cheap: the stock tree is 11.34 MB and gzips to
/// 927 KB, so the whole atlas source arrives in about three pages instead of thirty-one.
/// But nothing guarantees that an operator's files compress at all, so the ceiling has to
/// hold when they do not — and it does, because a 512 KiB chunk that refuses to compress
/// is still only ~683 KiB of base64, inside the wire cap that
/// <see cref="BridgeConfig.AssetBatchBytes"/>' deliberate factor of two leaves room for.
/// A design that leaned on the ratio would work on every tree anyone tested and fail on
/// the first one nobody did.
///
/// ── Two rules that are not negotiable here ────────────────────────────────────────
///
/// **1. The label set is this shard's, never the caller's.** This is the only family on
/// this link whose keys look like paths, and the website is the internet-facing component.
/// So nothing here joins a path that arrived on the wire: a fetch resolves its label
/// against the set <see cref="Enumerate"/> itself produced, and a label that is not in it
/// is refused — before any file is opened, and whatever it spells. The five groups are
/// fixed in code, the extensions are fixed in code, and the resolved path is checked to be
/// under the tree root even after all of that.
///
/// **2. A row re-declares its own address.** Each chunk carries its label, its index, its
/// byte offset and the hash of its own (uncompressed) bytes, and the manifest carries the
/// hash of the whole file. That is the §4.10 lesson on a fourth axis: a reassembly that
/// silently put chunk 3 where chunk 4 belongs would produce a file that parses — XML is
/// forgiving about what it skips — and a spawn atlas subtly missing a facet. Per-chunk
/// hashes make it a named error instead.
/// </summary>
public static class BridgeTree
{
/// <summary>The §5 key family this serves.</summary>
private const string Family = "tree";
/// <summary>
/// The five labelled groups `spawnAtlasSource.js` reads, and nothing else.
///
/// Fixed in code rather than configured, because a configurable list is a way for the
/// website to ask for a file this shard never meant to publish. An operator who wants
/// a different tree served wants a different feature.
/// </summary>
private static readonly string[] SingleFiles =
{
"Data/Regions.xml",
"Config/ChampionSpawns.xml"
};
private const string LocationsDir = "Data/Locations";
private const string SpawnsDir = "Spawns";
private const string DecorationDir = "Data/Decoration";
public static void Initialize()
{
if (!BridgeConfig.Enabled)
return;
// Its own consent, not the asset plane's (§10, phase 7). An operator who declines to
// serve their UO client still gets a spawn atlas, because these are their own files.
BridgeAssets.RegisterFamily(Family, ReplyFetch, ReplyManifest,
() => BridgeConfig.TreeEnabled,
"the shard's configuration tree is not served (Bridge.TreeEnabled is off)");
}
// ── the file set ─────────────────────────────────────────────────────────────────────
private sealed class TreeFile
{
public string Label;
public string Path;
public long Bytes;
public long MTime;
}
/// <summary>
/// Every atlas source file this shard has, tree-relative and forward-slashed.
///
/// The labels are `spawnAtlasSource.js`'s own, character for character, because they
/// are what the website keys its stored fingerprint on: the same tree read here and
/// read there has to produce the same label or every import looks like a change.
/// Forward slashes for the same reason — a Windows shard and a Linux one must agree.
/// </summary>
private static List<TreeFile> Enumerate()
{
string root = Core.BaseDirectory;
var files = new List<TreeFile>();
foreach (string label in SingleFiles)
Add(files, root, label);
foreach (string label in ListByExtension(root, LocationsDir, ".xml"))
Add(files, root, label);
foreach (string label in ListByExtension(root, SpawnsDir, ".xml"))
Add(files, root, label);
foreach (string label in ListTree(root, DecorationDir, ".cfg"))
Add(files, root, label);
return files;
}
private static void Add(List<TreeFile> files, string root, string label)
{
string path = Resolve(root, label);
if (path == null)
return;
try
{
var info = new FileInfo(path);
if (!info.Exists)
return;
files.Add(new TreeFile
{
Label = label,
Path = path,
Bytes = info.Length,
MTime = ToUnixMs(info.LastWriteTimeUtc)
});
}
catch (Exception e)
{
// A file the shard cannot stat is a file it cannot serve. Say so once, here,
// rather than as a refused row on every import pass forever.
Console.WriteLine("[Bridge] tree: cannot read {0}: {1}", label, e.Message);
}
}
/// <summary>One directory's files with the given extension, sorted, as labels.</summary>
private static List<string> ListByExtension(string root, string dir, string extension)
{
var labels = new List<string>();
string full = Path.Combine(root, dir.Replace('/', Path.DirectorySeparatorChar));
try
{
if (!Directory.Exists(full))
return labels;
foreach (string path in Directory.GetFiles(full))
{
string name = Path.GetFileName(path);
if (name.EndsWith(extension, StringComparison.OrdinalIgnoreCase))
labels.Add(dir + "/" + name);
}
}
catch (Exception e)
{
Console.WriteLine("[Bridge] tree: cannot list {0}: {1}", dir, e.Message);
}
labels.Sort(StringComparer.Ordinal);
return labels;
}
/// <summary>
/// One directory tree's files with the given extension, recursively.
///
/// Recursive because `Data/Decoration` nests two deep in places (`Magincia/Trammel`,
/// `Stygian Abyss/Ter Mur`, `Old/Britannia`), and the website's own reader says why
/// that matters: a flat read indexes a third of what the shard has, and the failure is
/// an authoring dropdown quietly missing whole expansions rather than an error anyone
/// would notice.
/// </summary>
private static List<string> ListTree(string root, string dir, string extension)
{
var labels = new List<string>();
string full = Path.Combine(root, dir.Replace('/', Path.DirectorySeparatorChar));
try
{
if (!Directory.Exists(full))
return labels;
foreach (string path in Directory.GetFiles(full, "*", SearchOption.AllDirectories))
{
if (!path.EndsWith(extension, StringComparison.OrdinalIgnoreCase))
continue;
string rel = path.Substring(full.Length).Replace('\\', '/').TrimStart('/');
if (rel.Length > 0)
labels.Add(dir + "/" + rel);
}
}
catch (Exception e)
{
Console.WriteLine("[Bridge] tree: cannot walk {0}: {1}", dir, e.Message);
}
labels.Sort(StringComparer.Ordinal);
return labels;
}
/// <summary>
/// A label to a path on this host, or null if it is not one this shard serves.
///
/// Rule 1 of the class doc lives here. The label has already been matched against the
/// enumerated set by the time a fetch calls this, and this still refuses anything with
/// a traversal segment, a drive or a root in it, and still checks that what
/// <c>Path.GetFullPath</c> produced is under the tree root. Three checks for one rule
/// because the cost of being wrong once is the website reading an arbitrary file off a
/// game server's disk.
/// </summary>
private static string Resolve(string root, string label)
{
if (String.IsNullOrEmpty(label) || label.IndexOf('\\') >= 0)
return null;
string[] segments = label.Split('/');
foreach (string segment in segments)
{
if (segment.Length == 0 || segment == "." || segment == "..")
return null;
}
if (Path.IsPathRooted(label))
return null;
try
{
string rootFull = Path.GetFullPath(root);
string full = Path.GetFullPath(Path.Combine(rootFull,
label.Replace('/', Path.DirectorySeparatorChar)));
if (!rootFull.EndsWith(Path.DirectorySeparatorChar.ToString(CultureInfo.InvariantCulture),
StringComparison.Ordinal))
{
rootFull += Path.DirectorySeparatorChar;
}
return full.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase) ? full : null;
}
catch
{
return null;
}
}
// ── the fingerprint ──────────────────────────────────────────────────────────────────
/// <summary>
/// What the whole tree currently is, in sixteen hex characters.
///
/// The same job <c>BridgeCatalog.SourceId</c> does for client files, and the same
/// reason: it goes on every page of a walk, and a page whose id differs from the
/// first's means the operator edited a spawn file while it was being read. Half of
/// what arrived then describes a tree that no longer exists and nothing later can tell
/// which half, so the website refuses the import outright rather than stitching one.
///
/// Built from (label, size, mtime) rather than from content hashes, because it is
/// computed on every page and hashing the tree's contents each time would spend a
/// tenth of a second per page to answer a question (size, mtime) answers for free.
/// The CONTENT hashes are still sent — once, per file, on the manifest — which is
/// where the website's own drift gate reads them from.
/// </summary>
private static string FingerprintOf(List<TreeFile> files)
{
var sb = new StringBuilder(256);
sb.Append(files.Count);
foreach (TreeFile file in files)
{
sb.Append('|').Append(file.Label)
.Append(':').Append(file.Bytes.ToString(CultureInfo.InvariantCulture))
.Append(':').Append(file.MTime.ToString(CultureInfo.InvariantCulture));
}
return BridgeAssets.Sha256Hex(Encoding.UTF8.GetBytes(sb.ToString())).Substring(0, 16);
}
// ── assets.manifest, for this family ─────────────────────────────────────────────────
/// <summary>
/// Worker thread. Every file this shard would serve, with its size, its content hash
/// and how many chunks it takes — and no bytes.
///
/// That separation is what makes the normal case free. The website stores these
/// hashes; on the next import it asks for this list again, compares, and fetches
/// nothing at all when nothing moved — which on a shard whose maps are not being
/// edited is every import.
///
/// A stock tree is 141 rows and fits in one page comfortably. It pages anyway, by the
/// same envelope as every other family, because the day a shard has three thousand
/// decoration files is not the day to discover this was the one walk that could not
/// end.
/// </summary>
private static void ReplyManifest(string reqId, string cursor)
{
List<TreeFile> files = Enumerate();
string fingerprint = FingerprintOf(files);
int from = ParseCursor(cursor);
if (from < 0 || from > files.Count)
from = 0;
var sb = BridgeJson.Begin("assets.manifest.ok");
sb.Str("reqId", reqId)
.Str("family", Family)
.Str("catalog", fingerprint)
.Num("chunkBytes", BridgeConfig.TreeChunkBytes)
.Num("total", files.Count)
.Num("from", from);
var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes);
int i = from;
for (; i < files.Count; i++)
{
TreeFile file = files[i];
string hash = HashFile(file.Path);
var item = new StringBuilder(256);
item.Append("{\"key\":");
BridgeJson.Text(item, Family + "/" + file.Label);
item.Append(",\"label\":");
BridgeJson.Text(item, file.Label);
item.Append(",\"bytes\":").Append(file.Bytes.ToString(CultureInfo.InvariantCulture));
item.Append(",\"mtime\":").Append(file.MTime.ToString(CultureInfo.InvariantCulture));
item.Append(",\"chunks\":").Append(
ChunkCount(file.Bytes).ToString(CultureInfo.InvariantCulture));
item.Append(",\"sha256\":");
BridgeJson.Text(item, hash);
item.Append('}');
if (!page.TryAdd(item.ToString(), "t:" + (i + 1).ToString(CultureInfo.InvariantCulture)))
break;
}
page.Close();
sb.Num("sent", page.Count);
BridgeLink.Emit(sb.End());
}
/// <summary>
/// How many chunks a file of this size takes.
///
/// **An empty file is one chunk, not none.** `Data/Locations` can legitimately hold an
/// empty file, and zero chunks would make it a manifest row the website could never
/// fetch: it would wait for content that has no address, and report the import
/// incomplete forever.
/// </summary>
private static int ChunkCount(long bytes)
{
long chunk = BridgeConfig.TreeChunkBytes;
long count = (bytes + chunk - 1) / chunk;
return count < 1 ? 1 : (int)count;
}
// ── assets.fetch, for this family ────────────────────────────────────────────────────
/// <summary>
/// Worker thread. The bytes for an explicit list of chunk keys.
///
/// Chunks are read with a seek rather than by holding the file, so the memory this
/// costs a running game server is one chunk regardless of how large an operator's
/// spawn tables are. A 4 MB file served eight times over is eight seeks and eight
/// 512 KiB reads — cheaper than caching it would be, and with no cache to invalidate
/// when the operator edits it mid-pass.
/// </summary>
private static void ReplyFetch(string reqId, List<string> keys, string expected, string cursor)
{
List<TreeFile> files = Enumerate();
string fingerprint = FingerprintOf(files);
// Shared with every other family on this plane, because an absent fingerprint and an
// empty one have to mean the same thing here and there — see
// `BridgeAssets.CatalogMismatch` for what treating them differently costs.
if (BridgeAssets.CatalogMismatch(expected, fingerprint))
{
// The tree moved between the manifest and this fetch. The same refusal the
// catalogue makes for a patched client, and for the same reason: these keys were
// chosen against a listing that no longer describes what is on disk.
BridgeAssets.Fail(reqId, "UNREADABLE",
"the shard's configuration tree changed since that manifest was read (catalog "
+ expected + " is now " + fingerprint + "); start the import again");
return;
}
var byLabel = new Dictionary<string, TreeFile>(StringComparer.Ordinal);
foreach (TreeFile file in files)
byLabel[file.Label] = file;
int from = ParseCursor(cursor);
if (from < 0 || from > keys.Count)
from = 0;
var sb = BridgeJson.Begin("assets.fetch.ok");
sb.Str("reqId", reqId)
.Str("family", Family)
.Str("catalog", fingerprint)
.Num("chunkBytes", BridgeConfig.TreeChunkBytes)
.Num("asked", keys.Count)
.Num("from", from);
var page = new BridgeAssets.PageBuilder(sb, "rows", BridgeConfig.AssetBatchBytes);
int i = from;
for (; i < keys.Count; i++)
{
string item = Render(byLabel, keys[i]);
if (!page.TryAdd(item, "t:" + (i + 1).ToString(CultureInfo.InvariantCulture)))
break;
}
page.Close();
sb.Num("sent", page.Count);
BridgeLink.Emit(sb.End());
}
/// <summary>
/// One key to one row.
///
/// A key this shard cannot serve is a row rather than a failed request, exactly as in
/// every other family, and `status` keeps the two kinds apart: `absent` is a file this
/// shard does not have (a tree with no `ChampionSpawns.xml` is a normal tree), and
/// `unsupported` is a key shape this family does not serve — which is a website bug,
/// and is counted separately so it cannot hide inside the expected gaps.
/// </summary>
private static string Render(Dictionary<string, TreeFile> byLabel, string key)
{
string label;
int chunk;
if (!ParseKey(key, out label, out chunk))
return Refusal(key, "unsupported", "not a tree chunk key (tree/<label>/c<n>)");
TreeFile file;
if (!byLabel.TryGetValue(label, out file))
{
// Rule 1: the label has to be one THIS shard enumerated. Anything else is refused
// here, before a path is built out of it, whatever it spells.
return Refusal(key, "absent", "this shard does not serve that file");
}
int chunks = ChunkCount(file.Bytes);
if (chunk < 0 || chunk >= chunks)
{
return Refusal(key, "unsupported",
"chunk " + chunk.ToString(CultureInfo.InvariantCulture) + " of "
+ chunks.ToString(CultureInfo.InvariantCulture));
}
long offset = (long)chunk * BridgeConfig.TreeChunkBytes;
byte[] raw;
try
{
raw = ReadChunk(file.Path, offset, BridgeConfig.TreeChunkBytes);
}
catch (Exception e)
{
Console.WriteLine("[Bridge] tree: cannot read {0} chunk {1}: {2}", label, chunk, e.Message);
return Refusal(key, "absent", e.GetType().Name);
}
byte[] packed;
try
{
packed = Gzip(raw);
}
catch (Exception e)
{
Console.WriteLine("[Bridge] tree: cannot compress {0} chunk {1}: {2}", label, chunk, e.Message);
return Refusal(key, "absent", e.GetType().Name);
}
var item = new StringBuilder(packed.Length * 2);
item.Append("{\"key\":");
BridgeJson.Text(item, key);
item.Append(",\"status\":\"ok\",\"label\":");
BridgeJson.Text(item, label);
item.Append(",\"chunk\":").Append(chunk.ToString(CultureInfo.InvariantCulture));
item.Append(",\"chunks\":").Append(chunks.ToString(CultureInfo.InvariantCulture));
item.Append(",\"offset\":").Append(offset.ToString(CultureInfo.InvariantCulture));
item.Append(",\"bytes\":").Append(raw.Length.ToString(CultureInfo.InvariantCulture));
item.Append(",\"sha256\":");
BridgeJson.Text(item, BridgeAssets.Sha256Hex(raw));
item.Append(",\"gzip\":");
BridgeJson.Text(item, Convert.ToBase64String(packed));
item.Append('}');
return item.ToString();
}
private static string Refusal(string key, string status, string reason)
{
var item = new StringBuilder(128);
item.Append("{\"key\":");
BridgeJson.Text(item, key);
item.Append(",\"status\":");
BridgeJson.Text(item, status);
item.Append(",\"reason\":");
BridgeJson.Text(item, reason);
item.Append('}');
return item.ToString();
}
/// <summary>
/// `tree/&lt;label&gt;/c&lt;n&gt;` into its label and chunk index.
///
/// The label itself contains slashes, so the chunk segment is taken off the END rather
/// than by counting segments from the front. That is unambiguous here and not by
/// luck: every label this family serves ends in `.xml` or `.cfg`, so no label's last
/// segment can be spelled `c` followed by digits.
/// </summary>
private static bool ParseKey(string key, out string label, out int chunk)
{
label = null;
chunk = -1;
if (String.IsNullOrEmpty(key))
return false;
string prefix = Family + "/";
if (!key.StartsWith(prefix, StringComparison.Ordinal))
return false;
int slash = key.LastIndexOf('/');
if (slash <= prefix.Length - 1)
return false;
string last = key.Substring(slash + 1);
if (last.Length < 2 || last[0] != 'c')
return false;
for (int i = 1; i < last.Length; i++)
{
if (last[i] < '0' || last[i] > '9')
return false;
}
if (!Int32.TryParse(last.Substring(1), NumberStyles.None, CultureInfo.InvariantCulture, out chunk))
return false;
label = key.Substring(prefix.Length, slash - prefix.Length);
return label.Length > 0;
}
private static int ParseCursor(string cursor)
{
if (String.IsNullOrEmpty(cursor) || !cursor.StartsWith("t:", StringComparison.Ordinal))
return 0;
int value;
return Int32.TryParse(cursor.Substring(2), NumberStyles.None,
CultureInfo.InvariantCulture, out value) ? value : 0;
}
// ── bytes ────────────────────────────────────────────────────────────────────────────
private static byte[] ReadChunk(string path, long offset, int length)
{
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read,
FileShare.ReadWrite, 1 << 16))
{
long remaining = stream.Length - offset;
if (remaining < 0)
remaining = 0;
if (remaining > length)
remaining = length;
var buffer = new byte[remaining];
stream.Seek(offset, SeekOrigin.Begin);
int filled = 0;
while (filled < buffer.Length)
{
int read = stream.Read(buffer, filled, buffer.Length - filled);
// A short read is not the end of the file here — the length was taken from the
// stream itself. Stopping on one would hand back a chunk whose declared length
// and real length disagree, which the website would only see as a hash
// mismatch on a file it cannot name a cause for.
if (read <= 0)
break;
filled += read;
}
if (filled == buffer.Length)
return buffer;
var exact = new byte[filled];
Buffer.BlockCopy(buffer, 0, exact, 0, filled);
return exact;
}
}
/// <summary>
/// A complete gzip member for exactly one empty chunk.
///
/// **`GZipStream` writes NOTHING for zero bytes of input**, on .NET Framework and on
/// Mono: the gzip header is emitted lazily on the first write, so a stream that is
/// opened and closed without one produces a zero-length buffer rather than the 20-byte
/// empty member. That is not a valid gzip stream, and the reader at the other end says
/// so — `zlib: unexpected end of file`.
///
/// It is not a hypothetical: **stock ServUO 57.4 ships two empty decoration files**
/// (`Felucca/ambitious solen queen quest.cfg` and
/// `Tokuno/terrible hatchlings quest.cfg`), so every import off an untouched tree hit
/// it. Worth knowing how it was found, because it says something about probes: an
/// offline harness reassembled all 141 files and reported success, since .NET's own
/// decompressor treats an empty stream as empty data and the chunk's declared length
/// (0) and hash (of nothing) both agreed with that. Only the live walk, through a
/// reader on a different runtime, disagreed.
///
/// The alternative — letting an empty chunk carry an empty payload and teaching the
/// reader to expect it — was rejected: it puts a special case on the wire, where every
/// future reader has to know it, instead of in the one place that builds the bytes.
/// Header (magic, deflate, no flags, no mtime, no XFL, unknown OS), one empty stored
/// block, then CRC32 and ISIZE of nothing.
/// </summary>
private static readonly byte[] EmptyGzip =
{
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
0x03, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
private static byte[] Gzip(byte[] raw)
{
if (raw.Length == 0)
return EmptyGzip;
using (var ms = new MemoryStream())
{
using (var gz = new GZipStream(ms, CompressionMode.Compress, true))
gz.Write(raw, 0, raw.Length);
return ms.ToArray();
}
}
/// <summary>
/// The content hash of one file, streamed.
///
/// Streamed rather than <c>File.ReadAllBytes</c> because this runs once per file per
/// manifest, and a stock tree's spawn files are 10 MB between them: reading them whole
/// would put that much through a game server's large object heap to produce 141 short
/// strings.
/// </summary>
private static string HashFile(string path)
{
try
{
using (var sha = System.Security.Cryptography.SHA256.Create())
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read,
FileShare.ReadWrite, 1 << 16))
{
var buffer = new byte[1 << 16];
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
sha.TransformBlock(buffer, 0, read, null, 0);
sha.TransformFinalBlock(buffer, 0, 0);
var sb = new StringBuilder(64);
foreach (byte b in sha.Hash)
sb.Append(b.ToString("x2", CultureInfo.InvariantCulture));
return sb.ToString();
}
}
catch (Exception e)
{
Console.WriteLine("[Bridge] tree: cannot hash {0}: {1}", path, e.Message);
return null;
}
}
private static long ToUnixMs(DateTime utc)
{
return (long)(utc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;
}
/// <summary>For `[Bridge] status`, the same one-line shape every other family reports.</summary>
public static string Status()
{
if (!BridgeConfig.TreeEnabled)
return "tree(disabled)";
List<TreeFile> files = Enumerate();
long bytes = 0;
foreach (TreeFile file in files)
bytes += file.Bytes;
return String.Format("tree(files={0} bytes={1} catalog={2})",
files.Count, bytes, FingerprintOf(files));
}
}
}