Merge pull request 'feat(sidecar): protocol 8 — the asset plane (Asset Bridge cutover, 1 of 5)' (#44) from edge into main

Reviewed-on: #44
Reviewed-by: Colby Whitlock <whitlocktech@gmail.com>

sidecar/src/main.rs

@@ -115,6 +115,14 @@ use tracing_subscriber::EnvFilter;
 ///     to every connected client. The dumb-forwarder property is doing real work here: the sidecar
 ///     does not know what an asset is, and must not learn.
 ///
+/// Phase 2 adds the first family that actually carries content: **`cliloc.table`**, served at
+/// `GET /cliloc`. It pages — the shard cuts at a byte budget and the caller echoes a cursor back —
+/// and the sidecar forwards it without keeping any of it, which matters more here than usual: the
+/// payload is five megabytes of EA's strings out of the operator's own client, and the one copy of
+/// it that should exist is the one the website imports. That phase also gave `assets.error` a
+/// `code`, so the status a refusal maps to stops depending on the wording of a human-facing
+/// sentence (see [`web::asset_error_status`]).
+///
 /// **No store migration**, again: nothing on this plane is an event, so nothing is persisted.
 pub const PROTOCOL_VERSION: u32 = 8;
 

sidecar/src/web.rs

@@ -141,6 +141,22 @@ pub async fn serve(addr: &str, state: AppState) -> anyhow::Result<()> {
         // the files on that host changed since the last import", and a cached answer to that is
         // worse than no answer.
         .route("/assets/sources", get(assets_sources))
+        // Stage 2 (phase 3): what the shard could serve, hashed but without the pixels, so the
+        // website can ask only for what changed. Paged.
+        .route("/assets/manifest", get(assets_manifest))
+        // The pixels, for an explicit list of keys. POST rather than GET because the list is the
+        // request -- a few hundred keys do not belong in a query string, and this is the one place
+        // on this link where a read takes a body.
+        .route("/assets/fetch", post(assets_fetch))
+        // Slug -> body id. The one asset-plane call the shard answers ON ITS CORE THREAD, because
+        // it resolves a class name by constructing the creature and reading its body; the sidecar
+        // neither knows nor cares, which is the point of forwarding verbatim.
+        .route("/assets/bodies", post(assets_bodies))
+        // The cliloc table (Protocol 8, phase 2): UO's id -> display-string map, read out of the
+        // shard's own client and paged. RPC for the same reason as the source gate, and one more:
+        // it is five megabytes of somebody else's copyrighted strings, which this process has no
+        // business holding a copy of. It forwards them and forgets them.
+        .route("/cliloc", get(cliloc_table))
         .route_layer(middleware::from_fn_with_state(state.clone(), gate));
 
     let app = Router::new()
@@ -1345,18 +1361,7 @@ fn respond_assets(result: Result<Value, RpcError>) -> (StatusCode, Json<Value>)
             if kind == BUSY_KIND {
                 (BUSY_STATUS, Json(value))
             } else if kind == "assets.error" {
-                let reason = value
-                    .get("reason")
-                    .and_then(|r| r.as_str())
-                    .unwrap_or("request rejected");
-
-                let code = if reason.contains("disabled") {
-                    StatusCode::FORBIDDEN
-                } else {
-                    StatusCode::BAD_REQUEST
-                };
-
-                (code, Json(value))
+                (asset_error_status(&value), Json(value))
             } else {
                 (StatusCode::OK, Json(value))
             }
@@ -1372,6 +1377,184 @@ fn respond_assets(result: Result<Value, RpcError>) -> (StatusCode, Json<Value>)
     }
 }
 
+/// The status behind one `assets.error`.
+///
+/// Phase 2 gave the frame a `code`, and the reason is worth stating: phase 1 decided between 403
+/// and 400 by looking for the word "disabled" **in the operator-facing sentence**. That works
+/// until someone improves the wording, at which point a refusal quietly becomes a bad request and
+/// an administrator goes hunting for a bug in a correctly-written call. The sentence is for a
+/// human; the code is for this function.
+///
+/// The substring check survives as a fallback because an overlay is deployed independently of the
+/// sidecar: a phase-1 shard paired with a phase-2 sidecar still sends the codeless frame, and it
+/// must keep getting its 403.
+fn asset_error_status(value: &Value) -> StatusCode {
+    match value.get("code").and_then(|c| c.as_str()) {
+        Some("DISABLED") => return StatusCode::FORBIDDEN,
+        // The shard has no such file. Not the caller's mistake and not a broken shard -- a client
+        // that does not carry what was asked for, which the website reports to its operator.
+        Some("NOT_FOUND") => return StatusCode::NOT_FOUND,
+        // It has the file and cannot decode it: truncated, hand-edited, or not what it claims to
+        // be. 422 rather than 400, because the request was fine and repeating it will not help.
+        Some("UNREADABLE") => return StatusCode::UNPROCESSABLE_ENTITY,
+        // The shard cannot do this right now (it could not start its asset worker, say). Same
+        // status as "no shard connected", because it means the same thing to a caller: come back.
+        Some("UNAVAILABLE") => return StatusCode::SERVICE_UNAVAILABLE,
+        Some("BAD_REQUEST") => return StatusCode::BAD_REQUEST,
+        _ => {}
+    }
+
+    let reason = value
+        .get("reason")
+        .and_then(|r| r.as_str())
+        .unwrap_or("request rejected");
+
+    if reason.contains("disabled") {
+        StatusCode::FORBIDDEN
+    } else {
+        StatusCode::BAD_REQUEST
+    }
+}
+
+/// The cliloc table, as the shard's own client holds it (docs/link/v8.md §9).
+///
+/// Until protocol 8 this table reached the website by hand: the operator installed UOFiddler,
+/// built a converter against its `Ultima.dll`, ran it over their client's compressed `Cliloc.enu`
+/// and copied the result to the web host. The shard could not help, because ServUO's bundled
+/// `Ultima.StringList` cannot read a modern client's file either. Phase 2 put the decompressor in
+/// the overlay, so the shard reads its own client and the operator installs nothing.
+///
+/// **Paged, and the caller drives the paging** -- `?cursor=` echoes back whatever the previous
+/// reply's `cursor` was, until a reply says `more: false`. The pages are cut by byte budget on the
+/// shard (512 KiB against the 1 MiB inbound line cap), so a stock English table arrives in about
+/// eleven of them. The sidecar keeps none of it: it has no opinion about what a cliloc is, and a
+/// cached copy of five megabytes of EA's strings is exactly what this process should not hold.
+///
+/// `?lang=` selects the file; it defaults to `enu` on the shard and the shard refuses anything its
+/// `Ultima.Files` cannot resolve, which is a 404 rather than a 400.
+async fn cliloc_table(
+    State(st): State<AppState>,
+    Query(q): Query<ClilocQuery>,
+) -> impl IntoResponse {
+    let req_id = st.rpc.next_req_id();
+    let mut cmd = json!({"kind": "cliloc.table", "reqId": req_id});
+
+    if let Some(lang) = q.lang.as_deref().filter(|s| !s.is_empty()) {
+        cmd["lang"] = json!(lang);
+    }
+
+    if let Some(cursor) = q.cursor.as_deref().filter(|s| !s.is_empty()) {
+        cmd["cursor"] = json!(cursor);
+    }
+
+    respond_assets(st.rpc.call(&st.shard, cmd, &req_id).await)
+}
+
+#[derive(Deserialize)]
+struct ClilocQuery {
+    lang: Option<String>,
+    cursor: Option<String>,
+}
+
+/// Stage 2 of the import gate (docs/link/v8.md §6, phase 3): every asset the shard could serve,
+/// with a hash and a size and **no pixels**.
+///
+/// That separation is the whole difference between an Update and a re-download. The website holds
+/// the hashes from last time, diffs this against them and asks `/assets/fetch` only for the keys
+/// that moved — which, on the normal restart-that-changed-nothing, is none of them.
+///
+/// **Paged, and the caller drives the paging**, same envelope as `/cliloc`: echo the previous
+/// reply's `cursor` until one says `more: false`, and read `cut` to learn *why* a page was the
+/// last — only `end` means the manifest is complete. This family pages on the shard's wall clock
+/// rather than on bytes, because its rows are tiny and building them means decoding hundreds of
+/// sprites, so expect several pages of a few hundred rows each.
+///
+/// `?family=` selects which asset family; phase 3 serves `body` and the shard refuses anything
+/// else by name rather than substituting a default.
+async fn assets_manifest(
+    State(st): State<AppState>,
+    Query(q): Query<ManifestQuery>,
+) -> impl IntoResponse {
+    let req_id = st.rpc.next_req_id();
+    let mut cmd = json!({"kind": "assets.manifest", "reqId": req_id});
+
+    if let Some(family) = q.family.as_deref().filter(|s| !s.is_empty()) {
+        cmd["family"] = json!(family);
+    }
+
+    if let Some(cursor) = q.cursor.as_deref().filter(|s| !s.is_empty()) {
+        cmd["cursor"] = json!(cursor);
+    }
+
+    respond_assets(st.rpc.call(&st.shard, cmd, &req_id).await)
+}
+
+#[derive(Deserialize)]
+struct ManifestQuery {
+    family: Option<String>,
+    cursor: Option<String>,
+}
+
+/// The bytes, for keys the caller names: `{"keys":[…],"catalog":<opt>,"cursor":<opt>}`.
+///
+/// Each row carries the sprite as base64 PNG. The shard encodes it, and that is deliberate rather
+/// than incidental: `System.Drawing` is already in its *decode* path (docs/link/v8.md §4.2), so
+/// PNG costs it no new dependency, while asking the website to encode would put an image encoder
+/// in Node and make the manifest's hash cover bytes nobody ever stores.
+///
+/// **`catalog` is the mid-import guard.** A manifest reply carries a `catalog` id derived from the
+/// client files themselves; passing it back here makes the shard refuse (422) if those files moved
+/// in between. Without it an operator who patched their client halfway through an import would get
+/// one asset set stitched out of two, with no error anywhere.
+///
+/// A key the shard cannot serve comes back as a **row** with a `status`, not as a failed request —
+/// a body this client has no art for is the expected answer for two thirds of the player bodies,
+/// and failing the whole page over one would make an import impossible on a stock client.
+async fn assets_fetch(State(st): State<AppState>, Json(body): Json<Value>) -> impl IntoResponse {
+    assets_call(&st, "assets.fetch", body).await
+}
+
+/// Slug → body id: `{"types":["GiantSpider", …]}` (docs/link/v8.md §8, phase 3).
+///
+/// The spawn atlas knows a creature by the class name in `Spawns/*.xml`; the client knows it by a
+/// body id; nothing in the ServUO tree declares the mapping as data. Only code running *inside*
+/// ServUO can answer it — construct the type, read `Body.BodyID`, delete it — which is why this is
+/// a request kind of its own rather than a step inside asset extraction: it runs on the shard's
+/// Core thread, while every decode on this plane runs off it.
+///
+/// **The shard caps the batch and refuses rather than truncates** a longer list, because every
+/// name in it costs a real constructor between two ticks of the world. Chunk the list; a 400 here
+/// names the cap.
+async fn assets_bodies(State(st): State<AppState>, Json(body): Json<Value>) -> impl IntoResponse {
+    assets_call(&st, "assets.bodies", body).await
+}
+
+/// `event_call`'s shape for the asset plane: take the caller's object verbatim, stamp `kind` and
+/// `reqId` on it, and map the reply through `respond_assets` rather than `respond_event`.
+///
+/// The two differ in exactly one way that matters, and it is the reason this is not `event_call`
+/// with a different string: `bridge.busy` is a **425** here, and it is the ordinary answer during
+/// an import rather than a rare collision. The shard serves one asset request at a time on purpose,
+/// so a caller that read busy as an error would abandon a healthy transfer.
+async fn assets_call(st: &AppState, kind: &str, body: Value) -> (StatusCode, Json<Value>) {
+    let mut obj = match body {
+        Value::Object(m) => m,
+        Value::Null => serde_json::Map::new(),
+        _ => {
+            return (
+                StatusCode::BAD_REQUEST,
+                Json(json!({"error": "body must be a JSON object"})),
+            )
+        }
+    };
+
+    let req_id = st.rpc.next_req_id();
+    obj.insert("kind".to_string(), json!(kind));
+    obj.insert("reqId".to_string(), json!(req_id));
+
+    respond_assets(st.rpc.call(&st.shard, Value::Object(obj), &req_id).await)
+}
+
 // ---- websocket ----
 
 async fn ws_upgrade(ws: WebSocketUpgrade, State(state): State<AppState>) -> impl IntoResponse {
@@ -1480,6 +1663,74 @@ mod tests {
         assert_eq!(respond_assets(Ok(value)).0, StatusCode::BAD_REQUEST);
     }
 
+    /// Phase 2's codes, each of which says something a 400 does not. `NOT_FOUND` is a client that
+    /// does not carry the file (an operator fact, not a bug); `UNREADABLE` is a file that is there
+    /// and cannot be decoded, where repeating the request cannot help; `UNAVAILABLE` is the shard
+    /// declining for now.
+    #[test]
+    fn an_asset_error_code_picks_the_status() {
+        let with = |code: &str| {
+            respond_assets(Ok(json!({
+                "kind": "assets.error",
+                "reqId": "r-1",
+                "code": code,
+                "reason": "…"
+            })))
+            .0
+        };
+
+        assert_eq!(with("DISABLED"), StatusCode::FORBIDDEN);
+        assert_eq!(with("NOT_FOUND"), StatusCode::NOT_FOUND);
+        assert_eq!(with("UNREADABLE"), StatusCode::UNPROCESSABLE_ENTITY);
+        assert_eq!(with("UNAVAILABLE"), StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(with("BAD_REQUEST"), StatusCode::BAD_REQUEST);
+        // An unknown code is not a reason to invent a status.
+        assert_eq!(with("SOMETHING_NEW"), StatusCode::BAD_REQUEST);
+    }
+
+    /// The overlay and the sidecar ship separately and an operator can run a phase-1 shard against
+    /// a phase-2 sidecar, so the codeless refusal must keep its 403. This is the whole reason the
+    /// substring check survives rather than being deleted with the code it preceded.
+    #[test]
+    fn a_codeless_disabled_refusal_is_still_a_403() {
+        let value = json!({
+            "kind": "assets.error",
+            "reqId": "r-9",
+            "reason": "asset extraction is disabled on this shard"
+        });
+
+        assert_eq!(respond_assets(Ok(value)).0, StatusCode::FORBIDDEN);
+    }
+
+    /// One page of the cliloc table. The assertion that matters is the envelope: `more` and
+    /// `cursor` reach the caller untouched, because the sidecar does not page this — the shard
+    /// cuts the pages and the website drives them, and a sidecar that "helpfully" assembled them
+    /// would be holding the whole table in memory to do it.
+    #[test]
+    fn a_cliloc_page_is_a_200_and_keeps_its_cursor() {
+        let value = json!({
+            "kind": "cliloc.table.ok",
+            "reqId": "r-2",
+            "lang": "enu",
+            "extractorVersion": 1,
+            "total": 67496,
+            "rows": [{"n": 1023721, "f": 0, "t": "quarter staff"}],
+            "more": true,
+            "cursor": "n:1023721",
+            "cut": "budget"
+        });
+
+        let (status, body) = respond_assets(Ok(value));
+
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(body.0.get("more").and_then(|v| v.as_bool()), Some(true));
+        assert_eq!(
+            body.0.get("cursor").and_then(|v| v.as_str()),
+            Some("n:1023721")
+        );
+        assert_eq!(body.0.get("total").and_then(|v| v.as_i64()), Some(67496));
+    }
+
     /// A source manifest comes back whole. Worth asserting because `respond_assets` sniffs `kind`
     /// and a family whose success kind ends in `.ok` sits one character away from the `.error`
     /// suffix the generic responder matches on -- which is exactly why this plane has its own