From bad19a668d0a8cc361d6cc6d535a0cbde57f6b9e Mon Sep 17 00:00:00 2001 From: wtclaude Date: Sun, 1 Nov 2026 01:30:10 -0600 Subject: [PATCH 1/9] fix(drill): staging 26341433's swap fields; a failed group no longer ends the harness (#20) The staging drill's run 47 failed on Rust staging 26341433: - Rust added BasePlayer.timeSinceAiming, BaseEntity.ServerPreProcessed and BaseNetworkable.ignoreNetworkLod to the swap's field list (D232): 55 -> 58. On edge now; main at the forced wipe of 2026-11-05. - Harness 0.7.4: place.set.each was refused and the refusal was thrown away; place.spawned failed; reading each[0] of an empty list then killed the coroutine, so the drill waited 45 minutes for a run that had stopped after 10. Place puts its spots on the navmesh, the set checks carry the refusal, a failed place.spawned ends the group, and any group that throws is a .exception FAIL line and the run goes on. - drill.js: rust-staging boots from empty Kits and RunicNPC data, so the write-once example kits match this Rust (s6.examples.fullCondition); the wait ends after 5 minutes without a new harness line; latest.log's tail goes into the issue when the harness did not finish. drill.test.js covers the stall logic. Refs #20, #21 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- README.md | 10 +++-- plugin/RunicNPC.cs | 3 ++ tools/RunicNpcTest.cs | 43 +++++++++++++++++---- tools/drill/drill.js | 81 +++++++++++++++++++++++++++++++-------- tools/drill/drill.test.js | 58 ++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 28 deletions(-) create mode 100644 tools/drill/drill.test.js diff --git a/README.md b/README.md index 891a161..0a892bb 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The API version moves when a call or a raised hook changes shape, not on every r ## After a Rust update: the swap's field list Our NPC is Rust's scientist with two components swapped, and the swap copies a fixed list of fields (D232), -generated from Rust's unmodified assembly. `rnpc.status` reports it as `swap fields: npc=55/55 brain=32/32 +generated from Rust's unmodified assembly. `rnpc.status` reports it as `swap fields: npc=58/58 brain=32/32 missing=- added=-`. On Carbon, where the assembly is unmodified, `added` names any field Rust has added since the list was made. To regenerate it: @@ -121,7 +121,7 @@ shows it, so the drill's live half and `rnpc.status` are what catch it. | Rust type | What RunicNPC depends on | Where (`plugin/RunicNPC.cs`) | Caught by | |---|---|---|---| -| `ScientistNPC` (prefabs `scientistnpc_*`) | Subclassed by `RunicNpcPlayer`; the swap replaces the prefab's component before spawn and copies `SwapNpcFields` (55 fields up its chain to `BaseNetworkable`) | `#region The swap`, `CreateNpc` | compile; field list | +| `ScientistNPC` (prefabs `scientistnpc_*`) | Subclassed by `RunicNpcPlayer`; the swap replaces the prefab's component before spawn and copies `SwapNpcFields` (58 fields up its chain to `BaseNetworkable`) | `#region The swap`, `CreateNpc` | compile; field list | | `ScientistBrain` | Subclassed by `RunicNpcBrain`; the swap copies `SwapBrainFields` (32 fields of `BaseAIBrain`) | `#region The swap` | compile; field list | | `ScientistNPC` / `HumanNPC` virtuals | `displayName`, `AttackerInfo(PlayerLifeStory.DeathInfo)`, `ShotTest(float)`, `TriggerDown()`, `Hurt(HitInfo)`, `OnDied(HitInfo)` overridden | `RunicNpcPlayer` | compile | | `IAIAttack`, `IAISenses` | Re-implemented on `RunicNpcPlayer`: `IsTargetInRange`, `EngagementRange`, `GetBestTarget`, `AttackTick`, `IsTarget`, `IsThreat`, `IsFriendly`. Rust's `HumanNPC.IsTarget` is not virtual, which is why | `RunicNpcPlayer` | compile; a changed meaning only at run time | @@ -153,8 +153,10 @@ the week. A run by hand always goes ahead. plugin against them. - **Live, once per build** (`tools/drill/drill.js live`): the `rust-staging` server (Oxide, a 3500 map) installs the same pair at start (`tools/drill/rig-start.sh`), loads RunicNPC, Kits and - `tools/RunicNpcTest.cs`, and runs `rnt.run all`. When both 6000-map rigs are running, `rust-carbon` is - stopped for it and started again. + `tools/RunicNpcTest.cs`, and runs `rnt.run all`. It starts the server from empty Kits and RunicNPC data + (their files are deleted first, so the example kits are written afresh) and gives up on the harness after + 5 minutes with no new output (45 minutes at most), putting the tail of the server's log in the issue. When + both 6000-map rigs are running, `rust-carbon` is stopped for it and started again. - **Report:** anything that fails or differs opens one issue for that build, labelled `staging-drill`. The last build checked is kept on the `drill` branch. diff --git a/plugin/RunicNPC.cs b/plugin/RunicNPC.cs index 67b29a8..90b1d1c 100644 --- a/plugin/RunicNPC.cs +++ b/plugin/RunicNPC.cs @@ -947,6 +947,7 @@ namespace Oxide.Plugins "BasePlayer.debugCommandedNpcs", "BasePlayer.GestureViewModel", "BasePlayer.lastPlayerVisibility", + "BasePlayer.timeSinceAiming", "BasePlayer.eyeHistory", "BasePlayer.tickHistory", "BasePlayer.fallDamageEffect", @@ -974,12 +975,14 @@ namespace Oxide.Plugins "BaseEntity.model", "BaseEntity.flags", "BaseEntity.HasBrain", + "BaseEntity.ServerPreProcessed", "BaseNetworkable.postNetworkUpdateComponents", "BaseNetworkable.prefabID", "BaseNetworkable.globalBroadcast", "BaseNetworkable.globalNetworkBehavior", "BaseNetworkable.globalBuildingBlock", "BaseNetworkable.networkRange", + "BaseNetworkable.ignoreNetworkLod", }; private static readonly string[] SwapBrainFields = { diff --git a/tools/RunicNpcTest.cs b/tools/RunicNpcTest.cs index a16a4a2..7e1ad01 100644 --- a/tools/RunicNpcTest.cs +++ b/tools/RunicNpcTest.cs @@ -45,7 +45,7 @@ namespace Oxide.Plugins /// a made-up user id (11400000001 and up), so IsNpc is false. They take no damage. /// /// - [Info("RunicNpcTest", "Runic Gateway", "0.7.3")] + [Info("RunicNpcTest", "Runic Gateway", "0.7.4")] [Description("RunicNPC stage 2 to 7 tests, the stage 6 spike and stage 9's cost check. A developer tool: never ship it.")] internal class RunicNpcTest : RustPlugin { @@ -753,7 +753,25 @@ namespace Oxide.Plugins continue; } - while (inner.MoveNext()) yield return inner.Current; + // A group that throws is a FAIL line, not the end of the run: yield cannot sit inside a try, so + // each step is taken inside one and the value is yielded outside it. + while (true) + { + object step; + try + { + if (!inner.MoveNext()) break; + step = inner.Current; + } + catch (Exception e) + { + Check($"{g}.exception", false, $"{e.GetType().Name}: {e.Message} at {(e.StackTrace ?? "").Split('\n').FirstOrDefault()?.Trim()}"); + break; + } + + yield return step; + } + Api("RunicNpc_DespawnOwner", Owner); ClearWorld(); yield return new WaitForSeconds(1f); @@ -1095,16 +1113,27 @@ namespace Oxide.Plugins private IEnumerator Place(Vector3 field) { - Vector3 a = field + new Vector3(-40, 0, 0); - Vector3 b = field + new Vector3(-40, 0, 30); - StandIn(field + new Vector3(-40, 0, 110), 8); - Check("place.set.each", Api("RunicNpc_SetPlacement", "t_each", Placement("t_each", a, 3, 20f, "each")) == null, ""); - Check("place.set.group", Api("RunicNpc_SetPlacement", "t_group", Placement("t_roam", b, 3, 20f, "group")) == null, ""); + // A roamer's spot must be on the navmesh when it is set (D219), so both spots are put there. + Vector3 a = OnMesh(field + new Vector3(-40, 0, 0)); + Vector3 b = OnMesh(field + new Vector3(-40, 0, 30)); + StandIn(OnMesh(field + new Vector3(-40, 0, 110)), 8); + string eachRefusal = Api("RunicNpc_SetPlacement", "t_each", Placement("t_each", a, 3, 20f, "each")); + Check("place.set.each", eachRefusal == null, eachRefusal ?? ""); + string groupRefusal = Api("RunicNpc_SetPlacement", "t_group", Placement("t_roam", b, 3, 20f, "group")); + Check("place.set.group", groupRefusal == null, groupRefusal ?? ""); yield return new WaitForSeconds(4f); List> each = List("placement:t_each"); List> group = List("placement:t_group"); Check("place.spawned", each.Count == 3 && group.Count == 3, $"each {each.Count}, group {group.Count}"); + if (each.Count != 3 || group.Count != 3) + { + // Everything below needs both sets alive. Take back what was set and end the group, so the run goes on. + Api("RunicNpc_RemovePlacement", "t_each"); + Api("RunicNpc_RemovePlacement", "t_group"); + Note("place: the placements did not spawn, so the rest of the group is skipped"); + yield break; + } // each: kill one; the other two stay, and it alone returns after ~20 s. var survivors = new HashSet(each.Skip(1).Select(x => (ulong)x["netId"])); diff --git a/tools/drill/drill.js b/tools/drill/drill.js index d7bf19e..9d52fdd 100644 --- a/tools/drill/drill.js +++ b/tools/drill/drill.js @@ -30,6 +30,20 @@ const BIG = (process.env.DRILL_BIG_RIGS || 'rust-oxide,rust-carbon').split(',') const STOPPABLE = process.env.DRILL_STOPPABLE || 'rust-carbon' const KITS_URL = 'https://umod.org/plugins/Kits.cs' const WAIT_DAYS = 3 // D317: how long a build waits for Oxide to catch up +const HARNESS_CAP_MS = 45 * 60000 // the whole harness run +const HARNESS_STALL_MS = 5 * 60000 // no new line in its file for this long: it has stopped (issue #20) + +// What a fresh server would not have (issue #20). RunicNPC writes its four example kits once, into +// Kits' own file, and never again, so a file kept from an older build keeps that build's kits; the drill +// starts from none of it. Files, never directories: a directory the panel's file API creates is not +// writable by the game (docs/runicnpc/PLAN.md §1.5), and the game makes these itself on its first boot. +const FRESH_DATA = [ + '/oxide/data/Kits/kits_data.json', + '/oxide/data/RunicNPC/profiles.json', + '/oxide/data/RunicNPC/placements.json', + '/oxide/data/RunicNPC/routes.json', + '/oxide/data/RunicNPC/state.json', +] const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) const now = () => new Date().toISOString() @@ -262,6 +276,7 @@ async function live(a) { await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs') await p.rm(rig, '/oxide/data/RunicNpcTest.json') await p.rm(rig, '/staging-drill.json') + for (const f of FRESH_DATA) await p.rm(rig, f) // already gone is fine: rm swallows it log(`starting ${RIG} on ${branch}`) const started = Date.now() @@ -310,9 +325,13 @@ async function live(a) { result.ran = true log('rnt.run all: running') - const lines = await waitHarness(p, rig, 45 * 60000) + const { lines, ended } = await waitHarness(p, rig, HARNESS_CAP_MS, HARNESS_STALL_MS) result.harness = summarise(lines) - if (!result.harness.done) finding(`The harness did not finish in 45 minutes (${result.harness.pass} pass, ${result.harness.fail} fail so far).`) + if (!result.harness.done) { + finding(unfinishedFinding(result.harness, lines, ended)) + // What the server said while it stalled (an exception in the harness's coroutine lands here, not in its file). + result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 40) + } for (const f of result.harness.failed) finding(`Harness: ${f}`) log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`) result.oxideErrors = await oxideErrors(p, rig) @@ -408,15 +427,38 @@ function readJsonText(t) { } } -async function waitHarness(p, id, timeoutMs) { - const end = Date.now() + timeoutMs +/** + * Where the harness stands, given its lines now and what was last seen of them ({ count, at }): done + * (`# done all:`), stalled (no new line for stallMs: it died without saying so, as in issue #20), or running. + */ +function harnessProgress(lines, seen, nowMs, stallMs) { + if (lines.some((l) => /^# done all:/.test(l))) return { state: 'done', seen } + if (lines.length > seen.count) return { state: 'running', seen: { count: lines.length, at: nowMs } } + if (nowMs - seen.at >= stallMs) return { state: 'stalled', seen } + return { state: 'running', seen } +} + +/** Waits for the harness; `ended` is done, stalled or capped (timeoutMs passed). */ +async function waitHarness(p, id, timeoutMs, stallMs) { + const start = Date.now() let lines = [] - while (Date.now() < end) { + let seen = { count: 0, at: start } + while (Date.now() < start + timeoutMs) { await sleep(30000) lines = readJsonText(await p.readOr(id, '/oxide/data/RunicNpcTest.json', null)) || lines - if (lines.some((l) => /^# done all:/.test(l))) break + const v = harnessProgress(lines, seen, Date.now(), stallMs) + seen = v.seen + if (v.state !== 'running') return { lines, ended: v.state } } - return lines + return { lines, ended: 'capped' } +} + +/** The finding for a harness that never wrote `# done all:`. */ +function unfinishedFinding(h, lines, ended) { + const so = `${h.pass} pass, ${h.fail} fail so far` + if (ended !== 'stalled') return `The harness did not finish in ${HARNESS_CAP_MS / 60000} minutes (${so}).` + const last = (lines[lines.length - 1] || 'no line at all').replace(/`/g, "'").slice(0, 200) + return `The harness stopped after \`${last}\` (no output for ${HARNESS_STALL_MS / 60000} minutes; ${so}).` } function summarise(lines) { @@ -578,14 +620,19 @@ async function fileIssue(title, body) { // ---------------------------------------------------------------- main -const a = args() -const commands = { window: window_, gate, live, report } -const command = commands[a._[0]] -if (!command) { - console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]') - process.exit(2) +// The pure parts, for drill.test.js; run as a script, it is the command line. +module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, HARNESS_STALL_MS } + +if (require.main === module) { + const a = args() + const commands = { window: window_, gate, live, report } + const command = commands[a._[0]] + if (!command) { + console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]') + process.exit(2) + } + command(a).catch((e) => { + console.error(e.message || e) + process.exit(1) + }) } -command(a).catch((e) => { - console.error(e.message || e) - process.exit(1) -}) diff --git a/tools/drill/drill.test.js b/tools/drill/drill.test.js new file mode 100644 index 0000000..2f4652c --- /dev/null +++ b/tools/drill/drill.test.js @@ -0,0 +1,58 @@ +// The drill's pure parts: how it decides the harness has stopped (issue #20), and what it starts clean. +// node --test tools/drill/drill.test.js + +const test = require('node:test') +const assert = require('node:assert') +const { harnessProgress, unfinishedFinding, FRESH_DATA, HARNESS_STALL_MS } = require('./drill.js') + +const MIN = 60000 +const T0 = 1000000 + +test('a harness that keeps writing is running, and its clock restarts on each new line', () => { + let v = harnessProgress(['# run all'], { count: 0, at: T0 }, T0 + 30000, HARNESS_STALL_MS) + assert.equal(v.state, 'running') + assert.deepEqual(v.seen, { count: 1, at: T0 + 30000 }) + // Four silent minutes, then a line: still running, and the clock restarts. + v = harnessProgress(['# run all', 'a: PASS '], v.seen, T0 + 30000 + 4 * MIN, HARNESS_STALL_MS) + assert.equal(v.state, 'running') + assert.equal(v.seen.at, T0 + 30000 + 4 * MIN) +}) + +test('no new line for five minutes is a stall; four minutes and a bit is not', () => { + const seen = { count: 2, at: T0 } + const lines = ['# run all', 'a: PASS '] + assert.equal(harnessProgress(lines, seen, T0 + 5 * MIN - 1, HARNESS_STALL_MS).state, 'running') + assert.equal(harnessProgress(lines, seen, T0 + 5 * MIN, HARNESS_STALL_MS).state, 'stalled') +}) + +test('a harness that never wrote a line stalls from when the wait began', () => { + assert.equal(harnessProgress([], { count: 0, at: T0 }, T0 + 5 * MIN, HARNESS_STALL_MS).state, 'stalled') +}) + +test('done wins, even when the file has been still for a long time', () => { + const lines = ['# run all', '# done all: 10 pass, 1 fail'] + assert.equal(harnessProgress(lines, { count: 2, at: T0 }, T0 + 30 * MIN, HARNESS_STALL_MS).state, 'done') +}) + +test('the stall finding names the last line and the counts', () => { + const lines = ['# run all', 'place.set.each: FAIL position: off the navmesh', 'place.spawned: FAIL each 0, group 3'] + const f = unfinishedFinding({ pass: 40, fail: 2 }, lines, 'stalled') + assert.equal(f, 'The harness stopped after `place.spawned: FAIL each 0, group 3` (no output for 5 minutes; 40 pass, 2 fail so far).') +}) + +test('a backtick in the last line cannot break the issue body, and a long line is cut', () => { + const f = unfinishedFinding({ pass: 0, fail: 0 }, ['x`y' + 'z'.repeat(500)], 'stalled') + assert.equal(f.split('`').length, 3) + assert.ok(f.length < 330) + assert.match(unfinishedFinding({ pass: 0, fail: 0 }, [], 'stalled'), /stopped after `no line at all`/) +}) + +test('a capped wait keeps the 45-minute message', () => { + assert.equal(unfinishedFinding({ pass: 3, fail: 1 }, ['# run all'], 'capped'), 'The harness did not finish in 45 minutes (3 pass, 1 fail so far).') +}) + +test('the clean start deletes files only: every path ends in .json and none is a directory', () => { + assert.ok(FRESH_DATA.includes('/oxide/data/Kits/kits_data.json')) + assert.ok(FRESH_DATA.includes('/oxide/data/RunicNPC/state.json')) + for (const f of FRESH_DATA) assert.match(f, /^\/oxide\/data\/(Kits|RunicNPC)\/[a-z_]+\.json$/) +}) -- 2.49.1 From 54fb1ece67ca54f677a76bfa30b564009a62690a Mon Sep 17 00:00:00 2001 From: wtclaude Date: Sun, 1 Nov 2026 01:35:45 -0600 Subject: [PATCH 2/9] ci(pr-checks): run the staging drill's tests (#20) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- .gitea/workflows/pr-checks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index 8c264ae..26ed66d 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -57,3 +57,7 @@ jobs: # not portable across the Node versions this project runs on. - name: Test the checker itself run: node --test scripts/checkPlugin.test.js + + # The staging drill's pure parts (tools/drill/drill.js): its stall limit and fresh-data list. + - name: Test the staging drill + run: node --test tools/drill/drill.test.js -- 2.49.1 From 4fb935d9dd8f47617b79ae43390a5218531eb761 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Sun, 1 Nov 2026 01:50:19 -0600 Subject: [PATCH 3/9] fix(drill): lay the harness's kit down before boot instead of reloading Kits (#22) Run 48 deleted Kits' file for a fresh start, and the harness's rnhrevolver kit, added after boot with `oxide.reload Kits` through the panel's command endpoint, never reached Kits: no reload or compile job followed, and every test profile was refused for want of the kit. The drill now replaces kits_data.json with one holding only that kit. Kits reads it at boot; RunicNPC's first load adds its example kits beside it (it skips kits already there) and settles its own Kits reload (D329). The add-after-boot stays as a fallback, now through the console's websocket, and is a finding when Kits does not load again. Refs #20, #22 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- README.md | 5 +++-- tools/drill/drill.js | 43 +++++++++++++++++++++++++++++---------- tools/drill/drill.test.js | 15 +++++++++++--- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0a892bb..2d3c296 100644 --- a/README.md +++ b/README.md @@ -153,8 +153,9 @@ the week. A run by hand always goes ahead. plugin against them. - **Live, once per build** (`tools/drill/drill.js live`): the `rust-staging` server (Oxide, a 3500 map) installs the same pair at start (`tools/drill/rig-start.sh`), loads RunicNPC, Kits and - `tools/RunicNpcTest.cs`, and runs `rnt.run all`. It starts the server from empty Kits and RunicNPC data - (their files are deleted first, so the example kits are written afresh) and gives up on the harness after + `tools/RunicNpcTest.cs`, and runs `rnt.run all`. It starts the server from fresh Kits and RunicNPC data + (RunicNPC's files are deleted and Kits' file holds only the harness's kit, so the example kits are written + afresh) and gives up on the harness after 5 minutes with no new output (45 minutes at most), putting the tail of the server's log in the issue. When both 6000-map rigs are running, `rust-carbon` is stopped for it and started again. - **Report:** anything that fails or differs opens one issue for that build, labelled `staging-drill`. diff --git a/tools/drill/drill.js b/tools/drill/drill.js index 9d52fdd..8d62e53 100644 --- a/tools/drill/drill.js +++ b/tools/drill/drill.js @@ -37,8 +37,10 @@ const HARNESS_STALL_MS = 5 * 60000 // no new line in its file for this long: it // Kits' own file, and never again, so a file kept from an older build keeps that build's kits; the drill // starts from none of it. Files, never directories: a directory the panel's file API creates is not // writable by the game (docs/runicnpc/PLAN.md §1.5), and the game makes these itself on its first boot. +// Kits' file is not deleted but replaced by one holding only the harness's kit (KITS_FILE, harnessKits): +// Kits reads it at boot and RunicNPC adds its examples beside it, so no reload is needed for it (#22). +const KITS_FILE = '/oxide/data/Kits/kits_data.json' const FRESH_DATA = [ - '/oxide/data/Kits/kits_data.json', '/oxide/data/RunicNPC/profiles.json', '/oxide/data/RunicNPC/placements.json', '/oxide/data/RunicNPC/routes.json', @@ -277,6 +279,7 @@ async function live(a) { await p.rm(rig, '/oxide/data/RunicNpcTest.json') await p.rm(rig, '/staging-drill.json') for (const f of FRESH_DATA) await p.rm(rig, f) // already gone is fine: rm swallows it + await p.write(rig, KITS_FILE, JSON.stringify(harnessKits(), null, 2)) log(`starting ${RIG} on ${branch}`) const started = Date.now() @@ -309,7 +312,7 @@ async function live(a) { } // The harness needs the hidden Kits kit rnhrevolver (tools/RunicNpcTest.cs). - await ensureKit(p, rig) + await ensureKit(p, rig, finding) await p.write(rig, '/oxide/plugins/RunicNpcTest.cs', fs.readFileSync(path.join(ROOT, 'tools', 'RunicNpcTest.cs'), 'utf8')) let accepted = false @@ -387,8 +390,18 @@ async function waitStatus(p, id, timeoutMs) { return last } -async function ensureKit(p, id) { - const file = '/oxide/data/Kits/kits_data.json' +/** Kits' data file holding only the harness's hidden kit, which the drill lays down before the start. */ +function harnessKits() { + return { _kits: { rnhrevolver: harnessKit() } } +} + +/** + * The harness's kit is laid down before the start (harnessKits), so it is normally there. If it is not + * (RunicNPC rewrote the file without it), it is added and Kits reloaded through the console's websocket: + * the panel's fire-and-forget command left no reload in run 48 (#22), so the reply is logged. + */ +async function ensureKit(p, id, finding) { + const file = KITS_FILE let data = null for (let i = 0; i < 12 && !data; i++) { data = readJsonText(await p.readOr(id, file, null)) @@ -396,9 +409,21 @@ async function ensureKit(p, id) { } if (!data) throw new Error(`Kits wrote no ${file}`) data._kits = data._kits || {} - if (data._kits.rnhrevolver) return + if (data._kits.rnhrevolver) { + log('the rnhrevolver kit is in Kits\' file, laid down before the start') + return + } + data._kits.rnhrevolver = harnessKit() + await p.write(id, file, JSON.stringify(data, null, 2)) + const reply = await p.console(id, 'oxide.reload Kits', 15) + const loaded = reply.some((l) => /Loaded plugin Kits/.test(l)) + log(`added the rnhrevolver kit and reloaded Kits: ${loaded ? 'Kits loaded again' : `no "Loaded plugin Kits" in 15 s (${reply.length} console line(s))`}`) + if (!loaded) finding('Kits did not reload after the drill added the harness\'s rnhrevolver kit; the harness\'s profiles may be refused.') +} + +function harnessKit() { const item = (Shortname, Amount, Position, Condition = 0) => ({ Shortname, Skin: 0, Amount, Condition, MaxCondition: Condition, Ammo: 0, Position, Frequency: -1 }) - data._kits.rnhrevolver = { + return { Name: 'rnhrevolver', Description: 'RunicNPC test harness: revolver + burlap (staging drill)', RequiredPermission: '', @@ -413,10 +438,6 @@ async function ensureKit(p, id) { WearItems: [item('burlap.shirt', 1, 0), item('burlap.trousers', 1, 1), item('hat.boonie', 1, 2)], BeltItems: [item('pistol.revolver', 1, 0, 100)], } - await p.write(id, file, JSON.stringify(data, null, 2)) - await p.command(id, 'oxide.reload Kits') - await sleep(10000) - log('added the rnhrevolver kit and reloaded Kits') } function readJsonText(t) { @@ -621,7 +642,7 @@ async function fileIssue(title, body) { // ---------------------------------------------------------------- main // The pure parts, for drill.test.js; run as a script, it is the command line. -module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, HARNESS_STALL_MS } +module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS } if (require.main === module) { const a = args() diff --git a/tools/drill/drill.test.js b/tools/drill/drill.test.js index 2f4652c..73ad312 100644 --- a/tools/drill/drill.test.js +++ b/tools/drill/drill.test.js @@ -3,7 +3,7 @@ const test = require('node:test') const assert = require('node:assert') -const { harnessProgress, unfinishedFinding, FRESH_DATA, HARNESS_STALL_MS } = require('./drill.js') +const { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS } = require('./drill.js') const MIN = 60000 const T0 = 1000000 @@ -52,7 +52,16 @@ test('a capped wait keeps the 45-minute message', () => { }) test('the clean start deletes files only: every path ends in .json and none is a directory', () => { - assert.ok(FRESH_DATA.includes('/oxide/data/Kits/kits_data.json')) assert.ok(FRESH_DATA.includes('/oxide/data/RunicNPC/state.json')) - for (const f of FRESH_DATA) assert.match(f, /^\/oxide\/data\/(Kits|RunicNPC)\/[a-z_]+\.json$/) + for (const f of FRESH_DATA) assert.match(f, /^\/oxide\/data\/RunicNPC\/[a-z_]+\.json$/) +}) + +test("Kits' file is replaced, not deleted, by one holding only the harness's kit (#22)", () => { + assert.ok(!FRESH_DATA.includes(KITS_FILE)) + assert.equal(KITS_FILE, '/oxide/data/Kits/kits_data.json') + const kits = harnessKits()._kits + assert.deepEqual(Object.keys(kits), ['rnhrevolver']) + assert.equal(kits.rnhrevolver.Name, 'rnhrevolver') + assert.equal(kits.rnhrevolver.IsHidden, true) + assert.deepEqual(kits.rnhrevolver.BeltItems.map((i) => i.Shortname), ['pistol.revolver']) }) -- 2.49.1 From cf3997bbce6947b9245c562686ad9786d2276082 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Sun, 1 Nov 2026 02:15:18 -0600 Subject: [PATCH 4/9] fix(drill): place spots found up to 15 m from the navmesh; count only this run's Oxide errors (#22) Run 49: place.set.each was refused "4.4 m from the navmesh", because OnMesh searched 3 m and returned the unsnapped point; Place now searches 15 m. The drill reported run 48's four error lines again: oxideErrors read the whole day's Oxide log, and now reads only what was added after the drill started the rig. Refs #20, #22 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- tools/RunicNpcTest.cs | 11 ++++++----- tools/drill/drill.js | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tools/RunicNpcTest.cs b/tools/RunicNpcTest.cs index 7e1ad01..f2fdb27 100644 --- a/tools/RunicNpcTest.cs +++ b/tools/RunicNpcTest.cs @@ -1113,9 +1113,10 @@ namespace Oxide.Plugins private IEnumerator Place(Vector3 field) { - // A roamer's spot must be on the navmesh when it is set (D219), so both spots are put there. - Vector3 a = OnMesh(field + new Vector3(-40, 0, 0)); - Vector3 b = OnMesh(field + new Vector3(-40, 0, 30)); + // A roamer's spot must be on the navmesh when it is set (D219), so both spots are put there; the + // wider search because a fixed offset from the field can land a few metres off it (run 49: 4.4 m). + Vector3 a = OnMesh(field + new Vector3(-40, 0, 0), 15f); + Vector3 b = OnMesh(field + new Vector3(-40, 0, 30), 15f); StandIn(OnMesh(field + new Vector3(-40, 0, 110)), 8); string eachRefusal = Api("RunicNpc_SetPlacement", "t_each", Placement("t_each", a, 3, 20f, "each")); Check("place.set.each", eachRefusal == null, eachRefusal ?? ""); @@ -1569,11 +1570,11 @@ namespace Oxide.Plugins return floor; } - private static Vector3 OnMesh(Vector3 p) + private static Vector3 OnMesh(Vector3 p, float radius = 3f) { p.y = TerrainMeta.HeightMap.GetHeight(p); NavMeshHit hit; - return SampleNavmesh(p, 3f, out hit) ? hit.position : p; + return SampleNavmesh(p, radius, out hit) ? hit.position : p; } private static JObject Placement(string profile, Vector3 at, int count, float respawn, string mode) => diff --git a/tools/drill/drill.js b/tools/drill/drill.js index 8d62e53..1ea36d3 100644 --- a/tools/drill/drill.js +++ b/tools/drill/drill.js @@ -281,6 +281,7 @@ async function live(a) { for (const f of FRESH_DATA) await p.rm(rig, f) // already gone is fine: rm swallows it await p.write(rig, KITS_FILE, JSON.stringify(harnessKits(), null, 2)) + const logMark = await oxideLogMark(p, rig) log(`starting ${RIG} on ${branch}`) const started = Date.now() await p.power(rig, 'start') @@ -298,7 +299,7 @@ async function live(a) { const status = await waitStatus(p, rig, 30 * 60000) if (!status) { finding('RunicNPC did not load: `rnpc.status` never answered.') - result.oxideErrors = await oxideErrors(p, rig) + result.oxideErrors = await oxideErrors(p, rig, logMark) return } result.status = status @@ -322,7 +323,7 @@ async function live(a) { } if (!accepted) { finding('The test harness did not load (`rnt.run all` was not answered).') - result.oxideErrors = await oxideErrors(p, rig) + result.oxideErrors = await oxideErrors(p, rig, logMark) return } result.ran = true @@ -337,7 +338,7 @@ async function live(a) { } for (const f of result.harness.failed) finding(`Harness: ${f}`) log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`) - result.oxideErrors = await oxideErrors(p, rig) + result.oxideErrors = await oxideErrors(p, rig, logMark) if (result.oxideErrors.length) finding(`RunicNPC logged ${result.oxideErrors.length} error line(s) in Oxide's log during the run.`) await p.command(rig, 'rnt.clear').catch(() => {}) } catch (e) { @@ -490,10 +491,17 @@ function summarise(lines) { } /** RunicNPC's and the harness's error lines from today's Oxide log. */ -async function oxideErrors(p, id) { +/** Where Oxide's log of today ends now, so oxideErrors reads only what a run adds (run 49 counted run 48's). */ +async function oxideLogMark(p, id) { + const day = now().slice(0, 10) + return { day, length: ((await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || '').length } +} + +async function oxideErrors(p, id, since = null) { const day = now().slice(0, 10) const text = (await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || '' return text + .slice(since && since.day === day ? since.length : 0) .split('\n') .filter((l) => /RunicNPC|RunicNpcTest/.test(l) && /\[Error\]|Exception|Failed|error CS/.test(l)) .slice(-40) -- 2.49.1 From 5c3a360b7059b65ed6b8321de6ccd77c320faf19 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Mon, 2 Nov 2026 04:24:14 -0600 Subject: [PATCH 5/9] fix(kit): find a kit's extra by its item definition when it has no held entity (#20) Rust's November 2026 build (staging 26344826) adds the server variable ItemModEntity.playerOnlyEntityBeltOnly, on by default: an item Rust marks player-only, the medical syringe among them, has a held entity only while it is in the belt. A kit puts its syringes in the main inventory, so the heal (D260) found no item whose held entity is a MedicalTool and never healed (s5.kit.heal on the staging drill: heals 0, health 75). FindHeld now asks the item definition's entity prefab when an item has no held entity, for every extra (heal, melee, flamethrower, grenade, rocket), and HasFuel reads the flamethrower's fuel type the same way. Holding the item moves it to the belt, where Rust makes its entity. A heal that cannot hold the item (a full belt) is now no heal, not a heal of an item never held (D331). On rust-staging: s5.kit.heal PASS, heals 2, syringes left 0, health 145. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- plugin/RunicNPC.cs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/plugin/RunicNPC.cs b/plugin/RunicNPC.cs index 90b1d1c..aa69d14 100644 --- a/plugin/RunicNPC.cs +++ b/plugin/RunicNPC.cs @@ -2784,13 +2784,29 @@ namespace Oxide.Plugins if (container == null) continue; foreach (Item item in container.itemList) - if (item != null && item.GetHeldEntity() is T) + if (item != null && HeldAs(item) != null) return item; } return null; } + /// + /// The item's held entity as a , or, when it has none, its definition's + /// entity prefab's. Since Rust's November 2026 build a "player-only" item (a syringe) has a held + /// entity only in the belt (ItemModEntity.playerOnlyEntityBeltOnly), and an extra waits in + /// the main inventory until moves it (issue #20). + /// + private static T HeldAs(Item item) where T : HeldEntity + { + BaseEntity held = item.GetHeldEntity(); + if (held != null) + return held as T; + ItemModEntity mod = item.info.GetComponent(); + GameObject prefab = mod != null && mod.entityPrefab != null && mod.entityPrefab.isValid ? mod.entityPrefab.Get() : null; + return prefab != null ? prefab.GetComponent() : null; + } + /// Puts an item in its hands; only the belt can be held, so it is moved there first. private static bool Hold(RunicNpcPlayer npc, Item item) { @@ -2914,7 +2930,8 @@ namespace Oxide.Plugins if (amount <= 0f) return false; - Hold(npc, item); + if (!Hold(npc, item)) + return false; // a full belt: nowhere to hold it. npc.SignalBroadcast(BaseEntity.Signal.Attack); item.UseItem(1); npc.Heal(amount); @@ -3011,8 +3028,10 @@ namespace Oxide.Plugins private static bool HasFuel(RunicNpcPlayer npc, Item flameItem) { - FlameThrower f = flameItem.GetHeldEntity() as FlameThrower; - return f != null && (f.ammo > 0 || (f.fuelType != null && npc.inventory.FindItemByItemName(f.fuelType.shortname) != null)); + // Off the belt it may have no held entity, so no tank (HeldAs): only the kit's fuel counts then. + FlameThrower held = flameItem.GetHeldEntity() as FlameThrower; + FlameThrower f = HeldAs(flameItem); + return f != null && ((held != null && held.ammo > 0) || (f.fuelType != null && npc.inventory.FindItemByItemName(f.fuelType.shortname) != null)); } /// -- 2.49.1 From df2a258f642ff0bd0be64e34892e3f40fa20d2c8 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Mon, 2 Nov 2026 04:24:15 -0600 Subject: [PATCH 6/9] fix(drill): stage 5's pairs stand in each other's sight; say how the leashed NPC came to fight (#20, #22) s5.fight.shootBack failed on every staging drill run because, on the drill's 3500 map (seed 1234), a cliff rock (cliff_hills_large_e_arid) stands between the fifth fight's victim and the bully 15 m north of it: the bully never saw the victim, so never shot it. In the open the victim answers within a second. Each fight's second NPC now stands where each one's eyes see the other's centre, both ways, as RunicNPC's own line of sight runs: north first, then the other seven ways round, with a note when it moves. s5.fight.leash failed once (run 49) and did not come back in three tries. The first moment the leashed NPC has a target, the harness now notes the target, its health, and how far it and the scientist are from its home, so the next failing run says whether it sensed, was shot, or heard an alarm. Harness 0.7.5. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- tools/RunicNpcTest.cs | 50 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/tools/RunicNpcTest.cs b/tools/RunicNpcTest.cs index f2fdb27..78bffaf 100644 --- a/tools/RunicNpcTest.cs +++ b/tools/RunicNpcTest.cs @@ -45,7 +45,7 @@ namespace Oxide.Plugins /// a made-up user id (11400000001 and up), so IsNpc is false. They take no damage. /// /// - [Info("RunicNpcTest", "Runic Gateway", "0.7.4")] + [Info("RunicNpcTest", "Runic Gateway", "0.7.5")] [Description("RunicNPC stage 2 to 7 tests, the stage 6 spike and stage 9's cost check. A developer tool: never ship it.")] internal class RunicNpcTest : RustPlugin { @@ -1753,6 +1753,31 @@ namespace Oxide.Plugins return false; } + /// + /// Where a stage 5 fight's second NPC stands: m from the first, on the navmesh, + /// with a clear line between their eyes. North first, as the fights were laid out; then the other seven + /// ways round. On the drill's 3500 map a cliff rock stood between the victim and the bully north of it, + /// so neither ever saw the other and s5.fight.shootBack failed on every run (issue #20). + /// + private Vector3 Partner(Vector3 at, float dist) + { + for (int k = 0; k < 8; k++) + { + float angle = k * Mathf.PI / 4f; + Vector3 p = OnMesh(at + new Vector3(Mathf.Sin(angle), 0f, Mathf.Cos(angle)) * dist); + NavMeshHit hit; + if (!SampleNavmesh(p, 1f, out hit)) continue; + // Each one's eyes to the other's centre, both ways, as RunicNPC's own line of sight runs. + if (Physics.Linecast(at + Vector3.up * 1.6f, p + Vector3.up * 0.9f, 1218519041, QueryTriggerInteraction.Ignore) || + Physics.Linecast(p + Vector3.up * 1.6f, at + Vector3.up * 0.9f, 1218519041, QueryTriggerInteraction.Ignore)) continue; + if (k > 0) Note($"s5: the partner {dist:0} m from {at} stands {k * 45}° round from north (no clear line north)"); + return p; + } + + Note($"s5: no clear line {dist:0} m round {at}; the partner stands north as before"); + return OnMesh(at + new Vector3(0, 0, dist)); + } + private int HorrorLeft() => BaseNetworkable.serverEntities.OfType().Count(c => !c.IsDestroyed && c.faction == BaseCombatEntity.Faction.Horror); private IEnumerator Stage5() @@ -1793,16 +1818,18 @@ namespace Oxide.Plugins Vector3 d = OnMesh(f + new Vector3(70, 0, 0)); Vector3 e = OnMesh(f + new Vector3(140, 0, 0)); + // Each fight's second NPC where the two can see each other (issue #20), found before anyone stands there. + Vector3 aTo = Partner(a, 15f), bTo = Partner(b, 15f), cTo = Partner(c, 15f), dTo = Partner(d, 25f), eTo = Partner(e, 15f); BasePlayer red = Spawn(a, "t_red"); - BasePlayer blue = Spawn(OnMesh(a + new Vector3(0, 0, 15)), "t_blue"); + BasePlayer blue = Spawn(aTo, "t_blue"); BasePlayer hunter = Spawn(b, "t_hunter"); - ScientistNPC prey = SpawnScientist(b + new Vector3(0, 0, 15)); + ScientistNPC prey = SpawnScientist(bTo); BasePlayer neutral = Spawn(c, "t_roam"); - ScientistNPC bystander = SpawnScientist(c + new Vector3(0, 0, 15)); + ScientistNPC bystander = SpawnScientist(cTo); BasePlayer leashed = Spawn(d, "t_leashed"); - ScientistNPC far = SpawnScientist(d + new Vector3(0, 0, 25)); + ScientistNPC far = SpawnScientist(dTo); BasePlayer victim = Spawn(e, "t_roam"); - BasePlayer bully = Spawn(OnMesh(e + new Vector3(0, 0, 15)), "t_bully"); + BasePlayer bully = Spawn(eTo, "t_bully"); var fighters = new Dictionary { { "red", red }, { "blue", blue }, { "hunter", hunter }, { "prey", prey }, { "neutral", neutral }, @@ -1823,7 +1850,16 @@ namespace Oxide.Plugins if (factionsAt < 0f && (Gone(red) || Gone(blue))) factionsAt = t; if (scientistAt < 0f && Gone(prey)) scientistAt = t; if (!Gone(neutral) && TargetOf(Describe(neutral)) != "none") neutralFought = true; - if (!Gone(leashed) && TargetOf(Describe(leashed)) != "none") leashedFought = true; + if (!leashedFought && !Gone(leashed) && TargetOf(Describe(leashed)) != "none") + { + leashedFought = true; + // Run 49 failed here once and it has not been seen since: say how it came to fight. A target its + // own sense found is within its chase (8 m) of home; a hurt one was shot; otherwise an ally's alarm. + Dictionary dl = Describe(leashed); + Vector3 leashHome = V(dl["home"]); + Note($"s5.fight.leash: fighting at {t} s, target {dl["target"]}, health {Convert.ToSingle(dl["health"]):0}/{Convert.ToSingle(dl["maxHealth"]):0}, " + + $"{Flat(V(dl["position"]), leashHome):0.0} m from home; the scientist {(Gone(far) ? "gone" : $"{Flat(far.transform.position, leashHome):0.0} m from its home, {far.health:0} hp")}"); + } if (!Gone(victim) && TargetOf(Describe(victim)) != "none") victimAnswered = true; foreach (BasePlayer x in new[] { red, blue, hunter }) { -- 2.49.1 From 512efa77cbee60f151126a51d5194997281b1b30 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Mon, 2 Nov 2026 06:15:16 -0600 Subject: [PATCH 7/9] =?UTF-8?q?ci(cutover):=20the=20wipe-day=20edge=20?= =?UTF-8?q?=E2=86=92=20main=20cutover,=20once=20five=20gates=20pass=20(D33?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workflow that merges edge into main on the forced-wipe weekend with no approval: hourly from Thursday 17:00 UTC to Sunday 23:00 UTC, through five gates in order, then a merge commit made with RUNICNPC_TOKEN (that step only) so release.yml cuts the release. 1. Rust's public build changed: dated the wipe day and naming the same depot manifest as staging (the staging build was promoted to live). 2. Oxide's public release caught up, RunicNPC compiles, the swap's fields match. 3. A Carbon boot on public Rust: rust-carbon proves the public manifest is installed, and rnpc.status says framework carbon, navmesh ready, missing=- added=-. 4. The drill's live half on public Rust with edge's plugin and harness. 5. PR checks, run fresh on the cutover PR's head just before the merge. Gates are kept in drill.json for one (wipe, public manifest, edge commit); any change runs them again. A failed Carbon or live attempt cools down for 3 hours. The merge is pinned to the commit the gates ran on. The cutover PR lists every `Closes #N` from the PRs merged into edge since the last cutover, so those issues close on the merge; afterwards each gets the release version. One comment a day while a gate blocks; at the deadline it hands back. The copy on main is only the trigger and step wiring; every step runs edge's code (a dry dispatch may name another branch, code_ref). Dispatches are dry unless dry_run is exactly false. rust-carbon is started again if the job left it stopped. drill.js only gains exports; its report already kept other keys, now tested. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- .gitea/workflows/cutover.yml | 267 ++++++++++ .gitea/workflows/pr-checks.yml | 5 + tools/drill/cutover.js | 871 +++++++++++++++++++++++++++++++++ tools/drill/cutover.test.js | 758 ++++++++++++++++++++++++++++ tools/drill/drill.js | 5 +- tools/drill/drill.test.js | 20 + 6 files changed, 1924 insertions(+), 2 deletions(-) create mode 100644 .gitea/workflows/cutover.yml create mode 100644 tools/drill/cutover.js create mode 100644 tools/drill/cutover.test.js diff --git a/.gitea/workflows/cutover.yml b/.gitea/workflows/cutover.yml new file mode 100644 index 0000000..06a7c94 --- /dev/null +++ b/.gitea/workflows/cutover.yml @@ -0,0 +1,267 @@ +# The wipe-day cutover (docs/runicnpc/PLAN.md stage 9c, D332): `edge` merges into `main` by itself. +# +# The org lead approves no merge on a wipe day. Rust changes on the monthly forced wipe, the first +# Thursday (D318), and RunicNPC must be proven on the public build before it is released. This job runs +# hourly from 17:00 UTC on that Thursday until Sunday 23:00 UTC and merges `edge` into `main` once five +# gates have passed, cheapest first (the decisions are pure functions in tools/drill/cutover.js, tested +# by cutover.test.js, which PR Checks runs): +# +# 1. public Rust's public build is dated the wipe day or later AND names the same depot manifest as +# Rust's current STAGING build (Facepunch promotes staging to public, so the two agree once the +# cut over from staging to live went through). The static half runs twice, BRANCH=public and +# BRANCH=staging; only the staging run's manifest is used. +# 2. oxide Oxide's public build has caught up (0 members behind), the swap's field list is unchanged +# and RunicNPC compiles. +# 3. carbon `rust-carbon` restarts on public Rust with edge's RunicNPC.cs: its appmanifest names the +# public manifest, and rnpc.status says framework=carbon, navmesh=ready, missing=- added=-. +# The rig is left as it was found (running stays running). +# 4. live the drill's live half (drill.js live --branch public) on `rust-staging`: every harness check +# passes, no finding. +# 5. checks `plugin-checks` is green on the cutover PR's head commit, which must be the commit the gates +# ran on. Never kept: it runs fresh, right before the merge. +# +# Each passed gate is kept in drill.json (the `cutover` key, on the `drill` branch) for that wipe, that +# public manifest and that `edge` commit; a change of any of them starts the gates again. A failed carbon or +# live attempt is not repeated for three hours for the same manifest and commit. As soon as gate 1 passes it +# opens ONE edge -> main PR (or adopts the open one: its title is never changed and only the block between +# and of its body is written) whose block carries a `Closes #N` +# line for every issue fixed by the PRs merged into `edge` since the last cutover (merging into `main`, the +# default branch, is what closes them). When a gate blocks it comments on that PR once a day; at Sunday +# 23:00 UTC it stops with a last comment and the org lead merges by hand, as before. The merge sends the +# head commit it checked, so Gitea refuses it if the PR moved. After the merge it waits for release.yml's +# release and comments the version on each issue that closed. +# +# Sequencing inside the job: `gates` runs 1-3 and stops in front of 4, saying live=true; the live half is +# its own step (drill.js live); `gates --after-live` records 4 and runs 5. The state is pushed to the +# `drill` branch after the first `gates` too, so a long live half that runs into the job timeout does not +# lose gate 3. A last always-run step starts `rust-carbon` again if it was running when the job began and is +# offline now (that server only, power `start` only). +# +# THE MAIN / EDGE CONTRACT. The copy of this file on `main` is the trigger and the step wiring (Gitea fires +# a schedule only from the default branch). Every step checks out `edge` and runs edge's scripts, plugin +# and harness: the gates and the decisions come from edge. The step ids and the output names (open, run, +# live, merge, pr, sha, merged) are a contract between the two: cutover.test.js reads this file and fails +# when one is used here but not written by tools/drill/cutover.js, so change both on edge and carry the +# new wiring to `main` in the same release. Because edge's code runs with RUNICNPC_TOKEN in the merge step, +# PR review into `edge` is the real gate: nothing reaches `edge` unreviewed. +# +# It shares the staging drill's concurrency group, so it never runs while the daily drill has `rust-staging`. +# +# Try it: Actions -> Wipe-day cutover -> Run workflow. A dispatch is a DRY RUN unless `dry_run` is +# explicitly false (anything else, or nothing, is dry): every gate runs and reports in the job summary (it +# does restart rust-carbon and use rust-staging); no PR is opened or edited, nothing is commented or merged, +# no state is written. `force` goes outside the wipe weekend. +# +# Secrets (Settings -> Actions -> Secrets, the repository's or the organisation's): +# PTERODACTYL_DRILL_KEY — the panel client key made for the drill (D309). Never printed. +# REGISTRY_USER / REGISTRY_TOKEN — as the drill: push the `drill` branch, open and edit the PR, comment. +# RUNICNPC_TOKEN — a personal access token locked to this repository, used by the merge step ALONE: a +# push made with the runner's own token would not start release.yml. + +name: Wipe-day cutover + +on: + schedule: + # Thursday from 17:00 UTC, then every hour of Friday, Saturday and Sunday. (Not one line with a day of + # the month as well: cron ORs the two.) The window step ends the runs that fall outside the weekend. + - cron: '11 17-23 * * 4' + - cron: '11 * * * 5,6,0' + workflow_dispatch: + inputs: + dry_run: + description: 'Run every gate and report; open, edit, comment and merge nothing (dry unless exactly false)' + type: boolean + default: true + force: + description: 'Work outside the wipe weekend' + type: boolean + default: false + code_ref: + description: 'Branch whose scripts and plugin a DRY run uses, to rehearse a change before it reaches edge (a real run always uses edge)' + type: string + default: 'edge' + +concurrency: + group: staging-drill + cancel-in-progress: false + +env: + GITEA_HOST: gitea.whitlocktech.com + REPO: RunicGateway/runicnpc-rust + +jobs: + cutover: + runs-on: ubuntu-latest + # Carbon's restart and navmesh, the live half (about an hour), the PR checks. + timeout-minutes: 180 + env: + WORK: ${{ github.workspace }}/cutover-work + STAGING_WORK: ${{ github.workspace }}/cutover-staging + # cutover.js treats a dispatch as dry unless this is exactly 'false'. + DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }} + steps: + # edge, always, except a dry dispatch naming another branch (code_ref). A real run on another branch could + # not merge anyway: the merge is pinned to the commit the gates ran on, which must be edge's head. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false' && github.event.inputs.code_ref || 'edge' }} + fetch-depth: 0 + + # The commit counts and the cut-off for the Closes list are read from git. + - name: main and the tags + run: git fetch origin main edge --tags + + # 22 or later: drill.js talks to the panel's console over the global WebSocket. + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: The wipe weekend? + id: window + run: node tools/drill/cutover.js window ${{ github.event.inputs.force == 'true' && '--force' || '' }} + + - name: Anything to merge? + id: pending + if: ${{ steps.window.outputs.open == 'true' }} + run: node tools/drill/cutover.js pending ${{ (github.event.inputs.force == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false')) && '--force' || '' }} + + # Whether rust-carbon is running now: gate 3 restarts it and the last step puts it back if needed. + - name: Note rust-carbon + if: ${{ steps.pending.outputs.run == 'true' }} + continue-on-error: true + env: + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: node tools/drill/cutover.js rig-note --work "$WORK" + + # tools/fieldlist and the compiler (Roslyn's csc.dll ships in the SDK). + - name: .NET SDK + if: ${{ steps.pending.outputs.run == 'true' }} + run: | + set -euo pipefail + curl -sSL -o "$RUNNER_TEMP/dotnet-install.sh" https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh + bash "$RUNNER_TEMP/dotnet-install.sh" --channel 9.0 --install-dir "$HOME/.dotnet" > /dev/null + echo "$HOME/.dotnet" >> "$GITHUB_PATH" + echo "DOTNET_ROOT=$HOME/.dotnet" >> "$GITHUB_ENV" + + - name: The drill branch (what the cutover has passed) + if: ${{ steps.pending.outputs.run == 'true' }} + run: | + set -euo pipefail + git config user.name "runicnpc-ci" + git config user.email "ci@whitlocktech.com" + rm -rf drill-state + git worktree prune + if git ls-remote --exit-code --heads origin drill > /dev/null 2>&1; then + git fetch origin drill + git worktree add -B drill drill-state origin/drill + else + # First run: a root commit with an empty tree, so the branch inherits no code history. + ROOT="$(git commit-tree "$(git hash-object -t tree /dev/null)" -m 'chore(drill): start the drill branch')" + git worktree add -B drill drill-state "$ROOT" + fi + [ -f drill-state/drill.json ] || echo '{}' > drill-state/drill.json + cat drill-state/drill.json + + - name: Static half (Rust's public build) + if: ${{ steps.pending.outputs.run == 'true' }} + env: + BRANCH: public + LC_ALL: C + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' + run: bash tools/drill/static.sh + + # Only this run's manifest is used (gate 1): the build Rust has on staging now. + - name: Static half (Rust's staging build, for its manifest) + if: ${{ steps.pending.outputs.run == 'true' }} + env: + BRANCH: staging + WORK: ${{ github.workspace }}/cutover-staging + LC_ALL: C + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' + run: bash tools/drill/static.sh + + - name: Gates 1-3 (public build, Oxide, Carbon) + id: gates + if: ${{ steps.pending.outputs.run == 'true' }} + env: + GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: >- + node tools/drill/cutover.js gates --state drill-state/drill.json --work "$WORK" --staging-work "$STAGING_WORK" + ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') && '--dry-run' || '' }} + ${{ github.event.inputs.force == 'true' && '--force' || '' }} + + - name: Keep the gates passed so far on the drill branch + if: ${{ always() && steps.pending.outputs.run == 'true' && !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') }} + env: + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -euo pipefail + CI_USER="$(printf '%s' "${REGISTRY_USER}" | tr -d '\r\n')" + CI_TOKEN="$(printf '%s' "${REGISTRY_TOKEN}" | tr -d '\r\n')" + git remote set-url origin "https://${CI_USER}:${CI_TOKEN}@${GITEA_HOST}/${REPO}.git" + cd drill-state + git add drill.json + if git diff --cached --quiet; then echo "no change"; exit 0; fi + node -e "const c=require('./drill.json').cutover||{};console.log('chore(drill): cutover '+(c.wipe||'?')+' '+(c.manifest||'?')+' [skip ci]')" > ../cutover-msg.txt + git commit -q -F ../cutover-msg.txt + git push origin drill + + - name: Gate 4, the drill again on live Rust (rust-staging) + id: live + if: ${{ steps.gates.outputs.live == 'true' }} + continue-on-error: true + env: + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: node tools/drill/drill.js live --work "$WORK" --branch public + + - name: Gates 4-5 (the live half, PR checks) + id: gates2 + if: ${{ always() && steps.gates.outputs.live == 'true' }} + env: + GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: >- + node tools/drill/cutover.js gates --after-live --state drill-state/drill.json --work "$WORK" --staging-work "$STAGING_WORK" + ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') && '--dry-run' || '' }} + ${{ github.event.inputs.force == 'true' && '--force' || '' }} + + # The only step that holds RUNICNPC_TOKEN. It merges the commit the gates ran on, or nothing. + - name: Merge edge into main + id: merge + if: ${{ (steps.gates2.outputs.merge == 'true' || steps.gates.outputs.merge == 'true') && !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') }} + env: + MERGE_TOKEN: ${{ secrets.RUNICNPC_TOKEN }} + run: node tools/drill/cutover.js merge --pr ${{ steps.gates2.outputs.pr || steps.gates.outputs.pr }} --sha ${{ steps.gates2.outputs.sha || steps.gates.outputs.sha }} --state drill-state/drill.json + + - name: Record the merge and the release + if: ${{ steps.merge.outputs.merged == 'true' }} + env: + GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: node tools/drill/cutover.js after --state drill-state/drill.json --pr ${{ steps.gates2.outputs.pr || steps.gates.outputs.pr }} + + - name: Keep the result on the drill branch + if: ${{ always() && steps.pending.outputs.run == 'true' && !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') }} + env: + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -euo pipefail + CI_USER="$(printf '%s' "${REGISTRY_USER}" | tr -d '\r\n')" + CI_TOKEN="$(printf '%s' "${REGISTRY_TOKEN}" | tr -d '\r\n')" + git remote set-url origin "https://${CI_USER}:${CI_TOKEN}@${GITEA_HOST}/${REPO}.git" + cd drill-state + git add drill.json + if git diff --cached --quiet; then echo "no change"; exit 0; fi + node -e "const c=require('./drill.json').cutover||{};console.log('chore(drill): cutover '+(c.wipe||'?')+' '+(c.manifest||'?')+' [skip ci]')" > ../cutover-msg.txt + git commit -q -F ../cutover-msg.txt + git push origin drill + + # Last: gate 3 and the live half stop and start rust-carbon; if it was running when the job began + # and is offline now, start it. That server only, power `start` only. + - name: Leave rust-carbon as found + if: ${{ always() && steps.pending.outputs.run == 'true' }} + continue-on-error: true + env: + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: node tools/drill/cutover.js rig-restore --work "$WORK" diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index 26ed66d..61e3eca 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -61,3 +61,8 @@ jobs: # The staging drill's pure parts (tools/drill/drill.js): its stall limit and fresh-data list. - name: Test the staging drill run: node --test tools/drill/drill.test.js + + # The wipe-day cutover's pure parts (tools/drill/cutover.js, D332): the weekend, the five gates' + # verdicts, the issues the PR closes and what the hourly run does next. + - name: Test the wipe-day cutover + run: node --test tools/drill/cutover.test.js diff --git a/tools/drill/cutover.js b/tools/drill/cutover.js new file mode 100644 index 0000000..85d8746 --- /dev/null +++ b/tools/drill/cutover.js @@ -0,0 +1,871 @@ +// The wipe-day cutover (docs/runicnpc/PLAN.md stage 9c, D332). Never shipped. +// +// .gitea/workflows/cutover.yml runs it hourly over the forced-wipe weekend. It merges `edge` into `main` +// once Rust's wipe update is live and RunicNPC is proven on it, so nobody approves a merge on a wipe day. +// +// node tools/drill/cutover.js window [--force] [--now ISO] in the wipe weekend? +// node tools/drill/cutover.js pending [--force] does edge have commits main lacks? +// (tools/drill/static.sh with BRANCH=public, then BRANCH=staging) the static halves +// node tools/drill/cutover.js rig-note --work is rust-carbon running now? +// node tools/drill/cutover.js gates --state --work [--staging-work ] [--dry-run] [--force] [--after-live] +// node tools/drill/drill.js live --work --branch public the live half (only when `gates` says) +// node tools/drill/cutover.js merge --pr --sha [--state f] MERGE_TOKEN only +// node tools/drill/cutover.js after --state --pr record it, comment the release +// node tools/drill/cutover.js rig-restore --work start rust-carbon if the job stopped it +// +// The five gates, cheapest first; it merges only when all pass (the later ones need the earlier ones): +// public Rust's public build is dated the wipe day or later AND names the same depot manifest as Rust's +// CURRENT staging build: Facepunch promotes the staging build to public, so the two agree once the +// cut over from staging to live went through. (static.sh runs a second time, BRANCH=staging, into +// its own work dir; only its manifest is used, and when that run fails this gate blocks.) +// oxide Oxide caught up, the swap's field list is unchanged, RunicNPC compiles (static.json). +// carbon rust-carbon restarts on public Rust with edge's RunicNPC.cs. Its appmanifest must name the public +// manifest (it really updated); then rnpc.status says framework=carbon, navmesh=ready, missing=- +// added=- (D232: only Carbon's unmodified assembly makes `added` mean anything). +// live drill.js live --branch public on rust-staging: every harness check passes, no finding. +// checks the cutover PR's combined status on its head commit is success and has a `plugin-checks` +// context, AND the PR head is the commit the gates ran on. +// +// The gates are tied to the code: drill.json keeps `edgeSha` (the commit this job checked out and ran the +// plugin and harness from) and the manifest; a change of either, or of the wipe, clears every gate. `checks` +// is never kept: it runs fresh, right before the merge, and the merge is sent `head_commit_id` so Gitea refuses +// it when the PR head moved. The state is written after EVERY gate result, before anything touches the PR. +// +// A carbon or live attempt that fails is not repeated for the same manifest and commit for COOL_DOWN_HOURS +// (3): the rigs are not restarted every hour. A new manifest or a new edge commit retries at once. +// +// How the gates are sequenced across the workflow's steps. `gates` is run twice at most: +// 1. `gates` runs public, oxide, carbon in this job and stops in front of live, saying +// `live=true`. A gate that fails ends the run (retried next hour, a cooling one +// after the cool-down); a gate that passed is kept in drill.json, never re-run. +// 2. `drill.js live` is its own workflow step; its output is a file (live.json). +// 3. `gates --after-live` reads live.json (the live gate), then runs checks. Both say `merge=true pr= +// sha=` when everything has passed. +// When live already passed (an earlier hour), step 1 goes on to checks by itself and the other two steps +// are skipped. The cutover PR is opened (or its marked block updated) as soon as the public gate has passed; +// an existing open edge to main PR is adopted: its title is never changed and only the block between +// and of its body is written (appended when absent). +// +// A dry run (--dry-run, or any workflow_dispatch whose dry_run input is not exactly 'false') runs every gate +// it can instead of stopping at the first failure, ignores what an earlier real run kept, and creates, edits, +// comments and merges nothing and writes no state. It does use the rigs: carbon restarts rust-carbon and live +// uses rust-staging. The table goes to the job summary. +// +// Every gate check below returns `true`, or a string saying why not. +// +// Environment: GITEA_TOKEN (the workflow's REGISTRY_TOKEN: read, open and edit the PR, comment) with +// GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them; PANEL_KEY / PANEL_URL as drill.js; +// MERGE_TOKEN (RUNICNPC_TOKEN) in the `merge` step only; GITHUB_EVENT_NAME + DRY_RUN_INPUT for the dry-run +// default. No token or key is ever printed. + +const fs = require('fs') +const path = require('path') +const { execFileSync } = require('child_process') + +const ROOT = path.resolve(__dirname, '..', '..') +const CARBON_RIG = process.env.CUTOVER_CARBON_RIG || 'rust-carbon' +const CARBON_PLUGIN = '/carbon/plugins/RunicNPC.cs' +const ACF = '/steamapps/appmanifest_258550.acf' +const LINUX_DEPOT = '258552' +const GATES = ['public', 'oxide', 'carbon', 'live', 'checks'] +const PERSISTED = GATES.slice(0, 4) // checks is never kept +const COOLING = ['carbon', 'live'] // the gates that cost a rig restart +const COOL_DOWN_HOURS = 3 +const GATE_LABELS = { + public: "1. Rust's public build is the promoted staging build, dated the wipe day", + oxide: '2. Oxide has caught up; RunicNPC compiles', + carbon: '3. A Carbon boot on public Rust', + live: '4. The drill again, on live Rust', + checks: '5. PR checks (plugin-checks) on the head commit', +} +const BLOCK_START = '' +const BLOCK_END = '' +const H = 3600000 +const CHECKS_WAIT_MS = 15 * 60000 +const RELEASE_WAIT_MS = 30 * 60000 + +// ---------------------------------------------------------------- pure parts (cutover.test.js) + +const day = (d) => d.toISOString().slice(0, 10) + +/** A reason in a PR body or a comment must not hold a closing keyword Gitea would act on: `#123` is broken up. */ +const neutralise = (text) => String(text == null ? '' : text).replace(/#(\d)/g, '#​$1') + +function firstThursday(y, m) { + const d = new Date(Date.UTC(y, m, 1)) + d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7)) + return d +} + +/** + * The forced wipe of `now`'s month (its first Thursday, D318) and the weekend that follows. `open` is the + * span the cutover works in: from the wipe Thursday 17:00 UTC to Sunday 23:00 UTC (`deadline`). `final` is + * the hour after it, in which the hourly run still starts once more so it can give up out loud. (nextWipe + * in drill.js cannot do this: on the Thursday itself it answers next month.) + */ +function wipeWeekend(now) { + const wipe = firstThursday(now.getUTCFullYear(), now.getUTCMonth()) + const t = now.getTime() + const start = wipe.getTime() + 17 * H + const deadline = new Date(wipe.getTime() + (3 * 24 + 23) * H) + return { open: t >= start && t < deadline.getTime(), final: t >= deadline.getTime() && t < deadline.getTime() + H, wipe, deadline } +} + +/** + * DepotDownloader's "Manifest N (10/31/2026 11:48:58)" date as UTC milliseconds: M/D/YYYY HH:mm:ss, 24 hours. + * An AM/PM form and ISO are accepted too; anything else is NaN (unparsable, so the gate blocks). + */ +function manifestTime(text) { + const s = String(text || '').trim() + const m = /^(\d{1,2})\/(\d{1,2})\/(\d{4}) (\d{1,2}):(\d{2}):(\d{2})(?: ([AP]M))?$/i.exec(s) + if (m) { + const [mo, d, y, hh, mi, ss] = [+m[1], +m[2], +m[3], +m[4], +m[5], +m[6]] + const ampm = m[7] && m[7].toUpperCase() + if (mi > 59 || ss > 59 || (ampm ? hh < 1 || hh > 12 : hh > 23)) return NaN + const h = ampm ? (hh % 12) + (ampm === 'PM' ? 12 : 0) : hh + const t = Date.UTC(y, mo - 1, d, h, mi, ss) + const back = new Date(t) + return back.getUTCMonth() === mo - 1 && back.getUTCDate() === d ? t : NaN + } + if (!/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(s)) return NaN + return Date.parse(/(Z|[+-]\d\d:?\d\d)$/.test(s) ? s : s + 'Z') +} + +/** The public depot's manifest is dated the wipe day (a Date, or YYYY-MM-DD) or later (UTC). */ +function publicChanged(stat, wipe) { + const t = manifestTime(stat && stat.manifestDate) + const w = wipe instanceof Date ? wipe.getTime() : Date.parse(wipe) + return Number.isFinite(t) && Number.isFinite(w) && t >= w +} + +function publicReason(stat, wipe) { + const date = stat && stat.manifestDate + if (!Number.isFinite(manifestTime(date))) return `the static half read no usable manifest date from Steam (${(stat && stat.note) || 'no note'})` + return `Rust's public build is manifest ${stat.manifest} of ${date}, from before the wipe (${wipe instanceof Date ? day(wipe) : wipe} 00:00 UTC)` +} + +/** Gate public: dated the wipe day or later, and the same manifest as Rust's current staging build. */ +function publicGate(stat, staging, wipe) { + if (!publicChanged(stat, wipe)) return publicReason(stat, wipe) + if (!staging || !staging.manifest) return `Rust's staging build could not be read (${(staging && staging.note) || 'no static.json'}), so the promotion to public cannot be confirmed` + if (staging.manifest !== stat.manifest) { + return `Rust's public build (manifest ${stat.manifest}) is not its current staging build (manifest ${staging.manifest}): the cut over from staging to live has not gone through` + } + return true +} + +/** Gate oxide: true, or why an Oxide server (or the plugin's swap) cannot be trusted on this build. */ +function oxideReady(stat) { + const s = stat || {} + if (s.managed !== 'ok') return `the static half could not fetch Rust's public build (${s.note || 'no static.json'})` + if (s.oxideBehind === -1 || s.oxideBehind === undefined) return "Oxide's public build could not be compared with Rust's" + if (s.oxideBehind !== 0) return `Oxide's public build is ${s.oxideBehind} member(s) behind` + if (s.compile !== 'ok') return `RunicNPC does not compile against public Rust with Oxide's build (compile ${s.compile})` + if (s.fields === 'differs') return "the swap's field list differs from public Rust's" + if (s.fields !== 'same') return `the swap's field list could not be checked (fields ${s.fields})` + return true +} + +/** Valve's text KeyValues (an appmanifest .acf) as nested objects. */ +function parseVdf(text) { + const root = {} + const stack = [root] + let key = null + for (const m of String(text || '').matchAll(/"((?:[^"\\]|\\.)*)"|([{}])/g)) { + const top = stack[stack.length - 1] + if (m[2] === '{') { + const o = {} + top[key] = o + stack.push(o) + key = null + } else if (m[2] === '}') { + if (stack.length > 1) stack.pop() + key = null + } else if (key === null) key = m[1] + else { + top[key] = m[1] + key = null + } + } + return root +} + +/** The manifest an appmanifest_258550.acf records for a depot (InstalledDepots, "258552", "manifest"), or null. */ +function acfDepotManifest(text, depot = LINUX_DEPOT) { + const state = parseVdf(text).AppState + const entry = state && state.InstalledDepots && state.InstalledDepots[depot] + return (entry && typeof entry === 'object' && entry.manifest) || null +} + +/** Gate carbon, first half: the rig's installed Linux depot is the public manifest. */ +function carbonBuild(acfText, publicManifest) { + const have = acfDepotManifest(acfText) + if (!have) return `the ${LINUX_DEPOT} manifest could not be read from ${ACF} on ${CARBON_RIG}` + if (have !== publicManifest) return `${CARBON_RIG} still runs manifest ${have} (public is ${publicManifest})` + return true +} + +/** Gate carbon, second half: the `rnpc.status` console lines (an array, or the text) of a Carbon server. */ +function carbonOk(statusLines) { + const lines = Array.isArray(statusLines) ? statusLines : String(statusLines || '').split('\n') + const line = lines.find((l) => /framework=/.test(l)) + if (!line) return 'rnpc.status never answered on Carbon' + const get = (k) => (new RegExp(`${k}=(\\S+)`).exec(line) || [])[1] + const framework = get('framework') + if (framework !== 'carbon') return `rnpc.status says framework=${framework}, not carbon` + if (get('navmesh') !== 'ready') return `the navmesh was not ready on Carbon (navmesh=${get('navmesh')})` + const missing = get('missing') + const added = get('added') + if (missing === undefined || added === undefined) return 'rnpc.status carried no swap-fields report' + if (missing !== '-') return `the swap's fields are missing on Carbon: ${missing}` + if (added !== '-') return `public Rust's Carbon assembly has fields the swap does not know: ${added}` + return true +} + +/** Gate live: live.json as drill.js live wrote it. */ +function liveOk(live) { + if (!live) return 'the live half wrote no live.json' + const first = (live.findings && live.findings[0]) || null + if (live.branch && live.branch !== 'public') return `live.json is from the ${live.branch} branch, not public` + if (!live.ran) return live.skipped || first || 'the live half did not run the harness' + if (!live.harness || !live.harness.done) return first || 'the harness did not finish' + if (live.harness.fail > 0) return `a harness check failed: ${(live.harness.failed && live.harness.failed[0]) || `${live.harness.fail} fail`}` + if (!(live.harness.pass > 0)) return 'the harness passed no check at all' + if (first) return first + return true +} + +/** Gate checks: where the cutover PR's combined commit status stands. { state: success|pending|failed, reason } */ +function checkState(combined) { + const statuses = (combined && combined.statuses) || [] + const plugin = statuses.filter((s) => String(s.context || '').includes('plugin-checks')) + const bad = statuses.find((s) => ['failure', 'error'].includes(s.status || s.state)) + if (bad) return { state: 'failed', reason: `${bad.context} is ${bad.status || bad.state}` } + if (!plugin.length) return { state: 'pending', reason: 'no plugin-checks status has been reported yet' } + if (combined.state !== 'success') return { state: 'pending', reason: `the PR's combined status is ${combined.state}` } + return { state: 'success', reason: '' } +} + +/** Issue numbers a PR body closes: a Gitea closing keyword directly before `#N`. Sorted, unique. */ +function collectCloses(prBodies) { + const found = new Set() + const re = /(? a - b) +} + +/** The cutover PR's description (the block). `gates` is [{ name, status, note }]. */ +function cutoverBody({ wipe, manifest, buildDate, closes = [], gates = [] }) { + return [ + 'The automated wipe-day cutover (D332, `docs/runicnpc/PLAN.md` stage 9c): `edge` merges into `main` once Rust\'s ' + + 'forced-wipe update is live and RunicNPC is proven on it, so no merge needs approving on a wipe day. ' + + 'The workflow merges this PR by itself when every gate below has passed; `release.yml` then cuts the release.', + '', + `Wipe: **${wipe}**. Rust's public build: manifest **${manifest}**, dated ${buildDate || '?'}.`, + '', + '| Gate | State | Detail |', + '| --- | --- | --- |', + ...gates.map((g) => `| ${g.name} | ${g.status} | ${neutralise(g.note || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`), + '', + closes.length ? 'Issues fixed by the PRs merged into `edge` since the last cutover:' : 'No issue is closed by this release.', + ...closes.map((n) => `Closes #${n}`), + '', + 'Opened by the runicnpc-rust cutover workflow (D332).', + ].join('\n') +} + +/** `existing` with the cutover block set to `content`: replaced between the markers, or appended when absent. */ +function withBlock(existing, content) { + const block = `${BLOCK_START}\n${content}\n${BLOCK_END}` + const text = String(existing || '') + const a = text.indexOf(BLOCK_START) + const b = text.indexOf(BLOCK_END, a + 1) + if (a >= 0 && b > a) return text.slice(0, a) + block + text.slice(b + BLOCK_END.length) + return text.trim() ? `${text.replace(/\s+$/, '')}\n\n${block}` : block +} + +const nextGate = (gates) => PERSISTED.find((g) => !(gates || {})[g]) + +/** A gate passed: the state with it kept (and its failure forgotten). */ +function pass(state, gate, now) { + const { [gate]: _gone, ...failed } = state.failed || {} + return { ...state, gates: { ...state.gates, [gate]: { at: now.toISOString() } }, failed } +} + +/** A cooling gate failed: when, and why. */ +function fail(state, gate, reason, now) { + return { ...state, failed: { ...state.failed, [gate]: { at: now.toISOString(), reason: String(reason) } } } +} + +/** What goes into drill.json: never `checks`. */ +function persistable(state, pr) { + const { checks: _checks, ...gates } = state.gates || {} + return { ...state, gates, pr: pr ? pr.number : null } +} + +/** + * What the hourly run does next, as data. `state` is the `cutover` object kept in drill.json: + * { wipe: 'YYYY-MM-DD', manifest, edgeSha, gates: { public, oxide, carbon, live: { at } }, failed: { carbon, live: { at, reason } }, + * pr, lastComment: 'YYYY-MM-DD', merged, gaveUp, mergeError } + * `obs`: { manifest, edgeSha, ahead (commits edge has that main lacks), openPr (number or null), forced, + * checksPassed (this run, never kept), blocked: { gate, reason } }. + * Returns { state, action, gate, reason, retryAt, write, comment, merge }: `action` is nothing | giveUp | run | + * blocked | cooling | merge; `write`, `comment` and `merge` say whether the CLI may persist the state, comment, + * merge (a dry run: none). + */ +function decide(state, obs, now, { dryRun = false } = {}) { + const w = wipeWeekend(now) + const wipe = day(w.wipe) + let s = state || {} + if (s.wipe !== wipe) s = { wipe, manifest: obs.manifest, edgeSha: obs.edgeSha, gates: {}, failed: {} } // a new wipe starts over + else if ((obs.manifest && s.manifest !== obs.manifest) || (obs.edgeSha && s.edgeSha !== obs.edgeSha)) { + // A new build, or new code: every gate again, and the failures that cooled them are forgotten. + const { mergeError: _e, ...rest } = s + s = { ...rest, manifest: obs.manifest || s.manifest, edgeSha: obs.edgeSha || s.edgeSha, gates: {}, failed: {} } + } + const { checks: _checks, ...gates } = s.gates || {} + s = { ...s, gates, failed: s.failed || {}, pr: obs.openPr || null } // the PR is kept while it is open + const out = { state: s, write: !dryRun, comment: false, merge: false } + + // One comment a day, none before the build is known to have changed (a quiet Thursday morning), none without a PR. + const hold = (action, gate, reason, retryAt) => { + const due = !dryRun && !!s.pr && !!s.gates.public && s.lastComment !== day(now) + return { ...out, state: due ? { ...s, lastComment: day(now) } : s, action, gate, reason, retryAt, comment: due } + } + + if (s.merged) return { ...out, action: 'nothing', reason: 'already merged' } + if (obs.ahead === 0 && !dryRun) return { ...out, action: 'nothing', reason: 'edge has no commits main lacks' } + if (now.getTime() >= w.deadline.getTime() && !obs.forced) { + if (s.gaveUp) return { ...out, action: 'nothing', reason: 'gave up already' } + return { ...out, state: { ...s, gaveUp: true }, action: 'giveUp', gate: nextGate(s.gates), comment: !dryRun && !!s.pr } + } + if (obs.blocked) { + const retryAt = COOLING.includes(obs.blocked.gate) ? new Date(now.getTime() + COOL_DOWN_HOURS * H) : null + return hold('blocked', obs.blocked.gate, obs.blocked.reason, retryAt) + } + const gate = nextGate(s.gates) + if (!gate) return obs.checksPassed ? { ...out, action: 'merge', merge: !dryRun } : { ...out, action: 'run', gate: 'checks' } + const failed = COOLING.includes(gate) && s.failed[gate] + if (failed && now.getTime() - Date.parse(failed.at) < COOL_DOWN_HOURS * H) { + return hold('cooling', gate, failed.reason, new Date(Date.parse(failed.at) + COOL_DOWN_HOURS * H)) + } + return { ...out, action: 'run', gate } +} + +/** The comment for a decision that asks for one. */ +function commentText(d, state = {}) { + const label = GATE_LABELS[d.gate] || d.gate + if (d.action === 'giveUp') { + const head = 'The wipe-day cutover (D332) stops here: the window closed (Sunday 23:00 UTC).' + const tail = 'Merge `edge` into `main` by hand when it is ready.' + if (state.mergeError) return `${head} Every gate passed; the merge failed: ${neutralise(state.mergeError)}. ${tail}` + if (!d.gate) return `${head} Every gate passed, but the PR checks and the merge did not complete. ${tail}` + return `${head} Gate "${label}" had not passed. ${tail}` + } + const when = d.retryAt ? ` It will not try this gate again before ${d.retryAt.toISOString().slice(0, 16).replace('T', ' ')} UTC.` : ' It tries again every hour until Sunday 23:00 UTC.' + return `The wipe-day cutover (D332) is waiting at gate "${label}": ${neutralise(d.reason)}.${when} This is the day's only comment.` +} + +/** Whether a dispatch is a dry run: unless its dry_run input is exactly 'false' it is. (A schedule never is.) */ +function isDry(a, env) { + return !!a['dry-run'] || (env.GITHUB_EVENT_NAME === 'workflow_dispatch' && env.DRY_RUN_INPUT !== 'false') +} + +// ---------------------------------------------------------------- plumbing + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +const log = (...a) => console.log(`[cutover ${new Date().toISOString().slice(11, 19)}]`, ...a) + +/** What the commands reach outside themselves for; cutover.test.js passes fakes. */ +function realDeps() { + return { + env: process.env, + now: () => new Date(), + sleep, + fetch: (u, o) => fetch(u, o), + git: (...a) => execFileSync('git', a, { cwd: ROOT, encoding: 'utf8' }).trim(), + panel: () => require('./drill.js').panel(), + } +} + +function args() { + const out = { _: [] } + const argv = process.argv.slice(2) + for (let i = 0; i < argv.length; i++) { + if (argv[i].startsWith('--')) { + const k = argv[i].slice(2) + out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true + } else out._.push(argv[i]) + } + return out +} + +const readJson = (f, fallback) => { + try { + return JSON.parse(fs.readFileSync(f, 'utf8')) + } catch { + return fallback + } +} + +const writeJson = (f, data) => fs.writeFileSync(f, JSON.stringify(data, null, 2) + '\n') + +const output = (env, name, value) => { + if (env.GITHUB_OUTPUT) fs.appendFileSync(env.GITHUB_OUTPUT, `${name}=${value}\n`) +} + +const summary = (env, md) => { + if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, md + '\n') +} + +function aheadCount(a, deps) { + if (a.ahead !== undefined && a.ahead !== true) return Number(a.ahead) + return Number(deps.git('rev-list', '--count', 'origin/main..origin/edge')) +} + +/** Gitea's REST API for this repo (the idiom of drill.js fileIssue). The token is never printed. */ +function gitea(token, deps) { + if (!token) throw new Error('no Gitea token in the environment') + const base = `${deps.env.GITHUB_SERVER_URL}/api/v1/repos/${deps.env.GITHUB_REPOSITORY}` + const headers = { Authorization: 'token ' + token, 'Content-Type': 'application/json', Accept: 'application/json' } + const call = async (method, p, data) => { + const r = await deps.fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined }) + const text = await r.text() + if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${text.slice(0, 200)}`) + return text ? JSON.parse(text) : null + } + return { + call, + async openPr() { + return (await call('GET', '/pulls?state=open&limit=50')).find((p) => p.head.ref === 'edge' && p.base.ref === 'main') || null + }, + } +} + +/** + * The `Closes #N` numbers of the PRs merged into edge since the last cutover: after the committer date of + * `git merge-base origin/main origin/edge`, the newest edge commit already in main. + */ +async function closesSinceCutOff(g, deps) { + let since = 0 + try { + since = Date.parse(deps.git('log', '-1', '--format=%cI', deps.git('merge-base', 'origin/main', 'origin/edge'))) || 0 + } catch {} + const bodies = [] + for (let page = 1; page <= 10; page++) { + const prs = await g.call('GET', `/pulls?state=closed&limit=50&page=${page}`) + for (const pr of prs) { + if (pr.merged && pr.base && pr.base.ref === 'edge' && Date.parse(pr.merged_at) > since) bodies.push(pr.body) + } + if (prs.length < 50) break + } + return collectCloses(bodies) +} + +// ---------------------------------------------------------------- window and pending + +function window_(a, deps = realDeps()) { + const now = a.now ? new Date(a.now) : deps.now() + const w = wipeWeekend(now) + const open = w.open || w.final || !!a.force + const verdict = w.open ? 'in the wipe weekend' : w.final ? 'the hour after it (the last run, to give up out loud)' : a.force ? 'outside the wipe weekend, but run by hand: going ahead' : 'outside the wipe weekend, nothing to do' + log(`the wipe is ${day(w.wipe)}, the cutover works until ${w.deadline.toISOString().slice(0, 16)}Z: ${verdict} (open=${open})`) + output(deps.env, 'open', open ? 'true' : 'false') +} + +function pending(a, deps = realDeps()) { + const ahead = aheadCount(a, deps) + const run = ahead > 0 || !!a.force + log(`edge is ${ahead} commit(s) ahead of main: ${run ? 'going ahead' : 'nothing to merge'}`) + output(deps.env, 'run', run ? 'true' : 'false') +} + +// ---------------------------------------------------------------- the rig + +async function carbonGate(ctx) { + const { waitRunning } = require('./drill.js') + const p = ctx.deps.panel() + const id = (await p.servers())[CARBON_RIG] + if (!id) return `no server named ${CARBON_RIG} on the panel` + await p.write(id, CARBON_PLUGIN, fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8')) + const was = await p.state(id) + log(`${CARBON_RIG} is ${was}: ${was === 'offline' ? 'starting' : 'restarting'} it (its egg updates Rust and Carbon)`) + await p.power(id, was === 'offline' ? 'start' : 'restart') + // `running` right after the signal is the old process: wait for it to leave first. + for (let i = 0; i < 60 && was !== 'offline' && (await p.state(id)) === 'running'; i++) await ctx.deps.sleep(5000) + if (!(await waitRunning(p, id, 45 * 60000))) return `${CARBON_RIG} did not come up in 45 minutes, or crashed on start` + // It must have updated to the public build, or everything after proves nothing. + const built = carbonBuild(await p.readOr(id, ACF, null), ctx.stat.manifest) + if (built !== true) return built + const lines = await carbonStatus(p, id, 30 * 60000, ctx.deps) + if (lines) log('rnpc.status:', lines.find((l) => l.includes('navmesh='))) + return carbonOk(lines || []) +} + +/** `rnpc.status` over the console until the navmesh is ready; the last answer if it never is. */ +async function carbonStatus(p, id, timeoutMs, deps) { + const end = Date.now() + timeoutMs + let last = null + while (Date.now() < end) { + try { + const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim()) + const at = lines.findIndex((l) => l.includes('RunicNPC ') && l.includes('api=')) + if (at >= 0) { + last = lines.slice(at, at + 14) + if (last.some((l) => l.includes('navmesh=ready'))) return last + } + } catch { + // the panel's console is not up yet + } + await deps.sleep(30000) + } + return last +} + +/** Writes whether rust-carbon is running now, to put it back if the job leaves it stopped. */ +async function rigNote(a, deps = realDeps()) { + const work = path.resolve(a.work || 'cutover-work') + const p = deps.panel() + const id = (await p.servers())[CARBON_RIG] + const state = id ? await p.state(id) : null + fs.mkdirSync(work, { recursive: true }) + writeJson(path.join(work, 'rig-carbon.json'), { name: CARBON_RIG, state }) + log(`${CARBON_RIG} is ${state || 'not on the panel'}`) +} + +/** Starts rust-carbon (power `start` on it alone) when it was running at the start of the job and is offline now. */ +async function rigRestore(a, deps = realDeps()) { + const rec = readJson(path.join(path.resolve(a.work || 'cutover-work'), 'rig-carbon.json'), null) + if (!rec || rec.state !== 'running') return log(`${CARBON_RIG} was not running at the start: nothing to put back`) + const p = deps.panel() + const id = (await p.servers())[CARBON_RIG] + if (!id) return log(`${CARBON_RIG} is not on the panel`) + const now = await p.state(id) + if (now !== 'offline') return log(`${CARBON_RIG} is ${now}: left as it is`) + await p.power(id, 'start') + log(`${CARBON_RIG} was running at the start and is offline: started it`) +} + +// ---------------------------------------------------------------- the gates + +async function checksGate(ctx) { + const wait = !ctx.dryRun + let pr = ctx.pr + if (!pr) return ctx.dryRun ? 'no cutover PR is open (a dry run opens none)' : 'no cutover PR is open' + const end = Date.now() + CHECKS_WAIT_MS + for (;;) { + pr = (await ctx.g.call('GET', `/pulls/${pr.number}`)) || pr + if (pr.head.sha !== ctx.edgeSha) return `the PR's head ${String(pr.head.sha).slice(0, 8)} is not the commit the gates ran on (${String(ctx.edgeSha).slice(0, 8)})` + const v = checkState(await ctx.g.call('GET', `/commits/${pr.head.sha}/status`)) + if (v.state === 'success') return true + if (v.state === 'failed') return v.reason + if (!wait || Date.now() >= end) return `${v.reason} (waited ${wait ? 15 : 0} minutes)` + await ctx.deps.sleep(30000) + } +} + +const RUNNERS = { + public: async (c) => publicGate(c.stat, c.staging, c.wipe), + oxide: async (c) => oxideReady(c.stat), + carbon: async (c) => carbonGate(c), + live: async (c) => liveOk(readJson(path.join(c.work, 'live.json'), null)), + checks: async (c) => checksGate(c), +} + +/** Runs one gate; whatever it throws is its reason, so a flaky API call blocks an hour, not the state. */ +async function runGate(gate, ctx) { + log(`gate ${gate}...`) + let r + try { + r = await RUNNERS[gate](ctx) + } catch (e) { + r = `the gate stopped: ${e.message}` + } + log(`gate ${gate}: ${r === true ? 'passed' : `NOT passed: ${r}`}`) + return r +} + +// ---------------------------------------------------------------- the PR + +function gateRows(state, rows) { + return GATES.map((g) => { + const done = state.gates[g] + const row = rows[g] + if (done) return { name: GATE_LABELS[g], status: 'passed', note: done.at.slice(0, 16).replace('T', ' ') + ' UTC' } + return { name: GATE_LABELS[g], status: row ? row.status : 'not yet', note: row ? row.detail : '' } + }) +} + +/** + * Opens the edge to main PR, or brings the marked block of the open one's body up to date. A PR it did not + * open keeps its title and the rest of its body. Not in a dry run, and never with a blank manifest. + */ +async function syncPr(ctx) { + if (ctx.dryRun) return ctx.pr + if (!ctx.stat.manifest) { + log('no manifest: the PR is left alone') + return ctx.pr + } + if (!ctx.closes) ctx.closes = await closesSinceCutOff(ctx.g, ctx.deps) + const content = cutoverBody({ wipe: ctx.wipeDate, manifest: ctx.stat.manifest, buildDate: ctx.stat.manifestDate, closes: ctx.closes, gates: gateRows(ctx.state, ctx.rows) }) + if (!ctx.pr) { + const title = `chore(release): RunicNPC cutover for the ${ctx.wipeDate} wipe, Rust ${ctx.stat.manifest}` + ctx.pr = await ctx.g.call('POST', '/pulls', { head: 'edge', base: 'main', title, body: withBlock('', content) }) + log(`opened PR #${ctx.pr.number}`) + } else { + const body = withBlock(ctx.pr.body, content) + if (body !== ctx.pr.body) { + ctx.pr = { ...ctx.pr, ...((await ctx.g.call('PATCH', `/pulls/${ctx.pr.number}`, { body })) || {}) } + log(`updated PR #${ctx.pr.number}`) + } + } + ctx.state = { ...ctx.state, pr: ctx.pr.number } + return ctx.pr +} + +// ---------------------------------------------------------------- gates (the command) + +async function gates(a, deps = realDeps()) { + const env = deps.env + const dryRun = isDry(a, env) + const afterLive = !!a['after-live'] + const work = path.resolve(a.work || 'cutover-work') + const file = readJson(a.state, {}) + const now = deps.now() + const w = wipeWeekend(now) + const stat = readJson(path.join(work, 'static.json'), {}) + const staging = readJson(path.join(path.resolve(a['staging-work'] || work + '-staging'), 'static.json'), null) + const g = gitea(env.GITEA_TOKEN, deps) + const prev = (afterLive && readJson(path.join(work, 'cutover-run.json'), null)) || { rows: {} } + + const pr = await g.openPr() + const edgeSha = a.edge || deps.git('rev-parse', 'HEAD') + const obs = { manifest: stat.manifest || '', edgeSha, ahead: aheadCount(a, deps), openPr: pr && pr.number, forced: !!a.force && !(w.open || w.final), checksPassed: !!(dryRun && prev.checks) } + // A dry run starts from nothing: it is there to try every gate, not to trust an earlier hour's. + let d = decide(dryRun ? undefined : file.cutover, obs, now, { dryRun }) + const ctx = { g, deps, stat, staging, work, now, dryRun, pr, edgeSha, wipe: w.wipe, wipeDate: day(w.wipe), state: d.state, rows: prev.rows, closes: null } + if (dryRun && prev.gates) ctx.state = { ...ctx.state, gates: prev.gates } // what the first half of this dry run passed + else for (const k of Object.keys(ctx.state.gates)) ctx.rows[k] = { status: 'passed earlier', detail: ctx.state.gates[k].at } + const note = (gate, r) => (ctx.rows[gate] = r === true ? { status: 'passed', detail: '' } : { status: 'failed', detail: r }) + // The state goes to the file after every gate result, before the PR is touched. + const persist = () => { + if (dryRun) return + file.cutover = persistable(ctx.state, ctx.pr) + writeJson(a.state, file) + } + const safely = async (what, fn) => { + try { + return await fn() + } catch (e) { + log(`${what} failed (the gates are kept): ${e.message}`) + } + } + + let blocked = null + let deferred = false + if (d.action === 'nothing') log(`nothing to do: ${d.reason}`) + else if (d.action === 'giveUp') log(`the cutover window closed with ${d.gate || 'every gate'} not done`) + else if (dryRun) { + // Every gate it can; live waits for its own step. + for (const gate of GATES) { + if (gate === 'live' && !afterLive) { + deferred = true + ctx.rows.live = { status: 'runs next', detail: 'drill.js live --branch public' } + } else if ((gate === 'live') === afterLive || (afterLive && gate === 'checks' && !prev.checks)) { + const r = await runGate(gate, ctx) + note(gate, r) + if (r === true) { + if (gate === 'checks') obs.checksPassed = true + else ctx.state = pass(ctx.state, gate, now) + } else if (!blocked) blocked = { gate, reason: r } + } + } + } else { + while (d.action === 'run') { + if (d.gate === 'live' && !afterLive) { + deferred = true + break + } + const r = await runGate(d.gate, ctx) + note(d.gate, r) + if (r !== true) { + blocked = { gate: d.gate, reason: r } + if (COOLING.includes(d.gate)) ctx.state = fail(ctx.state, d.gate, r, now) + persist() + break + } + if (d.gate === 'checks') obs.checksPassed = true + else { + ctx.state = pass(ctx.state, d.gate, now) + persist() + } + if (d.gate === 'public') { + await safely('opening the PR', () => syncPr(ctx)) + obs.openPr = ctx.pr ? ctx.pr.number : null + } + d = decide(ctx.state, obs, now, { dryRun }) + } + } + + // What the run ends in. + if (d.action !== 'nothing' && d.action !== 'giveUp') { + obs.openPr = ctx.pr ? ctx.pr.number : null + d = decide(ctx.state, blocked ? { ...obs, blocked } : obs, now, { dryRun }) + } + if (!dryRun && ctx.state.gates.public && d.action !== 'giveUp' && !ctx.state.merged) await safely('updating the PR', () => syncPr(ctx)) + const end = d.action + log(`${end}${d.gate ? ` (${d.gate})` : ''}${d.reason ? `: ${d.reason}` : ''}`) + + let finalState = d.state + if (d.comment && ctx.pr) { + const done = await safely('commenting', async () => { + await g.call('POST', `/issues/${ctx.pr.number}/comments`, { body: commentText(d, d.state) }) + return true + }) + if (done) log(`commented on PR #${ctx.pr.number}`) + else finalState = { ...finalState, lastComment: ctx.state.lastComment } // not said, so it may be said next hour + } + ctx.state = finalState + persist() + + fs.mkdirSync(work, { recursive: true }) + writeJson(path.join(work, 'cutover-run.json'), { rows: ctx.rows, gates: ctx.state.gates, checks: !!obs.checksPassed }) + output(env, 'live', deferred ? 'true' : 'false') + output(env, 'merge', d.action === 'merge' && d.merge ? 'true' : 'false') + output(env, 'sha', edgeSha) + if (ctx.pr) output(env, 'pr', ctx.pr.number) + if (!deferred) { + const rows = GATES.map((gate) => `| ${GATE_LABELS[gate]} | ${(ctx.rows[gate] || { status: 'not reached' }).status} | ${neutralise((ctx.rows[gate] || {}).detail || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`) + summary( + env, + [ + `## Wipe-day cutover${dryRun ? ' (dry run: nothing created, edited, commented or merged)' : ''}: ${end}`, + '', + `Wipe ${ctx.wipeDate}; Rust public manifest ${stat.manifest || '?'} of ${stat.manifestDate || '?'}; edge ${edgeSha.slice(0, 8)} is ${obs.ahead} commit(s) ahead of main.`, + '', + '| Gate | State | Detail |', + '| --- | --- | --- |', + ...rows, + d.action === 'merge' ? `\nEvery gate passed${dryRun ? ': a real run would merge now' : ''}.` : '', + ].join('\n') + ) + } + return d +} + +// ---------------------------------------------------------------- merge and after + +/** Records why the merge failed where the next hour (and a last comment) can read it. */ +function recordMergeError(stateFile, message) { + if (!stateFile || stateFile === true) return + const file = readJson(stateFile, {}) + file.cutover = { ...(file.cutover || {}), mergeError: String(message).slice(0, 300) } + writeJson(stateFile, file) +} + +async function merge(a, deps = realDeps()) { + const g = gitea(deps.env.MERGE_TOKEN, deps) + if (!a.pr || a.pr === true) throw new Error('--pr is required') + if (!a.sha || a.sha === true) throw new Error('--sha is required') + try { + const before = await g.call('GET', `/pulls/${a.pr}`) + if (before.head.sha !== a.sha) throw new Error(`PR #${a.pr}'s head ${String(before.head.sha).slice(0, 8)} is not the commit the gates ran on (${a.sha.slice(0, 8)})`) + // Not the squash: release.yml reads the feat/fix subjects of the commits that arrive. Never delete edge. + // head_commit_id: Gitea refuses the merge when the head moved since the gates. + await g.call('POST', `/pulls/${a.pr}/merge`, { Do: 'merge', delete_branch_after_merge: false, head_commit_id: a.sha }) + const pr = await g.call('GET', `/pulls/${a.pr}`) + if (!pr.merged) throw new Error(`PR #${a.pr} is not merged after the merge call`) + } catch (e) { + recordMergeError(a.state, e.message) + throw e + } + log(`merged PR #${a.pr}`) + output(deps.env, 'merged', 'true') +} + +async function after(a, deps = realDeps()) { + const g = gitea(deps.env.GITEA_TOKEN, deps) + const pr = await g.call('GET', `/pulls/${a.pr}`) + if (!pr.merged) return log(`PR #${a.pr} is not merged: nothing to record`) + const file = readJson(a.state, {}) + const { mergeError: _e, ...kept } = file.cutover || {} + file.cutover = { ...kept, merged: true, pr: pr.number } + writeJson(a.state, file) + + // release.yml cuts the release as the push to main arrives. + const mergedAt = Date.parse(pr.merged_at) + let release = null + for (const end = deps.now().getTime() + RELEASE_WAIT_MS; !release && deps.now().getTime() < end; ) { + const list = await g.call('GET', '/releases?draft=false&pre-release=false&limit=5') + release = (list || []).find((r) => Date.parse(r.created_at) > mergedAt) || null + if (!release) await deps.sleep(60000) + } + if (!release) return log('no release appeared in 30 minutes: no issue is commented') + log(`released as ${release.tag_name}`) + output(deps.env, 'release', release.tag_name) + + const text = `Released in ${release.tag_name} by the wipe cutover (D332).` + for (const n of collectCloses([pr.body])) { + const issue = await g.call('GET', `/issues/${n}`) + if (issue.state !== 'closed') { + log(`#${n} is not closed: no comment`) + continue + } + if ((await g.call('GET', `/issues/${n}/comments`)).some((c) => c.body === text)) continue + await g.call('POST', `/issues/${n}/comments`, { body: text }) + log(`commented on #${n}`) + } +} + +// ---------------------------------------------------------------- main + +module.exports = { + wipeWeekend, + publicChanged, + publicGate, + oxideReady, + parseVdf, + acfDepotManifest, + carbonBuild, + carbonOk, + liveOk, + checkState, + collectCloses, + cutoverBody, + withBlock, + decide, + commentText, + neutralise, + isDry, + pass, + fail, + persistable, + nextGate, + manifestTime, + gates, + merge, + after, + rigNote, + rigRestore, + checksGate, + closesSinceCutOff, + gitea, + GATES, + COOL_DOWN_HOURS, + BLOCK_START, + BLOCK_END, +} + +if (require.main === module) { + const a = args() + const commands = { window: window_, pending, gates, merge, after, 'rig-note': rigNote, 'rig-restore': rigRestore } + const command = commands[a._[0]] + if (!command) { + console.error('usage: node tools/drill/cutover.js window|pending|gates|merge|after|rig-note|rig-restore [--state f] [--work dir] [--pr n] [--sha s] [--dry-run] [--force] [--after-live]') + process.exit(2) + } + Promise.resolve(command(a)).catch((e) => { + console.error(e.message || e) + process.exit(1) + }) +} diff --git a/tools/drill/cutover.test.js b/tools/drill/cutover.test.js new file mode 100644 index 0000000..89ea734 --- /dev/null +++ b/tools/drill/cutover.test.js @@ -0,0 +1,758 @@ +// The wipe-day cutover (docs/runicnpc/PLAN.md D332): when it works, what each gate accepts, which issues the +// PR closes, what the hourly run does next, and the commands against a fake Gitea and a fake panel. +// node --test tools/drill/cutover.test.js + +const test = require('node:test') +const assert = require('node:assert') +const fs = require('fs') +const os = require('os') +const path = require('path') +const c = require('./cutover.js') +const { wipeWeekend, publicChanged, publicGate, oxideReady, carbonOk, carbonBuild, acfDepotManifest, liveOk, checkState, collectCloses, cutoverBody, withBlock, decide, commentText, neutralise, isDry, pass, fail, persistable, nextGate, manifestTime, GATES, COOL_DOWN_HOURS } = c + +const at = (s) => new Date(s) +const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'cutover-test-')) + +// ---- wipeWeekend: November 2026 has its wipe on Thursday the 5th, so the weekend ends Sunday the 8th. + +test('the weekend opens at 17:00 UTC on the wipe Thursday and not a minute before', () => { + assert.equal(wipeWeekend(at('2026-11-05T16:59:00Z')).open, false) + assert.equal(wipeWeekend(at('2026-11-05T17:00:00Z')).open, true) + assert.equal(wipeWeekend(at('2026-11-05T17:00:00Z')).wipe.toISOString(), '2026-11-05T00:00:00.000Z') +}) + +test('it closes at Sunday 23:00 UTC, with one more hour to give up in', () => { + assert.equal(wipeWeekend(at('2026-11-08T22:59:00Z')).open, true) + const late = wipeWeekend(at('2026-11-08T23:00:00Z')) + assert.equal(late.open, false) + assert.equal(late.final, true) + assert.equal(late.deadline.toISOString(), '2026-11-08T23:00:00.000Z') + assert.equal(wipeWeekend(at('2026-11-08T23:59:00Z')).final, true) + assert.equal(wipeWeekend(at('2026-11-09T00:00:00Z')).final, false) + assert.equal(wipeWeekend(at('2026-11-05T17:00:00Z')).final, false) +}) + +test('a Thursday that is not the first of the month is closed, as is the morning of the wipe', () => { + assert.equal(wipeWeekend(at('2026-11-12T18:00:00Z')).open, false) + assert.equal(wipeWeekend(at('2026-11-05T09:00:00Z')).open, false) + assert.equal(wipeWeekend(at('2026-11-06T12:00:00Z')).open, true) +}) + +test('a month whose first Thursday is the 7th ends its weekend on the 10th', () => { + const w = wipeWeekend(at('2026-05-09T12:00:00Z')) + assert.equal(w.wipe.toISOString(), '2026-05-07T00:00:00.000Z') + assert.equal(w.deadline.toISOString(), '2026-05-10T23:00:00.000Z') + assert.equal(w.open, true) + assert.equal(wipeWeekend(at('2026-05-10T23:00:00Z')).open, false) +}) + +test('when the 1st is the Thursday, the weekend is the 1st to the 4th', () => { + const w = wipeWeekend(at('2026-01-02T00:00:00Z')) + assert.equal(w.wipe.toISOString(), '2026-01-01T00:00:00.000Z') + assert.equal(w.deadline.toISOString(), '2026-01-04T23:00:00.000Z') + assert.equal(w.open, true) + assert.equal(wipeWeekend(at('2026-01-01T16:59:00Z')).open, false) +}) + +// ---- the manifest date, strictly + +test('manifestTime reads M/D/YYYY HH:mm:ss in 24 hours, with AM/PM and ISO fallbacks, and nothing else', () => { + assert.equal(manifestTime('10/31/2026 11:48:58'), Date.UTC(2026, 9, 31, 11, 48, 58)) + assert.equal(manifestTime('11/5/2026 13:00:00'), Date.UTC(2026, 10, 5, 13, 0, 0)) + assert.equal(manifestTime('11/5/2026 12:00:00 AM'), Date.UTC(2026, 10, 5, 0, 0, 0)) + assert.equal(manifestTime('11/5/2026 01:30:00 PM'), Date.UTC(2026, 10, 5, 13, 30, 0)) + assert.equal(manifestTime('2026-11-05T09:00:00Z'), Date.UTC(2026, 10, 5, 9, 0, 0)) + for (const bad of ['', undefined, '?', 'yesterday', '13/40/2026 01:00:00', '11/31/2026 01:00:00', '11/5/2026 25:00:00', '11/5/2026 13:00:00 PM', '5.11.2026 10:00:00', '11/5/26 10:00:00']) { + assert.ok(Number.isNaN(manifestTime(bad)), `${bad} should not parse`) + } +}) + +// ---- the gates + +test('publicChanged: the manifest must be dated the wipe day or later (UTC)', () => { + const wipe = at('2026-11-05T00:00:00Z') + assert.equal(publicChanged({ manifestDate: '11/4/2026 23:59:59' }, wipe), false) + assert.equal(publicChanged({ manifestDate: '11/5/2026 00:00:00' }, wipe), true) + assert.equal(publicChanged({ manifestDate: '10/31/2026 11:48:58' }, wipe), false) + assert.equal(publicChanged({ manifestDate: 'yesterday' }, wipe), false) + assert.equal(publicChanged(undefined, wipe), false) +}) + +test('publicGate: public must also be the build staging has now (the promotion went through)', () => { + const wipe = at('2026-11-05T00:00:00Z') + const pub = { manifest: '8140', manifestDate: '11/5/2026 11:48:58' } + assert.equal(publicGate(pub, { manifest: '8140' }, wipe), true) + assert.match(publicGate(pub, { manifest: '8150' }, wipe), /not its current staging build \(manifest 8150\)/) + assert.match(publicGate(pub, { note: 'no Assembly' }, wipe), /staging build could not be read/) + assert.match(publicGate(pub, null, wipe), /staging build could not be read/) + assert.match(publicGate({ manifest: '8100', manifestDate: '10/31/2026 11:48:58' }, { manifest: '8100' }, wipe), /from before the wipe/) + assert.match(publicGate({ manifest: '8140', manifestDate: 'garbage', note: 'n' }, { manifest: '8140' }, wipe), /no usable manifest date/) +}) + +test('oxideReady: true, or the reason', () => { + const ok = { managed: 'ok', oxideBehind: 0, compile: 'ok', fields: 'same' } + assert.equal(oxideReady(ok), true) + assert.match(oxideReady({ ...ok, oxideBehind: 12 }), /Oxide's public build is 12 member\(s\) behind/) + assert.match(oxideReady({ ...ok, fields: 'differs' }), /field list differs from public Rust's/) + assert.match(oxideReady({ ...ok, compile: 'errors' }), /does not compile/) + assert.match(oxideReady({ ...ok, managed: 'error', note: 'DepotDownloader could not be fetched' }), /could not fetch Rust's public build \(DepotDownloader/) + assert.match(oxideReady({ ...ok, oxideBehind: -1 }), /could not be compared/) + assert.match(oxideReady({ ...ok, fields: 'error' }), /could not be checked/) + assert.match(oxideReady(undefined), /could not fetch/) +}) + +const ACF = `"AppState" +{ +\t"appid"\t\t"258550" +\t"buildid"\t\t"22222222" +\t"InstalledDepots" +\t{ +\t\t"258551" +\t\t{ +\t\t\t"manifest"\t\t"111" +\t\t\t"size"\t\t"1" +\t\t} +\t\t"258552" +\t\t{ +\t\t\t"manifest"\t\t"8140123456789" +\t\t\t"size"\t\t"9000000000" +\t\t} +\t} +\t"SharedDepots" +\t{ +\t\t"228984"\t\t"228990" +\t} +} +` + +test('the appmanifest: the Linux depot\'s manifest, and null when it is not there', () => { + assert.equal(acfDepotManifest(ACF), '8140123456789') + assert.equal(acfDepotManifest(ACF, '258551'), '111') + assert.equal(acfDepotManifest(ACF, '999'), null) + assert.equal(acfDepotManifest(''), null) + assert.equal(acfDepotManifest(undefined), null) + assert.equal(acfDepotManifest('"AppState" { "InstalledDepots" { "258552" "oops" } }'), null) +}) + +test('carbonBuild: the rig must run the public manifest', () => { + assert.equal(carbonBuild(ACF, '8140123456789'), true) + assert.match(carbonBuild(ACF, '9999'), /rust-carbon still runs manifest 8140123456789 \(public is 9999\)/) + assert.match(carbonBuild(null, '9999'), /could not be read/) +}) + +const STATUS = 'framework=carbon navmesh=ready swap fields: npc=58/58 brain=32/32 missing=- added=-' + +test('carbonOk: a Carbon server with the navmesh ready and nothing missing or added', () => { + assert.equal(carbonOk(['RunicNPC 0.0.0 api=1 hooks=40 fired=3 silent=0', 'fired: x', STATUS, 'npcs=0']), true) + assert.equal(carbonOk(STATUS), true) +}) + +test('carbonOk: each way it can fail says why', () => { + assert.match(carbonOk([STATUS.replace('added=-', 'added=BasePlayer.x')]), /does not know: BasePlayer\.x/) + assert.match(carbonOk([STATUS.replace('missing=-', 'missing=BaseNpc.y,Brain.z')]), /missing on Carbon: BaseNpc\.y,Brain\.z/) + assert.match(carbonOk([STATUS.replace('framework=carbon', 'framework=oxide')]), /framework=oxide, not carbon/) + assert.match(carbonOk([STATUS.replace('navmesh=ready', 'navmesh=building')]), /navmesh=building/) + assert.match(carbonOk([]), /never answered/) + assert.match(carbonOk(['RunicNPC 0.0.0 api=1']), /never answered/) +}) + +test('liveOk: ran, harness done, some checks passed, no failed check and no finding', () => { + const good = { branch: 'public', ran: true, findings: [], harness: { done: true, pass: 266, fail: 0, failed: [] } } + assert.equal(liveOk(good), true) + assert.match(liveOk(null), /no live\.json/) + assert.match(liveOk({ ...good, ran: false, findings: ['The server did not start on Rust public 1'] }), /did not start/) + assert.match(liveOk({ ...good, ran: false, skipped: "Oxide's public build is 3 member(s) behind" }), /3 member/) + assert.match(liveOk({ ...good, harness: { done: false, pass: 5, fail: 0, failed: [] }, findings: ['The harness stopped after x'] }), /harness stopped after x/) + assert.match(liveOk({ ...good, harness: { done: true, pass: 5, fail: 1, failed: ['place.set: FAIL off the navmesh'] } }), /place\.set: FAIL off the navmesh/) + assert.match(liveOk({ ...good, findings: ['RunicNPC logged 2 error line(s) in Oxide\'s log during the run.'] }), /logged 2 error/) + assert.match(liveOk({ ...good, branch: 'staging' }), /staging branch/) + assert.match(liveOk({ ...good, harness: { done: true, pass: 0, fail: 0, failed: [] } }), /passed no check/) +}) + +test('checkState: success needs a plugin-checks context; a failure or a wait is said', () => { + const s = (context, status) => ({ context, status }) + assert.equal(checkState({ state: 'success', statuses: [s('PR Checks / plugin-checks (pull_request)', 'success')] }).state, 'success') + assert.equal(checkState({ state: 'success', statuses: [] }).state, 'pending') + assert.equal(checkState({ state: 'success', statuses: [s('other', 'success')] }).state, 'pending') + assert.equal(checkState({ state: 'pending', statuses: [s('PR Checks / plugin-checks (pull_request)', 'pending')] }).state, 'pending') + const failed = checkState({ state: 'failure', statuses: [s('PR Checks / plugin-checks (pull_request)', 'failure')] }) + assert.equal(failed.state, 'failed') + assert.match(failed.reason, /plugin-checks.*failure/) +}) + +// ---- what the PR closes, and what is written into it + +test('collectCloses reads closing keywords before #N, in any case', () => { + assert.deepEqual(collectCloses(['Closes #20', 'fixes #3', 'RESOLVED #9', 'Fixed #12.', 'close #4', 'Resolves: #8']), [3, 4, 8, 9, 12, 20]) +}) + +test('collectCloses takes several on a line, and dedupes and sorts', () => { + assert.deepEqual(collectCloses(['Closes #22, closes #20 and fixes #5', 'Closes #20\nCloses #5', 'closes #100']), [5, 20, 22, 100]) +}) + +test('collectCloses ignores references, bare numbers, prose and other repositories', () => { + assert.deepEqual(collectCloses(['Refs #21', 'see #21', '#21', 'This closes the loop on #21', 'prefix #3', 'unfixes #4', 'Closes RunicGateway/docs#9', '']), []) + assert.deepEqual(collectCloses([undefined, null]), []) +}) + +test('cutoverBody names the wipe and the build, lists a Closes line per issue, and ends with the disclosure', () => { + const body = cutoverBody({ + wipe: '2026-11-05', + manifest: '8140', + buildDate: '11/5/2026 11:48:58', + closes: [20, 22], + gates: [ + { name: '1. Rust public', status: 'passed', note: '2026-11-05 17:11 UTC' }, + { name: '2. Oxide', status: 'failed', note: 'a | b' }, + ], + }) + assert.match(body, /D332/) + assert.match(body, /manifest \*\*8140\*\*, dated 11\/5\/2026 11:48:58/) + assert.match(body, /^Closes #20$/m) + assert.match(body, /^Closes #22$/m) + assert.match(body, /\| 2\. Oxide \| failed \| a \/ b \|/) + assert.ok(body.endsWith('Opened by the runicnpc-rust cutover workflow (D332).')) + assert.match(cutoverBody({ wipe: 'w', manifest: 'm', closes: [], gates: [] }), /No issue is closed/) +}) + +test('a reason can never close an issue: #123 is broken up in the body and in comments', () => { + assert.equal(neutralise('fixes #123 and #4'), 'fixes #​123 and #​4') + const body = cutoverBody({ wipe: 'w', manifest: 'm', closes: [], gates: [{ name: 'g', status: 'failed', note: 'Closes #5: fixes #123' }] }) + assert.deepEqual(collectCloses([body]), []) + const text = commentText({ action: 'blocked', gate: 'oxide', reason: 'resolves #7' }) + assert.deepEqual(collectCloses([text]), []) + // The deliberate lines still close. + assert.deepEqual(collectCloses([cutoverBody({ wipe: 'w', manifest: 'm', closes: [9], gates: [] })]), [9]) +}) + +test('withBlock appends the marked block, replaces only what is between the markers, and is stable', () => { + const once = withBlock('A human wrote this.', 'one') + assert.equal(once, 'A human wrote this.\n\n\none\n') + const twice = withBlock(once + '\n\nAnd this after.', 'two') + assert.equal(twice, 'A human wrote this.\n\n\ntwo\n\n\nAnd this after.') + assert.equal(withBlock(twice, 'two'), twice) + assert.equal(withBlock('', 'x'), '\nx\n') +}) + +// ---- decide + +const THU = at('2026-11-05T18:11:00Z') +const obs = (more = {}) => ({ manifest: '8140', edgeSha: 'sha1', ahead: 3, openPr: null, ...more }) +const fresh = () => decide(undefined, obs(), THU) +const passed = (state, ...names) => names.reduce((s, g) => pass(s, g, THU), state) + +test('a first run starts at the public gate and asks to be written', () => { + const d = fresh() + assert.equal(d.action, 'run') + assert.equal(d.gate, 'public') + assert.equal(d.write, true) + assert.equal(d.state.wipe, '2026-11-05') + assert.equal(d.state.manifest, '8140') + assert.equal(d.state.edgeSha, 'sha1') +}) + +test('the gates run in order, checks comes last and is never kept, and then it merges', () => { + assert.deepEqual(GATES, ['public', 'oxide', 'carbon', 'live', 'checks']) + let s = fresh().state + for (const g of GATES.slice(0, 4)) { + const d = decide(s, obs(), THU) + assert.equal(d.gate, g) + s = pass(d.state, g, THU) + } + const checks = decide(s, obs(), THU) + assert.equal(checks.action, 'run') + assert.equal(checks.gate, 'checks') + const done = decide(s, obs({ checksPassed: true }), THU) + assert.equal(done.action, 'merge') + assert.equal(done.merge, true) + assert.equal(nextGate({ public: {}, carbon: {} }), 'oxide') + // A checks entry found in a stored state is dropped. + const old = decide({ ...s, gates: { ...s.gates, checks: { at: 'x' } } }, obs(), THU) + assert.equal(old.state.gates.checks, undefined) + assert.equal(old.gate, 'checks') + assert.equal(persistable({ ...s, gates: { ...s.gates, checks: {} } }, { number: 4 }).gates.checks, undefined) + assert.equal(persistable(s, { number: 4 }).pr, 4) +}) + +test('a passed gate is never run again for the same build and the same code', () => { + const s = passed(fresh().state, 'public', 'oxide', 'carbon') + const d = decide(s, obs(), at('2026-11-05T19:11:00Z')) + assert.equal(d.action, 'run') + assert.equal(d.gate, 'live') + assert.ok(d.state.gates.carbon) +}) + +test('a new public manifest starts the gates again, keeping the open PR', () => { + const s = { ...passed(fresh().state, 'public', 'oxide', 'carbon'), pr: 9 } + const d = decide(s, obs({ manifest: '8141', openPr: 9 }), at('2026-11-06T10:11:00Z')) + assert.equal(d.gate, 'public') + assert.deepEqual(d.state.gates, {}) + assert.equal(d.state.manifest, '8141') + assert.equal(d.state.pr, 9) + assert.equal(decide(s, obs({ manifest: '8141', openPr: null }), at('2026-11-06T10:11:00Z')).state.pr, null) +}) + +test('a new edge commit clears every gate and every cooled failure, so nothing proven on old code counts', () => { + const s = fail(passed(fresh().state, 'public', 'oxide'), 'carbon', 'x', THU) + const d = decide({ ...s, mergeError: 'old' }, obs({ edgeSha: 'sha2' }), at('2026-11-05T19:11:00Z')) + assert.equal(d.action, 'run') + assert.equal(d.gate, 'public') + assert.deepEqual(d.state.gates, {}) + assert.deepEqual(d.state.failed, {}) + assert.equal(d.state.edgeSha, 'sha2') + assert.equal(d.state.mergeError, undefined) + // A state written before edgeSha was kept counts as changed code. + const legacy = { wipe: '2026-11-05', manifest: '8140', gates: { public: { at: 'x' } } } + assert.equal(decide(legacy, obs(), THU).gate, 'public') +}) + +test('a new wipe forgets everything, comments and a give-up included', () => { + const s = { ...passed(fresh().state, 'public'), lastComment: '2026-11-05', gaveUp: true } + const d = decide(s, obs(), at('2026-12-03T18:00:00Z')) + assert.equal(d.action, 'run') + assert.equal(d.state.wipe, '2026-12-03') + assert.deepEqual(d.state.gates, {}) + assert.equal(d.state.lastComment, undefined) + assert.equal(d.state.gaveUp, undefined) +}) + +test('nothing to do when edge has no commit main lacks, or when it already merged', () => { + const d = decide(undefined, obs({ ahead: 0 }), THU) + assert.equal(d.action, 'nothing') + assert.match(d.reason, /no commits/) + assert.equal(decide({ ...fresh().state, merged: true }, obs(), THU).action, 'nothing') +}) + +test('the cool-down: a failed carbon or live is not tried again for three hours, a new commit retries at once', () => { + assert.equal(COOL_DOWN_HOURS, 3) + const base = passed(fresh().state, 'public', 'oxide') + const failedCarbon = fail(base, 'carbon', 'did not come up', THU) // failed at 18:11 + const at59 = decide(failedCarbon, obs(), at('2026-11-05T21:10:00Z')) // 2 h 59 min later + assert.equal(at59.action, 'cooling') + assert.equal(at59.gate, 'carbon') + assert.equal(at59.retryAt.toISOString(), '2026-11-05T21:11:00.000Z') + assert.match(at59.reason, /did not come up/) + assert.equal(decide(failedCarbon, obs(), at('2026-11-05T21:11:00Z')).action, 'run') // 3 h 00 + assert.equal(decide(failedCarbon, obs(), at('2026-11-05T21:11:00Z')).gate, 'carbon') + assert.equal(decide(failedCarbon, obs({ edgeSha: 'sha2' }), at('2026-11-05T19:00:00Z')).gate, 'public') + assert.equal(decide(failedCarbon, obs({ manifest: '8141' }), at('2026-11-05T19:00:00Z')).gate, 'public') + // The same rule for live; public and oxide are cheap and never cool. + const failedLive = fail(passed(base, 'carbon'), 'live', 'a check failed', THU) + assert.equal(decide(failedLive, obs(), at('2026-11-05T20:00:00Z')).action, 'cooling') + assert.equal(decide(failedLive, obs(), at('2026-11-05T21:11:00Z')).gate, 'live') + assert.equal(decide(fail(fresh().state, 'oxide', 'x', THU), obs(), at('2026-11-05T18:30:00Z')).gate, 'public') + // A pass forgets the failure. + assert.equal(pass(failedCarbon, 'carbon', THU).failed.carbon, undefined) +}) + +test('a cooling gate comments once a day and says when it will try again', () => { + const s = fail(passed(fresh().state, 'public', 'oxide'), 'carbon', 'did not come up', THU) + const d = decide(s, obs({ openPr: 9 }), at('2026-11-05T19:11:00Z')) + assert.equal(d.comment, true) + assert.match(commentText(d, d.state), /not try this gate again before 2026-11-05 21:11 UTC/) + assert.match(commentText(d, d.state), /3\. A Carbon boot/) + assert.equal(decide(d.state, obs({ openPr: 9 }), at('2026-11-05T20:11:00Z')).comment, false) +}) + +test('a blocked gate comments once a day, and only after the public gate passed', () => { + const blocked = { gate: 'oxide', reason: "Oxide's public build is 12 member(s) behind" } + const s = passed(fresh().state, 'public') + const first = decide(s, obs({ openPr: 9, blocked }), THU) + assert.equal(first.action, 'blocked') + assert.equal(first.comment, true) + assert.equal(first.state.lastComment, '2026-11-05') + assert.equal(decide(first.state, obs({ openPr: 9, blocked }), at('2026-11-05T19:11:00Z')).comment, false) + assert.equal(decide(first.state, obs({ openPr: 9, blocked }), at('2026-11-06T09:11:00Z')).comment, true) + assert.equal(decide(fresh().state, obs({ openPr: 9, blocked: { gate: 'public', reason: 'old' } }), THU).comment, false) + assert.equal(decide(s, obs({ openPr: null, blocked }), THU).comment, false) + // A blocked carbon says when it will be tried again. + const carbon = decide(s, obs({ openPr: 9, blocked: { gate: 'carbon', reason: 'x' } }), THU) + assert.equal(carbon.retryAt.toISOString(), '2026-11-05T21:11:00.000Z') +}) + +test('after the deadline it gives up once, with a last comment', () => { + const s = passed(fresh().state, 'public', 'oxide') + const late = at('2026-11-08T23:11:00Z') + const d = decide(s, obs({ openPr: 9 }), late) + assert.equal(d.action, 'giveUp') + assert.equal(d.gate, 'carbon') + assert.equal(d.comment, true) + assert.equal(d.state.gaveUp, true) + assert.match(commentText(d, d.state), /Gate "3\. A Carbon boot on public Rust" had not passed/) + const again = decide(d.state, obs({ openPr: 9 }), at('2026-11-08T23:40:00Z')) + assert.equal(again.action, 'nothing') + assert.equal(again.comment, false) + assert.equal(decide(s, obs({ openPr: null }), late).comment, false) + assert.equal(decide(s, obs({ openPr: 9, forced: true }), late).action, 'run') +}) + +test('giving up with every gate passed and the merge failed says so, never gate "undefined"', () => { + const s = { ...passed(fresh().state, 'public', 'oxide', 'carbon', 'live'), mergeError: 'POST /pulls/5/merge: 405 not mergeable' } + const d = decide(s, obs({ openPr: 5 }), at('2026-11-08T23:11:00Z')) + assert.equal(d.action, 'giveUp') + assert.equal(d.gate, undefined) + const text = commentText(d, d.state) + assert.match(text, /Every gate passed; the merge failed: POST \/pulls\/5\/merge: 405 not mergeable/) + assert.ok(!text.includes('undefined')) + assert.ok(!commentText(d, { ...d.state, mergeError: undefined }).includes('undefined')) +}) + +test('a dry run never asks to write, comment or merge', () => { + const dry = { dryRun: true } + assert.equal(decide(undefined, obs(), THU, dry).write, false) + const s = passed(fresh().state, 'public') + const b = decide(s, obs({ openPr: 9, blocked: { gate: 'oxide', reason: 'x' } }), THU, dry) + assert.equal(b.comment, false) + assert.equal(b.write, false) + assert.equal(b.state.lastComment, undefined) + const m = decide(passed(fresh().state, ...GATES.slice(0, 4)), obs({ checksPassed: true }), THU, dry) + assert.equal(m.action, 'merge') + assert.equal(m.merge, false) + const g = decide(s, obs({ openPr: 9 }), at('2026-11-08T23:11:00Z'), dry) + assert.equal(g.action, 'giveUp') + assert.equal(g.comment, false) + assert.equal(decide(undefined, obs({ ahead: 0 }), THU, dry).action, 'run') +}) + +test('a dispatch is dry unless its dry_run input is exactly false', () => { + const dispatch = (input) => ({ GITHUB_EVENT_NAME: 'workflow_dispatch', DRY_RUN_INPUT: input }) + assert.equal(isDry({}, dispatch(undefined)), true) + assert.equal(isDry({}, dispatch('')), true) + assert.equal(isDry({}, dispatch('true')), true) + assert.equal(isDry({}, dispatch('maybe')), true) + assert.equal(isDry({}, dispatch('false')), false) + assert.equal(isDry({}, { GITHUB_EVENT_NAME: 'schedule' }), false) + assert.equal(isDry({ 'dry-run': true }, { GITHUB_EVENT_NAME: 'schedule' }), true) +}) + +// ---- the commands, against a fake Gitea and a fake panel + +const GIT = (...a) => ({ 'rev-parse': 'sha1', 'rev-list': '3', 'merge-base': 'mb', log: '2026-11-01T00:00:00+00:00' }[a[0]] || '') + +/** deps with a fake fetch: handler(method, path, body) -> { status, json }; every call is recorded. */ +function fakeDeps({ now = '2026-11-05T18:11:00Z', handler, git = GIT, panel, env = {} }) { + const deps = { + calls: [], + at: now, + env: { GITHUB_SERVER_URL: 'http://gitea.test', GITHUB_REPOSITORY: 'o/r', GITEA_TOKEN: 'reg-token', MERGE_TOKEN: 'merge-token', ...env }, + now: () => new Date(deps.at), + sleep: async () => {}, + git: (...a) => git(...a), + panel, + fetch: async (url, o) => { + const p = url.replace('http://gitea.test/api/v1/repos/o/r', '') + const body = o.body ? JSON.parse(o.body) : undefined + deps.calls.push({ method: o.method, p, body, auth: o.headers.Authorization }) + const r = handler(o.method, p, body) || {} + const status = r.status || 200 + return { ok: status < 400, status, text: async () => (r.json === undefined ? '' : JSON.stringify(r.json)) } + }, + } + return deps +} + +const STAT = { branch: 'public', manifest: '8140', manifestDate: '11/5/2026 11:48:58', managed: 'ok', oxideBehind: 0, fields: 'same', compile: 'ok' } + +function rig(stat = STAT, staging = { branch: 'staging', manifest: '8140' }) { + const dir = tmp() + fs.mkdirSync(path.join(dir, 'work')) + fs.mkdirSync(path.join(dir, 'work-staging')) + fs.writeFileSync(path.join(dir, 'work', 'static.json'), JSON.stringify(stat)) + fs.writeFileSync(path.join(dir, 'work-staging', 'static.json'), JSON.stringify(staging)) + const state = path.join(dir, 'drill.json') + fs.writeFileSync(state, '{}') + const a = { state, work: path.join(dir, 'work'), 'staging-work': path.join(dir, 'work-staging') } + return { dir, a, state, read: () => JSON.parse(fs.readFileSync(state, 'utf8')) } +} + +/** A Gitea with one edge to main PR that exists once POSTed. */ +function fakeGitea() { + const g = { pr: null, comments: [] } + g.handler = (method, p, body) => { + if (method === 'GET' && p.startsWith('/pulls?state=open')) return { json: g.pr ? [g.pr] : [] } + if (method === 'GET' && p.startsWith('/pulls?state=closed')) return { json: [] } + if (method === 'POST' && p === '/pulls') { + if (g.failCreate) return { status: 500, json: { message: 'boom' } } + g.pr = { number: 5, head: { ref: 'edge', sha: 'sha1' }, base: { ref: 'main' }, ...body, head_: undefined } + g.pr.head = { ref: 'edge', sha: 'sha1' } + g.pr.base = { ref: 'main' } + return { json: g.pr } + } + if (method === 'PATCH' && p === '/pulls/5') { + g.pr = { ...g.pr, ...body } + return { json: g.pr } + } + if (method === 'POST' && p === '/issues/5/comments') { + g.comments.push(body.body) + return { json: {} } + } + return { json: [] } + } + return g +} + +test('gates in a dry run creates, edits, comments and writes nothing', async () => { + const r = rig() + const g = fakeGitea() + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { throw new Error('panel down') } }) }) + const d = await c.gates({ ...r.a, 'dry-run': true }, deps) + assert.ok(deps.calls.length > 0) + assert.ok(deps.calls.every((x) => x.method === 'GET'), JSON.stringify(deps.calls.filter((x) => x.method !== 'GET'))) + assert.equal(fs.readFileSync(r.state, 'utf8'), '{}') + assert.equal(d.comment, false) + assert.equal(g.pr, null) + assert.equal(g.comments.length, 0) + // It ran past the failed carbon gate to try the others. + const run = JSON.parse(fs.readFileSync(path.join(r.dir, 'work', 'cutover-run.json'), 'utf8')) + assert.equal(run.rows.public.status, 'passed') + assert.equal(run.rows.oxide.status, 'passed') + assert.equal(run.rows.carbon.status, 'failed') + assert.equal(run.rows.checks.status, 'failed') +}) + +test('a workflow_dispatch without dry_run=false is dry even without the flag', async () => { + const r = rig() + const g = fakeGitea() + const deps = fakeDeps({ handler: g.handler, env: { GITHUB_EVENT_NAME: 'workflow_dispatch' }, panel: () => ({ servers: async () => { throw new Error('down') } }) }) + await c.gates(r.a, deps) + assert.ok(deps.calls.every((x) => x.method === 'GET')) + assert.equal(fs.readFileSync(r.state, 'utf8'), '{}') +}) + +test('a real run keeps each gate, opens the PR, comments once, and cools a failed carbon for three hours', async () => { + const r = rig() + const g = fakeGitea() + let panelCalls = 0 + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { panelCalls++; throw new Error('panel down') } }) }) + + await c.gates(r.a, deps) // 18:11 + let s = r.read().cutover + assert.equal(s.edgeSha, 'sha1') + assert.deepEqual(Object.keys(s.gates).sort(), ['oxide', 'public']) + assert.match(s.failed.carbon.reason, /panel down/) + assert.equal(s.pr, 5) + assert.equal(s.lastComment, '2026-11-05') + assert.equal(panelCalls, 1) + assert.ok(g.pr.title.startsWith('chore(release): RunicNPC cutover for the 2026-11-05 wipe, Rust 8140')) + assert.ok(g.pr.body.includes('')) + assert.equal(g.comments.length, 1) + assert.match(g.comments[0], /not try this gate again before 2026-11-05 21:11 UTC/) + assert.ok(deps.calls.filter((x) => x.method !== 'GET').every((x) => x.auth === 'token reg-token')) + + deps.at = '2026-11-05T19:11:00Z' + await c.gates(r.a, deps) + assert.equal(panelCalls, 1, 'cooling: the rig is not touched') + deps.at = '2026-11-05T21:10:00Z' + await c.gates(r.a, deps) + assert.equal(panelCalls, 1, '2 h 59 min') + assert.equal(g.comments.length, 1, 'one comment a day') + deps.at = '2026-11-05T21:11:00Z' + await c.gates(r.a, deps) + assert.equal(panelCalls, 2, '3 h 00 min: it tries again') + + // New code: everything again, at once. + deps.git = (...a) => (a[0] === 'rev-parse' ? 'sha2' : GIT(...a)) + deps.at = '2026-11-05T21:30:00Z' + await c.gates(r.a, deps) + s = r.read().cutover + assert.equal(panelCalls, 3) + assert.equal(s.edgeSha, 'sha2') + assert.ok(s.gates.public && s.gates.oxide) +}) + +test('a PR call that fails never loses a gate that passed', async () => { + const r = rig() + const g = fakeGitea() + g.failCreate = true + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { throw new Error('panel down') } }) }) + await c.gates(r.a, deps) + const s = r.read().cutover + assert.ok(s.gates.public) + assert.ok(s.gates.oxide) + assert.equal(s.pr, null) +}) + +test('gate 1 blocks, and nothing else runs, when public is not what staging has', async () => { + const r = rig(STAT, { branch: 'staging', manifest: '8999' }) + const g = fakeGitea() + let panelCalls = 0 + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { panelCalls++; throw new Error('x') } }) }) + await c.gates(r.a, deps) + const s = r.read().cutover + assert.deepEqual(s.gates, {}) + assert.equal(panelCalls, 0) + assert.equal(g.pr, null) +}) + +test('checksGate refuses a PR whose head is not the commit the gates ran on', async () => { + const deps = fakeDeps({ handler: (m, p) => (p === '/pulls/3' ? { json: { number: 3, head: { sha: 'zzzzzzzz1' } } } : { json: {} }) }) + const ctx = { g: c.gitea('t', deps), deps, dryRun: false, edgeSha: 'aaaaaaaa1', pr: { number: 3 } } + assert.match(await c.checksGate(ctx), /head zzzzzzzz is not the commit the gates ran on \(aaaaaaaa\)/) + assert.ok(!deps.calls.some((x) => x.p.includes('/status'))) +}) + +test('merge sends Do=merge, keeps edge, names the head commit, and verifies it merged', async () => { + let merged = false + const deps = fakeDeps({ + handler: (method, p, body) => { + if (method === 'POST' && p === '/pulls/7/merge') { + merged = true + return {} + } + if (method === 'GET' && p === '/pulls/7') return { json: { number: 7, head: { sha: 'abc123' }, merged } } + return {} + }, + }) + await c.merge({ pr: '7', sha: 'abc123' }, deps) + const post = deps.calls.find((x) => x.method === 'POST') + assert.deepEqual(post.body, { Do: 'merge', delete_branch_after_merge: false, head_commit_id: 'abc123' }) + assert.equal(post.auth, 'token merge-token') + assert.equal(deps.calls[deps.calls.length - 1].method, 'GET') +}) + +test('merge refuses a moved head, and a merge that did not happen, and records why', async () => { + const r = rig() + fs.writeFileSync(r.state, JSON.stringify({ cutover: { wipe: '2026-11-05' } })) + const moved = fakeDeps({ handler: (m, p) => (p === '/pulls/7' ? { json: { head: { sha: 'other' }, merged: false } } : {}) }) + await assert.rejects(c.merge({ pr: '7', sha: 'abc123', state: r.state }, moved), /not the commit the gates ran on/) + assert.ok(!moved.calls.some((x) => x.method === 'POST')) + assert.match(r.read().cutover.mergeError, /not the commit the gates ran on/) + assert.equal(r.read().cutover.wipe, '2026-11-05') + + const unmerged = fakeDeps({ handler: (m, p) => (p === '/pulls/7' ? { json: { head: { sha: 'abc123' }, merged: false } } : {}) }) + await assert.rejects(c.merge({ pr: '7', sha: 'abc123' }, unmerged), /not merged after the merge call/) + const refused = fakeDeps({ handler: (m, p) => (m === 'POST' ? { status: 409, json: { message: 'head changed' } } : { json: { head: { sha: 'abc123' } } }) }) + await assert.rejects(c.merge({ pr: '7', sha: 'abc123' }, refused), /409/) + await assert.rejects(c.merge({ pr: '7' }, refused), /--sha/) +}) + +test('after records the merge, comments the release on each closed issue once, and is safe to repeat', async () => { + const r = rig() + fs.writeFileSync(r.state, JSON.stringify({ cutover: { wipe: '2026-11-05', mergeError: 'old', gates: {} } })) + const posted = {} + const deps = fakeDeps({ + handler: (method, p, body) => { + if (p === '/pulls/5') return { json: { number: 5, merged: true, merged_at: '2026-11-05T20:00:00Z', body: 'Closes #20\nCloses #22' } } + if (p.startsWith('/releases')) return { json: [{ tag_name: 'v1.2.0', created_at: '2026-11-05T20:05:00Z' }, { tag_name: 'v1.1.0', created_at: '2026-10-01T00:00:00Z' }] } + if (p === '/issues/20') return { json: { state: 'closed' } } + if (p === '/issues/22') return { json: { state: 'open' } } + if (method === 'GET' && p === '/issues/20/comments') return { json: (posted[20] || []).map((b) => ({ body: b })) } + if (method === 'POST' && p === '/issues/20/comments') { + posted[20] = [...(posted[20] || []), body.body] + return {} + } + return { json: [] } + }, + }) + await c.after({ state: r.state, pr: '5' }, deps) + await c.after({ state: r.state, pr: '5' }, deps) + assert.deepEqual(posted[20], ['Released in v1.2.0 by the wipe cutover (D332).']) + assert.equal(deps.calls.filter((x) => x.method === 'POST').length, 1) + assert.ok(!deps.calls.some((x) => x.p.startsWith('/issues/22/comments'))) + assert.equal(r.read().cutover.merged, true) + assert.equal(r.read().cutover.mergeError, undefined) +}) + +test('the Closes list is the PRs merged into edge after the newest edge commit already in main', async () => { + const prs = [ + { merged: true, base: { ref: 'edge' }, merged_at: '2026-10-20T00:00:00Z', body: 'Closes #1' }, + { merged: true, base: { ref: 'edge' }, merged_at: '2026-10-25T00:00:00Z', body: 'Closes #2, fixes #6' }, + { merged: true, base: { ref: 'main' }, merged_at: '2026-10-26T00:00:00Z', body: 'Closes #3' }, + { merged: false, base: { ref: 'edge' }, merged_at: null, body: 'Closes #4' }, + { merged: true, base: { ref: 'edge' }, merged_at: '2026-10-30T00:00:00Z', body: 'Refs #5' }, + ] + const asked = [] + const deps = fakeDeps({ + handler: (m, p) => (p.startsWith('/pulls?state=closed') ? { json: prs } : {}), + git: (...a) => { + asked.push(a.join(' ')) + if (a[0] === 'merge-base') return 'deadbeef' + if (a[0] === 'log') return '2026-10-22T09:00:00+00:00' + return '' + }, + }) + assert.deepEqual(await c.closesSinceCutOff(c.gitea('t', deps), deps), [2, 6]) + assert.ok(asked.includes('merge-base origin/main origin/edge')) + assert.ok(asked.includes('log -1 --format=%cI deadbeef')) +}) + +test('rust-carbon is started again only when it was running at the start and is offline now, and nothing else is touched', async () => { + const dir = tmp() + const powered = [] + const mkPanel = (state) => () => ({ + servers: async () => ({ 'rust-carbon': 'abc', 'rust-oxide': 'def' }), + state: async () => state, + power: async (id, signal) => powered.push([id, signal]), + }) + await c.rigNote({ work: dir }, { panel: mkPanel('running') }) + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'rig-carbon.json'), 'utf8')), { name: 'rust-carbon', state: 'running' }) + await c.rigRestore({ work: dir }, { panel: mkPanel('running') }) + await c.rigRestore({ work: dir }, { panel: mkPanel('starting') }) + assert.deepEqual(powered, []) + await c.rigRestore({ work: dir }, { panel: mkPanel('offline') }) + assert.deepEqual(powered, [['abc', 'start']]) + // It was off at the start: left off. + await c.rigNote({ work: dir }, { panel: mkPanel('offline') }) + await c.rigRestore({ work: dir }, { panel: mkPanel('offline') }) + assert.equal(powered.length, 1) +}) + +// ---- the workflow and the scripts agree + +const yml = fs.readFileSync(path.join(__dirname, '..', '..', '.gitea', 'workflows', 'cutover.yml'), 'utf8') +const src = fs.readFileSync(path.join(__dirname, 'cutover.js'), 'utf8') + +function stepsOf(text) { + return text + .split(/\n {6}- (?=name:|uses:)/) + .slice(1) + .map((b) => ({ id: (/^\s+id: (\w+)/m.exec(b) || [])[1], command: (/cutover\.js (\S+)/.exec(b) || /drill\.js (\S+)/.exec(b) || [])[1], script: /drill\.js/.test(b) && !/cutover\.js/.test(b) ? 'drill' : 'cutover', text: b })) +} + +const FUNCTIONS = { window: 'window_', pending: 'pending', gates: 'gates', merge: 'merge', after: 'after' } + +function written(command) { + const m = new RegExp(`\\n(?:async )?function ${FUNCTIONS[command]}\\(`).exec(src) + assert.ok(m, `cutover.js has no function for ${command}`) + const body = src.slice(m.index, src.indexOf('\n}\n', m.index)) + return new Set([...body.matchAll(/output\([^,]+,\s*'(\w+)'/g)].map((x) => x[1])) +} + +test('every steps..outputs. the workflow reads is written by the command that step runs', () => { + const steps = stepsOf(yml) + const used = [...yml.matchAll(/steps\.(\w+)\.outputs\.(\w+)/g)].map((m) => [m[1], m[2]]) + assert.ok(used.length >= 8) + for (const [id, name] of used) { + const step = steps.find((s) => s.id === id) + assert.ok(step, `steps.${id} is not a step of cutover.yml`) + assert.equal(step.script, 'cutover', `steps.${id}.outputs.${name}: ${id} runs drill.js, which writes no output the workflow reads`) + assert.ok(written(step.command).has(name), `cutover.js ${step.command} never writes output ${name} (read as steps.${id}.outputs.${name})`) + } +}) + +test('and the outputs the logic depends on are all read by the workflow', () => { + const steps = stepsOf(yml) + const needed = { window: ['open'], pending: ['run'], gates: ['live', 'merge', 'pr', 'sha'], merge: ['merged'] } + for (const [command, names] of Object.entries(needed)) { + const ids = steps.filter((s) => s.script === 'cutover' && s.command === command && s.id).map((s) => s.id) + assert.ok(ids.length > 0, `no step with an id runs cutover.js ${command}`) + const read = new Set([...yml.matchAll(/steps\.(\w+)\.outputs\.(\w+)/g)].filter((m) => ids.includes(m[1])).map((m) => m[2])) + for (const n of names) { + assert.ok(read.has(n), `cutover.yml never reads ${n} from the ${command} step`) + assert.ok(written(command).has(n), `cutover.js ${command} no longer writes ${n}`) + } + } +}) + +test('RUNICNPC_TOKEN is in the merge step alone, and a dispatch is dry unless dry_run is exactly false', () => { + const uses = yml.split('\n').filter((l) => !l.trim().startsWith('#') && l.includes('RUNICNPC_TOKEN')) + assert.equal(uses.length, 1) + const merge = stepsOf(yml).find((s) => s.text.split('\n').some((l) => !l.trim().startsWith('#') && l.includes('RUNICNPC_TOKEN'))) + assert.equal(merge.id, 'merge') + assert.match(merge.text, /MERGE_TOKEN: \$\{\{ secrets\.RUNICNPC_TOKEN \}\}/) + const code = yml.split('\n').filter((l) => !l.trim().startsWith('#')).join('\n') + assert.ok(!/dry_run\s*[=!]=\s*'true'/.test(code), 'dry_run must be compared with false, never true') + assert.ok((code.match(/dry_run != 'false'/g) || []).length >= 6) + for (const key of ['LC_ALL: C', "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1'"]) assert.equal(code.split(key).length - 1, 2, `${key} on both static halves`) + assert.match(code, /timeout-minutes: 180/) +}) diff --git a/tools/drill/drill.js b/tools/drill/drill.js index 1ea36d3..1ebeab7 100644 --- a/tools/drill/drill.js +++ b/tools/drill/drill.js @@ -649,8 +649,9 @@ async function fileIssue(title, body) { // ---------------------------------------------------------------- main -// The pure parts, for drill.test.js; run as a script, it is the command line. -module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS } +// The pure parts, for drill.test.js; run as a script, it is the command line. panel and waitRunning serve +// cutover.js (D332), and report is exported so a test can show it keeps the state keys it does not own. +module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS, panel, waitRunning, report } if (require.main === module) { const a = args() diff --git a/tools/drill/drill.test.js b/tools/drill/drill.test.js index 73ad312..dfbfe85 100644 --- a/tools/drill/drill.test.js +++ b/tools/drill/drill.test.js @@ -65,3 +65,23 @@ test("Kits' file is replaced, not deleted, by one holding only the harness's kit assert.equal(kits.rnhrevolver.IsHidden, true) assert.deepEqual(kits.rnhrevolver.BeltItems.map((i) => i.Shortname), ['pistol.revolver']) }) + +test('report keeps the state keys it does not own (the cutover, D332) and writes its own', async () => { + const fs = require('fs') + const os = require('os') + const path = require('path') + const { report } = require('./drill.js') + delete process.env.GITHUB_STEP_SUMMARY + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'drill-report-')) + const cutover = { wipe: '2026-11-05', manifest: '1', gates: { public: { at: '2026-11-05T17:11:00.000Z' } }, pr: 7 } + fs.writeFileSync(path.join(dir, 'drill.json'), JSON.stringify({ cutover })) + const stat = { branch: 'staging', manifest: '42', manifestDate: '10/31/2026 11:48:58', managed: 'ok', oxideBehind: 0, fields: 'same', compile: 'ok' } + fs.writeFileSync(path.join(dir, 'static.json'), JSON.stringify(stat)) + fs.writeFileSync(path.join(dir, 'live.json'), JSON.stringify({ branch: 'staging', ran: true, findings: [], harness: { done: true, pass: 3, fail: 0, failed: [] } })) + await report({ state: path.join(dir, 'drill.json'), work: dir }) + const after = JSON.parse(fs.readFileSync(path.join(dir, 'drill.json'), 'utf8')) + assert.deepEqual(after.cutover, cutover) + assert.equal(after.checked.manifest, '42') + assert.equal(after.lastRun.outcome, 'pass') + fs.rmSync(dir, { recursive: true, force: true }) +}) -- 2.49.1 From 29226ab450dcebded91d4583479c53773ac8b138 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Mon, 2 Nov 2026 07:08:47 -0600 Subject: [PATCH 8/9] ci(cutover): no Closes listed twice, room for the worst case, the checkout pinned (D332) From the second review: - A PR whose merge commit main already has is not listed again. edge's tip at a cutover is often that PR's merge commit, and its merged_at can be a moment after the cut-off's committer date, so its issues would have had the next release's comment too. - timeout-minutes 240: the worst case is about 200, and a timeout must never cut off the step that starts rust-carbon again. - A test pins the checkout: edge, except on a dry dispatch that names code_ref. Dry run (CI run 54, before these): every gate refused for the right reason. Public Rust is still the 22 October build, which lacks the three fields edge carries for November, and gates 1 to 4 each saw it; the panel key wrote to, restarted and read rust-carbon; the live half ran 266 pass, 0 fail; nothing was opened, edited, commented or merged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY --- .gitea/workflows/cutover.yml | 5 +++-- tools/drill/cutover.js | 12 +++++++++++- tools/drill/cutover.test.js | 27 ++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cutover.yml b/.gitea/workflows/cutover.yml index 06a7c94..4ec300c 100644 --- a/.gitea/workflows/cutover.yml +++ b/.gitea/workflows/cutover.yml @@ -92,8 +92,9 @@ env: jobs: cutover: runs-on: ubuntu-latest - # Carbon's restart and navmesh, the live half (about an hour), the PR checks. - timeout-minutes: 180 + # Carbon's restart and navmesh, the live half (about an hour), the PR checks: about 100 minutes, 200 at the + # worst. Room for that, so a timeout never cuts off the step that starts rust-carbon again. + timeout-minutes: 240 env: WORK: ${{ github.workspace }}/cutover-work STAGING_WORK: ${{ github.workspace }}/cutover-staging diff --git a/tools/drill/cutover.js b/tools/drill/cutover.js index 85d8746..0e8e13d 100644 --- a/tools/drill/cutover.js +++ b/tools/drill/cutover.js @@ -451,11 +451,21 @@ async function closesSinceCutOff(g, deps) { try { since = Date.parse(deps.git('log', '-1', '--format=%cI', deps.git('merge-base', 'origin/main', 'origin/edge'))) || 0 } catch {} + // A PR whose merge commit main already has shipped in an earlier cutover, even when its merged_at is a moment + // after the cut-off's committer date (edge's tip is often that very merge commit). + const shipped = (sha) => { + if (!sha) return false + try { + return /\borigin\/main\b/.test(deps.git('branch', '-r', '--contains', sha, '--list', 'origin/main')) + } catch { + return false + } + } const bodies = [] for (let page = 1; page <= 10; page++) { const prs = await g.call('GET', `/pulls?state=closed&limit=50&page=${page}`) for (const pr of prs) { - if (pr.merged && pr.base && pr.base.ref === 'edge' && Date.parse(pr.merged_at) > since) bodies.push(pr.body) + if (pr.merged && pr.base && pr.base.ref === 'edge' && Date.parse(pr.merged_at) > since && !shipped(pr.merge_commit_sha)) bodies.push(pr.body) } if (prs.length < 50) break } diff --git a/tools/drill/cutover.test.js b/tools/drill/cutover.test.js index 89ea734..5cda7b5 100644 --- a/tools/drill/cutover.test.js +++ b/tools/drill/cutover.test.js @@ -676,6 +676,24 @@ test('the Closes list is the PRs merged into edge after the newest edge commit a assert.ok(asked.includes('log -1 --format=%cI deadbeef')) }) +test('a PR whose merge commit main already has is not listed again, though merged a moment after the cut-off', async () => { + const prs = [ + // edge's tip at the last cutover: its merged_at is a second after the merge commit's committer date. + { merged: true, base: { ref: 'edge' }, merged_at: '2026-11-05T19:00:01Z', merge_commit_sha: 'shipped1', body: 'Closes #20' }, + { merged: true, base: { ref: 'edge' }, merged_at: '2026-11-20T00:00:00Z', merge_commit_sha: 'new1', body: 'Closes #30' }, + ] + const deps = fakeDeps({ + handler: (m, p) => (p.startsWith('/pulls?state=closed') ? { json: prs } : {}), + git: (...a) => { + if (a[0] === 'merge-base') return 'shipped1' + if (a[0] === 'log') return '2026-11-05T19:00:00+00:00' + if (a[0] === 'branch') return a[3] === 'shipped1' ? ' origin/main' : '' + return '' + }, + }) + assert.deepEqual(await c.closesSinceCutOff(c.gitea('t', deps), deps), [30]) +}) + test('rust-carbon is started again only when it was running at the start and is offline now, and nothing else is touched', async () => { const dir = tmp() const powered = [] @@ -744,6 +762,13 @@ test('and the outputs the logic depends on are all read by the workflow', () => } }) +test('the code checked out is edge, except on a dry dispatch that names code_ref', () => { + const refs = yml.split('\n').filter((l) => /^\s*ref:/.test(l)).map((l) => l.trim()) + assert.deepEqual(refs, [ + "ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false' && github.event.inputs.code_ref || 'edge' }}", + ]) +}) + test('RUNICNPC_TOKEN is in the merge step alone, and a dispatch is dry unless dry_run is exactly false', () => { const uses = yml.split('\n').filter((l) => !l.trim().startsWith('#') && l.includes('RUNICNPC_TOKEN')) assert.equal(uses.length, 1) @@ -754,5 +779,5 @@ test('RUNICNPC_TOKEN is in the merge step alone, and a dispatch is dry unless dr assert.ok(!/dry_run\s*[=!]=\s*'true'/.test(code), 'dry_run must be compared with false, never true') assert.ok((code.match(/dry_run != 'false'/g) || []).length >= 6) for (const key of ['LC_ALL: C', "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1'"]) assert.equal(code.split(key).length - 1, 2, `${key} on both static halves`) - assert.match(code, /timeout-minutes: 180/) + assert.match(code, /timeout-minutes: 240/) }) -- 2.49.1 From e5548472dac9f14833c09ddba4ab4e4fcf5e2a86 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Fri, 6 Nov 2026 04:37:50 -0600 Subject: [PATCH 9/9] fix(cutover): gate 1 asks Steam when public was set, not the manifest's date or staging (D332) The 2026-11-05 weekend never cut over: gate 1 blocked every hourly run. Both of its checks rested on assumptions the wipe disproved: - The wipe build went live Thursday evening, but its depot manifest is dated 11-04: the date DepotDownloader prints is when Facepunch built it. - Public never names staging's manifest. Facepunch promotes a hidden `release` branch ("Release, before public"), and staging had moved on before the wipe. Gate 1 now reads Steam's app info (steamcmd +app_info_print 258550, kept by static.sh as appinfo.txt with BRANCH=public): branches.public.timeupdated must be on the wipe day or later, and the Linux depot manifest Steam names for public must be the one the static half downloaded. The staging static half is gone from edge's cutover.yml; --staging-work is still accepted, since main's copy passes it until the cutover merge carries the new wiring over. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014HJEmgsXtLR5kLfh7f9y8P --- .gitea/workflows/cutover.yml | 24 ++--- tools/drill/cutover.js | 94 ++++++++++---------- tools/drill/cutover.test.js | 165 ++++++++++++++++++++++++++--------- tools/drill/static.sh | 38 +++++++- 4 files changed, 213 insertions(+), 108 deletions(-) diff --git a/.gitea/workflows/cutover.yml b/.gitea/workflows/cutover.yml index 4ec300c..4181a9e 100644 --- a/.gitea/workflows/cutover.yml +++ b/.gitea/workflows/cutover.yml @@ -6,10 +6,10 @@ # gates have passed, cheapest first (the decisions are pure functions in tools/drill/cutover.js, tested # by cutover.test.js, which PR Checks runs): # -# 1. public Rust's public build is dated the wipe day or later AND names the same depot manifest as -# Rust's current STAGING build (Facepunch promotes staging to public, so the two agree once the -# cut over from staging to live went through). The static half runs twice, BRANCH=public and -# BRANCH=staging; only the staging run's manifest is used. +# 1. public Steam set Rust's public branch to its build on the wipe day or later (steamcmd's app info, +# branches.public.timeupdated, which the static half keeps as appinfo.txt) AND the Linux depot +# manifest Steam names for public is the one the static half downloaded. (The first design used +# the manifest's date and staging's manifest; neither works, see tools/drill/cutover.js.) # 2. oxide Oxide's public build has caught up (0 members behind), the swap's field list is unchanged # and RunicNPC compiles. # 3. carbon `rust-carbon` restarts on public Rust with edge's RunicNPC.cs: its appmanifest names the @@ -97,7 +97,6 @@ jobs: timeout-minutes: 240 env: WORK: ${{ github.workspace }}/cutover-work - STAGING_WORK: ${{ github.workspace }}/cutover-staging # cutover.js treats a dispatch as dry unless this is exactly 'false'. DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }} steps: @@ -163,6 +162,7 @@ jobs: [ -f drill-state/drill.json ] || echo '{}' > drill-state/drill.json cat drill-state/drill.json + # With BRANCH=public it also keeps Steam's app info (appinfo.txt) for gate 1. - name: Static half (Rust's public build) if: ${{ steps.pending.outputs.run == 'true' }} env: @@ -171,16 +171,6 @@ jobs: DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' run: bash tools/drill/static.sh - # Only this run's manifest is used (gate 1): the build Rust has on staging now. - - name: Static half (Rust's staging build, for its manifest) - if: ${{ steps.pending.outputs.run == 'true' }} - env: - BRANCH: staging - WORK: ${{ github.workspace }}/cutover-staging - LC_ALL: C - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' - run: bash tools/drill/static.sh - - name: Gates 1-3 (public build, Oxide, Carbon) id: gates if: ${{ steps.pending.outputs.run == 'true' }} @@ -188,7 +178,7 @@ jobs: GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} run: >- - node tools/drill/cutover.js gates --state drill-state/drill.json --work "$WORK" --staging-work "$STAGING_WORK" + node tools/drill/cutover.js gates --state drill-state/drill.json --work "$WORK" ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') && '--dry-run' || '' }} ${{ github.event.inputs.force == 'true' && '--force' || '' }} @@ -223,7 +213,7 @@ jobs: env: GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} run: >- - node tools/drill/cutover.js gates --after-live --state drill-state/drill.json --work "$WORK" --staging-work "$STAGING_WORK" + node tools/drill/cutover.js gates --after-live --state drill-state/drill.json --work "$WORK" ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') && '--dry-run' || '' }} ${{ github.event.inputs.force == 'true' && '--force' || '' }} diff --git a/tools/drill/cutover.js b/tools/drill/cutover.js index 0e8e13d..da09d4b 100644 --- a/tools/drill/cutover.js +++ b/tools/drill/cutover.js @@ -5,19 +5,22 @@ // // node tools/drill/cutover.js window [--force] [--now ISO] in the wipe weekend? // node tools/drill/cutover.js pending [--force] does edge have commits main lacks? -// (tools/drill/static.sh with BRANCH=public, then BRANCH=staging) the static halves +// (tools/drill/static.sh with BRANCH=public) the static half + Steam's app info // node tools/drill/cutover.js rig-note --work is rust-carbon running now? -// node tools/drill/cutover.js gates --state --work [--staging-work ] [--dry-run] [--force] [--after-live] +// node tools/drill/cutover.js gates --state --work [--dry-run] [--force] [--after-live] +// (--staging-work is still accepted and ignored: main's wiring passes it) // node tools/drill/drill.js live --work --branch public the live half (only when `gates` says) // node tools/drill/cutover.js merge --pr --sha [--state f] MERGE_TOKEN only // node tools/drill/cutover.js after --state --pr record it, comment the release // node tools/drill/cutover.js rig-restore --work start rust-carbon if the job stopped it // // The five gates, cheapest first; it merges only when all pass (the later ones need the earlier ones): -// public Rust's public build is dated the wipe day or later AND names the same depot manifest as Rust's -// CURRENT staging build: Facepunch promotes the staging build to public, so the two agree once the -// cut over from staging to live went through. (static.sh runs a second time, BRANCH=staging, into -// its own work dir; only its manifest is used, and when that run fails this gate blocks.) +// public Steam set Rust's public branch to its build on the wipe day or later (branches.public.timeupdated +// in steamcmd's app info, which static.sh keeps as appinfo.txt), AND the Linux depot manifest Steam +// names for public is the one the static half downloaded. Not the depot manifest's date: that is +// when Facepunch BUILT it (the 2026-11-05 wipe build was dated 11-04). Not staging's manifest: +// public never shares one with staging (Facepunch promotes a hidden `release` branch, and staging +// has moved on by then). Both were the first design and blocked the whole 2026-11-05 weekend. // oxide Oxide caught up, the swap's field list is unchanged, RunicNPC compiles (static.json). // carbon rust-carbon restarts on public Rust with edge's RunicNPC.cs. Its appmanifest must name the public // manifest (it really updated); then rnpc.status says framework=carbon, navmesh=ready, missing=- @@ -72,7 +75,7 @@ const PERSISTED = GATES.slice(0, 4) // checks is never kept const COOLING = ['carbon', 'live'] // the gates that cost a rig restart const COOL_DOWN_HOURS = 3 const GATE_LABELS = { - public: "1. Rust's public build is the promoted staging build, dated the wipe day", + public: "1. Steam set Rust's public branch on the wipe day or later", oxide: '2. Oxide has caught up; RunicNPC compiles', carbon: '3. A Carbon boot on public Rust', live: '4. The drill again, on live Rust', @@ -112,44 +115,45 @@ function wipeWeekend(now) { } /** - * DepotDownloader's "Manifest N (10/31/2026 11:48:58)" date as UTC milliseconds: M/D/YYYY HH:mm:ss, 24 hours. - * An AM/PM form and ISO are accepted too; anything else is NaN (unparsable, so the gate blocks). + * One branch of Rust's dedicated server (app 258550) from steamcmd's `+app_info_print 258550` output (the + * appinfo.txt static.sh keeps; steamcmd's own lines come before the `"258550"` block and are skipped): + * `timeupdated`, when Steam set the branch to its build (Unix seconds); its `buildid`; and the manifest Steam + * names for the Linux depot on that branch (`gid`, or the older bare-string form). null when there is no + * 258550 block or no such branch. */ -function manifestTime(text) { - const s = String(text || '').trim() - const m = /^(\d{1,2})\/(\d{1,2})\/(\d{4}) (\d{1,2}):(\d{2}):(\d{2})(?: ([AP]M))?$/i.exec(s) - if (m) { - const [mo, d, y, hh, mi, ss] = [+m[1], +m[2], +m[3], +m[4], +m[5], +m[6]] - const ampm = m[7] && m[7].toUpperCase() - if (mi > 59 || ss > 59 || (ampm ? hh < 1 || hh > 12 : hh > 23)) return NaN - const h = ampm ? (hh % 12) + (ampm === 'PM' ? 12 : 0) : hh - const t = Date.UTC(y, mo - 1, d, h, mi, ss) - const back = new Date(t) - return back.getUTCMonth() === mo - 1 && back.getUTCDate() === d ? t : NaN +function steamBranch(text, branch = 'public', depot = LINUX_DEPOT) { + const s = String(text || '') + const at = s.search(/^"258550"[ \t]*\r?$/m) + if (at < 0) return null + const app = parseVdf(s.slice(at))['258550'] + const depots = app && typeof app === 'object' ? app.depots : null + const b = depots && depots.branches && depots.branches[branch] + if (!b || typeof b !== 'object') return null + const m = depots[depot] && depots[depot].manifests && depots[depot].manifests[branch] + return { + buildid: b.buildid || null, + timeupdated: b.timeupdated === undefined ? NaN : Number(b.timeupdated), + manifest: (m && typeof m === 'object' ? m.gid : m) || null, } - if (!/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(s)) return NaN - return Date.parse(/(Z|[+-]\d\d:?\d\d)$/.test(s) ? s : s + 'Z') } -/** The public depot's manifest is dated the wipe day (a Date, or YYYY-MM-DD) or later (UTC). */ -function publicChanged(stat, wipe) { - const t = manifestTime(stat && stat.manifestDate) +const iso = (ms) => new Date(ms).toISOString().slice(0, 16).replace('T', ' ') + ' UTC' + +/** + * Gate public: Steam set the public branch on the wipe day (a Date, or YYYY-MM-DD) or later, UTC, and the + * Linux depot manifest it names for public is the one the static half downloaded (`stat.manifest`): the two + * are read a minute apart, so a build pushed in between blocks one hour rather than tests the wrong build. + */ +function publicGate(stat, steam, wipe) { const w = wipe instanceof Date ? wipe.getTime() : Date.parse(wipe) - return Number.isFinite(t) && Number.isFinite(w) && t >= w -} - -function publicReason(stat, wipe) { - const date = stat && stat.manifestDate - if (!Number.isFinite(manifestTime(date))) return `the static half read no usable manifest date from Steam (${(stat && stat.note) || 'no note'})` - return `Rust's public build is manifest ${stat.manifest} of ${date}, from before the wipe (${wipe instanceof Date ? day(wipe) : wipe} 00:00 UTC)` -} - -/** Gate public: dated the wipe day or later, and the same manifest as Rust's current staging build. */ -function publicGate(stat, staging, wipe) { - if (!publicChanged(stat, wipe)) return publicReason(stat, wipe) - if (!staging || !staging.manifest) return `Rust's staging build could not be read (${(staging && staging.note) || 'no static.json'}), so the promotion to public cannot be confirmed` - if (staging.manifest !== stat.manifest) { - return `Rust's public build (manifest ${stat.manifest}) is not its current staging build (manifest ${staging.manifest}): the cut over from staging to live has not gone through` + if (!Number.isFinite(w)) return `no usable wipe date (${wipe})` + if (!steam) return `Steam's app info for Rust could not be read (${(stat && stat.steamNote) || 'no appinfo.txt'})` + const t = steam.timeupdated * 1000 + if (!Number.isFinite(t) || t <= 0) return "Steam's app info gives no time for Rust's public branch" + if (t < w) return `Steam last set Rust's public branch (build ${steam.buildid || '?'}) at ${iso(t)}, before the wipe (${day(new Date(w))} 00:00 UTC)` + if (!steam.manifest) return `Steam's app info names no ${LINUX_DEPOT} manifest for public` + if (steam.manifest !== (stat && stat.manifest)) { + return `Steam names manifest ${steam.manifest} for public, but the static half downloaded ${(stat && stat.manifest) || 'none'}: the next hour tries again` } return true } @@ -575,7 +579,7 @@ async function checksGate(ctx) { } const RUNNERS = { - public: async (c) => publicGate(c.stat, c.staging, c.wipe), + public: async (c) => publicGate(c.stat, c.steam, c.wipe), oxide: async (c) => oxideReady(c.stat), carbon: async (c) => carbonGate(c), live: async (c) => liveOk(readJson(path.join(c.work, 'live.json'), null)), @@ -644,7 +648,8 @@ async function gates(a, deps = realDeps()) { const now = deps.now() const w = wipeWeekend(now) const stat = readJson(path.join(work, 'static.json'), {}) - const staging = readJson(path.join(path.resolve(a['staging-work'] || work + '-staging'), 'static.json'), null) + const appinfo = path.join(work, 'appinfo.txt') + const steam = fs.existsSync(appinfo) ? steamBranch(fs.readFileSync(appinfo, 'utf8')) : null const g = gitea(env.GITEA_TOKEN, deps) const prev = (afterLive && readJson(path.join(work, 'cutover-run.json'), null)) || { rows: {} } @@ -653,7 +658,7 @@ async function gates(a, deps = realDeps()) { const obs = { manifest: stat.manifest || '', edgeSha, ahead: aheadCount(a, deps), openPr: pr && pr.number, forced: !!a.force && !(w.open || w.final), checksPassed: !!(dryRun && prev.checks) } // A dry run starts from nothing: it is there to try every gate, not to trust an earlier hour's. let d = decide(dryRun ? undefined : file.cutover, obs, now, { dryRun }) - const ctx = { g, deps, stat, staging, work, now, dryRun, pr, edgeSha, wipe: w.wipe, wipeDate: day(w.wipe), state: d.state, rows: prev.rows, closes: null } + const ctx = { g, deps, stat, steam, work, now, dryRun, pr, edgeSha, wipe: w.wipe, wipeDate: day(w.wipe), state: d.state, rows: prev.rows, closes: null } if (dryRun && prev.gates) ctx.state = { ...ctx.state, gates: prev.gates } // what the first half of this dry run passed else for (const k of Object.keys(ctx.state.gates)) ctx.rows[k] = { status: 'passed earlier', detail: ctx.state.gates[k].at } const note = (gate, r) => (ctx.rows[gate] = r === true ? { status: 'passed', detail: '' } : { status: 'failed', detail: r }) @@ -831,7 +836,7 @@ async function after(a, deps = realDeps()) { module.exports = { wipeWeekend, - publicChanged, + steamBranch, publicGate, oxideReady, parseVdf, @@ -851,7 +856,6 @@ module.exports = { fail, persistable, nextGate, - manifestTime, gates, merge, after, diff --git a/tools/drill/cutover.test.js b/tools/drill/cutover.test.js index 5cda7b5..54bac1c 100644 --- a/tools/drill/cutover.test.js +++ b/tools/drill/cutover.test.js @@ -8,7 +8,7 @@ const fs = require('fs') const os = require('os') const path = require('path') const c = require('./cutover.js') -const { wipeWeekend, publicChanged, publicGate, oxideReady, carbonOk, carbonBuild, acfDepotManifest, liveOk, checkState, collectCloses, cutoverBody, withBlock, decide, commentText, neutralise, isDry, pass, fail, persistable, nextGate, manifestTime, GATES, COOL_DOWN_HOURS } = c +const { wipeWeekend, steamBranch, publicGate, oxideReady, carbonOk, carbonBuild, acfDepotManifest, liveOk, checkState, collectCloses, cutoverBody, withBlock, decide, commentText, neutralise, isDry, pass, fail, persistable, nextGate, GATES, COOL_DOWN_HOURS } = c const at = (s) => new Date(s) const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'cutover-test-')) @@ -54,39 +54,108 @@ test('when the 1st is the Thursday, the weekend is the 1st to the 4th', () => { assert.equal(wipeWeekend(at('2026-01-01T16:59:00Z')).open, false) }) -// ---- the manifest date, strictly +// ---- Steam's app info (steamcmd +app_info_print 258550), cut down from the real one of 2026-11-06 -test('manifestTime reads M/D/YYYY HH:mm:ss in 24 hours, with AM/PM and ISO fallbacks, and nothing else', () => { - assert.equal(manifestTime('10/31/2026 11:48:58'), Date.UTC(2026, 9, 31, 11, 48, 58)) - assert.equal(manifestTime('11/5/2026 13:00:00'), Date.UTC(2026, 10, 5, 13, 0, 0)) - assert.equal(manifestTime('11/5/2026 12:00:00 AM'), Date.UTC(2026, 10, 5, 0, 0, 0)) - assert.equal(manifestTime('11/5/2026 01:30:00 PM'), Date.UTC(2026, 10, 5, 13, 30, 0)) - assert.equal(manifestTime('2026-11-05T09:00:00Z'), Date.UTC(2026, 10, 5, 9, 0, 0)) - for (const bad of ['', undefined, '?', 'yesterday', '13/40/2026 01:00:00', '11/31/2026 01:00:00', '11/5/2026 25:00:00', '11/5/2026 13:00:00 PM', '5.11.2026 10:00:00', '11/5/26 10:00:00']) { - assert.ok(Number.isNaN(manifestTime(bad)), `${bad} should not parse`) - } +const APPINFO = `Steam Console Client (c) Valve Corporation - version 1792617903 +-- type 'quit' to exit -- +Loading Steam API...OK + +Connecting anonymously to Steam Public...OK +Waiting for client config...OK +Waiting for user info...OK +AppID : 258550, change number : 40663829/40663829, last change : Fri Nov 6 04:32:39 2026 +"258550" +{ +\t"common" +\t{ +\t\t"name"\t\t"Rust Dedicated Server" +\t} +\t"depots" +\t{ +\t\t"258552" +\t\t{ +\t\t\t"config" +\t\t\t{ +\t\t\t\t"oslist"\t\t"linux" +\t\t\t} +\t\t\t"manifests" +\t\t\t{ +\t\t\t\t"public" +\t\t\t\t{ +\t\t\t\t\t"gid"\t\t"547616323393184824" +\t\t\t\t\t"size"\t\t"769819750" +\t\t\t\t} +\t\t\t\t"last-month" +\t\t\t\t{ +\t\t\t\t\t"gid"\t\t"1632958440323253420" +\t\t\t\t} +\t\t\t\t"staging" +\t\t\t\t{ +\t\t\t\t\t"gid"\t\t"4006809933087453123" +\t\t\t\t} +\t\t\t} +\t\t} +\t\t"branches" +\t\t{ +\t\t\t"public" +\t\t\t{ +\t\t\t\t"buildid"\t\t"26479619" +\t\t\t\t"timeupdated"\t\t"1793959612" +\t\t\t\t"timebuildupdated"\t\t"1793954045" +\t\t\t} +\t\t\t"release" +\t\t\t{ +\t\t\t\t"buildid"\t\t"26479619" +\t\t\t\t"description"\t\t"Release, before public" +\t\t\t\t"timeupdated"\t\t"1793954045" +\t\t\t} +\t\t\t"staging" +\t\t\t{ +\t\t\t\t"buildid"\t\t"26480085" +\t\t\t\t"timeupdated"\t\t"1793955490" +\t\t\t} +\t\t} +\t\t"privatebranches"\t\t"1" +\t} +} +Unloading Steam API...OK +` + +test("steamBranch reads a branch's set time, build id and Linux manifest, past steamcmd's own lines", () => { + assert.deepEqual(steamBranch(APPINFO), { buildid: '26479619', timeupdated: 1793959612, manifest: '547616323393184824' }) + assert.deepEqual(steamBranch(APPINFO, 'staging'), { buildid: '26480085', timeupdated: 1793955490, manifest: '4006809933087453123' }) + assert.deepEqual(steamBranch(APPINFO.replace(/\n/g, '\r\n')), { buildid: '26479619', timeupdated: 1793959612, manifest: '547616323393184824' }) + // The older form names the manifest as a bare string. + assert.equal(steamBranch(APPINFO.replace(/"public"\n\t\t\t\t\{\n\t\t\t\t\t"gid"\t\t"(\d+)"\n\t\t\t\t\t"size"\t\t"\d+"\n\t\t\t\t\}/, '"public"\t\t"$1"')).manifest, '547616323393184824') + assert.equal(steamBranch(APPINFO, 'nope'), null) + assert.equal(steamBranch('Loading Steam API...FAILED'), null) + assert.equal(steamBranch(''), null) + assert.equal(steamBranch(undefined), null) }) // ---- the gates -test('publicChanged: the manifest must be dated the wipe day or later (UTC)', () => { +test('publicGate: Steam set public on the wipe day or later, and names the manifest the static half has', () => { const wipe = at('2026-11-05T00:00:00Z') - assert.equal(publicChanged({ manifestDate: '11/4/2026 23:59:59' }, wipe), false) - assert.equal(publicChanged({ manifestDate: '11/5/2026 00:00:00' }, wipe), true) - assert.equal(publicChanged({ manifestDate: '10/31/2026 11:48:58' }, wipe), false) - assert.equal(publicChanged({ manifestDate: 'yesterday' }, wipe), false) - assert.equal(publicChanged(undefined, wipe), false) -}) - -test('publicGate: public must also be the build staging has now (the promotion went through)', () => { - const wipe = at('2026-11-05T00:00:00Z') - const pub = { manifest: '8140', manifestDate: '11/5/2026 11:48:58' } - assert.equal(publicGate(pub, { manifest: '8140' }, wipe), true) - assert.match(publicGate(pub, { manifest: '8150' }, wipe), /not its current staging build \(manifest 8150\)/) - assert.match(publicGate(pub, { note: 'no Assembly' }, wipe), /staging build could not be read/) - assert.match(publicGate(pub, null, wipe), /staging build could not be read/) - assert.match(publicGate({ manifest: '8100', manifestDate: '10/31/2026 11:48:58' }, { manifest: '8100' }, wipe), /from before the wipe/) - assert.match(publicGate({ manifest: '8140', manifestDate: 'garbage', note: 'n' }, { manifest: '8140' }, wipe), /no usable manifest date/) + const pub = { manifest: '547616323393184824', manifestDate: '11/06/2026 08:33:44' } + assert.equal(publicGate(pub, steamBranch(APPINFO), wipe), true) + // The 2026-11-05 case: the wipe build went live that evening, but its depot manifest was dated 11-04. + const thu = { buildid: '26427999', timeupdated: Date.UTC(2026, 10, 5, 18, 30) / 1000, manifest: '3647035647797862067' } + assert.equal(publicGate({ manifest: '3647035647797862067', manifestDate: '11/04/2026 12:57:41' }, thu, wipe), true) + assert.equal(publicGate(pub, { ...steamBranch(APPINFO), timeupdated: Date.UTC(2026, 10, 5) / 1000 }, wipe), true) + // Before the wipe: October's build still live. + const oct = { buildid: '26139947', timeupdated: Date.UTC(2026, 9, 22, 17, 40) / 1000, manifest: '1632958440323253420' } + assert.match(publicGate({ manifest: '1632958440323253420' }, oct, wipe), /at 2026-10-22 17:40 UTC, before the wipe \(2026-11-05 00:00 UTC\)/) + assert.match(publicGate(pub, { ...oct, timeupdated: Date.UTC(2026, 10, 4, 23, 59, 59) / 1000, manifest: pub.manifest }, wipe), /before the wipe/) + // Steam and DepotDownloader disagree (a push between the two reads): block this hour. + assert.match(publicGate({ manifest: '8150' }, steamBranch(APPINFO), wipe), /Steam names manifest 547616323393184824 for public, but the static half downloaded 8150/) + assert.match(publicGate({}, steamBranch(APPINFO), wipe), /downloaded none/) + assert.match(publicGate(pub, { ...steamBranch(APPINFO), manifest: null }, wipe), /names no 258552 manifest/) + assert.match(publicGate(pub, { ...steamBranch(APPINFO), timeupdated: NaN }, wipe), /gives no time/) + assert.match(publicGate({ ...pub, steamNote: 'steamcmd could not be fetched' }, null, wipe), /could not be read \(steamcmd could not be fetched\)/) + assert.match(publicGate(pub, null, wipe), /could not be read \(no appinfo.txt\)/) + assert.match(publicGate(pub, steamBranch(APPINFO), 'garbage'), /no usable wipe date/) + assert.equal(publicGate(pub, steamBranch(APPINFO), '2026-11-05'), true) }) test('oxideReady: true, or the reason', () => { @@ -453,14 +522,17 @@ function fakeDeps({ now = '2026-11-05T18:11:00Z', handler, git = GIT, panel, env const STAT = { branch: 'public', manifest: '8140', manifestDate: '11/5/2026 11:48:58', managed: 'ok', oxideBehind: 0, fields: 'same', compile: 'ok' } -function rig(stat = STAT, staging = { branch: 'staging', manifest: '8140' }) { +// Steam set public (2026-11-06 10:06 UTC) to the manifest STAT has. +const APPINFO_8140 = APPINFO.replace('"547616323393184824"', '"8140"') + +function rig(stat = STAT, appinfo = APPINFO_8140) { const dir = tmp() fs.mkdirSync(path.join(dir, 'work')) - fs.mkdirSync(path.join(dir, 'work-staging')) fs.writeFileSync(path.join(dir, 'work', 'static.json'), JSON.stringify(stat)) - fs.writeFileSync(path.join(dir, 'work-staging', 'static.json'), JSON.stringify(staging)) + if (appinfo !== null) fs.writeFileSync(path.join(dir, 'work', 'appinfo.txt'), appinfo) const state = path.join(dir, 'drill.json') fs.writeFileSync(state, '{}') + // main's cutover.yml still passes --staging-work; it is ignored. const a = { state, work: path.join(dir, 'work'), 'staging-work': path.join(dir, 'work-staging') } return { dir, a, state, read: () => JSON.parse(fs.readFileSync(state, 'utf8')) } } @@ -572,17 +644,23 @@ test('a PR call that fails never loses a gate that passed', async () => { assert.equal(s.pr, null) }) -test('gate 1 blocks, and nothing else runs, when public is not what staging has', async () => { - const r = rig(STAT, { branch: 'staging', manifest: '8999' }) - const g = fakeGitea() - let panelCalls = 0 - const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { panelCalls++; throw new Error('x') } }) }) - await c.gates(r.a, deps) - const s = r.read().cutover - assert.deepEqual(s.gates, {}) - assert.equal(panelCalls, 0) - assert.equal(g.pr, null) -}) +for (const [why, appinfo] of [ + ['Steam names another manifest for public', APPINFO], + ['Steam last set public before the wipe', APPINFO_8140.replace('"1793959612"', '"1792690223"')], + ['steamcmd left no appinfo.txt', null], +]) { + test(`gate 1 blocks, and nothing else runs, when ${why}`, async () => { + const r = rig(STAT, appinfo) + const g = fakeGitea() + let panelCalls = 0 + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { panelCalls++; throw new Error('x') } }) }) + await c.gates(r.a, deps) + const s = r.read().cutover + assert.deepEqual(s.gates, {}) + assert.equal(panelCalls, 0) + assert.equal(g.pr, null) + }) +} test('checksGate refuses a PR whose head is not the commit the gates ran on', async () => { const deps = fakeDeps({ handler: (m, p) => (p === '/pulls/3' ? { json: { number: 3, head: { sha: 'zzzzzzzz1' } } } : { json: {} }) }) @@ -778,6 +856,7 @@ test('RUNICNPC_TOKEN is in the merge step alone, and a dispatch is dry unless dr const code = yml.split('\n').filter((l) => !l.trim().startsWith('#')).join('\n') assert.ok(!/dry_run\s*[=!]=\s*'true'/.test(code), 'dry_run must be compared with false, never true') assert.ok((code.match(/dry_run != 'false'/g) || []).length >= 6) - for (const key of ['LC_ALL: C', "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1'"]) assert.equal(code.split(key).length - 1, 2, `${key} on both static halves`) + for (const key of ['LC_ALL: C', "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1'"]) assert.equal(code.split(key).length - 1, 1, `${key} on the static half`) + assert.ok(!/BRANCH: staging/.test(code), 'gate 1 reads Steam, not a staging static half') assert.match(code, /timeout-minutes: 240/) }) diff --git a/tools/drill/static.sh b/tools/drill/static.sh index 5cf7d47..cf8033c 100644 --- a/tools/drill/static.sh +++ b/tools/drill/static.sh @@ -35,7 +35,7 @@ DOTNET=${DOTNET:-dotnet} mkdir -p "$WORK" rm -rf "$WORK/depot" "$WORK/oxide" "$WORK/refs" "$WORK/fields" -rm -f "$WORK"/static.json "$WORK"/oxide-behind.txt "$WORK"/fields.diff "$WORK"/compile.txt +rm -f "$WORK"/static.json "$WORK"/oxide-behind.txt "$WORK"/fields.diff "$WORK"/compile.txt "$WORK"/appinfo.txt managed=error behind=-1 @@ -45,10 +45,11 @@ note="" oxide_modified="" manifest="" manifest_date="" +steam_note="" finish() { - printf '{"branch":"%s","manifest":"%s","manifestDate":"%s","managed":"%s","oxideBehind":%s,"fields":"%s","compile":"%s","oxideModified":"%s","note":"%s"}\n' \ - "$BRANCH" "$manifest" "$manifest_date" "$managed" "$behind" "$fields" "$compile" "$oxide_modified" "$note" > "$WORK/static.json" + printf '{"branch":"%s","manifest":"%s","manifestDate":"%s","managed":"%s","oxideBehind":%s,"fields":"%s","compile":"%s","oxideModified":"%s","note":"%s","steamNote":"%s"}\n' \ + "$BRANCH" "$manifest" "$manifest_date" "$managed" "$behind" "$fields" "$compile" "$oxide_modified" "$note" "$steam_note" > "$WORK/static.json" cat "$WORK/static.json" exit 0 } @@ -56,6 +57,37 @@ finish() { # Git Bash does not translate paths inside a response file; elsewhere this is the path as it is. native() { if command -v cygpath > /dev/null; then cygpath -m "$1"; else printf '%s' "$1"; fi; } +# ---- When Steam set the branch to its build (BRANCH=public only: the cutover's gate 1) ---- +# The depot manifest's date below is when Facepunch BUILT the build, not when it went live (the +# 2026-11-05 wipe build was dated 11-04). Steam's app info says when the branch was SET to it +# (depots.branches..timeupdated) and which Linux depot manifest that is. steamcmd's whole +# output goes to appinfo.txt; tools/drill/cutover.js reads it. A failure only leaves steamNote set, +# and gate 1 blocks. STEAMCMD points at another steamcmd (steamcmd.sh, or steamcmd.exe on Windows). + +steam_info() { + local sc=${STEAMCMD:-} i + if [ -z "$sc" ]; then + mkdir -p "$WORK/steamcmd" + curl -sSL --fail https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz | tar -xz -C "$WORK/steamcmd" \ + || { steam_note="steamcmd could not be fetched"; return; } + sc=$WORK/steamcmd/steamcmd.sh + fi + # steamcmd updates itself on its first start and then may print nothing, so it gets three tries. + # It is 32-bit: a bare runner image lacks its libgcc, which is installed after the first miss. + for i in 1 2 3; do + "$sc" +login anonymous +app_info_update 1 +app_info_print 258550 +quit > "$WORK/appinfo.txt" 2>&1 + grep -q '^"258550"' "$WORK/appinfo.txt" && return + if [ "$i" = 1 ] && command -v apt-get > /dev/null && [ "$(id -u)" = 0 ]; then + { apt-get update -qq && apt-get install -y -qq lib32gcc-s1; } > "$WORK/steamcmd-deps.txt" 2>&1 + fi + done + steam_note="steamcmd printed no app info: $(tail -2 "$WORK/appinfo.txt" | tr '\n"\\' ' ')" +} + +if [ "$BRANCH" = public ]; then + steam_info +fi + # ---- Rust's managed assemblies, from Steam (no login: the dedicated server is anonymous) ---- DD=${DEPOTDOWNLOADER:-} -- 2.49.1