diff --git a/.gitea/workflows/cutover.yml b/.gitea/workflows/cutover.yml new file mode 100644 index 0000000..4ec300c --- /dev/null +++ b/.gitea/workflows/cutover.yml @@ -0,0 +1,268 @@ +# The wipe-day cutover (docs/runicnpc/PLAN.md stage 9c, D332): `edge` merges into `main` by itself. +# +# The org lead approves no merge on a wipe day. Rust changes on the monthly forced wipe, the first +# Thursday (D318), and RunicNPC must be proven on the public build before it is released. This job runs +# hourly from 17:00 UTC on that Thursday until Sunday 23:00 UTC and merges `edge` into `main` once five +# gates have passed, cheapest first (the decisions are pure functions in tools/drill/cutover.js, tested +# by cutover.test.js, which PR Checks runs): +# +# 1. public Rust's public build is dated the wipe day or later AND names the same depot manifest as +# Rust's current STAGING build (Facepunch promotes staging to public, so the two agree once the +# cut over from staging to live went through). The static half runs twice, BRANCH=public and +# BRANCH=staging; only the staging run's manifest is used. +# 2. oxide Oxide's public build has caught up (0 members behind), the swap's field list is unchanged +# and RunicNPC compiles. +# 3. carbon `rust-carbon` restarts on public Rust with edge's RunicNPC.cs: its appmanifest names the +# public manifest, and rnpc.status says framework=carbon, navmesh=ready, missing=- added=-. +# The rig is left as it was found (running stays running). +# 4. live the drill's live half (drill.js live --branch public) on `rust-staging`: every harness check +# passes, no finding. +# 5. checks `plugin-checks` is green on the cutover PR's head commit, which must be the commit the gates +# ran on. Never kept: it runs fresh, right before the merge. +# +# Each passed gate is kept in drill.json (the `cutover` key, on the `drill` branch) for that wipe, that +# public manifest and that `edge` commit; a change of any of them starts the gates again. A failed carbon or +# live attempt is not repeated for three hours for the same manifest and commit. As soon as gate 1 passes it +# opens ONE edge -> main PR (or adopts the open one: its title is never changed and only the block between +# and of its body is written) whose block carries a `Closes #N` +# line for every issue fixed by the PRs merged into `edge` since the last cutover (merging into `main`, the +# default branch, is what closes them). When a gate blocks it comments on that PR once a day; at Sunday +# 23:00 UTC it stops with a last comment and the org lead merges by hand, as before. The merge sends the +# head commit it checked, so Gitea refuses it if the PR moved. After the merge it waits for release.yml's +# release and comments the version on each issue that closed. +# +# Sequencing inside the job: `gates` runs 1-3 and stops in front of 4, saying live=true; the live half is +# its own step (drill.js live); `gates --after-live` records 4 and runs 5. The state is pushed to the +# `drill` branch after the first `gates` too, so a long live half that runs into the job timeout does not +# lose gate 3. A last always-run step starts `rust-carbon` again if it was running when the job began and is +# offline now (that server only, power `start` only). +# +# THE MAIN / EDGE CONTRACT. The copy of this file on `main` is the trigger and the step wiring (Gitea fires +# a schedule only from the default branch). Every step checks out `edge` and runs edge's scripts, plugin +# and harness: the gates and the decisions come from edge. The step ids and the output names (open, run, +# live, merge, pr, sha, merged) are a contract between the two: cutover.test.js reads this file and fails +# when one is used here but not written by tools/drill/cutover.js, so change both on edge and carry the +# new wiring to `main` in the same release. Because edge's code runs with RUNICNPC_TOKEN in the merge step, +# PR review into `edge` is the real gate: nothing reaches `edge` unreviewed. +# +# It shares the staging drill's concurrency group, so it never runs while the daily drill has `rust-staging`. +# +# Try it: Actions -> Wipe-day cutover -> Run workflow. A dispatch is a DRY RUN unless `dry_run` is +# explicitly false (anything else, or nothing, is dry): every gate runs and reports in the job summary (it +# does restart rust-carbon and use rust-staging); no PR is opened or edited, nothing is commented or merged, +# no state is written. `force` goes outside the wipe weekend. +# +# Secrets (Settings -> Actions -> Secrets, the repository's or the organisation's): +# PTERODACTYL_DRILL_KEY — the panel client key made for the drill (D309). Never printed. +# REGISTRY_USER / REGISTRY_TOKEN — as the drill: push the `drill` branch, open and edit the PR, comment. +# RUNICNPC_TOKEN — a personal access token locked to this repository, used by the merge step ALONE: a +# push made with the runner's own token would not start release.yml. + +name: Wipe-day cutover + +on: + schedule: + # Thursday from 17:00 UTC, then every hour of Friday, Saturday and Sunday. (Not one line with a day of + # the month as well: cron ORs the two.) The window step ends the runs that fall outside the weekend. + - cron: '11 17-23 * * 4' + - cron: '11 * * * 5,6,0' + workflow_dispatch: + inputs: + dry_run: + description: 'Run every gate and report; open, edit, comment and merge nothing (dry unless exactly false)' + type: boolean + default: true + force: + description: 'Work outside the wipe weekend' + type: boolean + default: false + code_ref: + description: 'Branch whose scripts and plugin a DRY run uses, to rehearse a change before it reaches edge (a real run always uses edge)' + type: string + default: 'edge' + +concurrency: + group: staging-drill + cancel-in-progress: false + +env: + GITEA_HOST: gitea.whitlocktech.com + REPO: RunicGateway/runicnpc-rust + +jobs: + cutover: + runs-on: ubuntu-latest + # Carbon's restart and navmesh, the live half (about an hour), the PR checks: about 100 minutes, 200 at the + # worst. Room for that, so a timeout never cuts off the step that starts rust-carbon again. + timeout-minutes: 240 + env: + WORK: ${{ github.workspace }}/cutover-work + STAGING_WORK: ${{ github.workspace }}/cutover-staging + # cutover.js treats a dispatch as dry unless this is exactly 'false'. + DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }} + steps: + # edge, always, except a dry dispatch naming another branch (code_ref). A real run on another branch could + # not merge anyway: the merge is pinned to the commit the gates ran on, which must be edge's head. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false' && github.event.inputs.code_ref || 'edge' }} + fetch-depth: 0 + + # The commit counts and the cut-off for the Closes list are read from git. + - name: main and the tags + run: git fetch origin main edge --tags + + # 22 or later: drill.js talks to the panel's console over the global WebSocket. + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: The wipe weekend? + id: window + run: node tools/drill/cutover.js window ${{ github.event.inputs.force == 'true' && '--force' || '' }} + + - name: Anything to merge? + id: pending + if: ${{ steps.window.outputs.open == 'true' }} + run: node tools/drill/cutover.js pending ${{ (github.event.inputs.force == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false')) && '--force' || '' }} + + # Whether rust-carbon is running now: gate 3 restarts it and the last step puts it back if needed. + - name: Note rust-carbon + if: ${{ steps.pending.outputs.run == 'true' }} + continue-on-error: true + env: + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: node tools/drill/cutover.js rig-note --work "$WORK" + + # tools/fieldlist and the compiler (Roslyn's csc.dll ships in the SDK). + - name: .NET SDK + if: ${{ steps.pending.outputs.run == 'true' }} + run: | + set -euo pipefail + curl -sSL -o "$RUNNER_TEMP/dotnet-install.sh" https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh + bash "$RUNNER_TEMP/dotnet-install.sh" --channel 9.0 --install-dir "$HOME/.dotnet" > /dev/null + echo "$HOME/.dotnet" >> "$GITHUB_PATH" + echo "DOTNET_ROOT=$HOME/.dotnet" >> "$GITHUB_ENV" + + - name: The drill branch (what the cutover has passed) + if: ${{ steps.pending.outputs.run == 'true' }} + run: | + set -euo pipefail + git config user.name "runicnpc-ci" + git config user.email "ci@whitlocktech.com" + rm -rf drill-state + git worktree prune + if git ls-remote --exit-code --heads origin drill > /dev/null 2>&1; then + git fetch origin drill + git worktree add -B drill drill-state origin/drill + else + # First run: a root commit with an empty tree, so the branch inherits no code history. + ROOT="$(git commit-tree "$(git hash-object -t tree /dev/null)" -m 'chore(drill): start the drill branch')" + git worktree add -B drill drill-state "$ROOT" + fi + [ -f drill-state/drill.json ] || echo '{}' > drill-state/drill.json + cat drill-state/drill.json + + - name: Static half (Rust's public build) + if: ${{ steps.pending.outputs.run == 'true' }} + env: + BRANCH: public + LC_ALL: C + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' + run: bash tools/drill/static.sh + + # Only this run's manifest is used (gate 1): the build Rust has on staging now. + - name: Static half (Rust's staging build, for its manifest) + if: ${{ steps.pending.outputs.run == 'true' }} + env: + BRANCH: staging + WORK: ${{ github.workspace }}/cutover-staging + LC_ALL: C + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' + run: bash tools/drill/static.sh + + - name: Gates 1-3 (public build, Oxide, Carbon) + id: gates + if: ${{ steps.pending.outputs.run == 'true' }} + env: + GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: >- + node tools/drill/cutover.js gates --state drill-state/drill.json --work "$WORK" --staging-work "$STAGING_WORK" + ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') && '--dry-run' || '' }} + ${{ github.event.inputs.force == 'true' && '--force' || '' }} + + - name: Keep the gates passed so far on the drill branch + if: ${{ always() && steps.pending.outputs.run == 'true' && !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') }} + env: + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -euo pipefail + CI_USER="$(printf '%s' "${REGISTRY_USER}" | tr -d '\r\n')" + CI_TOKEN="$(printf '%s' "${REGISTRY_TOKEN}" | tr -d '\r\n')" + git remote set-url origin "https://${CI_USER}:${CI_TOKEN}@${GITEA_HOST}/${REPO}.git" + cd drill-state + git add drill.json + if git diff --cached --quiet; then echo "no change"; exit 0; fi + node -e "const c=require('./drill.json').cutover||{};console.log('chore(drill): cutover '+(c.wipe||'?')+' '+(c.manifest||'?')+' [skip ci]')" > ../cutover-msg.txt + git commit -q -F ../cutover-msg.txt + git push origin drill + + - name: Gate 4, the drill again on live Rust (rust-staging) + id: live + if: ${{ steps.gates.outputs.live == 'true' }} + continue-on-error: true + env: + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: node tools/drill/drill.js live --work "$WORK" --branch public + + - name: Gates 4-5 (the live half, PR checks) + id: gates2 + if: ${{ always() && steps.gates.outputs.live == 'true' }} + env: + GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: >- + node tools/drill/cutover.js gates --after-live --state drill-state/drill.json --work "$WORK" --staging-work "$STAGING_WORK" + ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') && '--dry-run' || '' }} + ${{ github.event.inputs.force == 'true' && '--force' || '' }} + + # The only step that holds RUNICNPC_TOKEN. It merges the commit the gates ran on, or nothing. + - name: Merge edge into main + id: merge + if: ${{ (steps.gates2.outputs.merge == 'true' || steps.gates.outputs.merge == 'true') && !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') }} + env: + MERGE_TOKEN: ${{ secrets.RUNICNPC_TOKEN }} + run: node tools/drill/cutover.js merge --pr ${{ steps.gates2.outputs.pr || steps.gates.outputs.pr }} --sha ${{ steps.gates2.outputs.sha || steps.gates.outputs.sha }} --state drill-state/drill.json + + - name: Record the merge and the release + if: ${{ steps.merge.outputs.merged == 'true' }} + env: + GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: node tools/drill/cutover.js after --state drill-state/drill.json --pr ${{ steps.gates2.outputs.pr || steps.gates.outputs.pr }} + + - name: Keep the result on the drill branch + if: ${{ always() && steps.pending.outputs.run == 'true' && !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false') }} + env: + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -euo pipefail + CI_USER="$(printf '%s' "${REGISTRY_USER}" | tr -d '\r\n')" + CI_TOKEN="$(printf '%s' "${REGISTRY_TOKEN}" | tr -d '\r\n')" + git remote set-url origin "https://${CI_USER}:${CI_TOKEN}@${GITEA_HOST}/${REPO}.git" + cd drill-state + git add drill.json + if git diff --cached --quiet; then echo "no change"; exit 0; fi + node -e "const c=require('./drill.json').cutover||{};console.log('chore(drill): cutover '+(c.wipe||'?')+' '+(c.manifest||'?')+' [skip ci]')" > ../cutover-msg.txt + git commit -q -F ../cutover-msg.txt + git push origin drill + + # Last: gate 3 and the live half stop and start rust-carbon; if it was running when the job began + # and is offline now, start it. That server only, power `start` only. + - name: Leave rust-carbon as found + if: ${{ always() && steps.pending.outputs.run == 'true' }} + continue-on-error: true + env: + PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }} + run: node tools/drill/cutover.js rig-restore --work "$WORK" diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index 26ed66d..61e3eca 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -61,3 +61,8 @@ jobs: # The staging drill's pure parts (tools/drill/drill.js): its stall limit and fresh-data list. - name: Test the staging drill run: node --test tools/drill/drill.test.js + + # The wipe-day cutover's pure parts (tools/drill/cutover.js, D332): the weekend, the five gates' + # verdicts, the issues the PR closes and what the hourly run does next. + - name: Test the wipe-day cutover + run: node --test tools/drill/cutover.test.js diff --git a/tools/drill/cutover.js b/tools/drill/cutover.js new file mode 100644 index 0000000..0e8e13d --- /dev/null +++ b/tools/drill/cutover.js @@ -0,0 +1,881 @@ +// The wipe-day cutover (docs/runicnpc/PLAN.md stage 9c, D332). Never shipped. +// +// .gitea/workflows/cutover.yml runs it hourly over the forced-wipe weekend. It merges `edge` into `main` +// once Rust's wipe update is live and RunicNPC is proven on it, so nobody approves a merge on a wipe day. +// +// node tools/drill/cutover.js window [--force] [--now ISO] in the wipe weekend? +// node tools/drill/cutover.js pending [--force] does edge have commits main lacks? +// (tools/drill/static.sh with BRANCH=public, then BRANCH=staging) the static halves +// node tools/drill/cutover.js rig-note --work is rust-carbon running now? +// node tools/drill/cutover.js gates --state --work [--staging-work ] [--dry-run] [--force] [--after-live] +// node tools/drill/drill.js live --work --branch public the live half (only when `gates` says) +// node tools/drill/cutover.js merge --pr --sha [--state f] MERGE_TOKEN only +// node tools/drill/cutover.js after --state --pr record it, comment the release +// node tools/drill/cutover.js rig-restore --work start rust-carbon if the job stopped it +// +// The five gates, cheapest first; it merges only when all pass (the later ones need the earlier ones): +// public Rust's public build is dated the wipe day or later AND names the same depot manifest as Rust's +// CURRENT staging build: Facepunch promotes the staging build to public, so the two agree once the +// cut over from staging to live went through. (static.sh runs a second time, BRANCH=staging, into +// its own work dir; only its manifest is used, and when that run fails this gate blocks.) +// oxide Oxide caught up, the swap's field list is unchanged, RunicNPC compiles (static.json). +// carbon rust-carbon restarts on public Rust with edge's RunicNPC.cs. Its appmanifest must name the public +// manifest (it really updated); then rnpc.status says framework=carbon, navmesh=ready, missing=- +// added=- (D232: only Carbon's unmodified assembly makes `added` mean anything). +// live drill.js live --branch public on rust-staging: every harness check passes, no finding. +// checks the cutover PR's combined status on its head commit is success and has a `plugin-checks` +// context, AND the PR head is the commit the gates ran on. +// +// The gates are tied to the code: drill.json keeps `edgeSha` (the commit this job checked out and ran the +// plugin and harness from) and the manifest; a change of either, or of the wipe, clears every gate. `checks` +// is never kept: it runs fresh, right before the merge, and the merge is sent `head_commit_id` so Gitea refuses +// it when the PR head moved. The state is written after EVERY gate result, before anything touches the PR. +// +// A carbon or live attempt that fails is not repeated for the same manifest and commit for COOL_DOWN_HOURS +// (3): the rigs are not restarted every hour. A new manifest or a new edge commit retries at once. +// +// How the gates are sequenced across the workflow's steps. `gates` is run twice at most: +// 1. `gates` runs public, oxide, carbon in this job and stops in front of live, saying +// `live=true`. A gate that fails ends the run (retried next hour, a cooling one +// after the cool-down); a gate that passed is kept in drill.json, never re-run. +// 2. `drill.js live` is its own workflow step; its output is a file (live.json). +// 3. `gates --after-live` reads live.json (the live gate), then runs checks. Both say `merge=true pr= +// sha=` when everything has passed. +// When live already passed (an earlier hour), step 1 goes on to checks by itself and the other two steps +// are skipped. The cutover PR is opened (or its marked block updated) as soon as the public gate has passed; +// an existing open edge to main PR is adopted: its title is never changed and only the block between +// and of its body is written (appended when absent). +// +// A dry run (--dry-run, or any workflow_dispatch whose dry_run input is not exactly 'false') runs every gate +// it can instead of stopping at the first failure, ignores what an earlier real run kept, and creates, edits, +// comments and merges nothing and writes no state. It does use the rigs: carbon restarts rust-carbon and live +// uses rust-staging. The table goes to the job summary. +// +// Every gate check below returns `true`, or a string saying why not. +// +// Environment: GITEA_TOKEN (the workflow's REGISTRY_TOKEN: read, open and edit the PR, comment) with +// GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them; PANEL_KEY / PANEL_URL as drill.js; +// MERGE_TOKEN (RUNICNPC_TOKEN) in the `merge` step only; GITHUB_EVENT_NAME + DRY_RUN_INPUT for the dry-run +// default. No token or key is ever printed. + +const fs = require('fs') +const path = require('path') +const { execFileSync } = require('child_process') + +const ROOT = path.resolve(__dirname, '..', '..') +const CARBON_RIG = process.env.CUTOVER_CARBON_RIG || 'rust-carbon' +const CARBON_PLUGIN = '/carbon/plugins/RunicNPC.cs' +const ACF = '/steamapps/appmanifest_258550.acf' +const LINUX_DEPOT = '258552' +const GATES = ['public', 'oxide', 'carbon', 'live', 'checks'] +const PERSISTED = GATES.slice(0, 4) // checks is never kept +const COOLING = ['carbon', 'live'] // the gates that cost a rig restart +const COOL_DOWN_HOURS = 3 +const GATE_LABELS = { + public: "1. Rust's public build is the promoted staging build, dated the wipe day", + oxide: '2. Oxide has caught up; RunicNPC compiles', + carbon: '3. A Carbon boot on public Rust', + live: '4. The drill again, on live Rust', + checks: '5. PR checks (plugin-checks) on the head commit', +} +const BLOCK_START = '' +const BLOCK_END = '' +const H = 3600000 +const CHECKS_WAIT_MS = 15 * 60000 +const RELEASE_WAIT_MS = 30 * 60000 + +// ---------------------------------------------------------------- pure parts (cutover.test.js) + +const day = (d) => d.toISOString().slice(0, 10) + +/** A reason in a PR body or a comment must not hold a closing keyword Gitea would act on: `#123` is broken up. */ +const neutralise = (text) => String(text == null ? '' : text).replace(/#(\d)/g, '#​$1') + +function firstThursday(y, m) { + const d = new Date(Date.UTC(y, m, 1)) + d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7)) + return d +} + +/** + * The forced wipe of `now`'s month (its first Thursday, D318) and the weekend that follows. `open` is the + * span the cutover works in: from the wipe Thursday 17:00 UTC to Sunday 23:00 UTC (`deadline`). `final` is + * the hour after it, in which the hourly run still starts once more so it can give up out loud. (nextWipe + * in drill.js cannot do this: on the Thursday itself it answers next month.) + */ +function wipeWeekend(now) { + const wipe = firstThursday(now.getUTCFullYear(), now.getUTCMonth()) + const t = now.getTime() + const start = wipe.getTime() + 17 * H + const deadline = new Date(wipe.getTime() + (3 * 24 + 23) * H) + return { open: t >= start && t < deadline.getTime(), final: t >= deadline.getTime() && t < deadline.getTime() + H, wipe, deadline } +} + +/** + * DepotDownloader's "Manifest N (10/31/2026 11:48:58)" date as UTC milliseconds: M/D/YYYY HH:mm:ss, 24 hours. + * An AM/PM form and ISO are accepted too; anything else is NaN (unparsable, so the gate blocks). + */ +function manifestTime(text) { + const s = String(text || '').trim() + const m = /^(\d{1,2})\/(\d{1,2})\/(\d{4}) (\d{1,2}):(\d{2}):(\d{2})(?: ([AP]M))?$/i.exec(s) + if (m) { + const [mo, d, y, hh, mi, ss] = [+m[1], +m[2], +m[3], +m[4], +m[5], +m[6]] + const ampm = m[7] && m[7].toUpperCase() + if (mi > 59 || ss > 59 || (ampm ? hh < 1 || hh > 12 : hh > 23)) return NaN + const h = ampm ? (hh % 12) + (ampm === 'PM' ? 12 : 0) : hh + const t = Date.UTC(y, mo - 1, d, h, mi, ss) + const back = new Date(t) + return back.getUTCMonth() === mo - 1 && back.getUTCDate() === d ? t : NaN + } + if (!/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(s)) return NaN + return Date.parse(/(Z|[+-]\d\d:?\d\d)$/.test(s) ? s : s + 'Z') +} + +/** The public depot's manifest is dated the wipe day (a Date, or YYYY-MM-DD) or later (UTC). */ +function publicChanged(stat, wipe) { + const t = manifestTime(stat && stat.manifestDate) + const w = wipe instanceof Date ? wipe.getTime() : Date.parse(wipe) + return Number.isFinite(t) && Number.isFinite(w) && t >= w +} + +function publicReason(stat, wipe) { + const date = stat && stat.manifestDate + if (!Number.isFinite(manifestTime(date))) return `the static half read no usable manifest date from Steam (${(stat && stat.note) || 'no note'})` + return `Rust's public build is manifest ${stat.manifest} of ${date}, from before the wipe (${wipe instanceof Date ? day(wipe) : wipe} 00:00 UTC)` +} + +/** Gate public: dated the wipe day or later, and the same manifest as Rust's current staging build. */ +function publicGate(stat, staging, wipe) { + if (!publicChanged(stat, wipe)) return publicReason(stat, wipe) + if (!staging || !staging.manifest) return `Rust's staging build could not be read (${(staging && staging.note) || 'no static.json'}), so the promotion to public cannot be confirmed` + if (staging.manifest !== stat.manifest) { + return `Rust's public build (manifest ${stat.manifest}) is not its current staging build (manifest ${staging.manifest}): the cut over from staging to live has not gone through` + } + return true +} + +/** Gate oxide: true, or why an Oxide server (or the plugin's swap) cannot be trusted on this build. */ +function oxideReady(stat) { + const s = stat || {} + if (s.managed !== 'ok') return `the static half could not fetch Rust's public build (${s.note || 'no static.json'})` + if (s.oxideBehind === -1 || s.oxideBehind === undefined) return "Oxide's public build could not be compared with Rust's" + if (s.oxideBehind !== 0) return `Oxide's public build is ${s.oxideBehind} member(s) behind` + if (s.compile !== 'ok') return `RunicNPC does not compile against public Rust with Oxide's build (compile ${s.compile})` + if (s.fields === 'differs') return "the swap's field list differs from public Rust's" + if (s.fields !== 'same') return `the swap's field list could not be checked (fields ${s.fields})` + return true +} + +/** Valve's text KeyValues (an appmanifest .acf) as nested objects. */ +function parseVdf(text) { + const root = {} + const stack = [root] + let key = null + for (const m of String(text || '').matchAll(/"((?:[^"\\]|\\.)*)"|([{}])/g)) { + const top = stack[stack.length - 1] + if (m[2] === '{') { + const o = {} + top[key] = o + stack.push(o) + key = null + } else if (m[2] === '}') { + if (stack.length > 1) stack.pop() + key = null + } else if (key === null) key = m[1] + else { + top[key] = m[1] + key = null + } + } + return root +} + +/** The manifest an appmanifest_258550.acf records for a depot (InstalledDepots, "258552", "manifest"), or null. */ +function acfDepotManifest(text, depot = LINUX_DEPOT) { + const state = parseVdf(text).AppState + const entry = state && state.InstalledDepots && state.InstalledDepots[depot] + return (entry && typeof entry === 'object' && entry.manifest) || null +} + +/** Gate carbon, first half: the rig's installed Linux depot is the public manifest. */ +function carbonBuild(acfText, publicManifest) { + const have = acfDepotManifest(acfText) + if (!have) return `the ${LINUX_DEPOT} manifest could not be read from ${ACF} on ${CARBON_RIG}` + if (have !== publicManifest) return `${CARBON_RIG} still runs manifest ${have} (public is ${publicManifest})` + return true +} + +/** Gate carbon, second half: the `rnpc.status` console lines (an array, or the text) of a Carbon server. */ +function carbonOk(statusLines) { + const lines = Array.isArray(statusLines) ? statusLines : String(statusLines || '').split('\n') + const line = lines.find((l) => /framework=/.test(l)) + if (!line) return 'rnpc.status never answered on Carbon' + const get = (k) => (new RegExp(`${k}=(\\S+)`).exec(line) || [])[1] + const framework = get('framework') + if (framework !== 'carbon') return `rnpc.status says framework=${framework}, not carbon` + if (get('navmesh') !== 'ready') return `the navmesh was not ready on Carbon (navmesh=${get('navmesh')})` + const missing = get('missing') + const added = get('added') + if (missing === undefined || added === undefined) return 'rnpc.status carried no swap-fields report' + if (missing !== '-') return `the swap's fields are missing on Carbon: ${missing}` + if (added !== '-') return `public Rust's Carbon assembly has fields the swap does not know: ${added}` + return true +} + +/** Gate live: live.json as drill.js live wrote it. */ +function liveOk(live) { + if (!live) return 'the live half wrote no live.json' + const first = (live.findings && live.findings[0]) || null + if (live.branch && live.branch !== 'public') return `live.json is from the ${live.branch} branch, not public` + if (!live.ran) return live.skipped || first || 'the live half did not run the harness' + if (!live.harness || !live.harness.done) return first || 'the harness did not finish' + if (live.harness.fail > 0) return `a harness check failed: ${(live.harness.failed && live.harness.failed[0]) || `${live.harness.fail} fail`}` + if (!(live.harness.pass > 0)) return 'the harness passed no check at all' + if (first) return first + return true +} + +/** Gate checks: where the cutover PR's combined commit status stands. { state: success|pending|failed, reason } */ +function checkState(combined) { + const statuses = (combined && combined.statuses) || [] + const plugin = statuses.filter((s) => String(s.context || '').includes('plugin-checks')) + const bad = statuses.find((s) => ['failure', 'error'].includes(s.status || s.state)) + if (bad) return { state: 'failed', reason: `${bad.context} is ${bad.status || bad.state}` } + if (!plugin.length) return { state: 'pending', reason: 'no plugin-checks status has been reported yet' } + if (combined.state !== 'success') return { state: 'pending', reason: `the PR's combined status is ${combined.state}` } + return { state: 'success', reason: '' } +} + +/** Issue numbers a PR body closes: a Gitea closing keyword directly before `#N`. Sorted, unique. */ +function collectCloses(prBodies) { + const found = new Set() + const re = /(? a - b) +} + +/** The cutover PR's description (the block). `gates` is [{ name, status, note }]. */ +function cutoverBody({ wipe, manifest, buildDate, closes = [], gates = [] }) { + return [ + 'The automated wipe-day cutover (D332, `docs/runicnpc/PLAN.md` stage 9c): `edge` merges into `main` once Rust\'s ' + + 'forced-wipe update is live and RunicNPC is proven on it, so no merge needs approving on a wipe day. ' + + 'The workflow merges this PR by itself when every gate below has passed; `release.yml` then cuts the release.', + '', + `Wipe: **${wipe}**. Rust's public build: manifest **${manifest}**, dated ${buildDate || '?'}.`, + '', + '| Gate | State | Detail |', + '| --- | --- | --- |', + ...gates.map((g) => `| ${g.name} | ${g.status} | ${neutralise(g.note || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`), + '', + closes.length ? 'Issues fixed by the PRs merged into `edge` since the last cutover:' : 'No issue is closed by this release.', + ...closes.map((n) => `Closes #${n}`), + '', + 'Opened by the runicnpc-rust cutover workflow (D332).', + ].join('\n') +} + +/** `existing` with the cutover block set to `content`: replaced between the markers, or appended when absent. */ +function withBlock(existing, content) { + const block = `${BLOCK_START}\n${content}\n${BLOCK_END}` + const text = String(existing || '') + const a = text.indexOf(BLOCK_START) + const b = text.indexOf(BLOCK_END, a + 1) + if (a >= 0 && b > a) return text.slice(0, a) + block + text.slice(b + BLOCK_END.length) + return text.trim() ? `${text.replace(/\s+$/, '')}\n\n${block}` : block +} + +const nextGate = (gates) => PERSISTED.find((g) => !(gates || {})[g]) + +/** A gate passed: the state with it kept (and its failure forgotten). */ +function pass(state, gate, now) { + const { [gate]: _gone, ...failed } = state.failed || {} + return { ...state, gates: { ...state.gates, [gate]: { at: now.toISOString() } }, failed } +} + +/** A cooling gate failed: when, and why. */ +function fail(state, gate, reason, now) { + return { ...state, failed: { ...state.failed, [gate]: { at: now.toISOString(), reason: String(reason) } } } +} + +/** What goes into drill.json: never `checks`. */ +function persistable(state, pr) { + const { checks: _checks, ...gates } = state.gates || {} + return { ...state, gates, pr: pr ? pr.number : null } +} + +/** + * What the hourly run does next, as data. `state` is the `cutover` object kept in drill.json: + * { wipe: 'YYYY-MM-DD', manifest, edgeSha, gates: { public, oxide, carbon, live: { at } }, failed: { carbon, live: { at, reason } }, + * pr, lastComment: 'YYYY-MM-DD', merged, gaveUp, mergeError } + * `obs`: { manifest, edgeSha, ahead (commits edge has that main lacks), openPr (number or null), forced, + * checksPassed (this run, never kept), blocked: { gate, reason } }. + * Returns { state, action, gate, reason, retryAt, write, comment, merge }: `action` is nothing | giveUp | run | + * blocked | cooling | merge; `write`, `comment` and `merge` say whether the CLI may persist the state, comment, + * merge (a dry run: none). + */ +function decide(state, obs, now, { dryRun = false } = {}) { + const w = wipeWeekend(now) + const wipe = day(w.wipe) + let s = state || {} + if (s.wipe !== wipe) s = { wipe, manifest: obs.manifest, edgeSha: obs.edgeSha, gates: {}, failed: {} } // a new wipe starts over + else if ((obs.manifest && s.manifest !== obs.manifest) || (obs.edgeSha && s.edgeSha !== obs.edgeSha)) { + // A new build, or new code: every gate again, and the failures that cooled them are forgotten. + const { mergeError: _e, ...rest } = s + s = { ...rest, manifest: obs.manifest || s.manifest, edgeSha: obs.edgeSha || s.edgeSha, gates: {}, failed: {} } + } + const { checks: _checks, ...gates } = s.gates || {} + s = { ...s, gates, failed: s.failed || {}, pr: obs.openPr || null } // the PR is kept while it is open + const out = { state: s, write: !dryRun, comment: false, merge: false } + + // One comment a day, none before the build is known to have changed (a quiet Thursday morning), none without a PR. + const hold = (action, gate, reason, retryAt) => { + const due = !dryRun && !!s.pr && !!s.gates.public && s.lastComment !== day(now) + return { ...out, state: due ? { ...s, lastComment: day(now) } : s, action, gate, reason, retryAt, comment: due } + } + + if (s.merged) return { ...out, action: 'nothing', reason: 'already merged' } + if (obs.ahead === 0 && !dryRun) return { ...out, action: 'nothing', reason: 'edge has no commits main lacks' } + if (now.getTime() >= w.deadline.getTime() && !obs.forced) { + if (s.gaveUp) return { ...out, action: 'nothing', reason: 'gave up already' } + return { ...out, state: { ...s, gaveUp: true }, action: 'giveUp', gate: nextGate(s.gates), comment: !dryRun && !!s.pr } + } + if (obs.blocked) { + const retryAt = COOLING.includes(obs.blocked.gate) ? new Date(now.getTime() + COOL_DOWN_HOURS * H) : null + return hold('blocked', obs.blocked.gate, obs.blocked.reason, retryAt) + } + const gate = nextGate(s.gates) + if (!gate) return obs.checksPassed ? { ...out, action: 'merge', merge: !dryRun } : { ...out, action: 'run', gate: 'checks' } + const failed = COOLING.includes(gate) && s.failed[gate] + if (failed && now.getTime() - Date.parse(failed.at) < COOL_DOWN_HOURS * H) { + return hold('cooling', gate, failed.reason, new Date(Date.parse(failed.at) + COOL_DOWN_HOURS * H)) + } + return { ...out, action: 'run', gate } +} + +/** The comment for a decision that asks for one. */ +function commentText(d, state = {}) { + const label = GATE_LABELS[d.gate] || d.gate + if (d.action === 'giveUp') { + const head = 'The wipe-day cutover (D332) stops here: the window closed (Sunday 23:00 UTC).' + const tail = 'Merge `edge` into `main` by hand when it is ready.' + if (state.mergeError) return `${head} Every gate passed; the merge failed: ${neutralise(state.mergeError)}. ${tail}` + if (!d.gate) return `${head} Every gate passed, but the PR checks and the merge did not complete. ${tail}` + return `${head} Gate "${label}" had not passed. ${tail}` + } + const when = d.retryAt ? ` It will not try this gate again before ${d.retryAt.toISOString().slice(0, 16).replace('T', ' ')} UTC.` : ' It tries again every hour until Sunday 23:00 UTC.' + return `The wipe-day cutover (D332) is waiting at gate "${label}": ${neutralise(d.reason)}.${when} This is the day's only comment.` +} + +/** Whether a dispatch is a dry run: unless its dry_run input is exactly 'false' it is. (A schedule never is.) */ +function isDry(a, env) { + return !!a['dry-run'] || (env.GITHUB_EVENT_NAME === 'workflow_dispatch' && env.DRY_RUN_INPUT !== 'false') +} + +// ---------------------------------------------------------------- plumbing + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +const log = (...a) => console.log(`[cutover ${new Date().toISOString().slice(11, 19)}]`, ...a) + +/** What the commands reach outside themselves for; cutover.test.js passes fakes. */ +function realDeps() { + return { + env: process.env, + now: () => new Date(), + sleep, + fetch: (u, o) => fetch(u, o), + git: (...a) => execFileSync('git', a, { cwd: ROOT, encoding: 'utf8' }).trim(), + panel: () => require('./drill.js').panel(), + } +} + +function args() { + const out = { _: [] } + const argv = process.argv.slice(2) + for (let i = 0; i < argv.length; i++) { + if (argv[i].startsWith('--')) { + const k = argv[i].slice(2) + out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true + } else out._.push(argv[i]) + } + return out +} + +const readJson = (f, fallback) => { + try { + return JSON.parse(fs.readFileSync(f, 'utf8')) + } catch { + return fallback + } +} + +const writeJson = (f, data) => fs.writeFileSync(f, JSON.stringify(data, null, 2) + '\n') + +const output = (env, name, value) => { + if (env.GITHUB_OUTPUT) fs.appendFileSync(env.GITHUB_OUTPUT, `${name}=${value}\n`) +} + +const summary = (env, md) => { + if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, md + '\n') +} + +function aheadCount(a, deps) { + if (a.ahead !== undefined && a.ahead !== true) return Number(a.ahead) + return Number(deps.git('rev-list', '--count', 'origin/main..origin/edge')) +} + +/** Gitea's REST API for this repo (the idiom of drill.js fileIssue). The token is never printed. */ +function gitea(token, deps) { + if (!token) throw new Error('no Gitea token in the environment') + const base = `${deps.env.GITHUB_SERVER_URL}/api/v1/repos/${deps.env.GITHUB_REPOSITORY}` + const headers = { Authorization: 'token ' + token, 'Content-Type': 'application/json', Accept: 'application/json' } + const call = async (method, p, data) => { + const r = await deps.fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined }) + const text = await r.text() + if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${text.slice(0, 200)}`) + return text ? JSON.parse(text) : null + } + return { + call, + async openPr() { + return (await call('GET', '/pulls?state=open&limit=50')).find((p) => p.head.ref === 'edge' && p.base.ref === 'main') || null + }, + } +} + +/** + * The `Closes #N` numbers of the PRs merged into edge since the last cutover: after the committer date of + * `git merge-base origin/main origin/edge`, the newest edge commit already in main. + */ +async function closesSinceCutOff(g, deps) { + let since = 0 + try { + since = Date.parse(deps.git('log', '-1', '--format=%cI', deps.git('merge-base', 'origin/main', 'origin/edge'))) || 0 + } catch {} + // A PR whose merge commit main already has shipped in an earlier cutover, even when its merged_at is a moment + // after the cut-off's committer date (edge's tip is often that very merge commit). + const shipped = (sha) => { + if (!sha) return false + try { + return /\borigin\/main\b/.test(deps.git('branch', '-r', '--contains', sha, '--list', 'origin/main')) + } catch { + return false + } + } + const bodies = [] + for (let page = 1; page <= 10; page++) { + const prs = await g.call('GET', `/pulls?state=closed&limit=50&page=${page}`) + for (const pr of prs) { + if (pr.merged && pr.base && pr.base.ref === 'edge' && Date.parse(pr.merged_at) > since && !shipped(pr.merge_commit_sha)) bodies.push(pr.body) + } + if (prs.length < 50) break + } + return collectCloses(bodies) +} + +// ---------------------------------------------------------------- window and pending + +function window_(a, deps = realDeps()) { + const now = a.now ? new Date(a.now) : deps.now() + const w = wipeWeekend(now) + const open = w.open || w.final || !!a.force + const verdict = w.open ? 'in the wipe weekend' : w.final ? 'the hour after it (the last run, to give up out loud)' : a.force ? 'outside the wipe weekend, but run by hand: going ahead' : 'outside the wipe weekend, nothing to do' + log(`the wipe is ${day(w.wipe)}, the cutover works until ${w.deadline.toISOString().slice(0, 16)}Z: ${verdict} (open=${open})`) + output(deps.env, 'open', open ? 'true' : 'false') +} + +function pending(a, deps = realDeps()) { + const ahead = aheadCount(a, deps) + const run = ahead > 0 || !!a.force + log(`edge is ${ahead} commit(s) ahead of main: ${run ? 'going ahead' : 'nothing to merge'}`) + output(deps.env, 'run', run ? 'true' : 'false') +} + +// ---------------------------------------------------------------- the rig + +async function carbonGate(ctx) { + const { waitRunning } = require('./drill.js') + const p = ctx.deps.panel() + const id = (await p.servers())[CARBON_RIG] + if (!id) return `no server named ${CARBON_RIG} on the panel` + await p.write(id, CARBON_PLUGIN, fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8')) + const was = await p.state(id) + log(`${CARBON_RIG} is ${was}: ${was === 'offline' ? 'starting' : 'restarting'} it (its egg updates Rust and Carbon)`) + await p.power(id, was === 'offline' ? 'start' : 'restart') + // `running` right after the signal is the old process: wait for it to leave first. + for (let i = 0; i < 60 && was !== 'offline' && (await p.state(id)) === 'running'; i++) await ctx.deps.sleep(5000) + if (!(await waitRunning(p, id, 45 * 60000))) return `${CARBON_RIG} did not come up in 45 minutes, or crashed on start` + // It must have updated to the public build, or everything after proves nothing. + const built = carbonBuild(await p.readOr(id, ACF, null), ctx.stat.manifest) + if (built !== true) return built + const lines = await carbonStatus(p, id, 30 * 60000, ctx.deps) + if (lines) log('rnpc.status:', lines.find((l) => l.includes('navmesh='))) + return carbonOk(lines || []) +} + +/** `rnpc.status` over the console until the navmesh is ready; the last answer if it never is. */ +async function carbonStatus(p, id, timeoutMs, deps) { + const end = Date.now() + timeoutMs + let last = null + while (Date.now() < end) { + try { + const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim()) + const at = lines.findIndex((l) => l.includes('RunicNPC ') && l.includes('api=')) + if (at >= 0) { + last = lines.slice(at, at + 14) + if (last.some((l) => l.includes('navmesh=ready'))) return last + } + } catch { + // the panel's console is not up yet + } + await deps.sleep(30000) + } + return last +} + +/** Writes whether rust-carbon is running now, to put it back if the job leaves it stopped. */ +async function rigNote(a, deps = realDeps()) { + const work = path.resolve(a.work || 'cutover-work') + const p = deps.panel() + const id = (await p.servers())[CARBON_RIG] + const state = id ? await p.state(id) : null + fs.mkdirSync(work, { recursive: true }) + writeJson(path.join(work, 'rig-carbon.json'), { name: CARBON_RIG, state }) + log(`${CARBON_RIG} is ${state || 'not on the panel'}`) +} + +/** Starts rust-carbon (power `start` on it alone) when it was running at the start of the job and is offline now. */ +async function rigRestore(a, deps = realDeps()) { + const rec = readJson(path.join(path.resolve(a.work || 'cutover-work'), 'rig-carbon.json'), null) + if (!rec || rec.state !== 'running') return log(`${CARBON_RIG} was not running at the start: nothing to put back`) + const p = deps.panel() + const id = (await p.servers())[CARBON_RIG] + if (!id) return log(`${CARBON_RIG} is not on the panel`) + const now = await p.state(id) + if (now !== 'offline') return log(`${CARBON_RIG} is ${now}: left as it is`) + await p.power(id, 'start') + log(`${CARBON_RIG} was running at the start and is offline: started it`) +} + +// ---------------------------------------------------------------- the gates + +async function checksGate(ctx) { + const wait = !ctx.dryRun + let pr = ctx.pr + if (!pr) return ctx.dryRun ? 'no cutover PR is open (a dry run opens none)' : 'no cutover PR is open' + const end = Date.now() + CHECKS_WAIT_MS + for (;;) { + pr = (await ctx.g.call('GET', `/pulls/${pr.number}`)) || pr + if (pr.head.sha !== ctx.edgeSha) return `the PR's head ${String(pr.head.sha).slice(0, 8)} is not the commit the gates ran on (${String(ctx.edgeSha).slice(0, 8)})` + const v = checkState(await ctx.g.call('GET', `/commits/${pr.head.sha}/status`)) + if (v.state === 'success') return true + if (v.state === 'failed') return v.reason + if (!wait || Date.now() >= end) return `${v.reason} (waited ${wait ? 15 : 0} minutes)` + await ctx.deps.sleep(30000) + } +} + +const RUNNERS = { + public: async (c) => publicGate(c.stat, c.staging, c.wipe), + oxide: async (c) => oxideReady(c.stat), + carbon: async (c) => carbonGate(c), + live: async (c) => liveOk(readJson(path.join(c.work, 'live.json'), null)), + checks: async (c) => checksGate(c), +} + +/** Runs one gate; whatever it throws is its reason, so a flaky API call blocks an hour, not the state. */ +async function runGate(gate, ctx) { + log(`gate ${gate}...`) + let r + try { + r = await RUNNERS[gate](ctx) + } catch (e) { + r = `the gate stopped: ${e.message}` + } + log(`gate ${gate}: ${r === true ? 'passed' : `NOT passed: ${r}`}`) + return r +} + +// ---------------------------------------------------------------- the PR + +function gateRows(state, rows) { + return GATES.map((g) => { + const done = state.gates[g] + const row = rows[g] + if (done) return { name: GATE_LABELS[g], status: 'passed', note: done.at.slice(0, 16).replace('T', ' ') + ' UTC' } + return { name: GATE_LABELS[g], status: row ? row.status : 'not yet', note: row ? row.detail : '' } + }) +} + +/** + * Opens the edge to main PR, or brings the marked block of the open one's body up to date. A PR it did not + * open keeps its title and the rest of its body. Not in a dry run, and never with a blank manifest. + */ +async function syncPr(ctx) { + if (ctx.dryRun) return ctx.pr + if (!ctx.stat.manifest) { + log('no manifest: the PR is left alone') + return ctx.pr + } + if (!ctx.closes) ctx.closes = await closesSinceCutOff(ctx.g, ctx.deps) + const content = cutoverBody({ wipe: ctx.wipeDate, manifest: ctx.stat.manifest, buildDate: ctx.stat.manifestDate, closes: ctx.closes, gates: gateRows(ctx.state, ctx.rows) }) + if (!ctx.pr) { + const title = `chore(release): RunicNPC cutover for the ${ctx.wipeDate} wipe, Rust ${ctx.stat.manifest}` + ctx.pr = await ctx.g.call('POST', '/pulls', { head: 'edge', base: 'main', title, body: withBlock('', content) }) + log(`opened PR #${ctx.pr.number}`) + } else { + const body = withBlock(ctx.pr.body, content) + if (body !== ctx.pr.body) { + ctx.pr = { ...ctx.pr, ...((await ctx.g.call('PATCH', `/pulls/${ctx.pr.number}`, { body })) || {}) } + log(`updated PR #${ctx.pr.number}`) + } + } + ctx.state = { ...ctx.state, pr: ctx.pr.number } + return ctx.pr +} + +// ---------------------------------------------------------------- gates (the command) + +async function gates(a, deps = realDeps()) { + const env = deps.env + const dryRun = isDry(a, env) + const afterLive = !!a['after-live'] + const work = path.resolve(a.work || 'cutover-work') + const file = readJson(a.state, {}) + const now = deps.now() + const w = wipeWeekend(now) + const stat = readJson(path.join(work, 'static.json'), {}) + const staging = readJson(path.join(path.resolve(a['staging-work'] || work + '-staging'), 'static.json'), null) + const g = gitea(env.GITEA_TOKEN, deps) + const prev = (afterLive && readJson(path.join(work, 'cutover-run.json'), null)) || { rows: {} } + + const pr = await g.openPr() + const edgeSha = a.edge || deps.git('rev-parse', 'HEAD') + const obs = { manifest: stat.manifest || '', edgeSha, ahead: aheadCount(a, deps), openPr: pr && pr.number, forced: !!a.force && !(w.open || w.final), checksPassed: !!(dryRun && prev.checks) } + // A dry run starts from nothing: it is there to try every gate, not to trust an earlier hour's. + let d = decide(dryRun ? undefined : file.cutover, obs, now, { dryRun }) + const ctx = { g, deps, stat, staging, work, now, dryRun, pr, edgeSha, wipe: w.wipe, wipeDate: day(w.wipe), state: d.state, rows: prev.rows, closes: null } + if (dryRun && prev.gates) ctx.state = { ...ctx.state, gates: prev.gates } // what the first half of this dry run passed + else for (const k of Object.keys(ctx.state.gates)) ctx.rows[k] = { status: 'passed earlier', detail: ctx.state.gates[k].at } + const note = (gate, r) => (ctx.rows[gate] = r === true ? { status: 'passed', detail: '' } : { status: 'failed', detail: r }) + // The state goes to the file after every gate result, before the PR is touched. + const persist = () => { + if (dryRun) return + file.cutover = persistable(ctx.state, ctx.pr) + writeJson(a.state, file) + } + const safely = async (what, fn) => { + try { + return await fn() + } catch (e) { + log(`${what} failed (the gates are kept): ${e.message}`) + } + } + + let blocked = null + let deferred = false + if (d.action === 'nothing') log(`nothing to do: ${d.reason}`) + else if (d.action === 'giveUp') log(`the cutover window closed with ${d.gate || 'every gate'} not done`) + else if (dryRun) { + // Every gate it can; live waits for its own step. + for (const gate of GATES) { + if (gate === 'live' && !afterLive) { + deferred = true + ctx.rows.live = { status: 'runs next', detail: 'drill.js live --branch public' } + } else if ((gate === 'live') === afterLive || (afterLive && gate === 'checks' && !prev.checks)) { + const r = await runGate(gate, ctx) + note(gate, r) + if (r === true) { + if (gate === 'checks') obs.checksPassed = true + else ctx.state = pass(ctx.state, gate, now) + } else if (!blocked) blocked = { gate, reason: r } + } + } + } else { + while (d.action === 'run') { + if (d.gate === 'live' && !afterLive) { + deferred = true + break + } + const r = await runGate(d.gate, ctx) + note(d.gate, r) + if (r !== true) { + blocked = { gate: d.gate, reason: r } + if (COOLING.includes(d.gate)) ctx.state = fail(ctx.state, d.gate, r, now) + persist() + break + } + if (d.gate === 'checks') obs.checksPassed = true + else { + ctx.state = pass(ctx.state, d.gate, now) + persist() + } + if (d.gate === 'public') { + await safely('opening the PR', () => syncPr(ctx)) + obs.openPr = ctx.pr ? ctx.pr.number : null + } + d = decide(ctx.state, obs, now, { dryRun }) + } + } + + // What the run ends in. + if (d.action !== 'nothing' && d.action !== 'giveUp') { + obs.openPr = ctx.pr ? ctx.pr.number : null + d = decide(ctx.state, blocked ? { ...obs, blocked } : obs, now, { dryRun }) + } + if (!dryRun && ctx.state.gates.public && d.action !== 'giveUp' && !ctx.state.merged) await safely('updating the PR', () => syncPr(ctx)) + const end = d.action + log(`${end}${d.gate ? ` (${d.gate})` : ''}${d.reason ? `: ${d.reason}` : ''}`) + + let finalState = d.state + if (d.comment && ctx.pr) { + const done = await safely('commenting', async () => { + await g.call('POST', `/issues/${ctx.pr.number}/comments`, { body: commentText(d, d.state) }) + return true + }) + if (done) log(`commented on PR #${ctx.pr.number}`) + else finalState = { ...finalState, lastComment: ctx.state.lastComment } // not said, so it may be said next hour + } + ctx.state = finalState + persist() + + fs.mkdirSync(work, { recursive: true }) + writeJson(path.join(work, 'cutover-run.json'), { rows: ctx.rows, gates: ctx.state.gates, checks: !!obs.checksPassed }) + output(env, 'live', deferred ? 'true' : 'false') + output(env, 'merge', d.action === 'merge' && d.merge ? 'true' : 'false') + output(env, 'sha', edgeSha) + if (ctx.pr) output(env, 'pr', ctx.pr.number) + if (!deferred) { + const rows = GATES.map((gate) => `| ${GATE_LABELS[gate]} | ${(ctx.rows[gate] || { status: 'not reached' }).status} | ${neutralise((ctx.rows[gate] || {}).detail || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`) + summary( + env, + [ + `## Wipe-day cutover${dryRun ? ' (dry run: nothing created, edited, commented or merged)' : ''}: ${end}`, + '', + `Wipe ${ctx.wipeDate}; Rust public manifest ${stat.manifest || '?'} of ${stat.manifestDate || '?'}; edge ${edgeSha.slice(0, 8)} is ${obs.ahead} commit(s) ahead of main.`, + '', + '| Gate | State | Detail |', + '| --- | --- | --- |', + ...rows, + d.action === 'merge' ? `\nEvery gate passed${dryRun ? ': a real run would merge now' : ''}.` : '', + ].join('\n') + ) + } + return d +} + +// ---------------------------------------------------------------- merge and after + +/** Records why the merge failed where the next hour (and a last comment) can read it. */ +function recordMergeError(stateFile, message) { + if (!stateFile || stateFile === true) return + const file = readJson(stateFile, {}) + file.cutover = { ...(file.cutover || {}), mergeError: String(message).slice(0, 300) } + writeJson(stateFile, file) +} + +async function merge(a, deps = realDeps()) { + const g = gitea(deps.env.MERGE_TOKEN, deps) + if (!a.pr || a.pr === true) throw new Error('--pr is required') + if (!a.sha || a.sha === true) throw new Error('--sha is required') + try { + const before = await g.call('GET', `/pulls/${a.pr}`) + if (before.head.sha !== a.sha) throw new Error(`PR #${a.pr}'s head ${String(before.head.sha).slice(0, 8)} is not the commit the gates ran on (${a.sha.slice(0, 8)})`) + // Not the squash: release.yml reads the feat/fix subjects of the commits that arrive. Never delete edge. + // head_commit_id: Gitea refuses the merge when the head moved since the gates. + await g.call('POST', `/pulls/${a.pr}/merge`, { Do: 'merge', delete_branch_after_merge: false, head_commit_id: a.sha }) + const pr = await g.call('GET', `/pulls/${a.pr}`) + if (!pr.merged) throw new Error(`PR #${a.pr} is not merged after the merge call`) + } catch (e) { + recordMergeError(a.state, e.message) + throw e + } + log(`merged PR #${a.pr}`) + output(deps.env, 'merged', 'true') +} + +async function after(a, deps = realDeps()) { + const g = gitea(deps.env.GITEA_TOKEN, deps) + const pr = await g.call('GET', `/pulls/${a.pr}`) + if (!pr.merged) return log(`PR #${a.pr} is not merged: nothing to record`) + const file = readJson(a.state, {}) + const { mergeError: _e, ...kept } = file.cutover || {} + file.cutover = { ...kept, merged: true, pr: pr.number } + writeJson(a.state, file) + + // release.yml cuts the release as the push to main arrives. + const mergedAt = Date.parse(pr.merged_at) + let release = null + for (const end = deps.now().getTime() + RELEASE_WAIT_MS; !release && deps.now().getTime() < end; ) { + const list = await g.call('GET', '/releases?draft=false&pre-release=false&limit=5') + release = (list || []).find((r) => Date.parse(r.created_at) > mergedAt) || null + if (!release) await deps.sleep(60000) + } + if (!release) return log('no release appeared in 30 minutes: no issue is commented') + log(`released as ${release.tag_name}`) + output(deps.env, 'release', release.tag_name) + + const text = `Released in ${release.tag_name} by the wipe cutover (D332).` + for (const n of collectCloses([pr.body])) { + const issue = await g.call('GET', `/issues/${n}`) + if (issue.state !== 'closed') { + log(`#${n} is not closed: no comment`) + continue + } + if ((await g.call('GET', `/issues/${n}/comments`)).some((c) => c.body === text)) continue + await g.call('POST', `/issues/${n}/comments`, { body: text }) + log(`commented on #${n}`) + } +} + +// ---------------------------------------------------------------- main + +module.exports = { + wipeWeekend, + publicChanged, + publicGate, + oxideReady, + parseVdf, + acfDepotManifest, + carbonBuild, + carbonOk, + liveOk, + checkState, + collectCloses, + cutoverBody, + withBlock, + decide, + commentText, + neutralise, + isDry, + pass, + fail, + persistable, + nextGate, + manifestTime, + gates, + merge, + after, + rigNote, + rigRestore, + checksGate, + closesSinceCutOff, + gitea, + GATES, + COOL_DOWN_HOURS, + BLOCK_START, + BLOCK_END, +} + +if (require.main === module) { + const a = args() + const commands = { window: window_, pending, gates, merge, after, 'rig-note': rigNote, 'rig-restore': rigRestore } + const command = commands[a._[0]] + if (!command) { + console.error('usage: node tools/drill/cutover.js window|pending|gates|merge|after|rig-note|rig-restore [--state f] [--work dir] [--pr n] [--sha s] [--dry-run] [--force] [--after-live]') + process.exit(2) + } + Promise.resolve(command(a)).catch((e) => { + console.error(e.message || e) + process.exit(1) + }) +} diff --git a/tools/drill/cutover.test.js b/tools/drill/cutover.test.js new file mode 100644 index 0000000..5cda7b5 --- /dev/null +++ b/tools/drill/cutover.test.js @@ -0,0 +1,783 @@ +// The wipe-day cutover (docs/runicnpc/PLAN.md D332): when it works, what each gate accepts, which issues the +// PR closes, what the hourly run does next, and the commands against a fake Gitea and a fake panel. +// node --test tools/drill/cutover.test.js + +const test = require('node:test') +const assert = require('node:assert') +const fs = require('fs') +const os = require('os') +const path = require('path') +const c = require('./cutover.js') +const { wipeWeekend, publicChanged, publicGate, oxideReady, carbonOk, carbonBuild, acfDepotManifest, liveOk, checkState, collectCloses, cutoverBody, withBlock, decide, commentText, neutralise, isDry, pass, fail, persistable, nextGate, manifestTime, GATES, COOL_DOWN_HOURS } = c + +const at = (s) => new Date(s) +const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'cutover-test-')) + +// ---- wipeWeekend: November 2026 has its wipe on Thursday the 5th, so the weekend ends Sunday the 8th. + +test('the weekend opens at 17:00 UTC on the wipe Thursday and not a minute before', () => { + assert.equal(wipeWeekend(at('2026-11-05T16:59:00Z')).open, false) + assert.equal(wipeWeekend(at('2026-11-05T17:00:00Z')).open, true) + assert.equal(wipeWeekend(at('2026-11-05T17:00:00Z')).wipe.toISOString(), '2026-11-05T00:00:00.000Z') +}) + +test('it closes at Sunday 23:00 UTC, with one more hour to give up in', () => { + assert.equal(wipeWeekend(at('2026-11-08T22:59:00Z')).open, true) + const late = wipeWeekend(at('2026-11-08T23:00:00Z')) + assert.equal(late.open, false) + assert.equal(late.final, true) + assert.equal(late.deadline.toISOString(), '2026-11-08T23:00:00.000Z') + assert.equal(wipeWeekend(at('2026-11-08T23:59:00Z')).final, true) + assert.equal(wipeWeekend(at('2026-11-09T00:00:00Z')).final, false) + assert.equal(wipeWeekend(at('2026-11-05T17:00:00Z')).final, false) +}) + +test('a Thursday that is not the first of the month is closed, as is the morning of the wipe', () => { + assert.equal(wipeWeekend(at('2026-11-12T18:00:00Z')).open, false) + assert.equal(wipeWeekend(at('2026-11-05T09:00:00Z')).open, false) + assert.equal(wipeWeekend(at('2026-11-06T12:00:00Z')).open, true) +}) + +test('a month whose first Thursday is the 7th ends its weekend on the 10th', () => { + const w = wipeWeekend(at('2026-05-09T12:00:00Z')) + assert.equal(w.wipe.toISOString(), '2026-05-07T00:00:00.000Z') + assert.equal(w.deadline.toISOString(), '2026-05-10T23:00:00.000Z') + assert.equal(w.open, true) + assert.equal(wipeWeekend(at('2026-05-10T23:00:00Z')).open, false) +}) + +test('when the 1st is the Thursday, the weekend is the 1st to the 4th', () => { + const w = wipeWeekend(at('2026-01-02T00:00:00Z')) + assert.equal(w.wipe.toISOString(), '2026-01-01T00:00:00.000Z') + assert.equal(w.deadline.toISOString(), '2026-01-04T23:00:00.000Z') + assert.equal(w.open, true) + assert.equal(wipeWeekend(at('2026-01-01T16:59:00Z')).open, false) +}) + +// ---- the manifest date, strictly + +test('manifestTime reads M/D/YYYY HH:mm:ss in 24 hours, with AM/PM and ISO fallbacks, and nothing else', () => { + assert.equal(manifestTime('10/31/2026 11:48:58'), Date.UTC(2026, 9, 31, 11, 48, 58)) + assert.equal(manifestTime('11/5/2026 13:00:00'), Date.UTC(2026, 10, 5, 13, 0, 0)) + assert.equal(manifestTime('11/5/2026 12:00:00 AM'), Date.UTC(2026, 10, 5, 0, 0, 0)) + assert.equal(manifestTime('11/5/2026 01:30:00 PM'), Date.UTC(2026, 10, 5, 13, 30, 0)) + assert.equal(manifestTime('2026-11-05T09:00:00Z'), Date.UTC(2026, 10, 5, 9, 0, 0)) + for (const bad of ['', undefined, '?', 'yesterday', '13/40/2026 01:00:00', '11/31/2026 01:00:00', '11/5/2026 25:00:00', '11/5/2026 13:00:00 PM', '5.11.2026 10:00:00', '11/5/26 10:00:00']) { + assert.ok(Number.isNaN(manifestTime(bad)), `${bad} should not parse`) + } +}) + +// ---- the gates + +test('publicChanged: the manifest must be dated the wipe day or later (UTC)', () => { + const wipe = at('2026-11-05T00:00:00Z') + assert.equal(publicChanged({ manifestDate: '11/4/2026 23:59:59' }, wipe), false) + assert.equal(publicChanged({ manifestDate: '11/5/2026 00:00:00' }, wipe), true) + assert.equal(publicChanged({ manifestDate: '10/31/2026 11:48:58' }, wipe), false) + assert.equal(publicChanged({ manifestDate: 'yesterday' }, wipe), false) + assert.equal(publicChanged(undefined, wipe), false) +}) + +test('publicGate: public must also be the build staging has now (the promotion went through)', () => { + const wipe = at('2026-11-05T00:00:00Z') + const pub = { manifest: '8140', manifestDate: '11/5/2026 11:48:58' } + assert.equal(publicGate(pub, { manifest: '8140' }, wipe), true) + assert.match(publicGate(pub, { manifest: '8150' }, wipe), /not its current staging build \(manifest 8150\)/) + assert.match(publicGate(pub, { note: 'no Assembly' }, wipe), /staging build could not be read/) + assert.match(publicGate(pub, null, wipe), /staging build could not be read/) + assert.match(publicGate({ manifest: '8100', manifestDate: '10/31/2026 11:48:58' }, { manifest: '8100' }, wipe), /from before the wipe/) + assert.match(publicGate({ manifest: '8140', manifestDate: 'garbage', note: 'n' }, { manifest: '8140' }, wipe), /no usable manifest date/) +}) + +test('oxideReady: true, or the reason', () => { + const ok = { managed: 'ok', oxideBehind: 0, compile: 'ok', fields: 'same' } + assert.equal(oxideReady(ok), true) + assert.match(oxideReady({ ...ok, oxideBehind: 12 }), /Oxide's public build is 12 member\(s\) behind/) + assert.match(oxideReady({ ...ok, fields: 'differs' }), /field list differs from public Rust's/) + assert.match(oxideReady({ ...ok, compile: 'errors' }), /does not compile/) + assert.match(oxideReady({ ...ok, managed: 'error', note: 'DepotDownloader could not be fetched' }), /could not fetch Rust's public build \(DepotDownloader/) + assert.match(oxideReady({ ...ok, oxideBehind: -1 }), /could not be compared/) + assert.match(oxideReady({ ...ok, fields: 'error' }), /could not be checked/) + assert.match(oxideReady(undefined), /could not fetch/) +}) + +const ACF = `"AppState" +{ +\t"appid"\t\t"258550" +\t"buildid"\t\t"22222222" +\t"InstalledDepots" +\t{ +\t\t"258551" +\t\t{ +\t\t\t"manifest"\t\t"111" +\t\t\t"size"\t\t"1" +\t\t} +\t\t"258552" +\t\t{ +\t\t\t"manifest"\t\t"8140123456789" +\t\t\t"size"\t\t"9000000000" +\t\t} +\t} +\t"SharedDepots" +\t{ +\t\t"228984"\t\t"228990" +\t} +} +` + +test('the appmanifest: the Linux depot\'s manifest, and null when it is not there', () => { + assert.equal(acfDepotManifest(ACF), '8140123456789') + assert.equal(acfDepotManifest(ACF, '258551'), '111') + assert.equal(acfDepotManifest(ACF, '999'), null) + assert.equal(acfDepotManifest(''), null) + assert.equal(acfDepotManifest(undefined), null) + assert.equal(acfDepotManifest('"AppState" { "InstalledDepots" { "258552" "oops" } }'), null) +}) + +test('carbonBuild: the rig must run the public manifest', () => { + assert.equal(carbonBuild(ACF, '8140123456789'), true) + assert.match(carbonBuild(ACF, '9999'), /rust-carbon still runs manifest 8140123456789 \(public is 9999\)/) + assert.match(carbonBuild(null, '9999'), /could not be read/) +}) + +const STATUS = 'framework=carbon navmesh=ready swap fields: npc=58/58 brain=32/32 missing=- added=-' + +test('carbonOk: a Carbon server with the navmesh ready and nothing missing or added', () => { + assert.equal(carbonOk(['RunicNPC 0.0.0 api=1 hooks=40 fired=3 silent=0', 'fired: x', STATUS, 'npcs=0']), true) + assert.equal(carbonOk(STATUS), true) +}) + +test('carbonOk: each way it can fail says why', () => { + assert.match(carbonOk([STATUS.replace('added=-', 'added=BasePlayer.x')]), /does not know: BasePlayer\.x/) + assert.match(carbonOk([STATUS.replace('missing=-', 'missing=BaseNpc.y,Brain.z')]), /missing on Carbon: BaseNpc\.y,Brain\.z/) + assert.match(carbonOk([STATUS.replace('framework=carbon', 'framework=oxide')]), /framework=oxide, not carbon/) + assert.match(carbonOk([STATUS.replace('navmesh=ready', 'navmesh=building')]), /navmesh=building/) + assert.match(carbonOk([]), /never answered/) + assert.match(carbonOk(['RunicNPC 0.0.0 api=1']), /never answered/) +}) + +test('liveOk: ran, harness done, some checks passed, no failed check and no finding', () => { + const good = { branch: 'public', ran: true, findings: [], harness: { done: true, pass: 266, fail: 0, failed: [] } } + assert.equal(liveOk(good), true) + assert.match(liveOk(null), /no live\.json/) + assert.match(liveOk({ ...good, ran: false, findings: ['The server did not start on Rust public 1'] }), /did not start/) + assert.match(liveOk({ ...good, ran: false, skipped: "Oxide's public build is 3 member(s) behind" }), /3 member/) + assert.match(liveOk({ ...good, harness: { done: false, pass: 5, fail: 0, failed: [] }, findings: ['The harness stopped after x'] }), /harness stopped after x/) + assert.match(liveOk({ ...good, harness: { done: true, pass: 5, fail: 1, failed: ['place.set: FAIL off the navmesh'] } }), /place\.set: FAIL off the navmesh/) + assert.match(liveOk({ ...good, findings: ['RunicNPC logged 2 error line(s) in Oxide\'s log during the run.'] }), /logged 2 error/) + assert.match(liveOk({ ...good, branch: 'staging' }), /staging branch/) + assert.match(liveOk({ ...good, harness: { done: true, pass: 0, fail: 0, failed: [] } }), /passed no check/) +}) + +test('checkState: success needs a plugin-checks context; a failure or a wait is said', () => { + const s = (context, status) => ({ context, status }) + assert.equal(checkState({ state: 'success', statuses: [s('PR Checks / plugin-checks (pull_request)', 'success')] }).state, 'success') + assert.equal(checkState({ state: 'success', statuses: [] }).state, 'pending') + assert.equal(checkState({ state: 'success', statuses: [s('other', 'success')] }).state, 'pending') + assert.equal(checkState({ state: 'pending', statuses: [s('PR Checks / plugin-checks (pull_request)', 'pending')] }).state, 'pending') + const failed = checkState({ state: 'failure', statuses: [s('PR Checks / plugin-checks (pull_request)', 'failure')] }) + assert.equal(failed.state, 'failed') + assert.match(failed.reason, /plugin-checks.*failure/) +}) + +// ---- what the PR closes, and what is written into it + +test('collectCloses reads closing keywords before #N, in any case', () => { + assert.deepEqual(collectCloses(['Closes #20', 'fixes #3', 'RESOLVED #9', 'Fixed #12.', 'close #4', 'Resolves: #8']), [3, 4, 8, 9, 12, 20]) +}) + +test('collectCloses takes several on a line, and dedupes and sorts', () => { + assert.deepEqual(collectCloses(['Closes #22, closes #20 and fixes #5', 'Closes #20\nCloses #5', 'closes #100']), [5, 20, 22, 100]) +}) + +test('collectCloses ignores references, bare numbers, prose and other repositories', () => { + assert.deepEqual(collectCloses(['Refs #21', 'see #21', '#21', 'This closes the loop on #21', 'prefix #3', 'unfixes #4', 'Closes RunicGateway/docs#9', '']), []) + assert.deepEqual(collectCloses([undefined, null]), []) +}) + +test('cutoverBody names the wipe and the build, lists a Closes line per issue, and ends with the disclosure', () => { + const body = cutoverBody({ + wipe: '2026-11-05', + manifest: '8140', + buildDate: '11/5/2026 11:48:58', + closes: [20, 22], + gates: [ + { name: '1. Rust public', status: 'passed', note: '2026-11-05 17:11 UTC' }, + { name: '2. Oxide', status: 'failed', note: 'a | b' }, + ], + }) + assert.match(body, /D332/) + assert.match(body, /manifest \*\*8140\*\*, dated 11\/5\/2026 11:48:58/) + assert.match(body, /^Closes #20$/m) + assert.match(body, /^Closes #22$/m) + assert.match(body, /\| 2\. Oxide \| failed \| a \/ b \|/) + assert.ok(body.endsWith('Opened by the runicnpc-rust cutover workflow (D332).')) + assert.match(cutoverBody({ wipe: 'w', manifest: 'm', closes: [], gates: [] }), /No issue is closed/) +}) + +test('a reason can never close an issue: #123 is broken up in the body and in comments', () => { + assert.equal(neutralise('fixes #123 and #4'), 'fixes #​123 and #​4') + const body = cutoverBody({ wipe: 'w', manifest: 'm', closes: [], gates: [{ name: 'g', status: 'failed', note: 'Closes #5: fixes #123' }] }) + assert.deepEqual(collectCloses([body]), []) + const text = commentText({ action: 'blocked', gate: 'oxide', reason: 'resolves #7' }) + assert.deepEqual(collectCloses([text]), []) + // The deliberate lines still close. + assert.deepEqual(collectCloses([cutoverBody({ wipe: 'w', manifest: 'm', closes: [9], gates: [] })]), [9]) +}) + +test('withBlock appends the marked block, replaces only what is between the markers, and is stable', () => { + const once = withBlock('A human wrote this.', 'one') + assert.equal(once, 'A human wrote this.\n\n\none\n') + const twice = withBlock(once + '\n\nAnd this after.', 'two') + assert.equal(twice, 'A human wrote this.\n\n\ntwo\n\n\nAnd this after.') + assert.equal(withBlock(twice, 'two'), twice) + assert.equal(withBlock('', 'x'), '\nx\n') +}) + +// ---- decide + +const THU = at('2026-11-05T18:11:00Z') +const obs = (more = {}) => ({ manifest: '8140', edgeSha: 'sha1', ahead: 3, openPr: null, ...more }) +const fresh = () => decide(undefined, obs(), THU) +const passed = (state, ...names) => names.reduce((s, g) => pass(s, g, THU), state) + +test('a first run starts at the public gate and asks to be written', () => { + const d = fresh() + assert.equal(d.action, 'run') + assert.equal(d.gate, 'public') + assert.equal(d.write, true) + assert.equal(d.state.wipe, '2026-11-05') + assert.equal(d.state.manifest, '8140') + assert.equal(d.state.edgeSha, 'sha1') +}) + +test('the gates run in order, checks comes last and is never kept, and then it merges', () => { + assert.deepEqual(GATES, ['public', 'oxide', 'carbon', 'live', 'checks']) + let s = fresh().state + for (const g of GATES.slice(0, 4)) { + const d = decide(s, obs(), THU) + assert.equal(d.gate, g) + s = pass(d.state, g, THU) + } + const checks = decide(s, obs(), THU) + assert.equal(checks.action, 'run') + assert.equal(checks.gate, 'checks') + const done = decide(s, obs({ checksPassed: true }), THU) + assert.equal(done.action, 'merge') + assert.equal(done.merge, true) + assert.equal(nextGate({ public: {}, carbon: {} }), 'oxide') + // A checks entry found in a stored state is dropped. + const old = decide({ ...s, gates: { ...s.gates, checks: { at: 'x' } } }, obs(), THU) + assert.equal(old.state.gates.checks, undefined) + assert.equal(old.gate, 'checks') + assert.equal(persistable({ ...s, gates: { ...s.gates, checks: {} } }, { number: 4 }).gates.checks, undefined) + assert.equal(persistable(s, { number: 4 }).pr, 4) +}) + +test('a passed gate is never run again for the same build and the same code', () => { + const s = passed(fresh().state, 'public', 'oxide', 'carbon') + const d = decide(s, obs(), at('2026-11-05T19:11:00Z')) + assert.equal(d.action, 'run') + assert.equal(d.gate, 'live') + assert.ok(d.state.gates.carbon) +}) + +test('a new public manifest starts the gates again, keeping the open PR', () => { + const s = { ...passed(fresh().state, 'public', 'oxide', 'carbon'), pr: 9 } + const d = decide(s, obs({ manifest: '8141', openPr: 9 }), at('2026-11-06T10:11:00Z')) + assert.equal(d.gate, 'public') + assert.deepEqual(d.state.gates, {}) + assert.equal(d.state.manifest, '8141') + assert.equal(d.state.pr, 9) + assert.equal(decide(s, obs({ manifest: '8141', openPr: null }), at('2026-11-06T10:11:00Z')).state.pr, null) +}) + +test('a new edge commit clears every gate and every cooled failure, so nothing proven on old code counts', () => { + const s = fail(passed(fresh().state, 'public', 'oxide'), 'carbon', 'x', THU) + const d = decide({ ...s, mergeError: 'old' }, obs({ edgeSha: 'sha2' }), at('2026-11-05T19:11:00Z')) + assert.equal(d.action, 'run') + assert.equal(d.gate, 'public') + assert.deepEqual(d.state.gates, {}) + assert.deepEqual(d.state.failed, {}) + assert.equal(d.state.edgeSha, 'sha2') + assert.equal(d.state.mergeError, undefined) + // A state written before edgeSha was kept counts as changed code. + const legacy = { wipe: '2026-11-05', manifest: '8140', gates: { public: { at: 'x' } } } + assert.equal(decide(legacy, obs(), THU).gate, 'public') +}) + +test('a new wipe forgets everything, comments and a give-up included', () => { + const s = { ...passed(fresh().state, 'public'), lastComment: '2026-11-05', gaveUp: true } + const d = decide(s, obs(), at('2026-12-03T18:00:00Z')) + assert.equal(d.action, 'run') + assert.equal(d.state.wipe, '2026-12-03') + assert.deepEqual(d.state.gates, {}) + assert.equal(d.state.lastComment, undefined) + assert.equal(d.state.gaveUp, undefined) +}) + +test('nothing to do when edge has no commit main lacks, or when it already merged', () => { + const d = decide(undefined, obs({ ahead: 0 }), THU) + assert.equal(d.action, 'nothing') + assert.match(d.reason, /no commits/) + assert.equal(decide({ ...fresh().state, merged: true }, obs(), THU).action, 'nothing') +}) + +test('the cool-down: a failed carbon or live is not tried again for three hours, a new commit retries at once', () => { + assert.equal(COOL_DOWN_HOURS, 3) + const base = passed(fresh().state, 'public', 'oxide') + const failedCarbon = fail(base, 'carbon', 'did not come up', THU) // failed at 18:11 + const at59 = decide(failedCarbon, obs(), at('2026-11-05T21:10:00Z')) // 2 h 59 min later + assert.equal(at59.action, 'cooling') + assert.equal(at59.gate, 'carbon') + assert.equal(at59.retryAt.toISOString(), '2026-11-05T21:11:00.000Z') + assert.match(at59.reason, /did not come up/) + assert.equal(decide(failedCarbon, obs(), at('2026-11-05T21:11:00Z')).action, 'run') // 3 h 00 + assert.equal(decide(failedCarbon, obs(), at('2026-11-05T21:11:00Z')).gate, 'carbon') + assert.equal(decide(failedCarbon, obs({ edgeSha: 'sha2' }), at('2026-11-05T19:00:00Z')).gate, 'public') + assert.equal(decide(failedCarbon, obs({ manifest: '8141' }), at('2026-11-05T19:00:00Z')).gate, 'public') + // The same rule for live; public and oxide are cheap and never cool. + const failedLive = fail(passed(base, 'carbon'), 'live', 'a check failed', THU) + assert.equal(decide(failedLive, obs(), at('2026-11-05T20:00:00Z')).action, 'cooling') + assert.equal(decide(failedLive, obs(), at('2026-11-05T21:11:00Z')).gate, 'live') + assert.equal(decide(fail(fresh().state, 'oxide', 'x', THU), obs(), at('2026-11-05T18:30:00Z')).gate, 'public') + // A pass forgets the failure. + assert.equal(pass(failedCarbon, 'carbon', THU).failed.carbon, undefined) +}) + +test('a cooling gate comments once a day and says when it will try again', () => { + const s = fail(passed(fresh().state, 'public', 'oxide'), 'carbon', 'did not come up', THU) + const d = decide(s, obs({ openPr: 9 }), at('2026-11-05T19:11:00Z')) + assert.equal(d.comment, true) + assert.match(commentText(d, d.state), /not try this gate again before 2026-11-05 21:11 UTC/) + assert.match(commentText(d, d.state), /3\. A Carbon boot/) + assert.equal(decide(d.state, obs({ openPr: 9 }), at('2026-11-05T20:11:00Z')).comment, false) +}) + +test('a blocked gate comments once a day, and only after the public gate passed', () => { + const blocked = { gate: 'oxide', reason: "Oxide's public build is 12 member(s) behind" } + const s = passed(fresh().state, 'public') + const first = decide(s, obs({ openPr: 9, blocked }), THU) + assert.equal(first.action, 'blocked') + assert.equal(first.comment, true) + assert.equal(first.state.lastComment, '2026-11-05') + assert.equal(decide(first.state, obs({ openPr: 9, blocked }), at('2026-11-05T19:11:00Z')).comment, false) + assert.equal(decide(first.state, obs({ openPr: 9, blocked }), at('2026-11-06T09:11:00Z')).comment, true) + assert.equal(decide(fresh().state, obs({ openPr: 9, blocked: { gate: 'public', reason: 'old' } }), THU).comment, false) + assert.equal(decide(s, obs({ openPr: null, blocked }), THU).comment, false) + // A blocked carbon says when it will be tried again. + const carbon = decide(s, obs({ openPr: 9, blocked: { gate: 'carbon', reason: 'x' } }), THU) + assert.equal(carbon.retryAt.toISOString(), '2026-11-05T21:11:00.000Z') +}) + +test('after the deadline it gives up once, with a last comment', () => { + const s = passed(fresh().state, 'public', 'oxide') + const late = at('2026-11-08T23:11:00Z') + const d = decide(s, obs({ openPr: 9 }), late) + assert.equal(d.action, 'giveUp') + assert.equal(d.gate, 'carbon') + assert.equal(d.comment, true) + assert.equal(d.state.gaveUp, true) + assert.match(commentText(d, d.state), /Gate "3\. A Carbon boot on public Rust" had not passed/) + const again = decide(d.state, obs({ openPr: 9 }), at('2026-11-08T23:40:00Z')) + assert.equal(again.action, 'nothing') + assert.equal(again.comment, false) + assert.equal(decide(s, obs({ openPr: null }), late).comment, false) + assert.equal(decide(s, obs({ openPr: 9, forced: true }), late).action, 'run') +}) + +test('giving up with every gate passed and the merge failed says so, never gate "undefined"', () => { + const s = { ...passed(fresh().state, 'public', 'oxide', 'carbon', 'live'), mergeError: 'POST /pulls/5/merge: 405 not mergeable' } + const d = decide(s, obs({ openPr: 5 }), at('2026-11-08T23:11:00Z')) + assert.equal(d.action, 'giveUp') + assert.equal(d.gate, undefined) + const text = commentText(d, d.state) + assert.match(text, /Every gate passed; the merge failed: POST \/pulls\/5\/merge: 405 not mergeable/) + assert.ok(!text.includes('undefined')) + assert.ok(!commentText(d, { ...d.state, mergeError: undefined }).includes('undefined')) +}) + +test('a dry run never asks to write, comment or merge', () => { + const dry = { dryRun: true } + assert.equal(decide(undefined, obs(), THU, dry).write, false) + const s = passed(fresh().state, 'public') + const b = decide(s, obs({ openPr: 9, blocked: { gate: 'oxide', reason: 'x' } }), THU, dry) + assert.equal(b.comment, false) + assert.equal(b.write, false) + assert.equal(b.state.lastComment, undefined) + const m = decide(passed(fresh().state, ...GATES.slice(0, 4)), obs({ checksPassed: true }), THU, dry) + assert.equal(m.action, 'merge') + assert.equal(m.merge, false) + const g = decide(s, obs({ openPr: 9 }), at('2026-11-08T23:11:00Z'), dry) + assert.equal(g.action, 'giveUp') + assert.equal(g.comment, false) + assert.equal(decide(undefined, obs({ ahead: 0 }), THU, dry).action, 'run') +}) + +test('a dispatch is dry unless its dry_run input is exactly false', () => { + const dispatch = (input) => ({ GITHUB_EVENT_NAME: 'workflow_dispatch', DRY_RUN_INPUT: input }) + assert.equal(isDry({}, dispatch(undefined)), true) + assert.equal(isDry({}, dispatch('')), true) + assert.equal(isDry({}, dispatch('true')), true) + assert.equal(isDry({}, dispatch('maybe')), true) + assert.equal(isDry({}, dispatch('false')), false) + assert.equal(isDry({}, { GITHUB_EVENT_NAME: 'schedule' }), false) + assert.equal(isDry({ 'dry-run': true }, { GITHUB_EVENT_NAME: 'schedule' }), true) +}) + +// ---- the commands, against a fake Gitea and a fake panel + +const GIT = (...a) => ({ 'rev-parse': 'sha1', 'rev-list': '3', 'merge-base': 'mb', log: '2026-11-01T00:00:00+00:00' }[a[0]] || '') + +/** deps with a fake fetch: handler(method, path, body) -> { status, json }; every call is recorded. */ +function fakeDeps({ now = '2026-11-05T18:11:00Z', handler, git = GIT, panel, env = {} }) { + const deps = { + calls: [], + at: now, + env: { GITHUB_SERVER_URL: 'http://gitea.test', GITHUB_REPOSITORY: 'o/r', GITEA_TOKEN: 'reg-token', MERGE_TOKEN: 'merge-token', ...env }, + now: () => new Date(deps.at), + sleep: async () => {}, + git: (...a) => git(...a), + panel, + fetch: async (url, o) => { + const p = url.replace('http://gitea.test/api/v1/repos/o/r', '') + const body = o.body ? JSON.parse(o.body) : undefined + deps.calls.push({ method: o.method, p, body, auth: o.headers.Authorization }) + const r = handler(o.method, p, body) || {} + const status = r.status || 200 + return { ok: status < 400, status, text: async () => (r.json === undefined ? '' : JSON.stringify(r.json)) } + }, + } + return deps +} + +const STAT = { branch: 'public', manifest: '8140', manifestDate: '11/5/2026 11:48:58', managed: 'ok', oxideBehind: 0, fields: 'same', compile: 'ok' } + +function rig(stat = STAT, staging = { branch: 'staging', manifest: '8140' }) { + const dir = tmp() + fs.mkdirSync(path.join(dir, 'work')) + fs.mkdirSync(path.join(dir, 'work-staging')) + fs.writeFileSync(path.join(dir, 'work', 'static.json'), JSON.stringify(stat)) + fs.writeFileSync(path.join(dir, 'work-staging', 'static.json'), JSON.stringify(staging)) + const state = path.join(dir, 'drill.json') + fs.writeFileSync(state, '{}') + const a = { state, work: path.join(dir, 'work'), 'staging-work': path.join(dir, 'work-staging') } + return { dir, a, state, read: () => JSON.parse(fs.readFileSync(state, 'utf8')) } +} + +/** A Gitea with one edge to main PR that exists once POSTed. */ +function fakeGitea() { + const g = { pr: null, comments: [] } + g.handler = (method, p, body) => { + if (method === 'GET' && p.startsWith('/pulls?state=open')) return { json: g.pr ? [g.pr] : [] } + if (method === 'GET' && p.startsWith('/pulls?state=closed')) return { json: [] } + if (method === 'POST' && p === '/pulls') { + if (g.failCreate) return { status: 500, json: { message: 'boom' } } + g.pr = { number: 5, head: { ref: 'edge', sha: 'sha1' }, base: { ref: 'main' }, ...body, head_: undefined } + g.pr.head = { ref: 'edge', sha: 'sha1' } + g.pr.base = { ref: 'main' } + return { json: g.pr } + } + if (method === 'PATCH' && p === '/pulls/5') { + g.pr = { ...g.pr, ...body } + return { json: g.pr } + } + if (method === 'POST' && p === '/issues/5/comments') { + g.comments.push(body.body) + return { json: {} } + } + return { json: [] } + } + return g +} + +test('gates in a dry run creates, edits, comments and writes nothing', async () => { + const r = rig() + const g = fakeGitea() + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { throw new Error('panel down') } }) }) + const d = await c.gates({ ...r.a, 'dry-run': true }, deps) + assert.ok(deps.calls.length > 0) + assert.ok(deps.calls.every((x) => x.method === 'GET'), JSON.stringify(deps.calls.filter((x) => x.method !== 'GET'))) + assert.equal(fs.readFileSync(r.state, 'utf8'), '{}') + assert.equal(d.comment, false) + assert.equal(g.pr, null) + assert.equal(g.comments.length, 0) + // It ran past the failed carbon gate to try the others. + const run = JSON.parse(fs.readFileSync(path.join(r.dir, 'work', 'cutover-run.json'), 'utf8')) + assert.equal(run.rows.public.status, 'passed') + assert.equal(run.rows.oxide.status, 'passed') + assert.equal(run.rows.carbon.status, 'failed') + assert.equal(run.rows.checks.status, 'failed') +}) + +test('a workflow_dispatch without dry_run=false is dry even without the flag', async () => { + const r = rig() + const g = fakeGitea() + const deps = fakeDeps({ handler: g.handler, env: { GITHUB_EVENT_NAME: 'workflow_dispatch' }, panel: () => ({ servers: async () => { throw new Error('down') } }) }) + await c.gates(r.a, deps) + assert.ok(deps.calls.every((x) => x.method === 'GET')) + assert.equal(fs.readFileSync(r.state, 'utf8'), '{}') +}) + +test('a real run keeps each gate, opens the PR, comments once, and cools a failed carbon for three hours', async () => { + const r = rig() + const g = fakeGitea() + let panelCalls = 0 + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { panelCalls++; throw new Error('panel down') } }) }) + + await c.gates(r.a, deps) // 18:11 + let s = r.read().cutover + assert.equal(s.edgeSha, 'sha1') + assert.deepEqual(Object.keys(s.gates).sort(), ['oxide', 'public']) + assert.match(s.failed.carbon.reason, /panel down/) + assert.equal(s.pr, 5) + assert.equal(s.lastComment, '2026-11-05') + assert.equal(panelCalls, 1) + assert.ok(g.pr.title.startsWith('chore(release): RunicNPC cutover for the 2026-11-05 wipe, Rust 8140')) + assert.ok(g.pr.body.includes('')) + assert.equal(g.comments.length, 1) + assert.match(g.comments[0], /not try this gate again before 2026-11-05 21:11 UTC/) + assert.ok(deps.calls.filter((x) => x.method !== 'GET').every((x) => x.auth === 'token reg-token')) + + deps.at = '2026-11-05T19:11:00Z' + await c.gates(r.a, deps) + assert.equal(panelCalls, 1, 'cooling: the rig is not touched') + deps.at = '2026-11-05T21:10:00Z' + await c.gates(r.a, deps) + assert.equal(panelCalls, 1, '2 h 59 min') + assert.equal(g.comments.length, 1, 'one comment a day') + deps.at = '2026-11-05T21:11:00Z' + await c.gates(r.a, deps) + assert.equal(panelCalls, 2, '3 h 00 min: it tries again') + + // New code: everything again, at once. + deps.git = (...a) => (a[0] === 'rev-parse' ? 'sha2' : GIT(...a)) + deps.at = '2026-11-05T21:30:00Z' + await c.gates(r.a, deps) + s = r.read().cutover + assert.equal(panelCalls, 3) + assert.equal(s.edgeSha, 'sha2') + assert.ok(s.gates.public && s.gates.oxide) +}) + +test('a PR call that fails never loses a gate that passed', async () => { + const r = rig() + const g = fakeGitea() + g.failCreate = true + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { throw new Error('panel down') } }) }) + await c.gates(r.a, deps) + const s = r.read().cutover + assert.ok(s.gates.public) + assert.ok(s.gates.oxide) + assert.equal(s.pr, null) +}) + +test('gate 1 blocks, and nothing else runs, when public is not what staging has', async () => { + const r = rig(STAT, { branch: 'staging', manifest: '8999' }) + const g = fakeGitea() + let panelCalls = 0 + const deps = fakeDeps({ handler: g.handler, panel: () => ({ servers: async () => { panelCalls++; throw new Error('x') } }) }) + await c.gates(r.a, deps) + const s = r.read().cutover + assert.deepEqual(s.gates, {}) + assert.equal(panelCalls, 0) + assert.equal(g.pr, null) +}) + +test('checksGate refuses a PR whose head is not the commit the gates ran on', async () => { + const deps = fakeDeps({ handler: (m, p) => (p === '/pulls/3' ? { json: { number: 3, head: { sha: 'zzzzzzzz1' } } } : { json: {} }) }) + const ctx = { g: c.gitea('t', deps), deps, dryRun: false, edgeSha: 'aaaaaaaa1', pr: { number: 3 } } + assert.match(await c.checksGate(ctx), /head zzzzzzzz is not the commit the gates ran on \(aaaaaaaa\)/) + assert.ok(!deps.calls.some((x) => x.p.includes('/status'))) +}) + +test('merge sends Do=merge, keeps edge, names the head commit, and verifies it merged', async () => { + let merged = false + const deps = fakeDeps({ + handler: (method, p, body) => { + if (method === 'POST' && p === '/pulls/7/merge') { + merged = true + return {} + } + if (method === 'GET' && p === '/pulls/7') return { json: { number: 7, head: { sha: 'abc123' }, merged } } + return {} + }, + }) + await c.merge({ pr: '7', sha: 'abc123' }, deps) + const post = deps.calls.find((x) => x.method === 'POST') + assert.deepEqual(post.body, { Do: 'merge', delete_branch_after_merge: false, head_commit_id: 'abc123' }) + assert.equal(post.auth, 'token merge-token') + assert.equal(deps.calls[deps.calls.length - 1].method, 'GET') +}) + +test('merge refuses a moved head, and a merge that did not happen, and records why', async () => { + const r = rig() + fs.writeFileSync(r.state, JSON.stringify({ cutover: { wipe: '2026-11-05' } })) + const moved = fakeDeps({ handler: (m, p) => (p === '/pulls/7' ? { json: { head: { sha: 'other' }, merged: false } } : {}) }) + await assert.rejects(c.merge({ pr: '7', sha: 'abc123', state: r.state }, moved), /not the commit the gates ran on/) + assert.ok(!moved.calls.some((x) => x.method === 'POST')) + assert.match(r.read().cutover.mergeError, /not the commit the gates ran on/) + assert.equal(r.read().cutover.wipe, '2026-11-05') + + const unmerged = fakeDeps({ handler: (m, p) => (p === '/pulls/7' ? { json: { head: { sha: 'abc123' }, merged: false } } : {}) }) + await assert.rejects(c.merge({ pr: '7', sha: 'abc123' }, unmerged), /not merged after the merge call/) + const refused = fakeDeps({ handler: (m, p) => (m === 'POST' ? { status: 409, json: { message: 'head changed' } } : { json: { head: { sha: 'abc123' } } }) }) + await assert.rejects(c.merge({ pr: '7', sha: 'abc123' }, refused), /409/) + await assert.rejects(c.merge({ pr: '7' }, refused), /--sha/) +}) + +test('after records the merge, comments the release on each closed issue once, and is safe to repeat', async () => { + const r = rig() + fs.writeFileSync(r.state, JSON.stringify({ cutover: { wipe: '2026-11-05', mergeError: 'old', gates: {} } })) + const posted = {} + const deps = fakeDeps({ + handler: (method, p, body) => { + if (p === '/pulls/5') return { json: { number: 5, merged: true, merged_at: '2026-11-05T20:00:00Z', body: 'Closes #20\nCloses #22' } } + if (p.startsWith('/releases')) return { json: [{ tag_name: 'v1.2.0', created_at: '2026-11-05T20:05:00Z' }, { tag_name: 'v1.1.0', created_at: '2026-10-01T00:00:00Z' }] } + if (p === '/issues/20') return { json: { state: 'closed' } } + if (p === '/issues/22') return { json: { state: 'open' } } + if (method === 'GET' && p === '/issues/20/comments') return { json: (posted[20] || []).map((b) => ({ body: b })) } + if (method === 'POST' && p === '/issues/20/comments') { + posted[20] = [...(posted[20] || []), body.body] + return {} + } + return { json: [] } + }, + }) + await c.after({ state: r.state, pr: '5' }, deps) + await c.after({ state: r.state, pr: '5' }, deps) + assert.deepEqual(posted[20], ['Released in v1.2.0 by the wipe cutover (D332).']) + assert.equal(deps.calls.filter((x) => x.method === 'POST').length, 1) + assert.ok(!deps.calls.some((x) => x.p.startsWith('/issues/22/comments'))) + assert.equal(r.read().cutover.merged, true) + assert.equal(r.read().cutover.mergeError, undefined) +}) + +test('the Closes list is the PRs merged into edge after the newest edge commit already in main', async () => { + const prs = [ + { merged: true, base: { ref: 'edge' }, merged_at: '2026-10-20T00:00:00Z', body: 'Closes #1' }, + { merged: true, base: { ref: 'edge' }, merged_at: '2026-10-25T00:00:00Z', body: 'Closes #2, fixes #6' }, + { merged: true, base: { ref: 'main' }, merged_at: '2026-10-26T00:00:00Z', body: 'Closes #3' }, + { merged: false, base: { ref: 'edge' }, merged_at: null, body: 'Closes #4' }, + { merged: true, base: { ref: 'edge' }, merged_at: '2026-10-30T00:00:00Z', body: 'Refs #5' }, + ] + const asked = [] + const deps = fakeDeps({ + handler: (m, p) => (p.startsWith('/pulls?state=closed') ? { json: prs } : {}), + git: (...a) => { + asked.push(a.join(' ')) + if (a[0] === 'merge-base') return 'deadbeef' + if (a[0] === 'log') return '2026-10-22T09:00:00+00:00' + return '' + }, + }) + assert.deepEqual(await c.closesSinceCutOff(c.gitea('t', deps), deps), [2, 6]) + assert.ok(asked.includes('merge-base origin/main origin/edge')) + assert.ok(asked.includes('log -1 --format=%cI deadbeef')) +}) + +test('a PR whose merge commit main already has is not listed again, though merged a moment after the cut-off', async () => { + const prs = [ + // edge's tip at the last cutover: its merged_at is a second after the merge commit's committer date. + { merged: true, base: { ref: 'edge' }, merged_at: '2026-11-05T19:00:01Z', merge_commit_sha: 'shipped1', body: 'Closes #20' }, + { merged: true, base: { ref: 'edge' }, merged_at: '2026-11-20T00:00:00Z', merge_commit_sha: 'new1', body: 'Closes #30' }, + ] + const deps = fakeDeps({ + handler: (m, p) => (p.startsWith('/pulls?state=closed') ? { json: prs } : {}), + git: (...a) => { + if (a[0] === 'merge-base') return 'shipped1' + if (a[0] === 'log') return '2026-11-05T19:00:00+00:00' + if (a[0] === 'branch') return a[3] === 'shipped1' ? ' origin/main' : '' + return '' + }, + }) + assert.deepEqual(await c.closesSinceCutOff(c.gitea('t', deps), deps), [30]) +}) + +test('rust-carbon is started again only when it was running at the start and is offline now, and nothing else is touched', async () => { + const dir = tmp() + const powered = [] + const mkPanel = (state) => () => ({ + servers: async () => ({ 'rust-carbon': 'abc', 'rust-oxide': 'def' }), + state: async () => state, + power: async (id, signal) => powered.push([id, signal]), + }) + await c.rigNote({ work: dir }, { panel: mkPanel('running') }) + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'rig-carbon.json'), 'utf8')), { name: 'rust-carbon', state: 'running' }) + await c.rigRestore({ work: dir }, { panel: mkPanel('running') }) + await c.rigRestore({ work: dir }, { panel: mkPanel('starting') }) + assert.deepEqual(powered, []) + await c.rigRestore({ work: dir }, { panel: mkPanel('offline') }) + assert.deepEqual(powered, [['abc', 'start']]) + // It was off at the start: left off. + await c.rigNote({ work: dir }, { panel: mkPanel('offline') }) + await c.rigRestore({ work: dir }, { panel: mkPanel('offline') }) + assert.equal(powered.length, 1) +}) + +// ---- the workflow and the scripts agree + +const yml = fs.readFileSync(path.join(__dirname, '..', '..', '.gitea', 'workflows', 'cutover.yml'), 'utf8') +const src = fs.readFileSync(path.join(__dirname, 'cutover.js'), 'utf8') + +function stepsOf(text) { + return text + .split(/\n {6}- (?=name:|uses:)/) + .slice(1) + .map((b) => ({ id: (/^\s+id: (\w+)/m.exec(b) || [])[1], command: (/cutover\.js (\S+)/.exec(b) || /drill\.js (\S+)/.exec(b) || [])[1], script: /drill\.js/.test(b) && !/cutover\.js/.test(b) ? 'drill' : 'cutover', text: b })) +} + +const FUNCTIONS = { window: 'window_', pending: 'pending', gates: 'gates', merge: 'merge', after: 'after' } + +function written(command) { + const m = new RegExp(`\\n(?:async )?function ${FUNCTIONS[command]}\\(`).exec(src) + assert.ok(m, `cutover.js has no function for ${command}`) + const body = src.slice(m.index, src.indexOf('\n}\n', m.index)) + return new Set([...body.matchAll(/output\([^,]+,\s*'(\w+)'/g)].map((x) => x[1])) +} + +test('every steps..outputs. the workflow reads is written by the command that step runs', () => { + const steps = stepsOf(yml) + const used = [...yml.matchAll(/steps\.(\w+)\.outputs\.(\w+)/g)].map((m) => [m[1], m[2]]) + assert.ok(used.length >= 8) + for (const [id, name] of used) { + const step = steps.find((s) => s.id === id) + assert.ok(step, `steps.${id} is not a step of cutover.yml`) + assert.equal(step.script, 'cutover', `steps.${id}.outputs.${name}: ${id} runs drill.js, which writes no output the workflow reads`) + assert.ok(written(step.command).has(name), `cutover.js ${step.command} never writes output ${name} (read as steps.${id}.outputs.${name})`) + } +}) + +test('and the outputs the logic depends on are all read by the workflow', () => { + const steps = stepsOf(yml) + const needed = { window: ['open'], pending: ['run'], gates: ['live', 'merge', 'pr', 'sha'], merge: ['merged'] } + for (const [command, names] of Object.entries(needed)) { + const ids = steps.filter((s) => s.script === 'cutover' && s.command === command && s.id).map((s) => s.id) + assert.ok(ids.length > 0, `no step with an id runs cutover.js ${command}`) + const read = new Set([...yml.matchAll(/steps\.(\w+)\.outputs\.(\w+)/g)].filter((m) => ids.includes(m[1])).map((m) => m[2])) + for (const n of names) { + assert.ok(read.has(n), `cutover.yml never reads ${n} from the ${command} step`) + assert.ok(written(command).has(n), `cutover.js ${command} no longer writes ${n}`) + } + } +}) + +test('the code checked out is edge, except on a dry dispatch that names code_ref', () => { + const refs = yml.split('\n').filter((l) => /^\s*ref:/.test(l)).map((l) => l.trim()) + assert.deepEqual(refs, [ + "ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'false' && github.event.inputs.code_ref || 'edge' }}", + ]) +}) + +test('RUNICNPC_TOKEN is in the merge step alone, and a dispatch is dry unless dry_run is exactly false', () => { + const uses = yml.split('\n').filter((l) => !l.trim().startsWith('#') && l.includes('RUNICNPC_TOKEN')) + assert.equal(uses.length, 1) + const merge = stepsOf(yml).find((s) => s.text.split('\n').some((l) => !l.trim().startsWith('#') && l.includes('RUNICNPC_TOKEN'))) + assert.equal(merge.id, 'merge') + assert.match(merge.text, /MERGE_TOKEN: \$\{\{ secrets\.RUNICNPC_TOKEN \}\}/) + const code = yml.split('\n').filter((l) => !l.trim().startsWith('#')).join('\n') + assert.ok(!/dry_run\s*[=!]=\s*'true'/.test(code), 'dry_run must be compared with false, never true') + assert.ok((code.match(/dry_run != 'false'/g) || []).length >= 6) + for (const key of ['LC_ALL: C', "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1'"]) assert.equal(code.split(key).length - 1, 2, `${key} on both static halves`) + assert.match(code, /timeout-minutes: 240/) +}) diff --git a/tools/drill/drill.js b/tools/drill/drill.js index 1ea36d3..1ebeab7 100644 --- a/tools/drill/drill.js +++ b/tools/drill/drill.js @@ -649,8 +649,9 @@ async function fileIssue(title, body) { // ---------------------------------------------------------------- main -// The pure parts, for drill.test.js; run as a script, it is the command line. -module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS } +// The pure parts, for drill.test.js; run as a script, it is the command line. panel and waitRunning serve +// cutover.js (D332), and report is exported so a test can show it keeps the state keys it does not own. +module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS, panel, waitRunning, report } if (require.main === module) { const a = args() diff --git a/tools/drill/drill.test.js b/tools/drill/drill.test.js index 73ad312..dfbfe85 100644 --- a/tools/drill/drill.test.js +++ b/tools/drill/drill.test.js @@ -65,3 +65,23 @@ test("Kits' file is replaced, not deleted, by one holding only the harness's kit assert.equal(kits.rnhrevolver.IsHidden, true) assert.deepEqual(kits.rnhrevolver.BeltItems.map((i) => i.Shortname), ['pistol.revolver']) }) + +test('report keeps the state keys it does not own (the cutover, D332) and writes its own', async () => { + const fs = require('fs') + const os = require('os') + const path = require('path') + const { report } = require('./drill.js') + delete process.env.GITHUB_STEP_SUMMARY + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'drill-report-')) + const cutover = { wipe: '2026-11-05', manifest: '1', gates: { public: { at: '2026-11-05T17:11:00.000Z' } }, pr: 7 } + fs.writeFileSync(path.join(dir, 'drill.json'), JSON.stringify({ cutover })) + const stat = { branch: 'staging', manifest: '42', manifestDate: '10/31/2026 11:48:58', managed: 'ok', oxideBehind: 0, fields: 'same', compile: 'ok' } + fs.writeFileSync(path.join(dir, 'static.json'), JSON.stringify(stat)) + fs.writeFileSync(path.join(dir, 'live.json'), JSON.stringify({ branch: 'staging', ran: true, findings: [], harness: { done: true, pass: 3, fail: 0, failed: [] } })) + await report({ state: path.join(dir, 'drill.json'), work: dir }) + const after = JSON.parse(fs.readFileSync(path.join(dir, 'drill.json'), 'utf8')) + assert.deepEqual(after.cutover, cutover) + assert.equal(after.checked.manifest, '42') + assert.equal(after.lastRun.outcome, 'pass') + fs.rmSync(dir, { recursive: true, force: true }) +})