// The wipe-day cutover (docs/runicnpc/PLAN.md stage 9c, D332). Never shipped.
//
// .gitea/workflows/cutover.yml runs it hourly over the forced-wipe weekend. It merges `edge` into `main`
// once Rust's wipe update is live and RunicNPC is proven on it, so nobody approves a merge on a wipe day.
//
// node tools/drill/cutover.js window [--force] [--now ISO] in the wipe weekend?
// node tools/drill/cutover.js pending [--force] does edge have commits main lacks?
// (tools/drill/static.sh with BRANCH=public) the static half + Steam's app info
// node tools/drill/cutover.js rig-note --work
is rust-carbon running now?
// node tools/drill/cutover.js gates --state --work [--dry-run] [--force] [--after-live]
// (--staging-work is still accepted and ignored: main's wiring passes it)
// node tools/drill/drill.js live --work --branch public the live half (only when `gates` says)
// node tools/drill/cutover.js merge --pr --sha [--state f] MERGE_TOKEN only
// node tools/drill/cutover.js after --state --pr record it, comment the release
// node tools/drill/cutover.js rig-restore --work start rust-carbon if the job stopped it
//
// The five gates, cheapest first; it merges only when all pass (the later ones need the earlier ones):
// public Steam set Rust's public branch to its build on the wipe day or later (branches.public.timeupdated
// in steamcmd's app info, which static.sh keeps as appinfo.txt), AND the Linux depot manifest Steam
// names for public is the one the static half downloaded. Not the depot manifest's date: that is
// when Facepunch BUILT it (the 2026-11-05 wipe build was dated 11-04). Not staging's manifest:
// public never shares one with staging (Facepunch promotes a hidden `release` branch, and staging
// has moved on by then). Both were the first design and blocked the whole 2026-11-05 weekend.
// oxide Oxide caught up, the swap's field list is unchanged, RunicNPC compiles (static.json).
// carbon rust-carbon restarts on public Rust with edge's RunicNPC.cs. Its appmanifest must name the public
// manifest (it really updated); then rnpc.status says framework=carbon, navmesh=ready, missing=-
// added=- (D232: only Carbon's unmodified assembly makes `added` mean anything).
// live drill.js live --branch public on rust-staging: every harness check passes, no finding.
// checks the cutover PR's combined status on its head commit is success and has a `plugin-checks`
// context, AND the PR head is the commit the gates ran on.
//
// The gates are tied to the code: drill.json keeps `edgeSha` (the commit this job checked out and ran the
// plugin and harness from) and the manifest; a change of either, or of the wipe, clears every gate. `checks`
// is never kept: it runs fresh, right before the merge, and the merge is sent `head_commit_id` so Gitea refuses
// it when the PR head moved. The state is written after EVERY gate result, before anything touches the PR.
//
// A carbon or live attempt that fails is not repeated for the same manifest and commit for COOL_DOWN_HOURS
// (3): the rigs are not restarted every hour. A new manifest or a new edge commit retries at once.
//
// How the gates are sequenced across the workflow's steps. `gates` is run twice at most:
// 1. `gates` runs public, oxide, carbon in this job and stops in front of live, saying
// `live=true`. A gate that fails ends the run (retried next hour, a cooling one
// after the cool-down); a gate that passed is kept in drill.json, never re-run.
// 2. `drill.js live` is its own workflow step; its output is a file (live.json).
// 3. `gates --after-live` reads live.json (the live gate), then runs checks. Both say `merge=true pr=
// sha=` when everything has passed.
// When live already passed (an earlier hour), step 1 goes on to checks by itself and the other two steps
// are skipped. The cutover PR is opened (or its marked block updated) as soon as the public gate has passed;
// an existing open edge to main PR is adopted: its title is never changed and only the block between
// and of its body is written (appended when absent).
//
// A dry run (--dry-run, or any workflow_dispatch whose dry_run input is not exactly 'false') runs every gate
// it can instead of stopping at the first failure, ignores what an earlier real run kept, and creates, edits,
// comments and merges nothing and writes no state. It does use the rigs: carbon restarts rust-carbon and live
// uses rust-staging. The table goes to the job summary.
//
// Every gate check below returns `true`, or a string saying why not.
//
// Environment: GITEA_TOKEN (the workflow's REGISTRY_TOKEN: read, open and edit the PR, comment) with
// GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them; PANEL_KEY / PANEL_URL as drill.js;
// MERGE_TOKEN (RUNICNPC_TOKEN) in the `merge` step only; GITHUB_EVENT_NAME + DRY_RUN_INPUT for the dry-run
// default. No token or key is ever printed.
const fs = require('fs')
const path = require('path')
const { execFileSync } = require('child_process')
const ROOT = path.resolve(__dirname, '..', '..')
const CARBON_RIG = process.env.CUTOVER_CARBON_RIG || 'rust-carbon'
const CARBON_PLUGIN = '/carbon/plugins/RunicNPC.cs'
const ACF = '/steamapps/appmanifest_258550.acf'
const LINUX_DEPOT = '258552'
const GATES = ['public', 'oxide', 'carbon', 'live', 'checks']
const PERSISTED = GATES.slice(0, 4) // checks is never kept
const COOLING = ['carbon', 'live'] // the gates that cost a rig restart
const COOL_DOWN_HOURS = 3
const GATE_LABELS = {
public: "1. Steam set Rust's public branch on the wipe day or later",
oxide: '2. Oxide has caught up; RunicNPC compiles',
carbon: '3. A Carbon boot on public Rust',
live: '4. The drill again, on live Rust',
checks: '5. PR checks (plugin-checks) on the head commit',
}
const BLOCK_START = ''
const BLOCK_END = ''
const H = 3600000
const CHECKS_WAIT_MS = 15 * 60000
const RELEASE_WAIT_MS = 30 * 60000
// ---------------------------------------------------------------- pure parts (cutover.test.js)
const day = (d) => d.toISOString().slice(0, 10)
/** A reason in a PR body or a comment must not hold a closing keyword Gitea would act on: `#123` is broken up. */
const neutralise = (text) => String(text == null ? '' : text).replace(/#(\d)/g, '#$1')
function firstThursday(y, m) {
const d = new Date(Date.UTC(y, m, 1))
d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7))
return d
}
/**
* The forced wipe of `now`'s month (its first Thursday, D318) and the weekend that follows. `open` is the
* span the cutover works in: from the wipe Thursday 17:00 UTC to Sunday 23:00 UTC (`deadline`). `final` is
* the hour after it, in which the hourly run still starts once more so it can give up out loud. (nextWipe
* in drill.js cannot do this: on the Thursday itself it answers next month.)
*/
function wipeWeekend(now) {
const wipe = firstThursday(now.getUTCFullYear(), now.getUTCMonth())
const t = now.getTime()
const start = wipe.getTime() + 17 * H
const deadline = new Date(wipe.getTime() + (3 * 24 + 23) * H)
return { open: t >= start && t < deadline.getTime(), final: t >= deadline.getTime() && t < deadline.getTime() + H, wipe, deadline }
}
/**
* One branch of Rust's dedicated server (app 258550) from steamcmd's `+app_info_print 258550` output (the
* appinfo.txt static.sh keeps; steamcmd's own lines come before the `"258550"` block and are skipped):
* `timeupdated`, when Steam set the branch to its build (Unix seconds); its `buildid`; and the manifest Steam
* names for the Linux depot on that branch (`gid`, or the older bare-string form). null when there is no
* 258550 block or no such branch.
*/
function steamBranch(text, branch = 'public', depot = LINUX_DEPOT) {
const s = String(text || '')
const at = s.search(/^"258550"[ \t]*\r?$/m)
if (at < 0) return null
const app = parseVdf(s.slice(at))['258550']
const depots = app && typeof app === 'object' ? app.depots : null
const b = depots && depots.branches && depots.branches[branch]
if (!b || typeof b !== 'object') return null
const m = depots[depot] && depots[depot].manifests && depots[depot].manifests[branch]
return {
buildid: b.buildid || null,
timeupdated: b.timeupdated === undefined ? NaN : Number(b.timeupdated),
manifest: (m && typeof m === 'object' ? m.gid : m) || null,
}
}
const iso = (ms) => new Date(ms).toISOString().slice(0, 16).replace('T', ' ') + ' UTC'
/**
* Gate public: Steam set the public branch on the wipe day (a Date, or YYYY-MM-DD) or later, UTC, and the
* Linux depot manifest it names for public is the one the static half downloaded (`stat.manifest`): the two
* are read a minute apart, so a build pushed in between blocks one hour rather than tests the wrong build.
*/
function publicGate(stat, steam, wipe) {
const w = wipe instanceof Date ? wipe.getTime() : Date.parse(wipe)
if (!Number.isFinite(w)) return `no usable wipe date (${wipe})`
if (!steam) return `Steam's app info for Rust could not be read (${(stat && stat.steamNote) || 'no appinfo.txt'})`
const t = steam.timeupdated * 1000
if (!Number.isFinite(t) || t <= 0) return "Steam's app info gives no time for Rust's public branch"
if (t < w) return `Steam last set Rust's public branch (build ${steam.buildid || '?'}) at ${iso(t)}, before the wipe (${day(new Date(w))} 00:00 UTC)`
if (!steam.manifest) return `Steam's app info names no ${LINUX_DEPOT} manifest for public`
if (steam.manifest !== (stat && stat.manifest)) {
return `Steam names manifest ${steam.manifest} for public, but the static half downloaded ${(stat && stat.manifest) || 'none'}: the next hour tries again`
}
return true
}
/** Gate oxide: true, or why an Oxide server (or the plugin's swap) cannot be trusted on this build. */
function oxideReady(stat) {
const s = stat || {}
if (s.managed !== 'ok') return `the static half could not fetch Rust's public build (${s.note || 'no static.json'})`
if (s.oxideBehind === -1 || s.oxideBehind === undefined) return "Oxide's public build could not be compared with Rust's"
if (s.oxideBehind !== 0) return `Oxide's public build is ${s.oxideBehind} member(s) behind`
if (s.compile !== 'ok') return `RunicNPC does not compile against public Rust with Oxide's build (compile ${s.compile})`
if (s.fields === 'differs') return "the swap's field list differs from public Rust's"
if (s.fields !== 'same') return `the swap's field list could not be checked (fields ${s.fields})`
return true
}
/** Valve's text KeyValues (an appmanifest .acf) as nested objects. */
function parseVdf(text) {
const root = {}
const stack = [root]
let key = null
for (const m of String(text || '').matchAll(/"((?:[^"\\]|\\.)*)"|([{}])/g)) {
const top = stack[stack.length - 1]
if (m[2] === '{') {
const o = {}
top[key] = o
stack.push(o)
key = null
} else if (m[2] === '}') {
if (stack.length > 1) stack.pop()
key = null
} else if (key === null) key = m[1]
else {
top[key] = m[1]
key = null
}
}
return root
}
/** The manifest an appmanifest_258550.acf records for a depot (InstalledDepots, "258552", "manifest"), or null. */
function acfDepotManifest(text, depot = LINUX_DEPOT) {
const state = parseVdf(text).AppState
const entry = state && state.InstalledDepots && state.InstalledDepots[depot]
return (entry && typeof entry === 'object' && entry.manifest) || null
}
/** Gate carbon, first half: the rig's installed Linux depot is the public manifest. */
function carbonBuild(acfText, publicManifest) {
const have = acfDepotManifest(acfText)
if (!have) return `the ${LINUX_DEPOT} manifest could not be read from ${ACF} on ${CARBON_RIG}`
if (have !== publicManifest) return `${CARBON_RIG} still runs manifest ${have} (public is ${publicManifest})`
return true
}
/** Gate carbon, second half: the `rnpc.status` console lines (an array, or the text) of a Carbon server. */
function carbonOk(statusLines) {
const lines = Array.isArray(statusLines) ? statusLines : String(statusLines || '').split('\n')
const line = lines.find((l) => /framework=/.test(l))
if (!line) return 'rnpc.status never answered on Carbon'
const get = (k) => (new RegExp(`${k}=(\\S+)`).exec(line) || [])[1]
const framework = get('framework')
if (framework !== 'carbon') return `rnpc.status says framework=${framework}, not carbon`
if (get('navmesh') !== 'ready') return `the navmesh was not ready on Carbon (navmesh=${get('navmesh')})`
const missing = get('missing')
const added = get('added')
if (missing === undefined || added === undefined) return 'rnpc.status carried no swap-fields report'
if (missing !== '-') return `the swap's fields are missing on Carbon: ${missing}`
if (added !== '-') return `public Rust's Carbon assembly has fields the swap does not know: ${added}`
return true
}
/** Gate live: live.json as drill.js live wrote it. */
function liveOk(live) {
if (!live) return 'the live half wrote no live.json'
const first = (live.findings && live.findings[0]) || null
if (live.branch && live.branch !== 'public') return `live.json is from the ${live.branch} branch, not public`
if (!live.ran) return live.skipped || first || 'the live half did not run the harness'
if (!live.harness || !live.harness.done) return first || 'the harness did not finish'
if (live.harness.fail > 0) return `a harness check failed: ${(live.harness.failed && live.harness.failed[0]) || `${live.harness.fail} fail`}`
if (!(live.harness.pass > 0)) return 'the harness passed no check at all'
if (first) return first
return true
}
/** Gate checks: where the cutover PR's combined commit status stands. { state: success|pending|failed, reason } */
function checkState(combined) {
const statuses = (combined && combined.statuses) || []
const plugin = statuses.filter((s) => String(s.context || '').includes('plugin-checks'))
const bad = statuses.find((s) => ['failure', 'error'].includes(s.status || s.state))
if (bad) return { state: 'failed', reason: `${bad.context} is ${bad.status || bad.state}` }
if (!plugin.length) return { state: 'pending', reason: 'no plugin-checks status has been reported yet' }
if (combined.state !== 'success') return { state: 'pending', reason: `the PR's combined status is ${combined.state}` }
return { state: 'success', reason: '' }
}
/** Issue numbers a PR body closes: a Gitea closing keyword directly before `#N`. Sorted, unique. */
function collectCloses(prBodies) {
const found = new Set()
const re = /(? a - b)
}
/** The cutover PR's description (the block). `gates` is [{ name, status, note }]. */
function cutoverBody({ wipe, manifest, buildDate, closes = [], gates = [] }) {
return [
'The automated wipe-day cutover (D332, `docs/runicnpc/PLAN.md` stage 9c): `edge` merges into `main` once Rust\'s ' +
'forced-wipe update is live and RunicNPC is proven on it, so no merge needs approving on a wipe day. ' +
'The workflow merges this PR by itself when every gate below has passed; `release.yml` then cuts the release.',
'',
`Wipe: **${wipe}**. Rust's public build: manifest **${manifest}**, dated ${buildDate || '?'}.`,
'',
'| Gate | State | Detail |',
'| --- | --- | --- |',
...gates.map((g) => `| ${g.name} | ${g.status} | ${neutralise(g.note || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`),
'',
closes.length ? 'Issues fixed by the PRs merged into `edge` since the last cutover:' : 'No issue is closed by this release.',
...closes.map((n) => `Closes #${n}`),
'',
'Opened by the runicnpc-rust cutover workflow (D332).',
].join('\n')
}
/** `existing` with the cutover block set to `content`: replaced between the markers, or appended when absent. */
function withBlock(existing, content) {
const block = `${BLOCK_START}\n${content}\n${BLOCK_END}`
const text = String(existing || '')
const a = text.indexOf(BLOCK_START)
const b = text.indexOf(BLOCK_END, a + 1)
if (a >= 0 && b > a) return text.slice(0, a) + block + text.slice(b + BLOCK_END.length)
return text.trim() ? `${text.replace(/\s+$/, '')}\n\n${block}` : block
}
const nextGate = (gates) => PERSISTED.find((g) => !(gates || {})[g])
/** A gate passed: the state with it kept (and its failure forgotten). */
function pass(state, gate, now) {
const { [gate]: _gone, ...failed } = state.failed || {}
return { ...state, gates: { ...state.gates, [gate]: { at: now.toISOString() } }, failed }
}
/** A cooling gate failed: when, and why. */
function fail(state, gate, reason, now) {
return { ...state, failed: { ...state.failed, [gate]: { at: now.toISOString(), reason: String(reason) } } }
}
/** What goes into drill.json: never `checks`. */
function persistable(state, pr) {
const { checks: _checks, ...gates } = state.gates || {}
return { ...state, gates, pr: pr ? pr.number : null }
}
/**
* What the hourly run does next, as data. `state` is the `cutover` object kept in drill.json:
* { wipe: 'YYYY-MM-DD', manifest, edgeSha, gates: { public, oxide, carbon, live: { at } }, failed: { carbon, live: { at, reason } },
* pr, lastComment: 'YYYY-MM-DD', merged, gaveUp, mergeError }
* `obs`: { manifest, edgeSha, ahead (commits edge has that main lacks), openPr (number or null), forced,
* checksPassed (this run, never kept), blocked: { gate, reason } }.
* Returns { state, action, gate, reason, retryAt, write, comment, merge }: `action` is nothing | giveUp | run |
* blocked | cooling | merge; `write`, `comment` and `merge` say whether the CLI may persist the state, comment,
* merge (a dry run: none).
*/
function decide(state, obs, now, { dryRun = false } = {}) {
const w = wipeWeekend(now)
const wipe = day(w.wipe)
let s = state || {}
if (s.wipe !== wipe) s = { wipe, manifest: obs.manifest, edgeSha: obs.edgeSha, gates: {}, failed: {} } // a new wipe starts over
else if ((obs.manifest && s.manifest !== obs.manifest) || (obs.edgeSha && s.edgeSha !== obs.edgeSha)) {
// A new build, or new code: every gate again, and the failures that cooled them are forgotten.
const { mergeError: _e, ...rest } = s
s = { ...rest, manifest: obs.manifest || s.manifest, edgeSha: obs.edgeSha || s.edgeSha, gates: {}, failed: {} }
}
const { checks: _checks, ...gates } = s.gates || {}
s = { ...s, gates, failed: s.failed || {}, pr: obs.openPr || null } // the PR is kept while it is open
const out = { state: s, write: !dryRun, comment: false, merge: false }
// One comment a day, none before the build is known to have changed (a quiet Thursday morning), none without a PR.
const hold = (action, gate, reason, retryAt) => {
const due = !dryRun && !!s.pr && !!s.gates.public && s.lastComment !== day(now)
return { ...out, state: due ? { ...s, lastComment: day(now) } : s, action, gate, reason, retryAt, comment: due }
}
if (s.merged) return { ...out, action: 'nothing', reason: 'already merged' }
if (obs.ahead === 0 && !dryRun) return { ...out, action: 'nothing', reason: 'edge has no commits main lacks' }
if (now.getTime() >= w.deadline.getTime() && !obs.forced) {
if (s.gaveUp) return { ...out, action: 'nothing', reason: 'gave up already' }
return { ...out, state: { ...s, gaveUp: true }, action: 'giveUp', gate: nextGate(s.gates), comment: !dryRun && !!s.pr }
}
if (obs.blocked) {
const retryAt = COOLING.includes(obs.blocked.gate) ? new Date(now.getTime() + COOL_DOWN_HOURS * H) : null
return hold('blocked', obs.blocked.gate, obs.blocked.reason, retryAt)
}
const gate = nextGate(s.gates)
if (!gate) return obs.checksPassed ? { ...out, action: 'merge', merge: !dryRun } : { ...out, action: 'run', gate: 'checks' }
const failed = COOLING.includes(gate) && s.failed[gate]
if (failed && now.getTime() - Date.parse(failed.at) < COOL_DOWN_HOURS * H) {
return hold('cooling', gate, failed.reason, new Date(Date.parse(failed.at) + COOL_DOWN_HOURS * H))
}
return { ...out, action: 'run', gate }
}
/** The comment for a decision that asks for one. */
function commentText(d, state = {}) {
const label = GATE_LABELS[d.gate] || d.gate
if (d.action === 'giveUp') {
const head = 'The wipe-day cutover (D332) stops here: the window closed (Sunday 23:00 UTC).'
const tail = 'Merge `edge` into `main` by hand when it is ready.'
if (state.mergeError) return `${head} Every gate passed; the merge failed: ${neutralise(state.mergeError)}. ${tail}`
if (!d.gate) return `${head} Every gate passed, but the PR checks and the merge did not complete. ${tail}`
return `${head} Gate "${label}" had not passed. ${tail}`
}
const when = d.retryAt ? ` It will not try this gate again before ${d.retryAt.toISOString().slice(0, 16).replace('T', ' ')} UTC.` : ' It tries again every hour until Sunday 23:00 UTC.'
return `The wipe-day cutover (D332) is waiting at gate "${label}": ${neutralise(d.reason)}.${when} This is the day's only comment.`
}
/** Whether a dispatch is a dry run: unless its dry_run input is exactly 'false' it is. (A schedule never is.) */
function isDry(a, env) {
return !!a['dry-run'] || (env.GITHUB_EVENT_NAME === 'workflow_dispatch' && env.DRY_RUN_INPUT !== 'false')
}
// ---------------------------------------------------------------- plumbing
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
const log = (...a) => console.log(`[cutover ${new Date().toISOString().slice(11, 19)}]`, ...a)
/** What the commands reach outside themselves for; cutover.test.js passes fakes. */
function realDeps() {
return {
env: process.env,
now: () => new Date(),
sleep,
fetch: (u, o) => fetch(u, o),
git: (...a) => execFileSync('git', a, { cwd: ROOT, encoding: 'utf8' }).trim(),
panel: () => require('./drill.js').panel(),
}
}
function args() {
const out = { _: [] }
const argv = process.argv.slice(2)
for (let i = 0; i < argv.length; i++) {
if (argv[i].startsWith('--')) {
const k = argv[i].slice(2)
out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true
} else out._.push(argv[i])
}
return out
}
const readJson = (f, fallback) => {
try {
return JSON.parse(fs.readFileSync(f, 'utf8'))
} catch {
return fallback
}
}
const writeJson = (f, data) => fs.writeFileSync(f, JSON.stringify(data, null, 2) + '\n')
const output = (env, name, value) => {
if (env.GITHUB_OUTPUT) fs.appendFileSync(env.GITHUB_OUTPUT, `${name}=${value}\n`)
}
const summary = (env, md) => {
if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, md + '\n')
}
function aheadCount(a, deps) {
if (a.ahead !== undefined && a.ahead !== true) return Number(a.ahead)
return Number(deps.git('rev-list', '--count', 'origin/main..origin/edge'))
}
/** Gitea's REST API for this repo (the idiom of drill.js fileIssue). The token is never printed. */
function gitea(token, deps) {
if (!token) throw new Error('no Gitea token in the environment')
const base = `${deps.env.GITHUB_SERVER_URL}/api/v1/repos/${deps.env.GITHUB_REPOSITORY}`
const headers = { Authorization: 'token ' + token, 'Content-Type': 'application/json', Accept: 'application/json' }
const call = async (method, p, data) => {
const r = await deps.fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined })
const text = await r.text()
if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${text.slice(0, 200)}`)
return text ? JSON.parse(text) : null
}
return {
call,
async openPr() {
return (await call('GET', '/pulls?state=open&limit=50')).find((p) => p.head.ref === 'edge' && p.base.ref === 'main') || null
},
}
}
/**
* The `Closes #N` numbers of the PRs merged into edge since the last cutover: after the committer date of
* `git merge-base origin/main origin/edge`, the newest edge commit already in main.
*/
async function closesSinceCutOff(g, deps) {
let since = 0
try {
since = Date.parse(deps.git('log', '-1', '--format=%cI', deps.git('merge-base', 'origin/main', 'origin/edge'))) || 0
} catch {}
// A PR whose merge commit main already has shipped in an earlier cutover, even when its merged_at is a moment
// after the cut-off's committer date (edge's tip is often that very merge commit).
const shipped = (sha) => {
if (!sha) return false
try {
return /\borigin\/main\b/.test(deps.git('branch', '-r', '--contains', sha, '--list', 'origin/main'))
} catch {
return false
}
}
const bodies = []
for (let page = 1; page <= 10; page++) {
const prs = await g.call('GET', `/pulls?state=closed&limit=50&page=${page}`)
for (const pr of prs) {
if (pr.merged && pr.base && pr.base.ref === 'edge' && Date.parse(pr.merged_at) > since && !shipped(pr.merge_commit_sha)) bodies.push(pr.body)
}
if (prs.length < 50) break
}
return collectCloses(bodies)
}
// ---------------------------------------------------------------- window and pending
function window_(a, deps = realDeps()) {
const now = a.now ? new Date(a.now) : deps.now()
const w = wipeWeekend(now)
const open = w.open || w.final || !!a.force
const verdict = w.open ? 'in the wipe weekend' : w.final ? 'the hour after it (the last run, to give up out loud)' : a.force ? 'outside the wipe weekend, but run by hand: going ahead' : 'outside the wipe weekend, nothing to do'
log(`the wipe is ${day(w.wipe)}, the cutover works until ${w.deadline.toISOString().slice(0, 16)}Z: ${verdict} (open=${open})`)
output(deps.env, 'open', open ? 'true' : 'false')
}
function pending(a, deps = realDeps()) {
const ahead = aheadCount(a, deps)
const run = ahead > 0 || !!a.force
log(`edge is ${ahead} commit(s) ahead of main: ${run ? 'going ahead' : 'nothing to merge'}`)
output(deps.env, 'run', run ? 'true' : 'false')
}
// ---------------------------------------------------------------- the rig
async function carbonGate(ctx) {
const { waitRunning } = require('./drill.js')
const p = ctx.deps.panel()
const id = (await p.servers())[CARBON_RIG]
if (!id) return `no server named ${CARBON_RIG} on the panel`
await p.write(id, CARBON_PLUGIN, fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8'))
const was = await p.state(id)
log(`${CARBON_RIG} is ${was}: ${was === 'offline' ? 'starting' : 'restarting'} it (its egg updates Rust and Carbon)`)
await p.power(id, was === 'offline' ? 'start' : 'restart')
// `running` right after the signal is the old process: wait for it to leave first.
for (let i = 0; i < 60 && was !== 'offline' && (await p.state(id)) === 'running'; i++) await ctx.deps.sleep(5000)
if (!(await waitRunning(p, id, 45 * 60000))) return `${CARBON_RIG} did not come up in 45 minutes, or crashed on start`
// It must have updated to the public build, or everything after proves nothing.
const built = carbonBuild(await p.readOr(id, ACF, null), ctx.stat.manifest)
if (built !== true) return built
const lines = await carbonStatus(p, id, 30 * 60000, ctx.deps)
if (lines) log('rnpc.status:', lines.find((l) => l.includes('navmesh=')))
return carbonOk(lines || [])
}
/** `rnpc.status` over the console until the navmesh is ready; the last answer if it never is. */
async function carbonStatus(p, id, timeoutMs, deps) {
const end = Date.now() + timeoutMs
let last = null
while (Date.now() < end) {
try {
const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim())
const at = lines.findIndex((l) => l.includes('RunicNPC ') && l.includes('api='))
if (at >= 0) {
last = lines.slice(at, at + 14)
if (last.some((l) => l.includes('navmesh=ready'))) return last
}
} catch {
// the panel's console is not up yet
}
await deps.sleep(30000)
}
return last
}
/** Writes whether rust-carbon is running now, to put it back if the job leaves it stopped. */
async function rigNote(a, deps = realDeps()) {
const work = path.resolve(a.work || 'cutover-work')
const p = deps.panel()
const id = (await p.servers())[CARBON_RIG]
const state = id ? await p.state(id) : null
fs.mkdirSync(work, { recursive: true })
writeJson(path.join(work, 'rig-carbon.json'), { name: CARBON_RIG, state })
log(`${CARBON_RIG} is ${state || 'not on the panel'}`)
}
/** Starts rust-carbon (power `start` on it alone) when it was running at the start of the job and is offline now. */
async function rigRestore(a, deps = realDeps()) {
const rec = readJson(path.join(path.resolve(a.work || 'cutover-work'), 'rig-carbon.json'), null)
if (!rec || rec.state !== 'running') return log(`${CARBON_RIG} was not running at the start: nothing to put back`)
const p = deps.panel()
const id = (await p.servers())[CARBON_RIG]
if (!id) return log(`${CARBON_RIG} is not on the panel`)
const now = await p.state(id)
if (now !== 'offline') return log(`${CARBON_RIG} is ${now}: left as it is`)
await p.power(id, 'start')
log(`${CARBON_RIG} was running at the start and is offline: started it`)
}
// ---------------------------------------------------------------- the gates
async function checksGate(ctx) {
const wait = !ctx.dryRun
let pr = ctx.pr
if (!pr) return ctx.dryRun ? 'no cutover PR is open (a dry run opens none)' : 'no cutover PR is open'
const end = Date.now() + CHECKS_WAIT_MS
for (;;) {
pr = (await ctx.g.call('GET', `/pulls/${pr.number}`)) || pr
if (pr.head.sha !== ctx.edgeSha) return `the PR's head ${String(pr.head.sha).slice(0, 8)} is not the commit the gates ran on (${String(ctx.edgeSha).slice(0, 8)})`
const v = checkState(await ctx.g.call('GET', `/commits/${pr.head.sha}/status`))
if (v.state === 'success') return true
if (v.state === 'failed') return v.reason
if (!wait || Date.now() >= end) return `${v.reason} (waited ${wait ? 15 : 0} minutes)`
await ctx.deps.sleep(30000)
}
}
const RUNNERS = {
public: async (c) => publicGate(c.stat, c.steam, c.wipe),
oxide: async (c) => oxideReady(c.stat),
carbon: async (c) => carbonGate(c),
live: async (c) => liveOk(readJson(path.join(c.work, 'live.json'), null)),
checks: async (c) => checksGate(c),
}
/** Runs one gate; whatever it throws is its reason, so a flaky API call blocks an hour, not the state. */
async function runGate(gate, ctx) {
log(`gate ${gate}...`)
let r
try {
r = await RUNNERS[gate](ctx)
} catch (e) {
r = `the gate stopped: ${e.message}`
}
log(`gate ${gate}: ${r === true ? 'passed' : `NOT passed: ${r}`}`)
return r
}
// ---------------------------------------------------------------- the PR
function gateRows(state, rows) {
return GATES.map((g) => {
const done = state.gates[g]
const row = rows[g]
if (done) return { name: GATE_LABELS[g], status: 'passed', note: done.at.slice(0, 16).replace('T', ' ') + ' UTC' }
return { name: GATE_LABELS[g], status: row ? row.status : 'not yet', note: row ? row.detail : '' }
})
}
/**
* Opens the edge to main PR, or brings the marked block of the open one's body up to date. A PR it did not
* open keeps its title and the rest of its body. Not in a dry run, and never with a blank manifest.
*/
async function syncPr(ctx) {
if (ctx.dryRun) return ctx.pr
if (!ctx.stat.manifest) {
log('no manifest: the PR is left alone')
return ctx.pr
}
if (!ctx.closes) ctx.closes = await closesSinceCutOff(ctx.g, ctx.deps)
const content = cutoverBody({ wipe: ctx.wipeDate, manifest: ctx.stat.manifest, buildDate: ctx.stat.manifestDate, closes: ctx.closes, gates: gateRows(ctx.state, ctx.rows) })
if (!ctx.pr) {
const title = `chore(release): RunicNPC cutover for the ${ctx.wipeDate} wipe, Rust ${ctx.stat.manifest}`
ctx.pr = await ctx.g.call('POST', '/pulls', { head: 'edge', base: 'main', title, body: withBlock('', content) })
log(`opened PR #${ctx.pr.number}`)
} else {
const body = withBlock(ctx.pr.body, content)
if (body !== ctx.pr.body) {
ctx.pr = { ...ctx.pr, ...((await ctx.g.call('PATCH', `/pulls/${ctx.pr.number}`, { body })) || {}) }
log(`updated PR #${ctx.pr.number}`)
}
}
ctx.state = { ...ctx.state, pr: ctx.pr.number }
return ctx.pr
}
// ---------------------------------------------------------------- gates (the command)
async function gates(a, deps = realDeps()) {
const env = deps.env
const dryRun = isDry(a, env)
const afterLive = !!a['after-live']
const work = path.resolve(a.work || 'cutover-work')
const file = readJson(a.state, {})
const now = deps.now()
const w = wipeWeekend(now)
const stat = readJson(path.join(work, 'static.json'), {})
const appinfo = path.join(work, 'appinfo.txt')
const steam = fs.existsSync(appinfo) ? steamBranch(fs.readFileSync(appinfo, 'utf8')) : null
const g = gitea(env.GITEA_TOKEN, deps)
const prev = (afterLive && readJson(path.join(work, 'cutover-run.json'), null)) || { rows: {} }
const pr = await g.openPr()
const edgeSha = a.edge || deps.git('rev-parse', 'HEAD')
const obs = { manifest: stat.manifest || '', edgeSha, ahead: aheadCount(a, deps), openPr: pr && pr.number, forced: !!a.force && !(w.open || w.final), checksPassed: !!(dryRun && prev.checks) }
// A dry run starts from nothing: it is there to try every gate, not to trust an earlier hour's.
let d = decide(dryRun ? undefined : file.cutover, obs, now, { dryRun })
const ctx = { g, deps, stat, steam, work, now, dryRun, pr, edgeSha, wipe: w.wipe, wipeDate: day(w.wipe), state: d.state, rows: prev.rows, closes: null }
if (dryRun && prev.gates) ctx.state = { ...ctx.state, gates: prev.gates } // what the first half of this dry run passed
else for (const k of Object.keys(ctx.state.gates)) ctx.rows[k] = { status: 'passed earlier', detail: ctx.state.gates[k].at }
const note = (gate, r) => (ctx.rows[gate] = r === true ? { status: 'passed', detail: '' } : { status: 'failed', detail: r })
// The state goes to the file after every gate result, before the PR is touched.
const persist = () => {
if (dryRun) return
file.cutover = persistable(ctx.state, ctx.pr)
writeJson(a.state, file)
}
const safely = async (what, fn) => {
try {
return await fn()
} catch (e) {
log(`${what} failed (the gates are kept): ${e.message}`)
}
}
let blocked = null
let deferred = false
if (d.action === 'nothing') log(`nothing to do: ${d.reason}`)
else if (d.action === 'giveUp') log(`the cutover window closed with ${d.gate || 'every gate'} not done`)
else if (dryRun) {
// Every gate it can; live waits for its own step.
for (const gate of GATES) {
if (gate === 'live' && !afterLive) {
deferred = true
ctx.rows.live = { status: 'runs next', detail: 'drill.js live --branch public' }
} else if ((gate === 'live') === afterLive || (afterLive && gate === 'checks' && !prev.checks)) {
const r = await runGate(gate, ctx)
note(gate, r)
if (r === true) {
if (gate === 'checks') obs.checksPassed = true
else ctx.state = pass(ctx.state, gate, now)
} else if (!blocked) blocked = { gate, reason: r }
}
}
} else {
while (d.action === 'run') {
if (d.gate === 'live' && !afterLive) {
deferred = true
break
}
const r = await runGate(d.gate, ctx)
note(d.gate, r)
if (r !== true) {
blocked = { gate: d.gate, reason: r }
if (COOLING.includes(d.gate)) ctx.state = fail(ctx.state, d.gate, r, now)
persist()
break
}
if (d.gate === 'checks') obs.checksPassed = true
else {
ctx.state = pass(ctx.state, d.gate, now)
persist()
}
if (d.gate === 'public') {
await safely('opening the PR', () => syncPr(ctx))
obs.openPr = ctx.pr ? ctx.pr.number : null
}
d = decide(ctx.state, obs, now, { dryRun })
}
}
// What the run ends in.
if (d.action !== 'nothing' && d.action !== 'giveUp') {
obs.openPr = ctx.pr ? ctx.pr.number : null
d = decide(ctx.state, blocked ? { ...obs, blocked } : obs, now, { dryRun })
}
if (!dryRun && ctx.state.gates.public && d.action !== 'giveUp' && !ctx.state.merged) await safely('updating the PR', () => syncPr(ctx))
const end = d.action
log(`${end}${d.gate ? ` (${d.gate})` : ''}${d.reason ? `: ${d.reason}` : ''}`)
let finalState = d.state
if (d.comment && ctx.pr) {
const done = await safely('commenting', async () => {
await g.call('POST', `/issues/${ctx.pr.number}/comments`, { body: commentText(d, d.state) })
return true
})
if (done) log(`commented on PR #${ctx.pr.number}`)
else finalState = { ...finalState, lastComment: ctx.state.lastComment } // not said, so it may be said next hour
}
ctx.state = finalState
persist()
fs.mkdirSync(work, { recursive: true })
writeJson(path.join(work, 'cutover-run.json'), { rows: ctx.rows, gates: ctx.state.gates, checks: !!obs.checksPassed })
output(env, 'live', deferred ? 'true' : 'false')
output(env, 'merge', d.action === 'merge' && d.merge ? 'true' : 'false')
output(env, 'sha', edgeSha)
if (ctx.pr) output(env, 'pr', ctx.pr.number)
if (!deferred) {
const rows = GATES.map((gate) => `| ${GATE_LABELS[gate]} | ${(ctx.rows[gate] || { status: 'not reached' }).status} | ${neutralise((ctx.rows[gate] || {}).detail || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`)
summary(
env,
[
`## Wipe-day cutover${dryRun ? ' (dry run: nothing created, edited, commented or merged)' : ''}: ${end}`,
'',
`Wipe ${ctx.wipeDate}; Rust public manifest ${stat.manifest || '?'} of ${stat.manifestDate || '?'}; edge ${edgeSha.slice(0, 8)} is ${obs.ahead} commit(s) ahead of main.`,
'',
'| Gate | State | Detail |',
'| --- | --- | --- |',
...rows,
d.action === 'merge' ? `\nEvery gate passed${dryRun ? ': a real run would merge now' : ''}.` : '',
].join('\n')
)
}
return d
}
// ---------------------------------------------------------------- merge and after
/** Records why the merge failed where the next hour (and a last comment) can read it. */
function recordMergeError(stateFile, message) {
if (!stateFile || stateFile === true) return
const file = readJson(stateFile, {})
file.cutover = { ...(file.cutover || {}), mergeError: String(message).slice(0, 300) }
writeJson(stateFile, file)
}
async function merge(a, deps = realDeps()) {
const g = gitea(deps.env.MERGE_TOKEN, deps)
if (!a.pr || a.pr === true) throw new Error('--pr is required')
if (!a.sha || a.sha === true) throw new Error('--sha is required')
try {
const before = await g.call('GET', `/pulls/${a.pr}`)
if (before.head.sha !== a.sha) throw new Error(`PR #${a.pr}'s head ${String(before.head.sha).slice(0, 8)} is not the commit the gates ran on (${a.sha.slice(0, 8)})`)
// Not the squash: release.yml reads the feat/fix subjects of the commits that arrive. Never delete edge.
// head_commit_id: Gitea refuses the merge when the head moved since the gates.
await g.call('POST', `/pulls/${a.pr}/merge`, { Do: 'merge', delete_branch_after_merge: false, head_commit_id: a.sha })
const pr = await g.call('GET', `/pulls/${a.pr}`)
if (!pr.merged) throw new Error(`PR #${a.pr} is not merged after the merge call`)
} catch (e) {
recordMergeError(a.state, e.message)
throw e
}
log(`merged PR #${a.pr}`)
output(deps.env, 'merged', 'true')
}
async function after(a, deps = realDeps()) {
const g = gitea(deps.env.GITEA_TOKEN, deps)
const pr = await g.call('GET', `/pulls/${a.pr}`)
if (!pr.merged) return log(`PR #${a.pr} is not merged: nothing to record`)
const file = readJson(a.state, {})
const { mergeError: _e, ...kept } = file.cutover || {}
file.cutover = { ...kept, merged: true, pr: pr.number }
writeJson(a.state, file)
// release.yml cuts the release as the push to main arrives.
const mergedAt = Date.parse(pr.merged_at)
let release = null
for (const end = deps.now().getTime() + RELEASE_WAIT_MS; !release && deps.now().getTime() < end; ) {
const list = await g.call('GET', '/releases?draft=false&pre-release=false&limit=5')
release = (list || []).find((r) => Date.parse(r.created_at) > mergedAt) || null
if (!release) await deps.sleep(60000)
}
if (!release) return log('no release appeared in 30 minutes: no issue is commented')
log(`released as ${release.tag_name}`)
output(deps.env, 'release', release.tag_name)
const text = `Released in ${release.tag_name} by the wipe cutover (D332).`
for (const n of collectCloses([pr.body])) {
const issue = await g.call('GET', `/issues/${n}`)
if (issue.state !== 'closed') {
log(`#${n} is not closed: no comment`)
continue
}
if ((await g.call('GET', `/issues/${n}/comments`)).some((c) => c.body === text)) continue
await g.call('POST', `/issues/${n}/comments`, { body: text })
log(`commented on #${n}`)
}
}
// ---------------------------------------------------------------- main
module.exports = {
wipeWeekend,
steamBranch,
publicGate,
oxideReady,
parseVdf,
acfDepotManifest,
carbonBuild,
carbonOk,
liveOk,
checkState,
collectCloses,
cutoverBody,
withBlock,
decide,
commentText,
neutralise,
isDry,
pass,
fail,
persistable,
nextGate,
gates,
merge,
after,
rigNote,
rigRestore,
checksGate,
closesSinceCutOff,
gitea,
GATES,
COOL_DOWN_HOURS,
BLOCK_START,
BLOCK_END,
}
if (require.main === module) {
const a = args()
const commands = { window: window_, pending, gates, merge, after, 'rig-note': rigNote, 'rig-restore': rigRestore }
const command = commands[a._[0]]
if (!command) {
console.error('usage: node tools/drill/cutover.js window|pending|gates|merge|after|rig-note|rig-restore [--state f] [--work dir] [--pr n] [--sha s] [--dry-run] [--force] [--after-live]')
process.exit(2)
}
Promise.resolve(command(a)).catch((e) => {
console.error(e.message || e)
process.exit(1)
})
}