// The wipe-day cutover (docs/runicnpc/PLAN.md stage 9c, D332). Never shipped. // // .gitea/workflows/cutover.yml runs it hourly over the forced-wipe weekend. It merges `edge` into `main` // once Rust's wipe update is live and RunicNPC is proven on it, so nobody approves a merge on a wipe day. // // node tools/drill/cutover.js window [--force] [--now ISO] in the wipe weekend? // node tools/drill/cutover.js pending [--force] does edge have commits main lacks? // (tools/drill/static.sh with BRANCH=public, then BRANCH=staging) the static halves // node tools/drill/cutover.js rig-note --work is rust-carbon running now? // node tools/drill/cutover.js gates --state --work [--staging-work ] [--dry-run] [--force] [--after-live] // node tools/drill/drill.js live --work --branch public the live half (only when `gates` says) // node tools/drill/cutover.js merge --pr --sha [--state f] MERGE_TOKEN only // node tools/drill/cutover.js after --state --pr record it, comment the release // node tools/drill/cutover.js rig-restore --work start rust-carbon if the job stopped it // // The five gates, cheapest first; it merges only when all pass (the later ones need the earlier ones): // public Rust's public build is dated the wipe day or later AND names the same depot manifest as Rust's // CURRENT staging build: Facepunch promotes the staging build to public, so the two agree once the // cut over from staging to live went through. (static.sh runs a second time, BRANCH=staging, into // its own work dir; only its manifest is used, and when that run fails this gate blocks.) // oxide Oxide caught up, the swap's field list is unchanged, RunicNPC compiles (static.json). // carbon rust-carbon restarts on public Rust with edge's RunicNPC.cs. Its appmanifest must name the public // manifest (it really updated); then rnpc.status says framework=carbon, navmesh=ready, missing=- // added=- (D232: only Carbon's unmodified assembly makes `added` mean anything). // live drill.js live --branch public on rust-staging: every harness check passes, no finding. // checks the cutover PR's combined status on its head commit is success and has a `plugin-checks` // context, AND the PR head is the commit the gates ran on. // // The gates are tied to the code: drill.json keeps `edgeSha` (the commit this job checked out and ran the // plugin and harness from) and the manifest; a change of either, or of the wipe, clears every gate. `checks` // is never kept: it runs fresh, right before the merge, and the merge is sent `head_commit_id` so Gitea refuses // it when the PR head moved. The state is written after EVERY gate result, before anything touches the PR. // // A carbon or live attempt that fails is not repeated for the same manifest and commit for COOL_DOWN_HOURS // (3): the rigs are not restarted every hour. A new manifest or a new edge commit retries at once. // // How the gates are sequenced across the workflow's steps. `gates` is run twice at most: // 1. `gates` runs public, oxide, carbon in this job and stops in front of live, saying // `live=true`. A gate that fails ends the run (retried next hour, a cooling one // after the cool-down); a gate that passed is kept in drill.json, never re-run. // 2. `drill.js live` is its own workflow step; its output is a file (live.json). // 3. `gates --after-live` reads live.json (the live gate), then runs checks. Both say `merge=true pr= // sha=` when everything has passed. // When live already passed (an earlier hour), step 1 goes on to checks by itself and the other two steps // are skipped. The cutover PR is opened (or its marked block updated) as soon as the public gate has passed; // an existing open edge to main PR is adopted: its title is never changed and only the block between // and of its body is written (appended when absent). // // A dry run (--dry-run, or any workflow_dispatch whose dry_run input is not exactly 'false') runs every gate // it can instead of stopping at the first failure, ignores what an earlier real run kept, and creates, edits, // comments and merges nothing and writes no state. It does use the rigs: carbon restarts rust-carbon and live // uses rust-staging. The table goes to the job summary. // // Every gate check below returns `true`, or a string saying why not. // // Environment: GITEA_TOKEN (the workflow's REGISTRY_TOKEN: read, open and edit the PR, comment) with // GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them; PANEL_KEY / PANEL_URL as drill.js; // MERGE_TOKEN (RUNICNPC_TOKEN) in the `merge` step only; GITHUB_EVENT_NAME + DRY_RUN_INPUT for the dry-run // default. No token or key is ever printed. const fs = require('fs') const path = require('path') const { execFileSync } = require('child_process') const ROOT = path.resolve(__dirname, '..', '..') const CARBON_RIG = process.env.CUTOVER_CARBON_RIG || 'rust-carbon' const CARBON_PLUGIN = '/carbon/plugins/RunicNPC.cs' const ACF = '/steamapps/appmanifest_258550.acf' const LINUX_DEPOT = '258552' const GATES = ['public', 'oxide', 'carbon', 'live', 'checks'] const PERSISTED = GATES.slice(0, 4) // checks is never kept const COOLING = ['carbon', 'live'] // the gates that cost a rig restart const COOL_DOWN_HOURS = 3 const GATE_LABELS = { public: "1. Rust's public build is the promoted staging build, dated the wipe day", oxide: '2. Oxide has caught up; RunicNPC compiles', carbon: '3. A Carbon boot on public Rust', live: '4. The drill again, on live Rust', checks: '5. PR checks (plugin-checks) on the head commit', } const BLOCK_START = '' const BLOCK_END = '' const H = 3600000 const CHECKS_WAIT_MS = 15 * 60000 const RELEASE_WAIT_MS = 30 * 60000 // ---------------------------------------------------------------- pure parts (cutover.test.js) const day = (d) => d.toISOString().slice(0, 10) /** A reason in a PR body or a comment must not hold a closing keyword Gitea would act on: `#123` is broken up. */ const neutralise = (text) => String(text == null ? '' : text).replace(/#(\d)/g, '#​$1') function firstThursday(y, m) { const d = new Date(Date.UTC(y, m, 1)) d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7)) return d } /** * The forced wipe of `now`'s month (its first Thursday, D318) and the weekend that follows. `open` is the * span the cutover works in: from the wipe Thursday 17:00 UTC to Sunday 23:00 UTC (`deadline`). `final` is * the hour after it, in which the hourly run still starts once more so it can give up out loud. (nextWipe * in drill.js cannot do this: on the Thursday itself it answers next month.) */ function wipeWeekend(now) { const wipe = firstThursday(now.getUTCFullYear(), now.getUTCMonth()) const t = now.getTime() const start = wipe.getTime() + 17 * H const deadline = new Date(wipe.getTime() + (3 * 24 + 23) * H) return { open: t >= start && t < deadline.getTime(), final: t >= deadline.getTime() && t < deadline.getTime() + H, wipe, deadline } } /** * DepotDownloader's "Manifest N (10/31/2026 11:48:58)" date as UTC milliseconds: M/D/YYYY HH:mm:ss, 24 hours. * An AM/PM form and ISO are accepted too; anything else is NaN (unparsable, so the gate blocks). */ function manifestTime(text) { const s = String(text || '').trim() const m = /^(\d{1,2})\/(\d{1,2})\/(\d{4}) (\d{1,2}):(\d{2}):(\d{2})(?: ([AP]M))?$/i.exec(s) if (m) { const [mo, d, y, hh, mi, ss] = [+m[1], +m[2], +m[3], +m[4], +m[5], +m[6]] const ampm = m[7] && m[7].toUpperCase() if (mi > 59 || ss > 59 || (ampm ? hh < 1 || hh > 12 : hh > 23)) return NaN const h = ampm ? (hh % 12) + (ampm === 'PM' ? 12 : 0) : hh const t = Date.UTC(y, mo - 1, d, h, mi, ss) const back = new Date(t) return back.getUTCMonth() === mo - 1 && back.getUTCDate() === d ? t : NaN } if (!/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(s)) return NaN return Date.parse(/(Z|[+-]\d\d:?\d\d)$/.test(s) ? s : s + 'Z') } /** The public depot's manifest is dated the wipe day (a Date, or YYYY-MM-DD) or later (UTC). */ function publicChanged(stat, wipe) { const t = manifestTime(stat && stat.manifestDate) const w = wipe instanceof Date ? wipe.getTime() : Date.parse(wipe) return Number.isFinite(t) && Number.isFinite(w) && t >= w } function publicReason(stat, wipe) { const date = stat && stat.manifestDate if (!Number.isFinite(manifestTime(date))) return `the static half read no usable manifest date from Steam (${(stat && stat.note) || 'no note'})` return `Rust's public build is manifest ${stat.manifest} of ${date}, from before the wipe (${wipe instanceof Date ? day(wipe) : wipe} 00:00 UTC)` } /** Gate public: dated the wipe day or later, and the same manifest as Rust's current staging build. */ function publicGate(stat, staging, wipe) { if (!publicChanged(stat, wipe)) return publicReason(stat, wipe) if (!staging || !staging.manifest) return `Rust's staging build could not be read (${(staging && staging.note) || 'no static.json'}), so the promotion to public cannot be confirmed` if (staging.manifest !== stat.manifest) { return `Rust's public build (manifest ${stat.manifest}) is not its current staging build (manifest ${staging.manifest}): the cut over from staging to live has not gone through` } return true } /** Gate oxide: true, or why an Oxide server (or the plugin's swap) cannot be trusted on this build. */ function oxideReady(stat) { const s = stat || {} if (s.managed !== 'ok') return `the static half could not fetch Rust's public build (${s.note || 'no static.json'})` if (s.oxideBehind === -1 || s.oxideBehind === undefined) return "Oxide's public build could not be compared with Rust's" if (s.oxideBehind !== 0) return `Oxide's public build is ${s.oxideBehind} member(s) behind` if (s.compile !== 'ok') return `RunicNPC does not compile against public Rust with Oxide's build (compile ${s.compile})` if (s.fields === 'differs') return "the swap's field list differs from public Rust's" if (s.fields !== 'same') return `the swap's field list could not be checked (fields ${s.fields})` return true } /** Valve's text KeyValues (an appmanifest .acf) as nested objects. */ function parseVdf(text) { const root = {} const stack = [root] let key = null for (const m of String(text || '').matchAll(/"((?:[^"\\]|\\.)*)"|([{}])/g)) { const top = stack[stack.length - 1] if (m[2] === '{') { const o = {} top[key] = o stack.push(o) key = null } else if (m[2] === '}') { if (stack.length > 1) stack.pop() key = null } else if (key === null) key = m[1] else { top[key] = m[1] key = null } } return root } /** The manifest an appmanifest_258550.acf records for a depot (InstalledDepots, "258552", "manifest"), or null. */ function acfDepotManifest(text, depot = LINUX_DEPOT) { const state = parseVdf(text).AppState const entry = state && state.InstalledDepots && state.InstalledDepots[depot] return (entry && typeof entry === 'object' && entry.manifest) || null } /** Gate carbon, first half: the rig's installed Linux depot is the public manifest. */ function carbonBuild(acfText, publicManifest) { const have = acfDepotManifest(acfText) if (!have) return `the ${LINUX_DEPOT} manifest could not be read from ${ACF} on ${CARBON_RIG}` if (have !== publicManifest) return `${CARBON_RIG} still runs manifest ${have} (public is ${publicManifest})` return true } /** Gate carbon, second half: the `rnpc.status` console lines (an array, or the text) of a Carbon server. */ function carbonOk(statusLines) { const lines = Array.isArray(statusLines) ? statusLines : String(statusLines || '').split('\n') const line = lines.find((l) => /framework=/.test(l)) if (!line) return 'rnpc.status never answered on Carbon' const get = (k) => (new RegExp(`${k}=(\\S+)`).exec(line) || [])[1] const framework = get('framework') if (framework !== 'carbon') return `rnpc.status says framework=${framework}, not carbon` if (get('navmesh') !== 'ready') return `the navmesh was not ready on Carbon (navmesh=${get('navmesh')})` const missing = get('missing') const added = get('added') if (missing === undefined || added === undefined) return 'rnpc.status carried no swap-fields report' if (missing !== '-') return `the swap's fields are missing on Carbon: ${missing}` if (added !== '-') return `public Rust's Carbon assembly has fields the swap does not know: ${added}` return true } /** Gate live: live.json as drill.js live wrote it. */ function liveOk(live) { if (!live) return 'the live half wrote no live.json' const first = (live.findings && live.findings[0]) || null if (live.branch && live.branch !== 'public') return `live.json is from the ${live.branch} branch, not public` if (!live.ran) return live.skipped || first || 'the live half did not run the harness' if (!live.harness || !live.harness.done) return first || 'the harness did not finish' if (live.harness.fail > 0) return `a harness check failed: ${(live.harness.failed && live.harness.failed[0]) || `${live.harness.fail} fail`}` if (!(live.harness.pass > 0)) return 'the harness passed no check at all' if (first) return first return true } /** Gate checks: where the cutover PR's combined commit status stands. { state: success|pending|failed, reason } */ function checkState(combined) { const statuses = (combined && combined.statuses) || [] const plugin = statuses.filter((s) => String(s.context || '').includes('plugin-checks')) const bad = statuses.find((s) => ['failure', 'error'].includes(s.status || s.state)) if (bad) return { state: 'failed', reason: `${bad.context} is ${bad.status || bad.state}` } if (!plugin.length) return { state: 'pending', reason: 'no plugin-checks status has been reported yet' } if (combined.state !== 'success') return { state: 'pending', reason: `the PR's combined status is ${combined.state}` } return { state: 'success', reason: '' } } /** Issue numbers a PR body closes: a Gitea closing keyword directly before `#N`. Sorted, unique. */ function collectCloses(prBodies) { const found = new Set() const re = /(? a - b) } /** The cutover PR's description (the block). `gates` is [{ name, status, note }]. */ function cutoverBody({ wipe, manifest, buildDate, closes = [], gates = [] }) { return [ 'The automated wipe-day cutover (D332, `docs/runicnpc/PLAN.md` stage 9c): `edge` merges into `main` once Rust\'s ' + 'forced-wipe update is live and RunicNPC is proven on it, so no merge needs approving on a wipe day. ' + 'The workflow merges this PR by itself when every gate below has passed; `release.yml` then cuts the release.', '', `Wipe: **${wipe}**. Rust's public build: manifest **${manifest}**, dated ${buildDate || '?'}.`, '', '| Gate | State | Detail |', '| --- | --- | --- |', ...gates.map((g) => `| ${g.name} | ${g.status} | ${neutralise(g.note || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`), '', closes.length ? 'Issues fixed by the PRs merged into `edge` since the last cutover:' : 'No issue is closed by this release.', ...closes.map((n) => `Closes #${n}`), '', 'Opened by the runicnpc-rust cutover workflow (D332).', ].join('\n') } /** `existing` with the cutover block set to `content`: replaced between the markers, or appended when absent. */ function withBlock(existing, content) { const block = `${BLOCK_START}\n${content}\n${BLOCK_END}` const text = String(existing || '') const a = text.indexOf(BLOCK_START) const b = text.indexOf(BLOCK_END, a + 1) if (a >= 0 && b > a) return text.slice(0, a) + block + text.slice(b + BLOCK_END.length) return text.trim() ? `${text.replace(/\s+$/, '')}\n\n${block}` : block } const nextGate = (gates) => PERSISTED.find((g) => !(gates || {})[g]) /** A gate passed: the state with it kept (and its failure forgotten). */ function pass(state, gate, now) { const { [gate]: _gone, ...failed } = state.failed || {} return { ...state, gates: { ...state.gates, [gate]: { at: now.toISOString() } }, failed } } /** A cooling gate failed: when, and why. */ function fail(state, gate, reason, now) { return { ...state, failed: { ...state.failed, [gate]: { at: now.toISOString(), reason: String(reason) } } } } /** What goes into drill.json: never `checks`. */ function persistable(state, pr) { const { checks: _checks, ...gates } = state.gates || {} return { ...state, gates, pr: pr ? pr.number : null } } /** * What the hourly run does next, as data. `state` is the `cutover` object kept in drill.json: * { wipe: 'YYYY-MM-DD', manifest, edgeSha, gates: { public, oxide, carbon, live: { at } }, failed: { carbon, live: { at, reason } }, * pr, lastComment: 'YYYY-MM-DD', merged, gaveUp, mergeError } * `obs`: { manifest, edgeSha, ahead (commits edge has that main lacks), openPr (number or null), forced, * checksPassed (this run, never kept), blocked: { gate, reason } }. * Returns { state, action, gate, reason, retryAt, write, comment, merge }: `action` is nothing | giveUp | run | * blocked | cooling | merge; `write`, `comment` and `merge` say whether the CLI may persist the state, comment, * merge (a dry run: none). */ function decide(state, obs, now, { dryRun = false } = {}) { const w = wipeWeekend(now) const wipe = day(w.wipe) let s = state || {} if (s.wipe !== wipe) s = { wipe, manifest: obs.manifest, edgeSha: obs.edgeSha, gates: {}, failed: {} } // a new wipe starts over else if ((obs.manifest && s.manifest !== obs.manifest) || (obs.edgeSha && s.edgeSha !== obs.edgeSha)) { // A new build, or new code: every gate again, and the failures that cooled them are forgotten. const { mergeError: _e, ...rest } = s s = { ...rest, manifest: obs.manifest || s.manifest, edgeSha: obs.edgeSha || s.edgeSha, gates: {}, failed: {} } } const { checks: _checks, ...gates } = s.gates || {} s = { ...s, gates, failed: s.failed || {}, pr: obs.openPr || null } // the PR is kept while it is open const out = { state: s, write: !dryRun, comment: false, merge: false } // One comment a day, none before the build is known to have changed (a quiet Thursday morning), none without a PR. const hold = (action, gate, reason, retryAt) => { const due = !dryRun && !!s.pr && !!s.gates.public && s.lastComment !== day(now) return { ...out, state: due ? { ...s, lastComment: day(now) } : s, action, gate, reason, retryAt, comment: due } } if (s.merged) return { ...out, action: 'nothing', reason: 'already merged' } if (obs.ahead === 0 && !dryRun) return { ...out, action: 'nothing', reason: 'edge has no commits main lacks' } if (now.getTime() >= w.deadline.getTime() && !obs.forced) { if (s.gaveUp) return { ...out, action: 'nothing', reason: 'gave up already' } return { ...out, state: { ...s, gaveUp: true }, action: 'giveUp', gate: nextGate(s.gates), comment: !dryRun && !!s.pr } } if (obs.blocked) { const retryAt = COOLING.includes(obs.blocked.gate) ? new Date(now.getTime() + COOL_DOWN_HOURS * H) : null return hold('blocked', obs.blocked.gate, obs.blocked.reason, retryAt) } const gate = nextGate(s.gates) if (!gate) return obs.checksPassed ? { ...out, action: 'merge', merge: !dryRun } : { ...out, action: 'run', gate: 'checks' } const failed = COOLING.includes(gate) && s.failed[gate] if (failed && now.getTime() - Date.parse(failed.at) < COOL_DOWN_HOURS * H) { return hold('cooling', gate, failed.reason, new Date(Date.parse(failed.at) + COOL_DOWN_HOURS * H)) } return { ...out, action: 'run', gate } } /** The comment for a decision that asks for one. */ function commentText(d, state = {}) { const label = GATE_LABELS[d.gate] || d.gate if (d.action === 'giveUp') { const head = 'The wipe-day cutover (D332) stops here: the window closed (Sunday 23:00 UTC).' const tail = 'Merge `edge` into `main` by hand when it is ready.' if (state.mergeError) return `${head} Every gate passed; the merge failed: ${neutralise(state.mergeError)}. ${tail}` if (!d.gate) return `${head} Every gate passed, but the PR checks and the merge did not complete. ${tail}` return `${head} Gate "${label}" had not passed. ${tail}` } const when = d.retryAt ? ` It will not try this gate again before ${d.retryAt.toISOString().slice(0, 16).replace('T', ' ')} UTC.` : ' It tries again every hour until Sunday 23:00 UTC.' return `The wipe-day cutover (D332) is waiting at gate "${label}": ${neutralise(d.reason)}.${when} This is the day's only comment.` } /** Whether a dispatch is a dry run: unless its dry_run input is exactly 'false' it is. (A schedule never is.) */ function isDry(a, env) { return !!a['dry-run'] || (env.GITHUB_EVENT_NAME === 'workflow_dispatch' && env.DRY_RUN_INPUT !== 'false') } // ---------------------------------------------------------------- plumbing const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) const log = (...a) => console.log(`[cutover ${new Date().toISOString().slice(11, 19)}]`, ...a) /** What the commands reach outside themselves for; cutover.test.js passes fakes. */ function realDeps() { return { env: process.env, now: () => new Date(), sleep, fetch: (u, o) => fetch(u, o), git: (...a) => execFileSync('git', a, { cwd: ROOT, encoding: 'utf8' }).trim(), panel: () => require('./drill.js').panel(), } } function args() { const out = { _: [] } const argv = process.argv.slice(2) for (let i = 0; i < argv.length; i++) { if (argv[i].startsWith('--')) { const k = argv[i].slice(2) out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true } else out._.push(argv[i]) } return out } const readJson = (f, fallback) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')) } catch { return fallback } } const writeJson = (f, data) => fs.writeFileSync(f, JSON.stringify(data, null, 2) + '\n') const output = (env, name, value) => { if (env.GITHUB_OUTPUT) fs.appendFileSync(env.GITHUB_OUTPUT, `${name}=${value}\n`) } const summary = (env, md) => { if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, md + '\n') } function aheadCount(a, deps) { if (a.ahead !== undefined && a.ahead !== true) return Number(a.ahead) return Number(deps.git('rev-list', '--count', 'origin/main..origin/edge')) } /** Gitea's REST API for this repo (the idiom of drill.js fileIssue). The token is never printed. */ function gitea(token, deps) { if (!token) throw new Error('no Gitea token in the environment') const base = `${deps.env.GITHUB_SERVER_URL}/api/v1/repos/${deps.env.GITHUB_REPOSITORY}` const headers = { Authorization: 'token ' + token, 'Content-Type': 'application/json', Accept: 'application/json' } const call = async (method, p, data) => { const r = await deps.fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined }) const text = await r.text() if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${text.slice(0, 200)}`) return text ? JSON.parse(text) : null } return { call, async openPr() { return (await call('GET', '/pulls?state=open&limit=50')).find((p) => p.head.ref === 'edge' && p.base.ref === 'main') || null }, } } /** * The `Closes #N` numbers of the PRs merged into edge since the last cutover: after the committer date of * `git merge-base origin/main origin/edge`, the newest edge commit already in main. */ async function closesSinceCutOff(g, deps) { let since = 0 try { since = Date.parse(deps.git('log', '-1', '--format=%cI', deps.git('merge-base', 'origin/main', 'origin/edge'))) || 0 } catch {} // A PR whose merge commit main already has shipped in an earlier cutover, even when its merged_at is a moment // after the cut-off's committer date (edge's tip is often that very merge commit). const shipped = (sha) => { if (!sha) return false try { return /\borigin\/main\b/.test(deps.git('branch', '-r', '--contains', sha, '--list', 'origin/main')) } catch { return false } } const bodies = [] for (let page = 1; page <= 10; page++) { const prs = await g.call('GET', `/pulls?state=closed&limit=50&page=${page}`) for (const pr of prs) { if (pr.merged && pr.base && pr.base.ref === 'edge' && Date.parse(pr.merged_at) > since && !shipped(pr.merge_commit_sha)) bodies.push(pr.body) } if (prs.length < 50) break } return collectCloses(bodies) } // ---------------------------------------------------------------- window and pending function window_(a, deps = realDeps()) { const now = a.now ? new Date(a.now) : deps.now() const w = wipeWeekend(now) const open = w.open || w.final || !!a.force const verdict = w.open ? 'in the wipe weekend' : w.final ? 'the hour after it (the last run, to give up out loud)' : a.force ? 'outside the wipe weekend, but run by hand: going ahead' : 'outside the wipe weekend, nothing to do' log(`the wipe is ${day(w.wipe)}, the cutover works until ${w.deadline.toISOString().slice(0, 16)}Z: ${verdict} (open=${open})`) output(deps.env, 'open', open ? 'true' : 'false') } function pending(a, deps = realDeps()) { const ahead = aheadCount(a, deps) const run = ahead > 0 || !!a.force log(`edge is ${ahead} commit(s) ahead of main: ${run ? 'going ahead' : 'nothing to merge'}`) output(deps.env, 'run', run ? 'true' : 'false') } // ---------------------------------------------------------------- the rig async function carbonGate(ctx) { const { waitRunning } = require('./drill.js') const p = ctx.deps.panel() const id = (await p.servers())[CARBON_RIG] if (!id) return `no server named ${CARBON_RIG} on the panel` await p.write(id, CARBON_PLUGIN, fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8')) const was = await p.state(id) log(`${CARBON_RIG} is ${was}: ${was === 'offline' ? 'starting' : 'restarting'} it (its egg updates Rust and Carbon)`) await p.power(id, was === 'offline' ? 'start' : 'restart') // `running` right after the signal is the old process: wait for it to leave first. for (let i = 0; i < 60 && was !== 'offline' && (await p.state(id)) === 'running'; i++) await ctx.deps.sleep(5000) if (!(await waitRunning(p, id, 45 * 60000))) return `${CARBON_RIG} did not come up in 45 minutes, or crashed on start` // It must have updated to the public build, or everything after proves nothing. const built = carbonBuild(await p.readOr(id, ACF, null), ctx.stat.manifest) if (built !== true) return built const lines = await carbonStatus(p, id, 30 * 60000, ctx.deps) if (lines) log('rnpc.status:', lines.find((l) => l.includes('navmesh='))) return carbonOk(lines || []) } /** `rnpc.status` over the console until the navmesh is ready; the last answer if it never is. */ async function carbonStatus(p, id, timeoutMs, deps) { const end = Date.now() + timeoutMs let last = null while (Date.now() < end) { try { const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim()) const at = lines.findIndex((l) => l.includes('RunicNPC ') && l.includes('api=')) if (at >= 0) { last = lines.slice(at, at + 14) if (last.some((l) => l.includes('navmesh=ready'))) return last } } catch { // the panel's console is not up yet } await deps.sleep(30000) } return last } /** Writes whether rust-carbon is running now, to put it back if the job leaves it stopped. */ async function rigNote(a, deps = realDeps()) { const work = path.resolve(a.work || 'cutover-work') const p = deps.panel() const id = (await p.servers())[CARBON_RIG] const state = id ? await p.state(id) : null fs.mkdirSync(work, { recursive: true }) writeJson(path.join(work, 'rig-carbon.json'), { name: CARBON_RIG, state }) log(`${CARBON_RIG} is ${state || 'not on the panel'}`) } /** Starts rust-carbon (power `start` on it alone) when it was running at the start of the job and is offline now. */ async function rigRestore(a, deps = realDeps()) { const rec = readJson(path.join(path.resolve(a.work || 'cutover-work'), 'rig-carbon.json'), null) if (!rec || rec.state !== 'running') return log(`${CARBON_RIG} was not running at the start: nothing to put back`) const p = deps.panel() const id = (await p.servers())[CARBON_RIG] if (!id) return log(`${CARBON_RIG} is not on the panel`) const now = await p.state(id) if (now !== 'offline') return log(`${CARBON_RIG} is ${now}: left as it is`) await p.power(id, 'start') log(`${CARBON_RIG} was running at the start and is offline: started it`) } // ---------------------------------------------------------------- the gates async function checksGate(ctx) { const wait = !ctx.dryRun let pr = ctx.pr if (!pr) return ctx.dryRun ? 'no cutover PR is open (a dry run opens none)' : 'no cutover PR is open' const end = Date.now() + CHECKS_WAIT_MS for (;;) { pr = (await ctx.g.call('GET', `/pulls/${pr.number}`)) || pr if (pr.head.sha !== ctx.edgeSha) return `the PR's head ${String(pr.head.sha).slice(0, 8)} is not the commit the gates ran on (${String(ctx.edgeSha).slice(0, 8)})` const v = checkState(await ctx.g.call('GET', `/commits/${pr.head.sha}/status`)) if (v.state === 'success') return true if (v.state === 'failed') return v.reason if (!wait || Date.now() >= end) return `${v.reason} (waited ${wait ? 15 : 0} minutes)` await ctx.deps.sleep(30000) } } const RUNNERS = { public: async (c) => publicGate(c.stat, c.staging, c.wipe), oxide: async (c) => oxideReady(c.stat), carbon: async (c) => carbonGate(c), live: async (c) => liveOk(readJson(path.join(c.work, 'live.json'), null)), checks: async (c) => checksGate(c), } /** Runs one gate; whatever it throws is its reason, so a flaky API call blocks an hour, not the state. */ async function runGate(gate, ctx) { log(`gate ${gate}...`) let r try { r = await RUNNERS[gate](ctx) } catch (e) { r = `the gate stopped: ${e.message}` } log(`gate ${gate}: ${r === true ? 'passed' : `NOT passed: ${r}`}`) return r } // ---------------------------------------------------------------- the PR function gateRows(state, rows) { return GATES.map((g) => { const done = state.gates[g] const row = rows[g] if (done) return { name: GATE_LABELS[g], status: 'passed', note: done.at.slice(0, 16).replace('T', ' ') + ' UTC' } return { name: GATE_LABELS[g], status: row ? row.status : 'not yet', note: row ? row.detail : '' } }) } /** * Opens the edge to main PR, or brings the marked block of the open one's body up to date. A PR it did not * open keeps its title and the rest of its body. Not in a dry run, and never with a blank manifest. */ async function syncPr(ctx) { if (ctx.dryRun) return ctx.pr if (!ctx.stat.manifest) { log('no manifest: the PR is left alone') return ctx.pr } if (!ctx.closes) ctx.closes = await closesSinceCutOff(ctx.g, ctx.deps) const content = cutoverBody({ wipe: ctx.wipeDate, manifest: ctx.stat.manifest, buildDate: ctx.stat.manifestDate, closes: ctx.closes, gates: gateRows(ctx.state, ctx.rows) }) if (!ctx.pr) { const title = `chore(release): RunicNPC cutover for the ${ctx.wipeDate} wipe, Rust ${ctx.stat.manifest}` ctx.pr = await ctx.g.call('POST', '/pulls', { head: 'edge', base: 'main', title, body: withBlock('', content) }) log(`opened PR #${ctx.pr.number}`) } else { const body = withBlock(ctx.pr.body, content) if (body !== ctx.pr.body) { ctx.pr = { ...ctx.pr, ...((await ctx.g.call('PATCH', `/pulls/${ctx.pr.number}`, { body })) || {}) } log(`updated PR #${ctx.pr.number}`) } } ctx.state = { ...ctx.state, pr: ctx.pr.number } return ctx.pr } // ---------------------------------------------------------------- gates (the command) async function gates(a, deps = realDeps()) { const env = deps.env const dryRun = isDry(a, env) const afterLive = !!a['after-live'] const work = path.resolve(a.work || 'cutover-work') const file = readJson(a.state, {}) const now = deps.now() const w = wipeWeekend(now) const stat = readJson(path.join(work, 'static.json'), {}) const staging = readJson(path.join(path.resolve(a['staging-work'] || work + '-staging'), 'static.json'), null) const g = gitea(env.GITEA_TOKEN, deps) const prev = (afterLive && readJson(path.join(work, 'cutover-run.json'), null)) || { rows: {} } const pr = await g.openPr() const edgeSha = a.edge || deps.git('rev-parse', 'HEAD') const obs = { manifest: stat.manifest || '', edgeSha, ahead: aheadCount(a, deps), openPr: pr && pr.number, forced: !!a.force && !(w.open || w.final), checksPassed: !!(dryRun && prev.checks) } // A dry run starts from nothing: it is there to try every gate, not to trust an earlier hour's. let d = decide(dryRun ? undefined : file.cutover, obs, now, { dryRun }) const ctx = { g, deps, stat, staging, work, now, dryRun, pr, edgeSha, wipe: w.wipe, wipeDate: day(w.wipe), state: d.state, rows: prev.rows, closes: null } if (dryRun && prev.gates) ctx.state = { ...ctx.state, gates: prev.gates } // what the first half of this dry run passed else for (const k of Object.keys(ctx.state.gates)) ctx.rows[k] = { status: 'passed earlier', detail: ctx.state.gates[k].at } const note = (gate, r) => (ctx.rows[gate] = r === true ? { status: 'passed', detail: '' } : { status: 'failed', detail: r }) // The state goes to the file after every gate result, before the PR is touched. const persist = () => { if (dryRun) return file.cutover = persistable(ctx.state, ctx.pr) writeJson(a.state, file) } const safely = async (what, fn) => { try { return await fn() } catch (e) { log(`${what} failed (the gates are kept): ${e.message}`) } } let blocked = null let deferred = false if (d.action === 'nothing') log(`nothing to do: ${d.reason}`) else if (d.action === 'giveUp') log(`the cutover window closed with ${d.gate || 'every gate'} not done`) else if (dryRun) { // Every gate it can; live waits for its own step. for (const gate of GATES) { if (gate === 'live' && !afterLive) { deferred = true ctx.rows.live = { status: 'runs next', detail: 'drill.js live --branch public' } } else if ((gate === 'live') === afterLive || (afterLive && gate === 'checks' && !prev.checks)) { const r = await runGate(gate, ctx) note(gate, r) if (r === true) { if (gate === 'checks') obs.checksPassed = true else ctx.state = pass(ctx.state, gate, now) } else if (!blocked) blocked = { gate, reason: r } } } } else { while (d.action === 'run') { if (d.gate === 'live' && !afterLive) { deferred = true break } const r = await runGate(d.gate, ctx) note(d.gate, r) if (r !== true) { blocked = { gate: d.gate, reason: r } if (COOLING.includes(d.gate)) ctx.state = fail(ctx.state, d.gate, r, now) persist() break } if (d.gate === 'checks') obs.checksPassed = true else { ctx.state = pass(ctx.state, d.gate, now) persist() } if (d.gate === 'public') { await safely('opening the PR', () => syncPr(ctx)) obs.openPr = ctx.pr ? ctx.pr.number : null } d = decide(ctx.state, obs, now, { dryRun }) } } // What the run ends in. if (d.action !== 'nothing' && d.action !== 'giveUp') { obs.openPr = ctx.pr ? ctx.pr.number : null d = decide(ctx.state, blocked ? { ...obs, blocked } : obs, now, { dryRun }) } if (!dryRun && ctx.state.gates.public && d.action !== 'giveUp' && !ctx.state.merged) await safely('updating the PR', () => syncPr(ctx)) const end = d.action log(`${end}${d.gate ? ` (${d.gate})` : ''}${d.reason ? `: ${d.reason}` : ''}`) let finalState = d.state if (d.comment && ctx.pr) { const done = await safely('commenting', async () => { await g.call('POST', `/issues/${ctx.pr.number}/comments`, { body: commentText(d, d.state) }) return true }) if (done) log(`commented on PR #${ctx.pr.number}`) else finalState = { ...finalState, lastComment: ctx.state.lastComment } // not said, so it may be said next hour } ctx.state = finalState persist() fs.mkdirSync(work, { recursive: true }) writeJson(path.join(work, 'cutover-run.json'), { rows: ctx.rows, gates: ctx.state.gates, checks: !!obs.checksPassed }) output(env, 'live', deferred ? 'true' : 'false') output(env, 'merge', d.action === 'merge' && d.merge ? 'true' : 'false') output(env, 'sha', edgeSha) if (ctx.pr) output(env, 'pr', ctx.pr.number) if (!deferred) { const rows = GATES.map((gate) => `| ${GATE_LABELS[gate]} | ${(ctx.rows[gate] || { status: 'not reached' }).status} | ${neutralise((ctx.rows[gate] || {}).detail || '').replace(/\|/g, '/').replace(/\r?\n/g, ' ')} |`) summary( env, [ `## Wipe-day cutover${dryRun ? ' (dry run: nothing created, edited, commented or merged)' : ''}: ${end}`, '', `Wipe ${ctx.wipeDate}; Rust public manifest ${stat.manifest || '?'} of ${stat.manifestDate || '?'}; edge ${edgeSha.slice(0, 8)} is ${obs.ahead} commit(s) ahead of main.`, '', '| Gate | State | Detail |', '| --- | --- | --- |', ...rows, d.action === 'merge' ? `\nEvery gate passed${dryRun ? ': a real run would merge now' : ''}.` : '', ].join('\n') ) } return d } // ---------------------------------------------------------------- merge and after /** Records why the merge failed where the next hour (and a last comment) can read it. */ function recordMergeError(stateFile, message) { if (!stateFile || stateFile === true) return const file = readJson(stateFile, {}) file.cutover = { ...(file.cutover || {}), mergeError: String(message).slice(0, 300) } writeJson(stateFile, file) } async function merge(a, deps = realDeps()) { const g = gitea(deps.env.MERGE_TOKEN, deps) if (!a.pr || a.pr === true) throw new Error('--pr is required') if (!a.sha || a.sha === true) throw new Error('--sha is required') try { const before = await g.call('GET', `/pulls/${a.pr}`) if (before.head.sha !== a.sha) throw new Error(`PR #${a.pr}'s head ${String(before.head.sha).slice(0, 8)} is not the commit the gates ran on (${a.sha.slice(0, 8)})`) // Not the squash: release.yml reads the feat/fix subjects of the commits that arrive. Never delete edge. // head_commit_id: Gitea refuses the merge when the head moved since the gates. await g.call('POST', `/pulls/${a.pr}/merge`, { Do: 'merge', delete_branch_after_merge: false, head_commit_id: a.sha }) const pr = await g.call('GET', `/pulls/${a.pr}`) if (!pr.merged) throw new Error(`PR #${a.pr} is not merged after the merge call`) } catch (e) { recordMergeError(a.state, e.message) throw e } log(`merged PR #${a.pr}`) output(deps.env, 'merged', 'true') } async function after(a, deps = realDeps()) { const g = gitea(deps.env.GITEA_TOKEN, deps) const pr = await g.call('GET', `/pulls/${a.pr}`) if (!pr.merged) return log(`PR #${a.pr} is not merged: nothing to record`) const file = readJson(a.state, {}) const { mergeError: _e, ...kept } = file.cutover || {} file.cutover = { ...kept, merged: true, pr: pr.number } writeJson(a.state, file) // release.yml cuts the release as the push to main arrives. const mergedAt = Date.parse(pr.merged_at) let release = null for (const end = deps.now().getTime() + RELEASE_WAIT_MS; !release && deps.now().getTime() < end; ) { const list = await g.call('GET', '/releases?draft=false&pre-release=false&limit=5') release = (list || []).find((r) => Date.parse(r.created_at) > mergedAt) || null if (!release) await deps.sleep(60000) } if (!release) return log('no release appeared in 30 minutes: no issue is commented') log(`released as ${release.tag_name}`) output(deps.env, 'release', release.tag_name) const text = `Released in ${release.tag_name} by the wipe cutover (D332).` for (const n of collectCloses([pr.body])) { const issue = await g.call('GET', `/issues/${n}`) if (issue.state !== 'closed') { log(`#${n} is not closed: no comment`) continue } if ((await g.call('GET', `/issues/${n}/comments`)).some((c) => c.body === text)) continue await g.call('POST', `/issues/${n}/comments`, { body: text }) log(`commented on #${n}`) } } // ---------------------------------------------------------------- main module.exports = { wipeWeekend, publicChanged, publicGate, oxideReady, parseVdf, acfDepotManifest, carbonBuild, carbonOk, liveOk, checkState, collectCloses, cutoverBody, withBlock, decide, commentText, neutralise, isDry, pass, fail, persistable, nextGate, manifestTime, gates, merge, after, rigNote, rigRestore, checksGate, closesSinceCutOff, gitea, GATES, COOL_DOWN_HOURS, BLOCK_START, BLOCK_END, } if (require.main === module) { const a = args() const commands = { window: window_, pending, gates, merge, after, 'rig-note': rigNote, 'rig-restore': rigRestore } const command = commands[a._[0]] if (!command) { console.error('usage: node tools/drill/cutover.js window|pending|gates|merge|after|rig-note|rig-restore [--state f] [--work dir] [--pr n] [--sha s] [--dry-run] [--force] [--after-live]') process.exit(2) } Promise.resolve(command(a)).catch((e) => { console.error(e.message || e) process.exit(1) }) }