// The staging drill (docs/runicnpc/PLAN.md stage 9c, D307-D309, D317, D318). Never shipped. // // .gitea/workflows/staging-drill.yml runs it daily, and it does something only in the week before the // monthly forced wipe (D318): // // node tools/drill/drill.js window [--today YYYY-MM-DD] in the drill week? // tools/drill/static.sh the static half // node tools/drill/drill.js gate --state --work a build not checked yet? // node tools/drill/drill.js live --work [--branch staging] the live half, on rust-staging // node tools/drill/drill.js report --state --work one issue per build; the state // // A build is the manifest of Rust's Linux depot, as Steam names it to static.sh. // // Environment: PANEL_KEY (or PANEL_KEY_FILE, a file with a `user: ptlc_…` line) is the drill's own // Pterodactyl client key; PANEL_URL the panel. The live half finds its servers by name: DRILL_RIG // (rust-staging) and the two 6000-map rigs, of which it stops DRILL_STOPPABLE (rust-carbon) when both // run, because the node has no memory for three (D309). It leaves every rig as it found it. // `report` needs GITEA_TOKEN, and GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them. // // The key is never printed, and neither is the server's console start line (it carries the RCON // password): only lines this script picks are ever written out. const fs = require('fs') const path = require('path') const ROOT = path.resolve(__dirname, '..', '..') const PANEL = (process.env.PANEL_URL || 'http://192.168.0.12').replace(/\/+$/, '') const RIG = process.env.DRILL_RIG || 'rust-staging' const BIG = (process.env.DRILL_BIG_RIGS || 'rust-oxide,rust-carbon').split(',') const STOPPABLE = process.env.DRILL_STOPPABLE || 'rust-carbon' const KITS_URL = 'https://umod.org/plugins/Kits.cs' const WAIT_DAYS = 3 // D317: how long a build waits for Oxide to catch up const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) const now = () => new Date().toISOString() const log = (...a) => console.log(`[drill ${now().slice(11, 19)}]`, ...a) function args() { const out = { _: [] } const argv = process.argv.slice(2) for (let i = 0; i < argv.length; i++) { if (argv[i].startsWith('--')) { const k = argv[i].slice(2) out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true } else out._.push(argv[i]) } return out } const readJson = (f, fallback) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')) } catch { return fallback } } function output(name, value) { if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`) } // ---------------------------------------------------------------- the panel function panelKey() { if (process.env.PANEL_KEY) return process.env.PANEL_KEY.trim() const file = process.env.PANEL_KEY_FILE if (!file) throw new Error('PANEL_KEY (or PANEL_KEY_FILE) is not set') const line = fs.readFileSync(file, 'utf8').split(/\r?\n/).find((l) => l.startsWith('user:')) if (!line) throw new Error(`${file} has no "user:" line`) return line.slice(5).trim() } function panel() { const key = panelKey() const headers = (type = 'application/json') => ({ Authorization: 'Bearer ' + key, Accept: 'application/json', 'Content-Type': type }) const api = async (method, p, body, type) => { const r = await fetch(`${PANEL}/api/client${p}`, { method, headers: headers(type), body }) const text = await r.text() if (!r.ok) throw new Error(`${method} ${p.split('?')[0]}: ${r.status} ${text.slice(0, 200)}`) return text } const p = { async servers() { const d = JSON.parse(await api('GET', '/?per_page=100')) return Object.fromEntries(d.data.map((s) => [s.attributes.name, s.attributes.identifier])) }, async state(id) { return JSON.parse(await api('GET', `/servers/${id}/resources`)).attributes.current_state }, power: (id, signal) => api('POST', `/servers/${id}/power`, JSON.stringify({ signal })), command: (id, command) => api('POST', `/servers/${id}/command`, JSON.stringify({ command })), read: (id, file) => api('GET', `/servers/${id}/files/contents?file=${encodeURIComponent(file)}`), async readOr(id, file, fallback = null) { try { return await p.read(id, file) } catch { return fallback } }, write: (id, file, content) => api('POST', `/servers/${id}/files/write?file=${encodeURIComponent(file)}`, content, 'text/plain'), async rm(id, file) { const dir = file.slice(0, file.lastIndexOf('/')) || '/' try { await api('POST', `/servers/${id}/files/delete`, JSON.stringify({ root: dir, files: [file.slice(file.lastIndexOf('/') + 1)] })) } catch { // already gone } }, /** Sends one console command and returns the console lines that follow, for `secs` seconds. */ async console(id, command, secs = 6) { const { data } = JSON.parse(await api('GET', `/servers/${id}/websocket`)) const strip = (s) => s.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '') return new Promise((resolve) => { const lines = [] let sent = false const ws = new WebSocket(data.socket, { headers: { Origin: PANEL } }) const done = () => { try { ws.close() } catch {} resolve(lines) } ws.onopen = () => ws.send(JSON.stringify({ event: 'auth', args: [data.token] })) ws.onmessage = (m) => { const msg = JSON.parse(m.data) if (msg.event === 'auth success' && !sent) { sent = true // The panel replays recent history on connect; let it pass, then send. setTimeout(() => { lines.length = 0 ws.send(JSON.stringify({ event: 'send command', args: [command] })) }, 800) setTimeout(done, secs * 1000) } else if (msg.event === 'console output' && sent) { lines.push(...strip(String(msg.args[0])).split('\n')) } } ws.onerror = () => done() }) }, async waitState(id, want, timeoutMs) { const end = Date.now() + timeoutMs while (Date.now() < end) { if ((await p.state(id)) === want) return true await sleep(5000) } return false }, /** Stops a server: `stop`, then `kill` if it has not stopped in time. */ async stop(id, graceMs = 180000) { if ((await p.state(id)) === 'offline') return await p.power(id, 'stop') if (!(await p.waitState(id, 'offline', graceMs))) { await p.power(id, 'kill') await p.waitState(id, 'offline', 60000) } }, } return p } // ---------------------------------------------------------------- window: the week before a forced wipe // Rust changes on the monthly forced wipe, the first Thursday (D318). The drill runs only in the seven // days before it, which leaves a week to fix what it finds; outside them the job stops at once. function nextWipe(today) { const firstThursday = (y, m) => { const d = new Date(Date.UTC(y, m, 1)) d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7)) return d } const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate()) const thisMonth = firstThursday(today.getUTCFullYear(), today.getUTCMonth()) return thisMonth.getTime() > t ? thisMonth : firstThursday(today.getUTCFullYear(), today.getUTCMonth() + 1) } function windowDays(today = new Date()) { const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate()) return Math.round((nextWipe(today).getTime() - t) / 86400000) } async function window_(a) { const days = windowDays(a.today ? new Date(a.today) : new Date()) const wipe = nextWipe(a.today ? new Date(a.today) : new Date()).toISOString().slice(0, 10) const open = days >= 1 && days <= 7 const verdict = open ? 'in the drill week' : a.force ? 'outside the drill week, but run by hand: going ahead' : 'outside the drill week, nothing to do' log(`the next forced wipe is ${wipe}, in ${days} day(s): ${verdict}`) output('open', open || a.force ? 'true' : 'false') } // ---------------------------------------------------------------- gate: is there anything to check? // A build is Steam's manifest of the Linux depot, which static.sh reads from Steam itself. async function gate(a) { const state = readJson(a.state, {}) const stat = readJson(path.join(path.resolve(a.work || 'drill-work'), 'static.json'), {}) const checked = state.checked && state.checked.manifest const isNew = !stat.manifest || stat.manifest !== checked log(`Rust ${stat.branch || 'staging'} is manifest ${stat.manifest || '?'} (${stat.manifestDate || '?'}); last checked ${checked || 'none'}`) output('run', isNew || a.force ? 'true' : 'false') if (!isNew && !a.force) log('already checked: nothing to do') } // ---------------------------------------------------------------- live: the harness on rust-staging async function live(a) { const work = path.resolve(a.work || 'drill-work') const branch = a.branch || 'staging' const result = { branch, at: now(), ran: false, findings: [], notes: [] } const save = () => fs.writeFileSync(path.join(work, 'live.json'), JSON.stringify(result, null, 2)) fs.mkdirSync(work, { recursive: true }) const stat = readJson(path.join(work, 'static.json'), null) if (!stat || stat.managed !== 'ok') { result.skipped = 'the static half did not run' save() return log(result.skipped) } if (stat.oxideBehind !== 0) { result.skipped = `Oxide's ${branch} build is ${stat.oxideBehind} member(s) behind Rust's, so an Oxide server cannot boot on it` save() return log(result.skipped) } const p = panel() const ids = await p.servers() const rig = ids[RIG] if (!rig) throw new Error(`no server named ${RIG} on the panel`) let restart = null const finding = (f) => { result.findings.push(f) log('FINDING', f) } try { // Room for it (D309): with both 6000-map rigs running the node's memory is full. const running = [] for (const name of BIG) if (ids[name] && (await p.state(ids[name])) === 'running') running.push(name) if (running.length === BIG.length && ids[STOPPABLE]) { log(`both ${BIG.join(' and ')} are running: saving and stopping ${STOPPABLE}`) await p.command(ids[STOPPABLE], 'server.save').catch(() => {}) await sleep(15000) await p.stop(ids[STOPPABLE]) restart = STOPPABLE result.notes.push(`${STOPPABLE} was stopped for the drill and started again afterwards`) } await p.stop(rig, 60000) // What runs: this commit's plugin and harness, and the startup that installs the branch. await p.write(rig, '/staging-drill.sh', fs.readFileSync(path.join(__dirname, 'rig-start.sh'), 'utf8').replace(/\r\n/g, '\n')) await p.write(rig, '/staging-drill.branch', branch + '\n') await p.write(rig, '/oxide/plugins/RunicNPC.cs', fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8')) const kits = await fetch(KITS_URL) if (!kits.ok) throw new Error(`Kits from uMod: ${kits.status}`) await p.write(rig, '/oxide/plugins/Kits.cs', await kits.text()) await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs') await p.rm(rig, '/oxide/data/RunicNpcTest.json') await p.rm(rig, '/staging-drill.json') log(`starting ${RIG} on ${branch}`) const started = Date.now() await p.power(rig, 'start') const up = await waitRunning(p, rig, 45 * 60000) result.installed = readJsonText(await p.readOr(rig, '/staging-drill.json', null)) if (!up) { finding(`The server did not start on Rust ${branch} ${result.installed?.buildid || ''} with Oxide's ${branch} build.`) result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 30) return } result.bootMinutes = Math.round((Date.now() - started) / 6000) / 10 log(`running after ${result.bootMinutes} min:`, JSON.stringify(result.installed)) // RunicNPC loaded, and the navmesh ready. const status = await waitStatus(p, rig, 30 * 60000) if (!status) { finding('RunicNPC did not load: `rnpc.status` never answered.') result.oxideErrors = await oxideErrors(p, rig) return } result.status = status log('rnpc.status:', status.find((l) => l.includes('navmesh='))) const fieldsLine = status.find((l) => l.includes('swap fields:')) || '' const missing = /missing=(\S+)/.exec(fieldsLine) if (missing && missing[1] !== '-') finding(`The swap's fields are missing on this build: ${missing[1]}`) if (!fieldsLine.includes('navmesh=ready')) { finding('The navmesh was not ready in 30 minutes.') return } // The harness needs the hidden Kits kit rnhrevolver (tools/RunicNpcTest.cs). await ensureKit(p, rig) await p.write(rig, '/oxide/plugins/RunicNpcTest.cs', fs.readFileSync(path.join(ROOT, 'tools', 'RunicNpcTest.cs'), 'utf8')) let accepted = false for (let i = 0; i < 18 && !accepted; i++) { await sleep(10000) accepted = (await p.console(rig, 'rnt.run all', 5)).some((l) => l.includes('rnt: running all')) } if (!accepted) { finding('The test harness did not load (`rnt.run all` was not answered).') result.oxideErrors = await oxideErrors(p, rig) return } result.ran = true log('rnt.run all: running') const lines = await waitHarness(p, rig, 45 * 60000) result.harness = summarise(lines) if (!result.harness.done) finding(`The harness did not finish in 45 minutes (${result.harness.pass} pass, ${result.harness.fail} fail so far).`) for (const f of result.harness.failed) finding(`Harness: ${f}`) log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`) result.oxideErrors = await oxideErrors(p, rig) if (result.oxideErrors.length) finding(`RunicNPC logged ${result.oxideErrors.length} error line(s) in Oxide's log during the run.`) await p.command(rig, 'rnt.clear').catch(() => {}) } catch (e) { finding(`The live half stopped: ${e.message}`) } finally { try { await p.stop(rig) await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs') } catch (e) { result.notes.push(`${RIG} could not be stopped cleanly: ${e.message}`) } if (restart) { try { await p.power(ids[restart], 'start') log(`started ${restart} again`) } catch (e) { result.findings.push(`${restart} was stopped for the drill and could NOT be started again: ${e.message}`) } } save() } } async function waitRunning(p, id, timeoutMs) { const end = Date.now() + timeoutMs let seenStarting = false while (Date.now() < end) { const s = await p.state(id) if (s === 'running') return true if (s === 'starting') seenStarting = true // Wings restarts a crashed server a few times; offline after starting is a crash it gave up on. if (s === 'offline' && seenStarting) return false await sleep(15000) } return false } async function waitStatus(p, id, timeoutMs) { const end = Date.now() + timeoutMs let last = null while (Date.now() < end) { const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim()) const at = lines.findIndex((l) => l.startsWith('RunicNPC ')) if (at >= 0) { last = lines.slice(at, at + 14) if (last.some((l) => l.includes('navmesh=ready'))) return last } await sleep(30000) } return last } async function ensureKit(p, id) { const file = '/oxide/data/Kits/kits_data.json' let data = null for (let i = 0; i < 12 && !data; i++) { data = readJsonText(await p.readOr(id, file, null)) if (!data) await sleep(5000) } if (!data) throw new Error(`Kits wrote no ${file}`) data._kits = data._kits || {} if (data._kits.rnhrevolver) return const item = (Shortname, Amount, Position, Condition = 0) => ({ Shortname, Skin: 0, Amount, Condition, MaxCondition: Condition, Ammo: 0, Position, Frequency: -1 }) data._kits.rnhrevolver = { Name: 'rnhrevolver', Description: 'RunicNPC test harness: revolver + burlap (staging drill)', RequiredPermission: '', MaximumUses: 0, RequiredAuth: 0, Cooldown: 0, Cost: 0, IsHidden: true, CopyPasteFile: '', KitImage: '', MainItems: [item('ammo.pistol', 120, 0)], WearItems: [item('burlap.shirt', 1, 0), item('burlap.trousers', 1, 1), item('hat.boonie', 1, 2)], BeltItems: [item('pistol.revolver', 1, 0, 100)], } await p.write(id, file, JSON.stringify(data, null, 2)) await p.command(id, 'oxide.reload Kits') await sleep(10000) log('added the rnhrevolver kit and reloaded Kits') } function readJsonText(t) { try { return t ? JSON.parse(t) : null } catch { return null } } async function waitHarness(p, id, timeoutMs) { const end = Date.now() + timeoutMs let lines = [] while (Date.now() < end) { await sleep(30000) lines = readJsonText(await p.readOr(id, '/oxide/data/RunicNpcTest.json', null)) || lines if (lines.some((l) => /^# done all:/.test(l))) break } return lines } function summarise(lines) { const failed = lines.filter((l) => / FAIL( |$)/.test(l)) const passed = lines.filter((l) => / PASS( |$)/.test(l)) const done = lines.find((l) => /^# done all:/.test(l)) return { done: !!done, summary: done || null, pass: passed.length, fail: failed.length, failed } } /** RunicNPC's and the harness's error lines from today's Oxide log. */ async function oxideErrors(p, id) { const day = now().slice(0, 10) const text = (await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || '' return text .split('\n') .filter((l) => /RunicNPC|RunicNpcTest/.test(l) && /\[Error\]|Exception|Failed|error CS/.test(l)) .slice(-40) } /** The console's last lines, without the startup line (it carries the RCON password). */ function safeTail(text, n) { return (text || '') .split('\n') .filter((l) => !/rcon\.password|\+rcon/i.test(l)) .slice(-n) } // ---------------------------------------------------------------- report: one issue per build async function report(a) { const work = path.resolve(a.work || 'drill-work') const statePath = a.state const state = readJson(statePath, {}) const stat = readJson(path.join(work, 'static.json'), null) const liv = readJson(path.join(work, 'live.json'), null) const branch = (stat && stat.branch) || 'staging' const manifest = (stat && stat.manifest) || '' const buildid = (liv && liv.installed && liv.installed.buildid) || '' // How a build is named: Steam's build id when the rig installed it, and always the manifest's date. const build = `${buildid ? `build ${buildid}, ` : ''}manifest ${manifest || '?'} of ${(stat && stat.manifestDate) || '?'}` const read = (f, n = 80) => { try { return fs.readFileSync(path.join(work, f), 'utf8').replace(/\r/g, '').split('\n').slice(0, n).join('\n').trim() } catch { return '' } } const findings = [] const sections = [] let outcome let markChecked = true if (!stat || stat.managed !== 'ok' || stat.fields === 'error' || stat.compile === 'error') { outcome = 'could not run' markChecked = false findings.push(`The static half could not run: ${(stat && stat.note) || 'no static.json'}`) if (stat && stat.fields === 'error') sections.push(['tools/fieldlist', read('fields.diff')]) } else if (stat.oxideBehind !== 0) { // A wait from an earlier drill week (or a run by hand outside one) starts over. const current = state.waiting && state.waiting.manifest === manifest && Date.now() - Date.parse(state.waiting.since) < 7 * 86400000 const waiting = current ? state.waiting : { manifest, since: now() } const days = (Date.now() - Date.parse(waiting.since)) / 86400000 if (days < WAIT_DAYS) { outcome = `waiting for Oxide (${stat.oxideBehind} behind, day ${Math.floor(days) + 1} of ${WAIT_DAYS})` if (branch === 'staging') { state.waiting = { ...waiting, behind: stat.oxideBehind, lastSeen: now() } markChecked = false } } else { outcome = 'Oxide has not caught up' findings.push( `Oxide's staging build is still ${stat.oxideBehind} member(s) behind Rust's staging build after ${WAIT_DAYS} days, ` + 'so an Oxide server cannot boot on it and the live half did not run. The static half ran against Oxide\'s older build.' ) sections.push(['What Oxide lacks (first 20)', read('oxide-behind.txt', 21)]) } } if (!outcome || outcome === 'Oxide has not caught up') { if (stat.fields === 'differs') { findings.push("The swap's field list differs from the one the plugin carries (D232): regenerate it with tools/fieldlist and check what changed.") sections.push(['Field list (carried → staging)', read('fields.diff', 120)]) } if (stat.compile === 'errors') { findings.push("RunicNPC does not compile against this build with Oxide's staging build.") sections.push(['Compiler output', read('compile.txt', 60)]) } if (!liv && stat.oxideBehind === 0) findings.push('The live half did not run: see the job log (is PTERODACTYL_DRILL_KEY set?).') if (liv && !liv.skipped) { findings.push(...liv.findings) if (liv.consoleTail && liv.consoleTail.length) sections.push(['Console, last lines', liv.consoleTail.join('\n')]) if (liv.oxideErrors && liv.oxideErrors.length) sections.push(["Oxide's log (RunicNPC lines)", liv.oxideErrors.join('\n')]) if (liv.status) sections.push(['rnpc.status', liv.status.join('\n')]) } if (!outcome) outcome = findings.length ? 'fail' : 'pass' if (branch === 'staging' && markChecked) delete state.waiting } const installed = liv && liv.installed ? `Rust ${liv.installed.branch} ${liv.installed.buildid}, Oxide build of ${liv.installed.oxideModified}` : '' const lines = [ `Static: Oxide behind ${stat ? stat.oxideBehind : '?'}, fields ${stat ? stat.fields : '?'}, compile ${stat ? stat.compile : '?'}.`, liv ? (liv.skipped ? `Live: skipped (${liv.skipped}).` : `Live: ${liv.harness ? `${liv.harness.pass} pass, ${liv.harness.fail} fail` : 'did not reach the harness'}${installed ? `, on ${installed}` : ''}${liv.bootMinutes ? `, booted in ${liv.bootMinutes} min` : ''}.`) : 'Live: did not run.', ] log(`${build} on ${branch}: ${outcome}`) lines.forEach((l) => log(l)) findings.forEach((f) => log('-', f)) if (process.env.GITHUB_STEP_SUMMARY) { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Staging drill: ${build}: ${outcome}\n\n${lines.join('\n\n')}\n\n${findings.map((f) => `- ${f}`).join('\n')}\n`) } // A rehearsal on another branch reports here and changes nothing. if (branch !== 'staging') return log('not the staging branch: no issue, no state') let issue = null if (findings.length) { const body = [ `The staging drill (docs/runicnpc/PLAN.md stage 9c) checked Rust's staging branch (**${build}**) on ${now().slice(0, 10)}: **${outcome}**.`, '', ...findings.map((f) => `- ${f}`), '', ...lines, ...sections.flatMap(([title, text]) => (text ? ['', `### ${title}`, '', '```', text, '```'] : [])), '', process.env.GITHUB_RUN_NUMBER ? `Run: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_NUMBER}` : '', ].join('\n') issue = await fileIssue(`Staging drill: Rust staging manifest ${manifest} (${(stat && stat.manifestDate) || '?'})`, body) } if (markChecked) state.checked = { manifest, buildid, at: now(), outcome, issue, oxideBehind: stat ? stat.oxideBehind : null } state.lastRun = { at: now(), manifest, buildid, outcome, issue } fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n') output('outcome', outcome) } async function fileIssue(title, body) { const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}` const headers = { Authorization: 'token ' + process.env.GITEA_TOKEN, 'Content-Type': 'application/json', Accept: 'application/json' } const call = async (method, p, data) => { const r = await fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined }) if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${(await r.text()).slice(0, 200)}`) return r.json() } const open = await call('GET', `/issues?state=open&type=issues&labels=staging-drill&limit=50`) const same = open.find((i) => i.title === title) if (same) { await call('POST', `/issues/${same.number}/comments`, { body }) log(`commented on #${same.number}`) return same.number } const labels = await call('GET', '/labels?limit=100') let label = labels.find((l) => l.name === 'staging-drill') if (!label) label = await call('POST', '/labels', { name: 'staging-drill', color: '#d93f0b', description: "Rust's staging branch breaks RunicNPC (the daily staging drill)" }) const created = await call('POST', '/issues', { title, body, labels: [label.id] }) log(`opened #${created.number}`) return created.number } // ---------------------------------------------------------------- main const a = args() const commands = { window: window_, gate, live, report } const command = commands[a._[0]] if (!command) { console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]') process.exit(2) } command(a).catch((e) => { console.error(e.message || e) process.exit(1) })