wtclaude
c25f73c525
A Gitea job that checks each new Rust staging build in the seven days before the monthly forced wipe: - tools/drill/static.sh: staging's managed assemblies (DepotDownloader) and Oxide's staging build; whether Oxide has caught up (fieldlist --oxide-behind), the swap's field list compared, and the plugin compiled. - tools/drill/drill.js: window (D318), gate (one check per depot manifest), live (rust-staging runs rnt.run all; rust-carbon is stopped and restarted when both 6000-map rigs run, D309) and report (one issue per build, state on the drill branch; a build waits up to three days for Oxide, D317). - tools/drill/rig-start.sh: the rig's startup; installs the branch and Oxide's build for it, and refuses to start a mismatched pair. - RunicNpcTest 0.7.2: stage 5's five fights look for a field with room for their 280 m line (D319), and the spawn check names what did not spawn. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY
589 lines
25 KiB
JavaScript
589 lines
25 KiB
JavaScript
// The staging drill (docs/runicnpc/PLAN.md stage 9c, D307-D309, D317, D318). Never shipped.
|
|
//
|
|
// .gitea/workflows/staging-drill.yml runs it daily, and it does something only in the week before the
|
|
// monthly forced wipe (D318):
|
|
//
|
|
// node tools/drill/drill.js window [--today YYYY-MM-DD] in the drill week?
|
|
// tools/drill/static.sh the static half
|
|
// node tools/drill/drill.js gate --state <drill.json> --work <dir> a build not checked yet?
|
|
// node tools/drill/drill.js live --work <dir> [--branch staging] the live half, on rust-staging
|
|
// node tools/drill/drill.js report --state <drill.json> --work <dir> one issue per build; the state
|
|
//
|
|
// A build is the manifest of Rust's Linux depot, as Steam names it to static.sh.
|
|
//
|
|
// Environment: PANEL_KEY (or PANEL_KEY_FILE, a file with a `user: ptlc_…` line) is the drill's own
|
|
// Pterodactyl client key; PANEL_URL the panel. The live half finds its servers by name: DRILL_RIG
|
|
// (rust-staging) and the two 6000-map rigs, of which it stops DRILL_STOPPABLE (rust-carbon) when both
|
|
// run, because the node has no memory for three (D309). It leaves every rig as it found it.
|
|
// `report` needs GITEA_TOKEN, and GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them.
|
|
//
|
|
// The key is never printed, and neither is the server's console start line (it carries the RCON
|
|
// password): only lines this script picks are ever written out.
|
|
|
|
const fs = require('fs')
|
|
const path = require('path')
|
|
|
|
const ROOT = path.resolve(__dirname, '..', '..')
|
|
const PANEL = (process.env.PANEL_URL || 'http://192.168.0.12').replace(/\/+$/, '')
|
|
const RIG = process.env.DRILL_RIG || 'rust-staging'
|
|
const BIG = (process.env.DRILL_BIG_RIGS || 'rust-oxide,rust-carbon').split(',')
|
|
const STOPPABLE = process.env.DRILL_STOPPABLE || 'rust-carbon'
|
|
const KITS_URL = 'https://umod.org/plugins/Kits.cs'
|
|
const WAIT_DAYS = 3 // D317: how long a build waits for Oxide to catch up
|
|
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
|
const now = () => new Date().toISOString()
|
|
const log = (...a) => console.log(`[drill ${now().slice(11, 19)}]`, ...a)
|
|
|
|
function args() {
|
|
const out = { _: [] }
|
|
const argv = process.argv.slice(2)
|
|
for (let i = 0; i < argv.length; i++) {
|
|
if (argv[i].startsWith('--')) {
|
|
const k = argv[i].slice(2)
|
|
out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true
|
|
} else out._.push(argv[i])
|
|
}
|
|
return out
|
|
}
|
|
|
|
const readJson = (f, fallback) => {
|
|
try {
|
|
return JSON.parse(fs.readFileSync(f, 'utf8'))
|
|
} catch {
|
|
return fallback
|
|
}
|
|
}
|
|
|
|
function output(name, value) {
|
|
if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`)
|
|
}
|
|
|
|
// ---------------------------------------------------------------- the panel
|
|
|
|
function panelKey() {
|
|
if (process.env.PANEL_KEY) return process.env.PANEL_KEY.trim()
|
|
const file = process.env.PANEL_KEY_FILE
|
|
if (!file) throw new Error('PANEL_KEY (or PANEL_KEY_FILE) is not set')
|
|
const line = fs.readFileSync(file, 'utf8').split(/\r?\n/).find((l) => l.startsWith('user:'))
|
|
if (!line) throw new Error(`${file} has no "user:" line`)
|
|
return line.slice(5).trim()
|
|
}
|
|
|
|
function panel() {
|
|
const key = panelKey()
|
|
const headers = (type = 'application/json') => ({ Authorization: 'Bearer ' + key, Accept: 'application/json', 'Content-Type': type })
|
|
const api = async (method, p, body, type) => {
|
|
const r = await fetch(`${PANEL}/api/client${p}`, { method, headers: headers(type), body })
|
|
const text = await r.text()
|
|
if (!r.ok) throw new Error(`${method} ${p.split('?')[0]}: ${r.status} ${text.slice(0, 200)}`)
|
|
return text
|
|
}
|
|
|
|
const p = {
|
|
async servers() {
|
|
const d = JSON.parse(await api('GET', '/?per_page=100'))
|
|
return Object.fromEntries(d.data.map((s) => [s.attributes.name, s.attributes.identifier]))
|
|
},
|
|
async state(id) {
|
|
return JSON.parse(await api('GET', `/servers/${id}/resources`)).attributes.current_state
|
|
},
|
|
power: (id, signal) => api('POST', `/servers/${id}/power`, JSON.stringify({ signal })),
|
|
command: (id, command) => api('POST', `/servers/${id}/command`, JSON.stringify({ command })),
|
|
read: (id, file) => api('GET', `/servers/${id}/files/contents?file=${encodeURIComponent(file)}`),
|
|
async readOr(id, file, fallback = null) {
|
|
try {
|
|
return await p.read(id, file)
|
|
} catch {
|
|
return fallback
|
|
}
|
|
},
|
|
write: (id, file, content) => api('POST', `/servers/${id}/files/write?file=${encodeURIComponent(file)}`, content, 'text/plain'),
|
|
async rm(id, file) {
|
|
const dir = file.slice(0, file.lastIndexOf('/')) || '/'
|
|
try {
|
|
await api('POST', `/servers/${id}/files/delete`, JSON.stringify({ root: dir, files: [file.slice(file.lastIndexOf('/') + 1)] }))
|
|
} catch {
|
|
// already gone
|
|
}
|
|
},
|
|
|
|
/** Sends one console command and returns the console lines that follow, for `secs` seconds. */
|
|
async console(id, command, secs = 6) {
|
|
const { data } = JSON.parse(await api('GET', `/servers/${id}/websocket`))
|
|
const strip = (s) => s.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '')
|
|
return new Promise((resolve) => {
|
|
const lines = []
|
|
let sent = false
|
|
const ws = new WebSocket(data.socket, { headers: { Origin: PANEL } })
|
|
const done = () => {
|
|
try {
|
|
ws.close()
|
|
} catch {}
|
|
resolve(lines)
|
|
}
|
|
ws.onopen = () => ws.send(JSON.stringify({ event: 'auth', args: [data.token] }))
|
|
ws.onmessage = (m) => {
|
|
const msg = JSON.parse(m.data)
|
|
if (msg.event === 'auth success' && !sent) {
|
|
sent = true
|
|
// The panel replays recent history on connect; let it pass, then send.
|
|
setTimeout(() => {
|
|
lines.length = 0
|
|
ws.send(JSON.stringify({ event: 'send command', args: [command] }))
|
|
}, 800)
|
|
setTimeout(done, secs * 1000)
|
|
} else if (msg.event === 'console output' && sent) {
|
|
lines.push(...strip(String(msg.args[0])).split('\n'))
|
|
}
|
|
}
|
|
ws.onerror = () => done()
|
|
})
|
|
},
|
|
|
|
async waitState(id, want, timeoutMs) {
|
|
const end = Date.now() + timeoutMs
|
|
while (Date.now() < end) {
|
|
if ((await p.state(id)) === want) return true
|
|
await sleep(5000)
|
|
}
|
|
return false
|
|
},
|
|
|
|
/** Stops a server: `stop`, then `kill` if it has not stopped in time. */
|
|
async stop(id, graceMs = 180000) {
|
|
if ((await p.state(id)) === 'offline') return
|
|
await p.power(id, 'stop')
|
|
if (!(await p.waitState(id, 'offline', graceMs))) {
|
|
await p.power(id, 'kill')
|
|
await p.waitState(id, 'offline', 60000)
|
|
}
|
|
},
|
|
}
|
|
return p
|
|
}
|
|
|
|
// ---------------------------------------------------------------- window: the week before a forced wipe
|
|
|
|
// Rust changes on the monthly forced wipe, the first Thursday (D318). The drill runs only in the seven
|
|
// days before it, which leaves a week to fix what it finds; outside them the job stops at once.
|
|
function nextWipe(today) {
|
|
const firstThursday = (y, m) => {
|
|
const d = new Date(Date.UTC(y, m, 1))
|
|
d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7))
|
|
return d
|
|
}
|
|
const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate())
|
|
const thisMonth = firstThursday(today.getUTCFullYear(), today.getUTCMonth())
|
|
return thisMonth.getTime() > t ? thisMonth : firstThursday(today.getUTCFullYear(), today.getUTCMonth() + 1)
|
|
}
|
|
|
|
function windowDays(today = new Date()) {
|
|
const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate())
|
|
return Math.round((nextWipe(today).getTime() - t) / 86400000)
|
|
}
|
|
|
|
async function window_(a) {
|
|
const days = windowDays(a.today ? new Date(a.today) : new Date())
|
|
const wipe = nextWipe(a.today ? new Date(a.today) : new Date()).toISOString().slice(0, 10)
|
|
const open = days >= 1 && days <= 7
|
|
log(`the next forced wipe is ${wipe}, in ${days} day(s): ${open ? 'in the drill week' : 'outside the drill week, nothing to do'}`)
|
|
output('open', open || a.force ? 'true' : 'false')
|
|
}
|
|
|
|
// ---------------------------------------------------------------- gate: is there anything to check?
|
|
|
|
// A build is Steam's manifest of the Linux depot, which static.sh reads from Steam itself.
|
|
async function gate(a) {
|
|
const state = readJson(a.state, {})
|
|
const stat = readJson(path.join(path.resolve(a.work || 'drill-work'), 'static.json'), {})
|
|
const checked = state.checked && state.checked.manifest
|
|
const isNew = !stat.manifest || stat.manifest !== checked
|
|
log(`Rust ${stat.branch || 'staging'} is manifest ${stat.manifest || '?'} (${stat.manifestDate || '?'}); last checked ${checked || 'none'}`)
|
|
output('run', isNew || a.force ? 'true' : 'false')
|
|
if (!isNew && !a.force) log('already checked: nothing to do')
|
|
}
|
|
|
|
// ---------------------------------------------------------------- live: the harness on rust-staging
|
|
|
|
async function live(a) {
|
|
const work = path.resolve(a.work || 'drill-work')
|
|
const branch = a.branch || 'staging'
|
|
const result = { branch, at: now(), ran: false, findings: [], notes: [] }
|
|
const save = () => fs.writeFileSync(path.join(work, 'live.json'), JSON.stringify(result, null, 2))
|
|
fs.mkdirSync(work, { recursive: true })
|
|
|
|
const stat = readJson(path.join(work, 'static.json'), null)
|
|
if (!stat || stat.managed !== 'ok') {
|
|
result.skipped = 'the static half did not run'
|
|
save()
|
|
return log(result.skipped)
|
|
}
|
|
if (stat.oxideBehind !== 0) {
|
|
result.skipped = `Oxide's ${branch} build is ${stat.oxideBehind} member(s) behind Rust's, so an Oxide server cannot boot on it`
|
|
save()
|
|
return log(result.skipped)
|
|
}
|
|
|
|
const p = panel()
|
|
const ids = await p.servers()
|
|
const rig = ids[RIG]
|
|
if (!rig) throw new Error(`no server named ${RIG} on the panel`)
|
|
|
|
let restart = null
|
|
const finding = (f) => {
|
|
result.findings.push(f)
|
|
log('FINDING', f)
|
|
}
|
|
|
|
try {
|
|
// Room for it (D309): with both 6000-map rigs running the node's memory is full.
|
|
const running = []
|
|
for (const name of BIG) if (ids[name] && (await p.state(ids[name])) === 'running') running.push(name)
|
|
if (running.length === BIG.length && ids[STOPPABLE]) {
|
|
log(`both ${BIG.join(' and ')} are running: saving and stopping ${STOPPABLE}`)
|
|
await p.command(ids[STOPPABLE], 'server.save').catch(() => {})
|
|
await sleep(15000)
|
|
await p.stop(ids[STOPPABLE])
|
|
restart = STOPPABLE
|
|
result.notes.push(`${STOPPABLE} was stopped for the drill and started again afterwards`)
|
|
}
|
|
|
|
await p.stop(rig, 60000)
|
|
|
|
// What runs: this commit's plugin and harness, and the startup that installs the branch.
|
|
await p.write(rig, '/staging-drill.sh', fs.readFileSync(path.join(__dirname, 'rig-start.sh'), 'utf8').replace(/\r\n/g, '\n'))
|
|
await p.write(rig, '/staging-drill.branch', branch + '\n')
|
|
await p.write(rig, '/oxide/plugins/RunicNPC.cs', fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8'))
|
|
const kits = await fetch(KITS_URL)
|
|
if (!kits.ok) throw new Error(`Kits from uMod: ${kits.status}`)
|
|
await p.write(rig, '/oxide/plugins/Kits.cs', await kits.text())
|
|
await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
|
|
await p.rm(rig, '/oxide/data/RunicNpcTest.json')
|
|
await p.rm(rig, '/staging-drill.json')
|
|
|
|
log(`starting ${RIG} on ${branch}`)
|
|
const started = Date.now()
|
|
await p.power(rig, 'start')
|
|
const up = await waitRunning(p, rig, 45 * 60000)
|
|
result.installed = readJsonText(await p.readOr(rig, '/staging-drill.json', null))
|
|
if (!up) {
|
|
finding(`The server did not start on Rust ${branch} ${result.installed?.buildid || ''} with Oxide's ${branch} build.`)
|
|
result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 30)
|
|
return
|
|
}
|
|
result.bootMinutes = Math.round((Date.now() - started) / 6000) / 10
|
|
log(`running after ${result.bootMinutes} min:`, JSON.stringify(result.installed))
|
|
|
|
// RunicNPC loaded, and the navmesh ready.
|
|
const status = await waitStatus(p, rig, 30 * 60000)
|
|
if (!status) {
|
|
finding('RunicNPC did not load: `rnpc.status` never answered.')
|
|
result.oxideErrors = await oxideErrors(p, rig)
|
|
return
|
|
}
|
|
result.status = status
|
|
log('rnpc.status:', status.find((l) => l.includes('navmesh=')))
|
|
const fieldsLine = status.find((l) => l.includes('swap fields:')) || ''
|
|
const missing = /missing=(\S+)/.exec(fieldsLine)
|
|
if (missing && missing[1] !== '-') finding(`The swap's fields are missing on this build: ${missing[1]}`)
|
|
if (!fieldsLine.includes('navmesh=ready')) {
|
|
finding('The navmesh was not ready in 30 minutes.')
|
|
return
|
|
}
|
|
|
|
// The harness needs the hidden Kits kit rnhrevolver (tools/RunicNpcTest.cs).
|
|
await ensureKit(p, rig)
|
|
|
|
await p.write(rig, '/oxide/plugins/RunicNpcTest.cs', fs.readFileSync(path.join(ROOT, 'tools', 'RunicNpcTest.cs'), 'utf8'))
|
|
let accepted = false
|
|
for (let i = 0; i < 18 && !accepted; i++) {
|
|
await sleep(10000)
|
|
accepted = (await p.console(rig, 'rnt.run all', 5)).some((l) => l.includes('rnt: running all'))
|
|
}
|
|
if (!accepted) {
|
|
finding('The test harness did not load (`rnt.run all` was not answered).')
|
|
result.oxideErrors = await oxideErrors(p, rig)
|
|
return
|
|
}
|
|
result.ran = true
|
|
log('rnt.run all: running')
|
|
|
|
const lines = await waitHarness(p, rig, 45 * 60000)
|
|
result.harness = summarise(lines)
|
|
if (!result.harness.done) finding(`The harness did not finish in 45 minutes (${result.harness.pass} pass, ${result.harness.fail} fail so far).`)
|
|
for (const f of result.harness.failed) finding(`Harness: ${f}`)
|
|
log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`)
|
|
result.oxideErrors = await oxideErrors(p, rig)
|
|
if (result.oxideErrors.length) finding(`RunicNPC logged ${result.oxideErrors.length} error line(s) in Oxide's log during the run.`)
|
|
await p.command(rig, 'rnt.clear').catch(() => {})
|
|
} catch (e) {
|
|
finding(`The live half stopped: ${e.message}`)
|
|
} finally {
|
|
try {
|
|
await p.stop(rig)
|
|
await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
|
|
} catch (e) {
|
|
result.notes.push(`${RIG} could not be stopped cleanly: ${e.message}`)
|
|
}
|
|
if (restart) {
|
|
try {
|
|
await p.power(ids[restart], 'start')
|
|
log(`started ${restart} again`)
|
|
} catch (e) {
|
|
result.findings.push(`${restart} was stopped for the drill and could NOT be started again: ${e.message}`)
|
|
}
|
|
}
|
|
save()
|
|
}
|
|
}
|
|
|
|
async function waitRunning(p, id, timeoutMs) {
|
|
const end = Date.now() + timeoutMs
|
|
let seenStarting = false
|
|
while (Date.now() < end) {
|
|
const s = await p.state(id)
|
|
if (s === 'running') return true
|
|
if (s === 'starting') seenStarting = true
|
|
// Wings restarts a crashed server a few times; offline after starting is a crash it gave up on.
|
|
if (s === 'offline' && seenStarting) return false
|
|
await sleep(15000)
|
|
}
|
|
return false
|
|
}
|
|
|
|
async function waitStatus(p, id, timeoutMs) {
|
|
const end = Date.now() + timeoutMs
|
|
let last = null
|
|
while (Date.now() < end) {
|
|
const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim())
|
|
const at = lines.findIndex((l) => l.startsWith('RunicNPC '))
|
|
if (at >= 0) {
|
|
last = lines.slice(at, at + 14)
|
|
if (last.some((l) => l.includes('navmesh=ready'))) return last
|
|
}
|
|
await sleep(30000)
|
|
}
|
|
return last
|
|
}
|
|
|
|
async function ensureKit(p, id) {
|
|
const file = '/oxide/data/Kits/kits_data.json'
|
|
let data = null
|
|
for (let i = 0; i < 12 && !data; i++) {
|
|
data = readJsonText(await p.readOr(id, file, null))
|
|
if (!data) await sleep(5000)
|
|
}
|
|
if (!data) throw new Error(`Kits wrote no ${file}`)
|
|
data._kits = data._kits || {}
|
|
if (data._kits.rnhrevolver) return
|
|
const item = (Shortname, Amount, Position, Condition = 0) => ({ Shortname, Skin: 0, Amount, Condition, MaxCondition: Condition, Ammo: 0, Position, Frequency: -1 })
|
|
data._kits.rnhrevolver = {
|
|
Name: 'rnhrevolver',
|
|
Description: 'RunicNPC test harness: revolver + burlap (staging drill)',
|
|
RequiredPermission: '',
|
|
MaximumUses: 0,
|
|
RequiredAuth: 0,
|
|
Cooldown: 0,
|
|
Cost: 0,
|
|
IsHidden: true,
|
|
CopyPasteFile: '',
|
|
KitImage: '',
|
|
MainItems: [item('ammo.pistol', 120, 0)],
|
|
WearItems: [item('burlap.shirt', 1, 0), item('burlap.trousers', 1, 1), item('hat.boonie', 1, 2)],
|
|
BeltItems: [item('pistol.revolver', 1, 0, 100)],
|
|
}
|
|
await p.write(id, file, JSON.stringify(data, null, 2))
|
|
await p.command(id, 'oxide.reload Kits')
|
|
await sleep(10000)
|
|
log('added the rnhrevolver kit and reloaded Kits')
|
|
}
|
|
|
|
function readJsonText(t) {
|
|
try {
|
|
return t ? JSON.parse(t) : null
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
async function waitHarness(p, id, timeoutMs) {
|
|
const end = Date.now() + timeoutMs
|
|
let lines = []
|
|
while (Date.now() < end) {
|
|
await sleep(30000)
|
|
lines = readJsonText(await p.readOr(id, '/oxide/data/RunicNpcTest.json', null)) || lines
|
|
if (lines.some((l) => /^# done all:/.test(l))) break
|
|
}
|
|
return lines
|
|
}
|
|
|
|
function summarise(lines) {
|
|
const failed = lines.filter((l) => / FAIL( |$)/.test(l))
|
|
const passed = lines.filter((l) => / PASS( |$)/.test(l))
|
|
const done = lines.find((l) => /^# done all:/.test(l))
|
|
return { done: !!done, summary: done || null, pass: passed.length, fail: failed.length, failed }
|
|
}
|
|
|
|
/** RunicNPC's and the harness's error lines from today's Oxide log. */
|
|
async function oxideErrors(p, id) {
|
|
const day = now().slice(0, 10)
|
|
const text = (await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || ''
|
|
return text
|
|
.split('\n')
|
|
.filter((l) => /RunicNPC|RunicNpcTest/.test(l) && /\[Error\]|Exception|Failed|error CS/.test(l))
|
|
.slice(-40)
|
|
}
|
|
|
|
/** The console's last lines, without the startup line (it carries the RCON password). */
|
|
function safeTail(text, n) {
|
|
return (text || '')
|
|
.split('\n')
|
|
.filter((l) => !/rcon\.password|\+rcon/i.test(l))
|
|
.slice(-n)
|
|
}
|
|
|
|
// ---------------------------------------------------------------- report: one issue per build
|
|
|
|
async function report(a) {
|
|
const work = path.resolve(a.work || 'drill-work')
|
|
const statePath = a.state
|
|
const state = readJson(statePath, {})
|
|
const stat = readJson(path.join(work, 'static.json'), null)
|
|
const liv = readJson(path.join(work, 'live.json'), null)
|
|
const branch = (stat && stat.branch) || 'staging'
|
|
const manifest = (stat && stat.manifest) || ''
|
|
const buildid = (liv && liv.installed && liv.installed.buildid) || ''
|
|
// How a build is named: Steam's build id when the rig installed it, and always the manifest's date.
|
|
const build = `${buildid ? `build ${buildid}, ` : ''}manifest ${manifest || '?'} of ${(stat && stat.manifestDate) || '?'}`
|
|
const read = (f, n = 80) => {
|
|
try {
|
|
return fs.readFileSync(path.join(work, f), 'utf8').replace(/\r/g, '').split('\n').slice(0, n).join('\n').trim()
|
|
} catch {
|
|
return ''
|
|
}
|
|
}
|
|
|
|
const findings = []
|
|
const sections = []
|
|
let outcome
|
|
let markChecked = true
|
|
|
|
if (!stat || stat.managed !== 'ok' || stat.fields === 'error' || stat.compile === 'error') {
|
|
outcome = 'could not run'
|
|
markChecked = false
|
|
findings.push(`The static half could not run: ${(stat && stat.note) || 'no static.json'}`)
|
|
if (stat && stat.fields === 'error') sections.push(['tools/fieldlist', read('fields.diff')])
|
|
} else if (stat.oxideBehind !== 0) {
|
|
const waiting = state.waiting && state.waiting.manifest === manifest ? state.waiting : { manifest, since: now() }
|
|
const days = (Date.now() - Date.parse(waiting.since)) / 86400000
|
|
if (days < WAIT_DAYS) {
|
|
outcome = `waiting for Oxide (${stat.oxideBehind} behind, day ${Math.floor(days) + 1} of ${WAIT_DAYS})`
|
|
if (branch === 'staging') {
|
|
state.waiting = { ...waiting, behind: stat.oxideBehind, lastSeen: now() }
|
|
markChecked = false
|
|
}
|
|
} else {
|
|
outcome = 'Oxide has not caught up'
|
|
findings.push(
|
|
`Oxide's staging build is still ${stat.oxideBehind} member(s) behind Rust's staging build after ${WAIT_DAYS} days, ` +
|
|
'so an Oxide server cannot boot on it and the live half did not run. The static half ran against Oxide\'s older build.'
|
|
)
|
|
sections.push(['What Oxide lacks (first 20)', read('oxide-behind.txt', 21)])
|
|
}
|
|
}
|
|
|
|
if (!outcome || outcome === 'Oxide has not caught up') {
|
|
if (stat.fields === 'differs') {
|
|
findings.push("The swap's field list differs from the one the plugin carries (D232): regenerate it with tools/fieldlist and check what changed.")
|
|
sections.push(['Field list (carried → staging)', read('fields.diff', 120)])
|
|
}
|
|
if (stat.compile === 'errors') {
|
|
findings.push("RunicNPC does not compile against this build with Oxide's staging build.")
|
|
sections.push(['Compiler output', read('compile.txt', 60)])
|
|
}
|
|
if (!liv && stat.oxideBehind === 0) findings.push('The live half did not run: see the job log (is PTERODACTYL_DRILL_KEY set?).')
|
|
if (liv && !liv.skipped) {
|
|
findings.push(...liv.findings)
|
|
if (liv.consoleTail && liv.consoleTail.length) sections.push(['Console, last lines', liv.consoleTail.join('\n')])
|
|
if (liv.oxideErrors && liv.oxideErrors.length) sections.push(["Oxide's log (RunicNPC lines)", liv.oxideErrors.join('\n')])
|
|
if (liv.status) sections.push(['rnpc.status', liv.status.join('\n')])
|
|
}
|
|
if (!outcome) outcome = findings.length ? 'fail' : 'pass'
|
|
if (branch === 'staging' && markChecked) delete state.waiting
|
|
}
|
|
|
|
const installed = liv && liv.installed ? `Rust ${liv.installed.branch} ${liv.installed.buildid}, Oxide build of ${liv.installed.oxideModified}` : ''
|
|
const lines = [
|
|
`Static: Oxide behind ${stat ? stat.oxideBehind : '?'}, fields ${stat ? stat.fields : '?'}, compile ${stat ? stat.compile : '?'}.`,
|
|
liv ? (liv.skipped ? `Live: skipped (${liv.skipped}).` : `Live: ${liv.harness ? `${liv.harness.pass} pass, ${liv.harness.fail} fail` : 'did not reach the harness'}${installed ? `, on ${installed}` : ''}${liv.bootMinutes ? `, booted in ${liv.bootMinutes} min` : ''}.`) : 'Live: did not run.',
|
|
]
|
|
log(`${build} on ${branch}: ${outcome}`)
|
|
lines.forEach((l) => log(l))
|
|
findings.forEach((f) => log('-', f))
|
|
if (process.env.GITHUB_STEP_SUMMARY) {
|
|
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Staging drill: ${build}: ${outcome}\n\n${lines.join('\n\n')}\n\n${findings.map((f) => `- ${f}`).join('\n')}\n`)
|
|
}
|
|
|
|
// A rehearsal on another branch reports here and changes nothing.
|
|
if (branch !== 'staging') return log('not the staging branch: no issue, no state')
|
|
|
|
let issue = null
|
|
if (findings.length) {
|
|
const body = [
|
|
`The staging drill (docs/runicnpc/PLAN.md stage 9c) checked Rust's staging branch (**${build}**) on ${now().slice(0, 10)}: **${outcome}**.`,
|
|
'',
|
|
...findings.map((f) => `- ${f}`),
|
|
'',
|
|
...lines,
|
|
...sections.flatMap(([title, text]) => (text ? ['', `### ${title}`, '', '```', text, '```'] : [])),
|
|
'',
|
|
process.env.GITHUB_RUN_NUMBER ? `Run: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_NUMBER}` : '',
|
|
].join('\n')
|
|
issue = await fileIssue(`Staging drill: Rust staging manifest ${manifest} (${(stat && stat.manifestDate) || '?'})`, body)
|
|
}
|
|
|
|
if (markChecked) state.checked = { manifest, buildid, at: now(), outcome, issue, oxideBehind: stat ? stat.oxideBehind : null }
|
|
state.lastRun = { at: now(), manifest, buildid, outcome, issue }
|
|
fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n')
|
|
output('outcome', outcome)
|
|
}
|
|
|
|
async function fileIssue(title, body) {
|
|
const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}`
|
|
const headers = { Authorization: 'token ' + process.env.GITEA_TOKEN, 'Content-Type': 'application/json', Accept: 'application/json' }
|
|
const call = async (method, p, data) => {
|
|
const r = await fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined })
|
|
if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${(await r.text()).slice(0, 200)}`)
|
|
return r.json()
|
|
}
|
|
|
|
const open = await call('GET', `/issues?state=open&type=issues&labels=staging-drill&limit=50`)
|
|
const same = open.find((i) => i.title === title)
|
|
if (same) {
|
|
await call('POST', `/issues/${same.number}/comments`, { body })
|
|
log(`commented on #${same.number}`)
|
|
return same.number
|
|
}
|
|
const labels = await call('GET', '/labels?limit=100')
|
|
let label = labels.find((l) => l.name === 'staging-drill')
|
|
if (!label) label = await call('POST', '/labels', { name: 'staging-drill', color: '#d93f0b', description: "Rust's staging branch breaks RunicNPC (the daily staging drill)" })
|
|
const created = await call('POST', '/issues', { title, body, labels: [label.id] })
|
|
log(`opened #${created.number}`)
|
|
return created.number
|
|
}
|
|
|
|
// ---------------------------------------------------------------- main
|
|
|
|
const a = args()
|
|
const commands = { window: window_, gate, live, report }
|
|
const command = commands[a._[0]]
|
|
if (!command) {
|
|
console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]')
|
|
process.exit(2)
|
|
}
|
|
command(a).catch((e) => {
|
|
console.error(e.message || e)
|
|
process.exit(1)
|
|
})
|