runicnpc-rust/tools/drill/drill.js

// The staging drill (docs/runicnpc/PLAN.md stage 9c, D307-D309, D317, D318). Never shipped.
//
// .gitea/workflows/staging-drill.yml runs it daily, and it does something only in the week before the
// monthly forced wipe (D318):
//
//   node tools/drill/drill.js window [--today YYYY-MM-DD]                in the drill week?
//   tools/drill/static.sh                                                the static half
//   node tools/drill/drill.js gate   --state <drill.json> --work <dir>   a build not checked yet?
//   node tools/drill/drill.js live   --work <dir> [--branch staging]     the live half, on rust-staging
//   node tools/drill/drill.js report --state <drill.json> --work <dir>   one issue per build; the state
//
// A build is the manifest of Rust's Linux depot, as Steam names it to static.sh.
//
// Environment: PANEL_KEY (or PANEL_KEY_FILE, a file with a `user: ptlc_…` line) is the drill's own
// Pterodactyl client key; PANEL_URL the panel. The live half finds its servers by name: DRILL_RIG
// (rust-staging) and the two 6000-map rigs, of which it stops DRILL_STOPPABLE (rust-carbon) when both
// run, because the node has no memory for three (D309). It leaves every rig as it found it.
// `report` needs GITEA_TOKEN, and GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them.
//
// The key is never printed, and neither is the server's console start line (it carries the RCON
// password): only lines this script picks are ever written out.

const fs = require('fs')
const path = require('path')

const ROOT = path.resolve(__dirname, '..', '..')
const PANEL = (process.env.PANEL_URL || 'http://192.168.0.12').replace(/\/+$/, '')
const RIG = process.env.DRILL_RIG || 'rust-staging'
const BIG = (process.env.DRILL_BIG_RIGS || 'rust-oxide,rust-carbon').split(',')
const STOPPABLE = process.env.DRILL_STOPPABLE || 'rust-carbon'
const KITS_URL = 'https://umod.org/plugins/Kits.cs'
const WAIT_DAYS = 3 // D317: how long a build waits for Oxide to catch up
const HARNESS_CAP_MS = 45 * 60000 // the whole harness run
const HARNESS_STALL_MS = 5 * 60000 // no new line in its file for this long: it has stopped (issue #20)

// What a fresh server would not have (issue #20). RunicNPC writes its four example kits once, into
// Kits' own file, and never again, so a file kept from an older build keeps that build's kits; the drill
// starts from none of it. Files, never directories: a directory the panel's file API creates is not
// writable by the game (docs/runicnpc/PLAN.md §1.5), and the game makes these itself on its first boot.
// Kits' file is not deleted but replaced by one holding only the harness's kit (KITS_FILE, harnessKits):
// Kits reads it at boot and RunicNPC adds its examples beside it, so no reload is needed for it (#22).
const KITS_FILE = '/oxide/data/Kits/kits_data.json'
const FRESH_DATA = [
  '/oxide/data/RunicNPC/profiles.json',
  '/oxide/data/RunicNPC/placements.json',
  '/oxide/data/RunicNPC/routes.json',
  '/oxide/data/RunicNPC/state.json',
]

const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
const now = () => new Date().toISOString()
const log = (...a) => console.log(`[drill ${now().slice(11, 19)}]`, ...a)

function args() {
  const out = { _: [] }
  const argv = process.argv.slice(2)
  for (let i = 0; i < argv.length; i++) {
    if (argv[i].startsWith('--')) {
      const k = argv[i].slice(2)
      out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true
    } else out._.push(argv[i])
  }
  return out
}

const readJson = (f, fallback) => {
  try {
    return JSON.parse(fs.readFileSync(f, 'utf8'))
  } catch {
    return fallback
  }
}

function output(name, value) {
  if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`)
}

// ---------------------------------------------------------------- the panel

function panelKey() {
  if (process.env.PANEL_KEY) return process.env.PANEL_KEY.trim()
  const file = process.env.PANEL_KEY_FILE
  if (!file) throw new Error('PANEL_KEY (or PANEL_KEY_FILE) is not set')
  const line = fs.readFileSync(file, 'utf8').split(/\r?\n/).find((l) => l.startsWith('user:'))
  if (!line) throw new Error(`${file} has no "user:" line`)
  return line.slice(5).trim()
}

function panel() {
  const key = panelKey()
  const headers = (type = 'application/json') => ({ Authorization: 'Bearer ' + key, Accept: 'application/json', 'Content-Type': type })
  const api = async (method, p, body, type) => {
    const r = await fetch(`${PANEL}/api/client${p}`, { method, headers: headers(type), body })
    const text = await r.text()
    if (!r.ok) throw new Error(`${method} ${p.split('?')[0]}: ${r.status} ${text.slice(0, 200)}`)
    return text
  }

  const p = {
    async servers() {
      const d = JSON.parse(await api('GET', '/?per_page=100'))
      return Object.fromEntries(d.data.map((s) => [s.attributes.name, s.attributes.identifier]))
    },
    async state(id) {
      return JSON.parse(await api('GET', `/servers/${id}/resources`)).attributes.current_state
    },
    power: (id, signal) => api('POST', `/servers/${id}/power`, JSON.stringify({ signal })),
    command: (id, command) => api('POST', `/servers/${id}/command`, JSON.stringify({ command })),
    read: (id, file) => api('GET', `/servers/${id}/files/contents?file=${encodeURIComponent(file)}`),
    async readOr(id, file, fallback = null) {
      try {
        return await p.read(id, file)
      } catch {
        return fallback
      }
    },
    write: (id, file, content) => api('POST', `/servers/${id}/files/write?file=${encodeURIComponent(file)}`, content, 'text/plain'),
    async rm(id, file) {
      const dir = file.slice(0, file.lastIndexOf('/')) || '/'
      try {
        await api('POST', `/servers/${id}/files/delete`, JSON.stringify({ root: dir, files: [file.slice(file.lastIndexOf('/') + 1)] }))
      } catch {
        // already gone
      }
    },

    /** Sends one console command and returns the console lines that follow, for `secs` seconds. */
    async console(id, command, secs = 6) {
      const { data } = JSON.parse(await api('GET', `/servers/${id}/websocket`))
      const strip = (s) => s.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '')
      return new Promise((resolve) => {
        const lines = []
        let sent = false
        const ws = new WebSocket(data.socket, { headers: { Origin: PANEL } })
        const done = () => {
          try {
            ws.close()
          } catch {}
          resolve(lines)
        }
        ws.onopen = () => ws.send(JSON.stringify({ event: 'auth', args: [data.token] }))
        ws.onmessage = (m) => {
          const msg = JSON.parse(m.data)
          if (msg.event === 'auth success' && !sent) {
            sent = true
            // The panel replays recent history on connect; let it pass, then send.
            setTimeout(() => {
              lines.length = 0
              ws.send(JSON.stringify({ event: 'send command', args: [command] }))
            }, 800)
            setTimeout(done, secs * 1000)
          } else if (msg.event === 'console output' && sent) {
            lines.push(...strip(String(msg.args[0])).split('\n'))
          }
        }
        ws.onerror = () => done()
      })
    },

    async waitState(id, want, timeoutMs) {
      const end = Date.now() + timeoutMs
      while (Date.now() < end) {
        if ((await p.state(id)) === want) return true
        await sleep(5000)
      }
      return false
    },

    /** Stops a server: `stop`, then `kill` if it has not stopped in time. */
    async stop(id, graceMs = 180000) {
      if ((await p.state(id)) === 'offline') return
      await p.power(id, 'stop')
      if (!(await p.waitState(id, 'offline', graceMs))) {
        await p.power(id, 'kill')
        await p.waitState(id, 'offline', 60000)
      }
    },
  }
  return p
}

// ---------------------------------------------------------------- window: the week before a forced wipe

// Rust changes on the monthly forced wipe, the first Thursday (D318). The drill runs only in the seven
// days before it, which leaves a week to fix what it finds; outside them the job stops at once.
function nextWipe(today) {
  const firstThursday = (y, m) => {
    const d = new Date(Date.UTC(y, m, 1))
    d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7))
    return d
  }
  const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate())
  const thisMonth = firstThursday(today.getUTCFullYear(), today.getUTCMonth())
  return thisMonth.getTime() > t ? thisMonth : firstThursday(today.getUTCFullYear(), today.getUTCMonth() + 1)
}

function windowDays(today = new Date()) {
  const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate())
  return Math.round((nextWipe(today).getTime() - t) / 86400000)
}

async function window_(a) {
  const days = windowDays(a.today ? new Date(a.today) : new Date())
  const wipe = nextWipe(a.today ? new Date(a.today) : new Date()).toISOString().slice(0, 10)
  const open = days >= 1 && days <= 7
  const verdict = open ? 'in the drill week' : a.force ? 'outside the drill week, but run by hand: going ahead' : 'outside the drill week, nothing to do'
  log(`the next forced wipe is ${wipe}, in ${days} day(s): ${verdict}`)
  output('open', open || a.force ? 'true' : 'false')
}

// ---------------------------------------------------------------- gate: is there anything to check?

// A build is Steam's manifest of the Linux depot, which static.sh reads from Steam itself.
async function gate(a) {
  const state = readJson(a.state, {})
  const stat = readJson(path.join(path.resolve(a.work || 'drill-work'), 'static.json'), {})
  const checked = state.checked && state.checked.manifest
  const isNew = !stat.manifest || stat.manifest !== checked
  log(`Rust ${stat.branch || 'staging'} is manifest ${stat.manifest || '?'} (${stat.manifestDate || '?'}); last checked ${checked || 'none'}`)
  output('run', isNew || a.force ? 'true' : 'false')
  if (!isNew && !a.force) log('already checked: nothing to do')
}

// ---------------------------------------------------------------- live: the harness on rust-staging

async function live(a) {
  const work = path.resolve(a.work || 'drill-work')
  const branch = a.branch || 'staging'
  const result = { branch, at: now(), ran: false, findings: [], notes: [] }
  const save = () => fs.writeFileSync(path.join(work, 'live.json'), JSON.stringify(result, null, 2))
  fs.mkdirSync(work, { recursive: true })

  const stat = readJson(path.join(work, 'static.json'), null)
  if (!stat || stat.managed !== 'ok') {
    result.skipped = 'the static half did not run'
    save()
    return log(result.skipped)
  }
  if (stat.oxideBehind !== 0) {
    result.skipped = `Oxide's ${branch} build is ${stat.oxideBehind} member(s) behind Rust's, so an Oxide server cannot boot on it`
    save()
    return log(result.skipped)
  }

  const p = panel()
  const ids = await p.servers()
  const rig = ids[RIG]
  if (!rig) throw new Error(`no server named ${RIG} on the panel`)

  let restart = null
  const finding = (f) => {
    result.findings.push(f)
    log('FINDING', f)
  }

  try {
    // Room for it (D309): with both 6000-map rigs running the node's memory is full.
    const running = []
    for (const name of BIG) if (ids[name] && (await p.state(ids[name])) === 'running') running.push(name)
    if (running.length === BIG.length && ids[STOPPABLE]) {
      log(`both ${BIG.join(' and ')} are running: saving and stopping ${STOPPABLE}`)
      await p.command(ids[STOPPABLE], 'server.save').catch(() => {})
      await sleep(15000)
      await p.stop(ids[STOPPABLE])
      restart = STOPPABLE
      result.notes.push(`${STOPPABLE} was stopped for the drill and started again afterwards`)
    }

    await p.stop(rig, 60000)

    // What runs: this commit's plugin and harness, and the startup that installs the branch.
    await p.write(rig, '/staging-drill.sh', fs.readFileSync(path.join(__dirname, 'rig-start.sh'), 'utf8').replace(/\r\n/g, '\n'))
    await p.write(rig, '/staging-drill.branch', branch + '\n')
    await p.write(rig, '/oxide/plugins/RunicNPC.cs', fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8'))
    const kits = await fetch(KITS_URL)
    if (!kits.ok) throw new Error(`Kits from uMod: ${kits.status}`)
    await p.write(rig, '/oxide/plugins/Kits.cs', await kits.text())
    await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
    await p.rm(rig, '/oxide/data/RunicNpcTest.json')
    await p.rm(rig, '/staging-drill.json')
    for (const f of FRESH_DATA) await p.rm(rig, f) // already gone is fine: rm swallows it
    await p.write(rig, KITS_FILE, JSON.stringify(harnessKits(), null, 2))

    const logMark = await oxideLogMark(p, rig)
    log(`starting ${RIG} on ${branch}`)
    const started = Date.now()
    await p.power(rig, 'start')
    const up = await waitRunning(p, rig, 45 * 60000)
    result.installed = readJsonText(await p.readOr(rig, '/staging-drill.json', null))
    if (!up) {
      finding(`The server did not start on Rust ${branch} ${result.installed?.buildid || ''} with Oxide's ${branch} build.`)
      result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 30)
      return
    }
    result.bootMinutes = Math.round((Date.now() - started) / 6000) / 10
    log(`running after ${result.bootMinutes} min:`, JSON.stringify(result.installed))

    // RunicNPC loaded, and the navmesh ready.
    const status = await waitStatus(p, rig, 30 * 60000)
    if (!status) {
      finding('RunicNPC did not load: `rnpc.status` never answered.')
      result.oxideErrors = await oxideErrors(p, rig, logMark)
      return
    }
    result.status = status
    log('rnpc.status:', status.find((l) => l.includes('navmesh=')))
    const fieldsLine = status.find((l) => l.includes('swap fields:')) || ''
    const missing = /missing=(\S+)/.exec(fieldsLine)
    if (missing && missing[1] !== '-') finding(`The swap's fields are missing on this build: ${missing[1]}`)
    if (!fieldsLine.includes('navmesh=ready')) {
      finding('The navmesh was not ready in 30 minutes.')
      return
    }

    // The harness needs the hidden Kits kit rnhrevolver (tools/RunicNpcTest.cs).
    await ensureKit(p, rig, finding)

    await p.write(rig, '/oxide/plugins/RunicNpcTest.cs', fs.readFileSync(path.join(ROOT, 'tools', 'RunicNpcTest.cs'), 'utf8'))
    let accepted = false
    for (let i = 0; i < 18 && !accepted; i++) {
      await sleep(10000)
      accepted = (await p.console(rig, 'rnt.run all', 5)).some((l) => l.includes('rnt: running all'))
    }
    if (!accepted) {
      finding('The test harness did not load (`rnt.run all` was not answered).')
      result.oxideErrors = await oxideErrors(p, rig, logMark)
      return
    }
    result.ran = true
    log('rnt.run all: running')

    const { lines, ended } = await waitHarness(p, rig, HARNESS_CAP_MS, HARNESS_STALL_MS)
    result.harness = summarise(lines)
    if (!result.harness.done) {
      finding(unfinishedFinding(result.harness, lines, ended))
      // What the server said while it stalled (an exception in the harness's coroutine lands here, not in its file).
      result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 40)
    }
    for (const f of result.harness.failed) finding(`Harness: ${f}`)
    log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`)
    result.oxideErrors = await oxideErrors(p, rig, logMark)
    if (result.oxideErrors.length) finding(`RunicNPC logged ${result.oxideErrors.length} error line(s) in Oxide's log during the run.`)
    await p.command(rig, 'rnt.clear').catch(() => {})
  } catch (e) {
    finding(`The live half stopped: ${e.message}`)
  } finally {
    try {
      await p.stop(rig)
      await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
    } catch (e) {
      result.notes.push(`${RIG} could not be stopped cleanly: ${e.message}`)
    }
    if (restart) {
      try {
        await p.power(ids[restart], 'start')
        log(`started ${restart} again`)
      } catch (e) {
        result.findings.push(`${restart} was stopped for the drill and could NOT be started again: ${e.message}`)
      }
    }
    save()
  }
}

async function waitRunning(p, id, timeoutMs) {
  const end = Date.now() + timeoutMs
  let seenStarting = false
  while (Date.now() < end) {
    const s = await p.state(id)
    if (s === 'running') return true
    if (s === 'starting') seenStarting = true
    // Wings restarts a crashed server a few times; offline after starting is a crash it gave up on.
    if (s === 'offline' && seenStarting) return false
    await sleep(15000)
  }
  return false
}

async function waitStatus(p, id, timeoutMs) {
  const end = Date.now() + timeoutMs
  let last = null
  while (Date.now() < end) {
    const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim())
    const at = lines.findIndex((l) => l.startsWith('RunicNPC '))
    if (at >= 0) {
      last = lines.slice(at, at + 14)
      if (last.some((l) => l.includes('navmesh=ready'))) return last
    }
    await sleep(30000)
  }
  return last
}

/** Kits' data file holding only the harness's hidden kit, which the drill lays down before the start. */
function harnessKits() {
  return { _kits: { rnhrevolver: harnessKit() } }
}

/**
 * The harness's kit is laid down before the start (harnessKits), so it is normally there. If it is not
 * (RunicNPC rewrote the file without it), it is added and Kits reloaded through the console's websocket:
 * the panel's fire-and-forget command left no reload in run 48 (#22), so the reply is logged.
 */
async function ensureKit(p, id, finding) {
  const file = KITS_FILE
  let data = null
  for (let i = 0; i < 12 && !data; i++) {
    data = readJsonText(await p.readOr(id, file, null))
    if (!data) await sleep(5000)
  }
  if (!data) throw new Error(`Kits wrote no ${file}`)
  data._kits = data._kits || {}
  if (data._kits.rnhrevolver) {
    log('the rnhrevolver kit is in Kits\' file, laid down before the start')
    return
  }
  data._kits.rnhrevolver = harnessKit()
  await p.write(id, file, JSON.stringify(data, null, 2))
  const reply = await p.console(id, 'oxide.reload Kits', 15)
  const loaded = reply.some((l) => /Loaded plugin Kits/.test(l))
  log(`added the rnhrevolver kit and reloaded Kits: ${loaded ? 'Kits loaded again' : `no "Loaded plugin Kits" in 15 s (${reply.length} console line(s))`}`)
  if (!loaded) finding('Kits did not reload after the drill added the harness\'s rnhrevolver kit; the harness\'s profiles may be refused.')
}

function harnessKit() {
  const item = (Shortname, Amount, Position, Condition = 0) => ({ Shortname, Skin: 0, Amount, Condition, MaxCondition: Condition, Ammo: 0, Position, Frequency: -1 })
  return {
    Name: 'rnhrevolver',
    Description: 'RunicNPC test harness: revolver + burlap (staging drill)',
    RequiredPermission: '',
    MaximumUses: 0,
    RequiredAuth: 0,
    Cooldown: 0,
    Cost: 0,
    IsHidden: true,
    CopyPasteFile: '',
    KitImage: '',
    MainItems: [item('ammo.pistol', 120, 0)],
    WearItems: [item('burlap.shirt', 1, 0), item('burlap.trousers', 1, 1), item('hat.boonie', 1, 2)],
    BeltItems: [item('pistol.revolver', 1, 0, 100)],
  }
}

function readJsonText(t) {
  try {
    return t ? JSON.parse(t) : null
  } catch {
    return null
  }
}

/**
 * Where the harness stands, given its lines now and what was last seen of them ({ count, at }): done
 * (`# done all:`), stalled (no new line for stallMs: it died without saying so, as in issue #20), or running.
 */
function harnessProgress(lines, seen, nowMs, stallMs) {
  if (lines.some((l) => /^# done all:/.test(l))) return { state: 'done', seen }
  if (lines.length > seen.count) return { state: 'running', seen: { count: lines.length, at: nowMs } }
  if (nowMs - seen.at >= stallMs) return { state: 'stalled', seen }
  return { state: 'running', seen }
}

/** Waits for the harness; `ended` is done, stalled or capped (timeoutMs passed). */
async function waitHarness(p, id, timeoutMs, stallMs) {
  const start = Date.now()
  let lines = []
  let seen = { count: 0, at: start }
  while (Date.now() < start + timeoutMs) {
    await sleep(30000)
    lines = readJsonText(await p.readOr(id, '/oxide/data/RunicNpcTest.json', null)) || lines
    const v = harnessProgress(lines, seen, Date.now(), stallMs)
    seen = v.seen
    if (v.state !== 'running') return { lines, ended: v.state }
  }
  return { lines, ended: 'capped' }
}

/** The finding for a harness that never wrote `# done all:`. */
function unfinishedFinding(h, lines, ended) {
  const so = `${h.pass} pass, ${h.fail} fail so far`
  if (ended !== 'stalled') return `The harness did not finish in ${HARNESS_CAP_MS / 60000} minutes (${so}).`
  const last = (lines[lines.length - 1] || 'no line at all').replace(/`/g, "'").slice(0, 200)
  return `The harness stopped after \`${last}\` (no output for ${HARNESS_STALL_MS / 60000} minutes; ${so}).`
}

function summarise(lines) {
  const failed = lines.filter((l) => / FAIL( |$)/.test(l))
  const passed = lines.filter((l) => / PASS( |$)/.test(l))
  const done = lines.find((l) => /^# done all:/.test(l))
  return { done: !!done, summary: done || null, pass: passed.length, fail: failed.length, failed }
}

/** RunicNPC's and the harness's error lines from today's Oxide log. */
/** Where Oxide's log of today ends now, so oxideErrors reads only what a run adds (run 49 counted run 48's). */
async function oxideLogMark(p, id) {
  const day = now().slice(0, 10)
  return { day, length: ((await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || '').length }
}

async function oxideErrors(p, id, since = null) {
  const day = now().slice(0, 10)
  const text = (await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || ''
  return text
    .slice(since && since.day === day ? since.length : 0)
    .split('\n')
    .filter((l) => /RunicNPC|RunicNpcTest/.test(l) && /\[Error\]|Exception|Failed|error CS/.test(l))
    .slice(-40)
}

/** The console's last lines, without the startup line (it carries the RCON password). */
function safeTail(text, n) {
  return (text || '')
    .split('\n')
    .filter((l) => !/rcon\.password|\+rcon/i.test(l))
    .slice(-n)
}

// ---------------------------------------------------------------- report: one issue per build

async function report(a) {
  const work = path.resolve(a.work || 'drill-work')
  const statePath = a.state
  const state = readJson(statePath, {})
  const stat = readJson(path.join(work, 'static.json'), null)
  const liv = readJson(path.join(work, 'live.json'), null)
  const branch = (stat && stat.branch) || 'staging'
  const manifest = (stat && stat.manifest) || ''
  const buildid = (liv && liv.installed && liv.installed.buildid) || ''
  // How a build is named: Steam's build id when the rig installed it, and always the manifest's date.
  const build = `${buildid ? `build ${buildid}, ` : ''}manifest ${manifest || '?'} of ${(stat && stat.manifestDate) || '?'}`
  const read = (f, n = 80) => {
    try {
      return fs.readFileSync(path.join(work, f), 'utf8').replace(/\r/g, '').split('\n').slice(0, n).join('\n').trim()
    } catch {
      return ''
    }
  }

  const findings = []
  const sections = []
  let outcome
  let markChecked = true

  if (!stat || stat.managed !== 'ok' || stat.fields === 'error' || stat.compile === 'error') {
    outcome = 'could not run'
    markChecked = false
    findings.push(`The static half could not run: ${(stat && stat.note) || 'no static.json'}`)
    if (stat && stat.fields === 'error') sections.push(['tools/fieldlist', read('fields.diff')])
  } else if (stat.oxideBehind !== 0) {
    // A wait from an earlier drill week (or a run by hand outside one) starts over.
    const current = state.waiting && state.waiting.manifest === manifest && Date.now() - Date.parse(state.waiting.since) < 7 * 86400000
    const waiting = current ? state.waiting : { manifest, since: now() }
    const days = (Date.now() - Date.parse(waiting.since)) / 86400000
    if (days < WAIT_DAYS) {
      outcome = `waiting for Oxide (${stat.oxideBehind} behind, day ${Math.floor(days) + 1} of ${WAIT_DAYS})`
      if (branch === 'staging') {
        state.waiting = { ...waiting, behind: stat.oxideBehind, lastSeen: now() }
        markChecked = false
      }
    } else {
      outcome = 'Oxide has not caught up'
      findings.push(
        `Oxide's staging build is still ${stat.oxideBehind} member(s) behind Rust's staging build after ${WAIT_DAYS} days, ` +
          'so an Oxide server cannot boot on it and the live half did not run. The static half ran against Oxide\'s older build.'
      )
      sections.push(['What Oxide lacks (first 20)', read('oxide-behind.txt', 21)])
    }
  }

  if (!outcome || outcome === 'Oxide has not caught up') {
    if (stat.fields === 'differs') {
      findings.push("The swap's field list differs from the one the plugin carries (D232): regenerate it with tools/fieldlist and check what changed.")
      sections.push(['Field list (carried → staging)', read('fields.diff', 120)])
    }
    if (stat.compile === 'errors') {
      findings.push("RunicNPC does not compile against this build with Oxide's staging build.")
      sections.push(['Compiler output', read('compile.txt', 60)])
    }
    if (!liv && stat.oxideBehind === 0) findings.push('The live half did not run: see the job log (is PTERODACTYL_DRILL_KEY set?).')
    if (liv && !liv.skipped) {
      findings.push(...liv.findings)
      if (liv.consoleTail && liv.consoleTail.length) sections.push(['Console, last lines', liv.consoleTail.join('\n')])
      if (liv.oxideErrors && liv.oxideErrors.length) sections.push(["Oxide's log (RunicNPC lines)", liv.oxideErrors.join('\n')])
      if (liv.status) sections.push(['rnpc.status', liv.status.join('\n')])
    }
    if (!outcome) outcome = findings.length ? 'fail' : 'pass'
    if (branch === 'staging' && markChecked) delete state.waiting
  }

  const installed = liv && liv.installed ? `Rust ${liv.installed.branch} ${liv.installed.buildid}, Oxide build of ${liv.installed.oxideModified}` : ''
  const lines = [
    `Static: Oxide behind ${stat ? stat.oxideBehind : '?'}, fields ${stat ? stat.fields : '?'}, compile ${stat ? stat.compile : '?'}.`,
    liv ? (liv.skipped ? `Live: skipped (${liv.skipped}).` : `Live: ${liv.harness ? `${liv.harness.pass} pass, ${liv.harness.fail} fail` : 'did not reach the harness'}${installed ? `, on ${installed}` : ''}${liv.bootMinutes ? `, booted in ${liv.bootMinutes} min` : ''}.`) : 'Live: did not run.',
  ]
  log(`${build} on ${branch}: ${outcome}`)
  lines.forEach((l) => log(l))
  findings.forEach((f) => log('-', f))
  if (process.env.GITHUB_STEP_SUMMARY) {
    fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Staging drill: ${build}: ${outcome}\n\n${lines.join('\n\n')}\n\n${findings.map((f) => `- ${f}`).join('\n')}\n`)
  }

  // A rehearsal on another branch reports here and changes nothing.
  if (branch !== 'staging') return log('not the staging branch: no issue, no state')

  let issue = null
  if (findings.length) {
    const body = [
      `The staging drill (docs/runicnpc/PLAN.md stage 9c) checked Rust's staging branch (**${build}**) on ${now().slice(0, 10)}: **${outcome}**.`,
      '',
      ...findings.map((f) => `- ${f}`),
      '',
      ...lines,
      ...sections.flatMap(([title, text]) => (text ? ['', `### ${title}`, '', '```', text, '```'] : [])),
      '',
      process.env.GITHUB_RUN_NUMBER ? `Run: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_NUMBER}` : '',
    ].join('\n')
    issue = await fileIssue(`Staging drill: Rust staging manifest ${manifest} (${(stat && stat.manifestDate) || '?'})`, body)
  }

  if (markChecked) state.checked = { manifest, buildid, at: now(), outcome, issue, oxideBehind: stat ? stat.oxideBehind : null }
  state.lastRun = { at: now(), manifest, buildid, outcome, issue }
  fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n')
  output('outcome', outcome)
}

async function fileIssue(title, body) {
  const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}`
  const headers = { Authorization: 'token ' + process.env.GITEA_TOKEN, 'Content-Type': 'application/json', Accept: 'application/json' }
  const call = async (method, p, data) => {
    const r = await fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined })
    if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${(await r.text()).slice(0, 200)}`)
    return r.json()
  }

  const open = await call('GET', `/issues?state=open&type=issues&labels=staging-drill&limit=50`)
  const same = open.find((i) => i.title === title)
  if (same) {
    await call('POST', `/issues/${same.number}/comments`, { body })
    log(`commented on #${same.number}`)
    return same.number
  }
  const labels = await call('GET', '/labels?limit=100')
  let label = labels.find((l) => l.name === 'staging-drill')
  if (!label) label = await call('POST', '/labels', { name: 'staging-drill', color: '#d93f0b', description: "Rust's staging branch breaks RunicNPC (the daily staging drill)" })
  const created = await call('POST', '/issues', { title, body, labels: [label.id] })
  log(`opened #${created.number}`)
  return created.number
}

// ---------------------------------------------------------------- main

// The pure parts, for drill.test.js; run as a script, it is the command line. panel and waitRunning serve
// cutover.js (D332), and report is exported so a test can show it keeps the state keys it does not own.
module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, KITS_FILE, harnessKits, HARNESS_STALL_MS, panel, waitRunning, report }

if (require.main === module) {
  const a = args()
  const commands = { window: window_, gate, live, report }
  const command = commands[a._[0]]
  if (!command) {
    console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]')
    process.exit(2)
  }
  command(a).catch((e) => {
    console.error(e.message || e)
    process.exit(1)
  })
}