fix(drill): staging 26341433's swap fields; a failed group no longer ends the harness (#20)

The staging drill's run 47 failed on Rust staging 26341433:

- Rust added BasePlayer.timeSinceAiming, BaseEntity.ServerPreProcessed and
  BaseNetworkable.ignoreNetworkLod to the swap's field list (D232): 55 -> 58.
  On edge now; main at the forced wipe of 2026-11-05.
- Harness 0.7.4: place.set.each was refused and the refusal was thrown
  away; place.spawned failed; reading each[0] of an empty list then killed
  the coroutine, so the drill waited 45 minutes for a run that had stopped
  after 10. Place puts its spots on the navmesh, the set checks carry the
  refusal, a failed place.spawned ends the group, and any group that throws
  is a <group>.exception FAIL line and the run goes on.
- drill.js: rust-staging boots from empty Kits and RunicNPC data, so the
  write-once example kits match this Rust (s6.examples.fullCondition); the
  wait ends after 5 minutes without a new harness line; latest.log's tail
  goes into the issue when the harness did not finish. drill.test.js
  covers the stall logic.

Refs #20, #21

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY

tools/drill/drill.js

@@ -30,6 +30,20 @@ const BIG = (process.env.DRILL_BIG_RIGS || 'rust-oxide,rust-carbon').split(',')
 const STOPPABLE = process.env.DRILL_STOPPABLE || 'rust-carbon'
 const KITS_URL = 'https://umod.org/plugins/Kits.cs'
 const WAIT_DAYS = 3 // D317: how long a build waits for Oxide to catch up
+const HARNESS_CAP_MS = 45 * 60000 // the whole harness run
+const HARNESS_STALL_MS = 5 * 60000 // no new line in its file for this long: it has stopped (issue #20)
+
+// What a fresh server would not have (issue #20). RunicNPC writes its four example kits once, into
+// Kits' own file, and never again, so a file kept from an older build keeps that build's kits; the drill
+// starts from none of it. Files, never directories: a directory the panel's file API creates is not
+// writable by the game (docs/runicnpc/PLAN.md §1.5), and the game makes these itself on its first boot.
+const FRESH_DATA = [
+  '/oxide/data/Kits/kits_data.json',
+  '/oxide/data/RunicNPC/profiles.json',
+  '/oxide/data/RunicNPC/placements.json',
+  '/oxide/data/RunicNPC/routes.json',
+  '/oxide/data/RunicNPC/state.json',
+]
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
 const now = () => new Date().toISOString()
@@ -262,6 +276,7 @@ async function live(a) {
     await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
     await p.rm(rig, '/oxide/data/RunicNpcTest.json')
     await p.rm(rig, '/staging-drill.json')
+    for (const f of FRESH_DATA) await p.rm(rig, f) // already gone is fine: rm swallows it
 
     log(`starting ${RIG} on ${branch}`)
     const started = Date.now()
@@ -310,9 +325,13 @@ async function live(a) {
     result.ran = true
     log('rnt.run all: running')
 
-    const lines = await waitHarness(p, rig, 45 * 60000)
+    const { lines, ended } = await waitHarness(p, rig, HARNESS_CAP_MS, HARNESS_STALL_MS)
     result.harness = summarise(lines)
-    if (!result.harness.done) finding(`The harness did not finish in 45 minutes (${result.harness.pass} pass, ${result.harness.fail} fail so far).`)
+    if (!result.harness.done) {
+      finding(unfinishedFinding(result.harness, lines, ended))
+      // What the server said while it stalled (an exception in the harness's coroutine lands here, not in its file).
+      result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 40)
+    }
     for (const f of result.harness.failed) finding(`Harness: ${f}`)
     log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`)
     result.oxideErrors = await oxideErrors(p, rig)
@@ -408,15 +427,38 @@ function readJsonText(t) {
   }
 }
 
-async function waitHarness(p, id, timeoutMs) {
-  const end = Date.now() + timeoutMs
+/**
+ * Where the harness stands, given its lines now and what was last seen of them ({ count, at }): done
+ * (`# done all:`), stalled (no new line for stallMs: it died without saying so, as in issue #20), or running.
+ */
+function harnessProgress(lines, seen, nowMs, stallMs) {
+  if (lines.some((l) => /^# done all:/.test(l))) return { state: 'done', seen }
+  if (lines.length > seen.count) return { state: 'running', seen: { count: lines.length, at: nowMs } }
+  if (nowMs - seen.at >= stallMs) return { state: 'stalled', seen }
+  return { state: 'running', seen }
+}
+
+/** Waits for the harness; `ended` is done, stalled or capped (timeoutMs passed). */
+async function waitHarness(p, id, timeoutMs, stallMs) {
+  const start = Date.now()
   let lines = []
-  while (Date.now() < end) {
+  let seen = { count: 0, at: start }
+  while (Date.now() < start + timeoutMs) {
     await sleep(30000)
     lines = readJsonText(await p.readOr(id, '/oxide/data/RunicNpcTest.json', null)) || lines
-    if (lines.some((l) => /^# done all:/.test(l))) break
+    const v = harnessProgress(lines, seen, Date.now(), stallMs)
+    seen = v.seen
+    if (v.state !== 'running') return { lines, ended: v.state }
   }
-  return lines
+  return { lines, ended: 'capped' }
+}
+
+/** The finding for a harness that never wrote `# done all:`. */
+function unfinishedFinding(h, lines, ended) {
+  const so = `${h.pass} pass, ${h.fail} fail so far`
+  if (ended !== 'stalled') return `The harness did not finish in ${HARNESS_CAP_MS / 60000} minutes (${so}).`
+  const last = (lines[lines.length - 1] || 'no line at all').replace(/`/g, "'").slice(0, 200)
+  return `The harness stopped after \`${last}\` (no output for ${HARNESS_STALL_MS / 60000} minutes; ${so}).`
 }
 
 function summarise(lines) {
@@ -578,14 +620,19 @@ async function fileIssue(title, body) {
 
 // ---------------------------------------------------------------- main
 
-const a = args()
-const commands = { window: window_, gate, live, report }
-const command = commands[a._[0]]
-if (!command) {
-  console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]')
-  process.exit(2)
+// The pure parts, for drill.test.js; run as a script, it is the command line.
+module.exports = { harnessProgress, unfinishedFinding, FRESH_DATA, HARNESS_STALL_MS }
+
+if (require.main === module) {
+  const a = args()
+  const commands = { window: window_, gate, live, report }
+  const command = commands[a._[0]]
+  if (!command) {
+    console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]')
+    process.exit(2)
+  }
+  command(a).catch((e) => {
+    console.error(e.message || e)
+    process.exit(1)
+  })
 }
-command(a).catch((e) => {
-  console.error(e.message || e)
-  process.exit(1)
-})