Merge pull request 'feat(drill): the staging drill, the week before each forced wipe (stage 9c, D307–D309, D317–D319)' (#15) from feat/stage-9c into edge
Reviewed-on: #15
This commit is contained in:
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Shell scripts run on Linux (the CI runner, the staging rig): a CRLF checkout breaks them.
|
||||
*.sh text eol=lf
|
||||
149
.gitea/workflows/staging-drill.yml
Normal file
149
.gitea/workflows/staging-drill.yml
Normal file
@@ -0,0 +1,149 @@
|
||||
# The staging drill (docs/runicnpc/PLAN.md stage 9c, D307-D309, D317, D318).
|
||||
#
|
||||
# Rust changes under RunicNPC on every forced wipe, and what breaks first is the subclass swap and
|
||||
# the brain (README "The update surface"). Facepunch publishes each build on Steam's `staging`
|
||||
# branch days before it ships. The forced wipe is monthly, the first Thursday, so the job does
|
||||
# something only in the seven days before it, which leaves a week to fix what it finds (D318). It
|
||||
# is scheduled daily and stops at its first step outside that week. In the week, it checks every
|
||||
# new staging build:
|
||||
#
|
||||
# window (tools/drill/drill.js window): the week before a forced wipe? (A run by hand always is.)
|
||||
# static half (tools/drill/static.sh, about a minute, no server):
|
||||
# Rust's managed assemblies from Steam's staging branch and Oxide's staging build →
|
||||
# has Oxide caught up · the swap's field list regenerated and compared · RunicNPC compiled.
|
||||
# gate (tools/drill/drill.js gate): stop here when this build (the manifest of Rust's Linux
|
||||
# depot) was already checked.
|
||||
# live half (drill.js live): the `rust-staging` server installs the same pair at start, then
|
||||
# RunicNPC, Kits and the test harness are loaded and `rnt.run all` runs. When both 6000-map rigs
|
||||
# are running, `rust-carbon` is stopped for it and started again (D309). Skipped while Oxide is
|
||||
# behind Rust: an Oxide server on that pair dies at boot.
|
||||
# report (drill.js report): anything that fails or differs opens ONE issue for the build, labelled
|
||||
# `staging-drill` (a later run on the same build comments on it). The result is kept on the
|
||||
# `drill` branch (drill.json), like the installer's `bundles` branch.
|
||||
#
|
||||
# While Oxide's staging build is behind Rust's, a build waits up to three days, opening nothing;
|
||||
# then it is reported as "Oxide has not caught up" (D317).
|
||||
#
|
||||
# Run by hand with `branch: public` to rehearse it on a pair known to match: it reports in the job
|
||||
# summary only, and changes no state and opens no issue.
|
||||
#
|
||||
# The schedule runs only from the default branch (`main`), so it starts with the stage 9 cutover;
|
||||
# before then, run it by hand on `edge`.
|
||||
#
|
||||
# Secrets (Settings → Actions → Secrets, the organisation's):
|
||||
# PTERODACTYL_DRILL_KEY — the panel client key made for this job alone (D309). Never printed.
|
||||
# REGISTRY_USER / REGISTRY_TOKEN — as release.yml: push the `drill` branch and open issues.
|
||||
|
||||
name: Staging drill
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '23 10 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: 'Steam branch to drill (public = a rehearsal: job summary only)'
|
||||
default: staging
|
||||
force:
|
||||
description: 'Check the build again even if it was checked'
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: staging-drill
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITEA_HOST: gitea.whitlocktech.com
|
||||
REPO: RunicGateway/runicnpc-rust
|
||||
|
||||
jobs:
|
||||
drill:
|
||||
runs-on: ubuntu-latest
|
||||
# Rust's update, a 3500 map's boot and navmesh (about 18 minutes when a new Rust build
|
||||
# regenerates the map), and the harness: under an hour.
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
BRANCH: ${{ github.event.inputs.branch || 'staging' }}
|
||||
WORK: ${{ github.workspace }}/drill-work
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# 22 or later: drill.js talks to the panel's console over the global WebSocket.
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: The week before a forced wipe?
|
||||
id: window
|
||||
run: node tools/drill/drill.js window ${{ github.event_name == 'workflow_dispatch' && '--force' || '' }}
|
||||
|
||||
# tools/fieldlist and the compiler (Roslyn's csc.dll ships in the SDK).
|
||||
- name: .NET SDK
|
||||
if: ${{ steps.window.outputs.open == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -sSL -o "$RUNNER_TEMP/dotnet-install.sh" https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh
|
||||
bash "$RUNNER_TEMP/dotnet-install.sh" --channel 9.0 --install-dir "$HOME/.dotnet" > /dev/null
|
||||
echo "$HOME/.dotnet" >> "$GITHUB_PATH"
|
||||
echo "DOTNET_ROOT=$HOME/.dotnet" >> "$GITHUB_ENV"
|
||||
|
||||
- name: The drill branch (the last build checked)
|
||||
if: ${{ steps.window.outputs.open == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "runicnpc-ci"
|
||||
git config user.email "ci@whitlocktech.com"
|
||||
rm -rf drill-state
|
||||
git worktree prune
|
||||
if git ls-remote --exit-code --heads origin drill > /dev/null 2>&1; then
|
||||
git fetch origin drill
|
||||
git worktree add -B drill drill-state origin/drill
|
||||
else
|
||||
# First run: a root commit with an empty tree, so the branch inherits no code history.
|
||||
ROOT="$(git commit-tree "$(git hash-object -t tree /dev/null)" -m 'chore(drill): start the drill branch')"
|
||||
git worktree add -B drill drill-state "$ROOT"
|
||||
fi
|
||||
[ -f drill-state/drill.json ] || echo '{}' > drill-state/drill.json
|
||||
cat drill-state/drill.json
|
||||
|
||||
- name: Static half
|
||||
if: ${{ steps.window.outputs.open == 'true' }}
|
||||
run: bash tools/drill/static.sh
|
||||
|
||||
- name: A build not checked yet?
|
||||
id: gate
|
||||
if: ${{ steps.window.outputs.open == 'true' }}
|
||||
run: node tools/drill/drill.js gate --state drill-state/drill.json --work "$WORK" ${{ github.event.inputs.force == 'true' && '--force' || '' }}
|
||||
|
||||
- name: Live half (rust-staging)
|
||||
id: live
|
||||
if: ${{ steps.gate.outputs.run == 'true' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
PANEL_KEY: ${{ secrets.PTERODACTYL_DRILL_KEY }}
|
||||
run: node tools/drill/drill.js live --work "$WORK" --branch "$BRANCH"
|
||||
|
||||
- name: Report
|
||||
id: report
|
||||
if: ${{ always() && steps.gate.outputs.run == 'true' }}
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: node tools/drill/drill.js report --state drill-state/drill.json --work "$WORK"
|
||||
|
||||
- name: Keep the result on the drill branch
|
||||
if: ${{ always() && steps.gate.outputs.run == 'true' && env.BRANCH == 'staging' }}
|
||||
env:
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CI_USER="$(printf '%s' "${REGISTRY_USER}" | tr -d '\r\n')"
|
||||
CI_TOKEN="$(printf '%s' "${REGISTRY_TOKEN}" | tr -d '\r\n')"
|
||||
git remote set-url origin "https://${CI_USER}:${CI_TOKEN}@${GITEA_HOST}/${REPO}.git"
|
||||
cd drill-state
|
||||
git add -A
|
||||
if git diff --cached --quiet; then echo "no change"; exit 0; fi
|
||||
node -e "const s=require('./drill.json').lastRun||{};console.log('chore(drill): '+(s.manifest||'?')+' '+(s.outcome||'')+' [skip ci]')" > ../drill-msg.txt
|
||||
git commit -q -F ../drill-msg.txt
|
||||
git push origin drill
|
||||
38
README.md
38
README.md
@@ -92,7 +92,8 @@ The API version moves when a call or a raised hook changes shape, not on every r
|
||||
| `plugin/RunicNPC.cs` | The plugin. The only file a server gets. |
|
||||
| `plugin.toml` | Its declarations: API version, framework floors, required plugins. The release copies them into the manifest. |
|
||||
| `scripts/checkPlugin.js` | The static checks run on every pull request and again before a release (see its header). |
|
||||
| `tools/` | Developer scaffolding for the test rigs, never shipped: the panel scripts (see `tools/rigs.example.json`); `RunicNpcHarness.cs`, the stage 1 and 5 measurement plugin; `RunicNpcTest.cs`, the test harness (`rnt.run all`, then `rnt.after` after a reload or restart) and stage 9's performance check (`rnt.cost`, below); and `fieldlist/` with `managed.js`, which regenerate the swap's field list (below). |
|
||||
| `tools/` | Developer scaffolding for the test rigs, never shipped: the panel scripts (see `tools/rigs.example.json`); `RunicNpcHarness.cs`, the stage 1 and 5 measurement plugin; `RunicNpcTest.cs`, the test harness (`rnt.run all`, then `rnt.after` after a reload or restart) and stage 9's performance check (`rnt.cost`, below); and `fieldlist/` with `managed.js`, which regenerate the swap's field list (below). `drill/` is the staging drill's (below). |
|
||||
| `.gitea/workflows/staging-drill.yml` | The staging drill, the week before each forced wipe. |
|
||||
|
||||
## After a Rust update: the swap's field list
|
||||
|
||||
@@ -112,7 +113,7 @@ fields Rust itself serialises.
|
||||
## The update surface: what a Rust update can break
|
||||
|
||||
Everything RunicNPC takes from Rust's own classes for the swap and the brain, in one place. A Rust update
|
||||
that changes any of it breaks RunicNPC; the staging drill (PLAN.md stage 9, D308) checks exactly this list
|
||||
that changes any of it breaks RunicNPC; the staging drill (PLAN.md stage 9c, below) checks exactly this list
|
||||
against every new staging build. **Caught by** says how: *compile* means the plugin no longer compiles
|
||||
against the new assembly; *field list* means `tools/fieldlist`'s regenerated list differs from the
|
||||
plugin's (and `rnpc.status` says `missing=` or `added=` at run time); *run time* means only a running server
|
||||
@@ -138,6 +139,39 @@ The **hooks** RunicNPC listens to are listed by `rnpc.status`, which names any t
|
||||
that stays silent after an update has probably been renamed (neither framework reports a hook that matches
|
||||
nothing).
|
||||
|
||||
## The staging drill
|
||||
|
||||
Facepunch puts each Rust build on Steam's `staging` branch days before it ships, and Rust changes on the
|
||||
monthly forced wipe, the first Thursday. `.gitea/workflows/staging-drill.yml` works in the seven days
|
||||
before each forced wipe, which leaves a week to fix what it finds, and checks every new staging build in
|
||||
that week (PLAN.md stage 9c, D307–D309, D317, D318). It is scheduled daily and stops at once outside
|
||||
the week. A run by hand always goes ahead.
|
||||
|
||||
- **Static, about a minute, no server** (`tools/drill/static.sh`): Rust's managed assemblies
|
||||
from the staging branch and Oxide's staging build. It asks whether Oxide has caught up with Rust
|
||||
(`tools/fieldlist --oxide-behind`), regenerates the swap's field list and compares it, and compiles the
|
||||
plugin against them.
|
||||
- **Live, once per build** (`tools/drill/drill.js live`): the `rust-staging` server (Oxide, a 3500 map)
|
||||
installs the same pair at start (`tools/drill/rig-start.sh`), loads RunicNPC, Kits and
|
||||
`tools/RunicNpcTest.cs`, and runs `rnt.run all`. When both 6000-map rigs are running, `rust-carbon` is
|
||||
stopped for it and started again.
|
||||
- **Report:** anything that fails or differs opens one issue for that build, labelled `staging-drill`.
|
||||
The last build checked is kept on the `drill` branch.
|
||||
|
||||
A build is the manifest of Rust's Linux depot, as Steam names it. Oxide's build for a branch carries its
|
||||
own patched `Assembly-CSharp.dll`, and while it is older than Rust's, an Oxide server on that pair dies
|
||||
at boot. So a build waits, opening nothing, until Oxide catches up, for up to three days. After that it is
|
||||
reported as "Oxide has not caught up". To rehearse it on a pair known to match, run the workflow by hand
|
||||
with `branch: public`: it reports in the job summary only.
|
||||
|
||||
By hand, from Git Bash or Linux:
|
||||
|
||||
```bash
|
||||
BRANCH=public DEPOTDOWNLOADER=/path/to/DepotDownloader bash tools/drill/static.sh # → drill-work/static.json
|
||||
PANEL_KEY_FILE=/path/to/key node tools/drill/drill.js live --branch public # → drill-work/live.json
|
||||
node tools/drill/drill.js report --state /tmp/drill.json # public: prints only
|
||||
```
|
||||
|
||||
## Performance: `rnt.cost`
|
||||
|
||||
Stage 9's check (D306): 100 of RunicNPC's NPCs, idle and fighting 20 stand-in players, against the same
|
||||
|
||||
@@ -45,7 +45,7 @@ namespace Oxide.Plugins
|
||||
/// a made-up user id (11400000001 and up), so <c>IsNpc</c> is false. They take no damage.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Info("RunicNpcTest", "Runic Gateway", "0.7.0")]
|
||||
[Info("RunicNpcTest", "Runic Gateway", "0.7.2")]
|
||||
[Description("RunicNPC stage 2 to 7 tests, the stage 6 spike and stage 9's cost check. A developer tool: never ship it.")]
|
||||
internal class RunicNpcTest : RustPlugin
|
||||
{
|
||||
@@ -147,11 +147,17 @@ namespace Oxide.Plugins
|
||||
private static MonumentInfo[] Monuments() => _monuments ?? (_monuments = UnityEngine.Object.FindObjectsOfType<MonumentInfo>());
|
||||
|
||||
/// <summary>A flat, dry spot on the navmesh far from every monument; fixed seed, so reruns land in the same field.</summary>
|
||||
private static bool FindField(float clear, int seed, out Vector3 at)
|
||||
/// <summary>
|
||||
/// An open field on the navmesh. With <paramref name="line"/>, the points every <c>line / 2</c> m out to
|
||||
/// ±<paramref name="line"/> along x, and 25 m past each, must be on it too: stage 5's five fights stand
|
||||
/// that far apart. On a 6000 map the first field had room; on the staging drill's maps it is rare (D319).
|
||||
/// </summary>
|
||||
private static bool FindField(float clear, int seed, out Vector3 at, float line = 0f)
|
||||
{
|
||||
var rng = new System.Random(seed);
|
||||
float half = World.Size / 2f - 300f;
|
||||
for (int i = 0; i < 4000; i++)
|
||||
// The line is rare on a small map: 5 samples in 20000 have it on the drill's 3500 map (D319).
|
||||
for (int i = 0; i < (line > 0f ? 20000 : 4000); i++)
|
||||
{
|
||||
var p = new Vector3((float)(rng.NextDouble() * 2 - 1) * half, 0f, (float)(rng.NextDouble() * 2 - 1) * half);
|
||||
p.y = TerrainMeta.HeightMap.GetHeight(p);
|
||||
@@ -168,6 +174,16 @@ namespace Oxide.Plugins
|
||||
open = SampleNavmesh(hit.position + new Vector3(Mathf.Cos(a), 0f, Mathf.Sin(a)) * 35f, 2f, out h2);
|
||||
}
|
||||
|
||||
for (int k = -2; k <= 2 && open && line > 0f; k++)
|
||||
{
|
||||
Vector3 q = hit.position + new Vector3(k * line / 2f, 0f, 0f);
|
||||
Vector3 r = q + new Vector3(0f, 0f, 25f);
|
||||
q.y = TerrainMeta.HeightMap.GetHeight(q);
|
||||
r.y = TerrainMeta.HeightMap.GetHeight(r);
|
||||
NavMeshHit h2;
|
||||
open = SampleNavmesh(q, 3f, out h2) && SampleNavmesh(r, 3f, out h2);
|
||||
}
|
||||
|
||||
if (!open) continue;
|
||||
at = hit.position;
|
||||
return true;
|
||||
@@ -1710,7 +1726,7 @@ namespace Oxide.Plugins
|
||||
private IEnumerator Stage5()
|
||||
{
|
||||
Vector3 f;
|
||||
if (!FindField(300f, 2025, out f))
|
||||
if (!FindField(300f, 2025, out f, 140f))
|
||||
{
|
||||
Check("s5.setup.field", false, "no second open field on the navmesh");
|
||||
yield break;
|
||||
@@ -1755,7 +1771,13 @@ namespace Oxide.Plugins
|
||||
ScientistNPC far = SpawnScientist(d + new Vector3(0, 0, 25));
|
||||
BasePlayer victim = Spawn(e, "t_roam");
|
||||
BasePlayer bully = Spawn(OnMesh(e + new Vector3(0, 0, 15)), "t_bully");
|
||||
Check("s5.fight.spawned", red && blue && hunter && prey && neutral && bystander && leashed && far && victim && bully, "");
|
||||
var fighters = new Dictionary<string, BaseEntity>
|
||||
{
|
||||
{ "red", red }, { "blue", blue }, { "hunter", hunter }, { "prey", prey }, { "neutral", neutral },
|
||||
{ "bystander", bystander }, { "leashed", leashed }, { "far", far }, { "victim", victim }, { "bully", bully },
|
||||
};
|
||||
string[] unspawned = fighters.Where(kv => kv.Value == null).Select(kv => kv.Key).ToArray();
|
||||
Check("s5.fight.spawned", unspawned.Length == 0, unspawned.Length == 0 ? "" : $"not spawned: {string.Join(", ", unspawned)} (field {f}, map {World.Size})");
|
||||
if (!(red && blue && hunter && prey && neutral && bystander && leashed && far && victim && bully)) yield break;
|
||||
|
||||
float farHealth = far.health, bystanderHealth = bystander.health, preyHealth = prey.health;
|
||||
|
||||
591
tools/drill/drill.js
Normal file
591
tools/drill/drill.js
Normal file
@@ -0,0 +1,591 @@
|
||||
// The staging drill (docs/runicnpc/PLAN.md stage 9c, D307-D309, D317, D318). Never shipped.
|
||||
//
|
||||
// .gitea/workflows/staging-drill.yml runs it daily, and it does something only in the week before the
|
||||
// monthly forced wipe (D318):
|
||||
//
|
||||
// node tools/drill/drill.js window [--today YYYY-MM-DD] in the drill week?
|
||||
// tools/drill/static.sh the static half
|
||||
// node tools/drill/drill.js gate --state <drill.json> --work <dir> a build not checked yet?
|
||||
// node tools/drill/drill.js live --work <dir> [--branch staging] the live half, on rust-staging
|
||||
// node tools/drill/drill.js report --state <drill.json> --work <dir> one issue per build; the state
|
||||
//
|
||||
// A build is the manifest of Rust's Linux depot, as Steam names it to static.sh.
|
||||
//
|
||||
// Environment: PANEL_KEY (or PANEL_KEY_FILE, a file with a `user: ptlc_…` line) is the drill's own
|
||||
// Pterodactyl client key; PANEL_URL the panel. The live half finds its servers by name: DRILL_RIG
|
||||
// (rust-staging) and the two 6000-map rigs, of which it stops DRILL_STOPPABLE (rust-carbon) when both
|
||||
// run, because the node has no memory for three (D309). It leaves every rig as it found it.
|
||||
// `report` needs GITEA_TOKEN, and GITHUB_SERVER_URL + GITHUB_REPOSITORY as Gitea Actions sets them.
|
||||
//
|
||||
// The key is never printed, and neither is the server's console start line (it carries the RCON
|
||||
// password): only lines this script picks are ever written out.
|
||||
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..', '..')
|
||||
const PANEL = (process.env.PANEL_URL || 'http://192.168.0.12').replace(/\/+$/, '')
|
||||
const RIG = process.env.DRILL_RIG || 'rust-staging'
|
||||
const BIG = (process.env.DRILL_BIG_RIGS || 'rust-oxide,rust-carbon').split(',')
|
||||
const STOPPABLE = process.env.DRILL_STOPPABLE || 'rust-carbon'
|
||||
const KITS_URL = 'https://umod.org/plugins/Kits.cs'
|
||||
const WAIT_DAYS = 3 // D317: how long a build waits for Oxide to catch up
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||||
const now = () => new Date().toISOString()
|
||||
const log = (...a) => console.log(`[drill ${now().slice(11, 19)}]`, ...a)
|
||||
|
||||
function args() {
|
||||
const out = { _: [] }
|
||||
const argv = process.argv.slice(2)
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i].startsWith('--')) {
|
||||
const k = argv[i].slice(2)
|
||||
out[k] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true
|
||||
} else out._.push(argv[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const readJson = (f, fallback) => {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(f, 'utf8'))
|
||||
} catch {
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
function output(name, value) {
|
||||
if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the panel
|
||||
|
||||
function panelKey() {
|
||||
if (process.env.PANEL_KEY) return process.env.PANEL_KEY.trim()
|
||||
const file = process.env.PANEL_KEY_FILE
|
||||
if (!file) throw new Error('PANEL_KEY (or PANEL_KEY_FILE) is not set')
|
||||
const line = fs.readFileSync(file, 'utf8').split(/\r?\n/).find((l) => l.startsWith('user:'))
|
||||
if (!line) throw new Error(`${file} has no "user:" line`)
|
||||
return line.slice(5).trim()
|
||||
}
|
||||
|
||||
function panel() {
|
||||
const key = panelKey()
|
||||
const headers = (type = 'application/json') => ({ Authorization: 'Bearer ' + key, Accept: 'application/json', 'Content-Type': type })
|
||||
const api = async (method, p, body, type) => {
|
||||
const r = await fetch(`${PANEL}/api/client${p}`, { method, headers: headers(type), body })
|
||||
const text = await r.text()
|
||||
if (!r.ok) throw new Error(`${method} ${p.split('?')[0]}: ${r.status} ${text.slice(0, 200)}`)
|
||||
return text
|
||||
}
|
||||
|
||||
const p = {
|
||||
async servers() {
|
||||
const d = JSON.parse(await api('GET', '/?per_page=100'))
|
||||
return Object.fromEntries(d.data.map((s) => [s.attributes.name, s.attributes.identifier]))
|
||||
},
|
||||
async state(id) {
|
||||
return JSON.parse(await api('GET', `/servers/${id}/resources`)).attributes.current_state
|
||||
},
|
||||
power: (id, signal) => api('POST', `/servers/${id}/power`, JSON.stringify({ signal })),
|
||||
command: (id, command) => api('POST', `/servers/${id}/command`, JSON.stringify({ command })),
|
||||
read: (id, file) => api('GET', `/servers/${id}/files/contents?file=${encodeURIComponent(file)}`),
|
||||
async readOr(id, file, fallback = null) {
|
||||
try {
|
||||
return await p.read(id, file)
|
||||
} catch {
|
||||
return fallback
|
||||
}
|
||||
},
|
||||
write: (id, file, content) => api('POST', `/servers/${id}/files/write?file=${encodeURIComponent(file)}`, content, 'text/plain'),
|
||||
async rm(id, file) {
|
||||
const dir = file.slice(0, file.lastIndexOf('/')) || '/'
|
||||
try {
|
||||
await api('POST', `/servers/${id}/files/delete`, JSON.stringify({ root: dir, files: [file.slice(file.lastIndexOf('/') + 1)] }))
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
},
|
||||
|
||||
/** Sends one console command and returns the console lines that follow, for `secs` seconds. */
|
||||
async console(id, command, secs = 6) {
|
||||
const { data } = JSON.parse(await api('GET', `/servers/${id}/websocket`))
|
||||
const strip = (s) => s.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '')
|
||||
return new Promise((resolve) => {
|
||||
const lines = []
|
||||
let sent = false
|
||||
const ws = new WebSocket(data.socket, { headers: { Origin: PANEL } })
|
||||
const done = () => {
|
||||
try {
|
||||
ws.close()
|
||||
} catch {}
|
||||
resolve(lines)
|
||||
}
|
||||
ws.onopen = () => ws.send(JSON.stringify({ event: 'auth', args: [data.token] }))
|
||||
ws.onmessage = (m) => {
|
||||
const msg = JSON.parse(m.data)
|
||||
if (msg.event === 'auth success' && !sent) {
|
||||
sent = true
|
||||
// The panel replays recent history on connect; let it pass, then send.
|
||||
setTimeout(() => {
|
||||
lines.length = 0
|
||||
ws.send(JSON.stringify({ event: 'send command', args: [command] }))
|
||||
}, 800)
|
||||
setTimeout(done, secs * 1000)
|
||||
} else if (msg.event === 'console output' && sent) {
|
||||
lines.push(...strip(String(msg.args[0])).split('\n'))
|
||||
}
|
||||
}
|
||||
ws.onerror = () => done()
|
||||
})
|
||||
},
|
||||
|
||||
async waitState(id, want, timeoutMs) {
|
||||
const end = Date.now() + timeoutMs
|
||||
while (Date.now() < end) {
|
||||
if ((await p.state(id)) === want) return true
|
||||
await sleep(5000)
|
||||
}
|
||||
return false
|
||||
},
|
||||
|
||||
/** Stops a server: `stop`, then `kill` if it has not stopped in time. */
|
||||
async stop(id, graceMs = 180000) {
|
||||
if ((await p.state(id)) === 'offline') return
|
||||
await p.power(id, 'stop')
|
||||
if (!(await p.waitState(id, 'offline', graceMs))) {
|
||||
await p.power(id, 'kill')
|
||||
await p.waitState(id, 'offline', 60000)
|
||||
}
|
||||
},
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- window: the week before a forced wipe
|
||||
|
||||
// Rust changes on the monthly forced wipe, the first Thursday (D318). The drill runs only in the seven
|
||||
// days before it, which leaves a week to fix what it finds; outside them the job stops at once.
|
||||
function nextWipe(today) {
|
||||
const firstThursday = (y, m) => {
|
||||
const d = new Date(Date.UTC(y, m, 1))
|
||||
d.setUTCDate(1 + ((4 - d.getUTCDay() + 7) % 7))
|
||||
return d
|
||||
}
|
||||
const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate())
|
||||
const thisMonth = firstThursday(today.getUTCFullYear(), today.getUTCMonth())
|
||||
return thisMonth.getTime() > t ? thisMonth : firstThursday(today.getUTCFullYear(), today.getUTCMonth() + 1)
|
||||
}
|
||||
|
||||
function windowDays(today = new Date()) {
|
||||
const t = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate())
|
||||
return Math.round((nextWipe(today).getTime() - t) / 86400000)
|
||||
}
|
||||
|
||||
async function window_(a) {
|
||||
const days = windowDays(a.today ? new Date(a.today) : new Date())
|
||||
const wipe = nextWipe(a.today ? new Date(a.today) : new Date()).toISOString().slice(0, 10)
|
||||
const open = days >= 1 && days <= 7
|
||||
const verdict = open ? 'in the drill week' : a.force ? 'outside the drill week, but run by hand: going ahead' : 'outside the drill week, nothing to do'
|
||||
log(`the next forced wipe is ${wipe}, in ${days} day(s): ${verdict}`)
|
||||
output('open', open || a.force ? 'true' : 'false')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- gate: is there anything to check?
|
||||
|
||||
// A build is Steam's manifest of the Linux depot, which static.sh reads from Steam itself.
|
||||
async function gate(a) {
|
||||
const state = readJson(a.state, {})
|
||||
const stat = readJson(path.join(path.resolve(a.work || 'drill-work'), 'static.json'), {})
|
||||
const checked = state.checked && state.checked.manifest
|
||||
const isNew = !stat.manifest || stat.manifest !== checked
|
||||
log(`Rust ${stat.branch || 'staging'} is manifest ${stat.manifest || '?'} (${stat.manifestDate || '?'}); last checked ${checked || 'none'}`)
|
||||
output('run', isNew || a.force ? 'true' : 'false')
|
||||
if (!isNew && !a.force) log('already checked: nothing to do')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- live: the harness on rust-staging
|
||||
|
||||
async function live(a) {
|
||||
const work = path.resolve(a.work || 'drill-work')
|
||||
const branch = a.branch || 'staging'
|
||||
const result = { branch, at: now(), ran: false, findings: [], notes: [] }
|
||||
const save = () => fs.writeFileSync(path.join(work, 'live.json'), JSON.stringify(result, null, 2))
|
||||
fs.mkdirSync(work, { recursive: true })
|
||||
|
||||
const stat = readJson(path.join(work, 'static.json'), null)
|
||||
if (!stat || stat.managed !== 'ok') {
|
||||
result.skipped = 'the static half did not run'
|
||||
save()
|
||||
return log(result.skipped)
|
||||
}
|
||||
if (stat.oxideBehind !== 0) {
|
||||
result.skipped = `Oxide's ${branch} build is ${stat.oxideBehind} member(s) behind Rust's, so an Oxide server cannot boot on it`
|
||||
save()
|
||||
return log(result.skipped)
|
||||
}
|
||||
|
||||
const p = panel()
|
||||
const ids = await p.servers()
|
||||
const rig = ids[RIG]
|
||||
if (!rig) throw new Error(`no server named ${RIG} on the panel`)
|
||||
|
||||
let restart = null
|
||||
const finding = (f) => {
|
||||
result.findings.push(f)
|
||||
log('FINDING', f)
|
||||
}
|
||||
|
||||
try {
|
||||
// Room for it (D309): with both 6000-map rigs running the node's memory is full.
|
||||
const running = []
|
||||
for (const name of BIG) if (ids[name] && (await p.state(ids[name])) === 'running') running.push(name)
|
||||
if (running.length === BIG.length && ids[STOPPABLE]) {
|
||||
log(`both ${BIG.join(' and ')} are running: saving and stopping ${STOPPABLE}`)
|
||||
await p.command(ids[STOPPABLE], 'server.save').catch(() => {})
|
||||
await sleep(15000)
|
||||
await p.stop(ids[STOPPABLE])
|
||||
restart = STOPPABLE
|
||||
result.notes.push(`${STOPPABLE} was stopped for the drill and started again afterwards`)
|
||||
}
|
||||
|
||||
await p.stop(rig, 60000)
|
||||
|
||||
// What runs: this commit's plugin and harness, and the startup that installs the branch.
|
||||
await p.write(rig, '/staging-drill.sh', fs.readFileSync(path.join(__dirname, 'rig-start.sh'), 'utf8').replace(/\r\n/g, '\n'))
|
||||
await p.write(rig, '/staging-drill.branch', branch + '\n')
|
||||
await p.write(rig, '/oxide/plugins/RunicNPC.cs', fs.readFileSync(path.join(ROOT, 'plugin', 'RunicNPC.cs'), 'utf8'))
|
||||
const kits = await fetch(KITS_URL)
|
||||
if (!kits.ok) throw new Error(`Kits from uMod: ${kits.status}`)
|
||||
await p.write(rig, '/oxide/plugins/Kits.cs', await kits.text())
|
||||
await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
|
||||
await p.rm(rig, '/oxide/data/RunicNpcTest.json')
|
||||
await p.rm(rig, '/staging-drill.json')
|
||||
|
||||
log(`starting ${RIG} on ${branch}`)
|
||||
const started = Date.now()
|
||||
await p.power(rig, 'start')
|
||||
const up = await waitRunning(p, rig, 45 * 60000)
|
||||
result.installed = readJsonText(await p.readOr(rig, '/staging-drill.json', null))
|
||||
if (!up) {
|
||||
finding(`The server did not start on Rust ${branch} ${result.installed?.buildid || ''} with Oxide's ${branch} build.`)
|
||||
result.consoleTail = safeTail(await p.readOr(rig, '/latest.log', ''), 30)
|
||||
return
|
||||
}
|
||||
result.bootMinutes = Math.round((Date.now() - started) / 6000) / 10
|
||||
log(`running after ${result.bootMinutes} min:`, JSON.stringify(result.installed))
|
||||
|
||||
// RunicNPC loaded, and the navmesh ready.
|
||||
const status = await waitStatus(p, rig, 30 * 60000)
|
||||
if (!status) {
|
||||
finding('RunicNPC did not load: `rnpc.status` never answered.')
|
||||
result.oxideErrors = await oxideErrors(p, rig)
|
||||
return
|
||||
}
|
||||
result.status = status
|
||||
log('rnpc.status:', status.find((l) => l.includes('navmesh=')))
|
||||
const fieldsLine = status.find((l) => l.includes('swap fields:')) || ''
|
||||
const missing = /missing=(\S+)/.exec(fieldsLine)
|
||||
if (missing && missing[1] !== '-') finding(`The swap's fields are missing on this build: ${missing[1]}`)
|
||||
if (!fieldsLine.includes('navmesh=ready')) {
|
||||
finding('The navmesh was not ready in 30 minutes.')
|
||||
return
|
||||
}
|
||||
|
||||
// The harness needs the hidden Kits kit rnhrevolver (tools/RunicNpcTest.cs).
|
||||
await ensureKit(p, rig)
|
||||
|
||||
await p.write(rig, '/oxide/plugins/RunicNpcTest.cs', fs.readFileSync(path.join(ROOT, 'tools', 'RunicNpcTest.cs'), 'utf8'))
|
||||
let accepted = false
|
||||
for (let i = 0; i < 18 && !accepted; i++) {
|
||||
await sleep(10000)
|
||||
accepted = (await p.console(rig, 'rnt.run all', 5)).some((l) => l.includes('rnt: running all'))
|
||||
}
|
||||
if (!accepted) {
|
||||
finding('The test harness did not load (`rnt.run all` was not answered).')
|
||||
result.oxideErrors = await oxideErrors(p, rig)
|
||||
return
|
||||
}
|
||||
result.ran = true
|
||||
log('rnt.run all: running')
|
||||
|
||||
const lines = await waitHarness(p, rig, 45 * 60000)
|
||||
result.harness = summarise(lines)
|
||||
if (!result.harness.done) finding(`The harness did not finish in 45 minutes (${result.harness.pass} pass, ${result.harness.fail} fail so far).`)
|
||||
for (const f of result.harness.failed) finding(`Harness: ${f}`)
|
||||
log(`harness: ${result.harness.pass} pass, ${result.harness.fail} fail`)
|
||||
result.oxideErrors = await oxideErrors(p, rig)
|
||||
if (result.oxideErrors.length) finding(`RunicNPC logged ${result.oxideErrors.length} error line(s) in Oxide's log during the run.`)
|
||||
await p.command(rig, 'rnt.clear').catch(() => {})
|
||||
} catch (e) {
|
||||
finding(`The live half stopped: ${e.message}`)
|
||||
} finally {
|
||||
try {
|
||||
await p.stop(rig)
|
||||
await p.rm(rig, '/oxide/plugins/RunicNpcTest.cs')
|
||||
} catch (e) {
|
||||
result.notes.push(`${RIG} could not be stopped cleanly: ${e.message}`)
|
||||
}
|
||||
if (restart) {
|
||||
try {
|
||||
await p.power(ids[restart], 'start')
|
||||
log(`started ${restart} again`)
|
||||
} catch (e) {
|
||||
result.findings.push(`${restart} was stopped for the drill and could NOT be started again: ${e.message}`)
|
||||
}
|
||||
}
|
||||
save()
|
||||
}
|
||||
}
|
||||
|
||||
async function waitRunning(p, id, timeoutMs) {
|
||||
const end = Date.now() + timeoutMs
|
||||
let seenStarting = false
|
||||
while (Date.now() < end) {
|
||||
const s = await p.state(id)
|
||||
if (s === 'running') return true
|
||||
if (s === 'starting') seenStarting = true
|
||||
// Wings restarts a crashed server a few times; offline after starting is a crash it gave up on.
|
||||
if (s === 'offline' && seenStarting) return false
|
||||
await sleep(15000)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
async function waitStatus(p, id, timeoutMs) {
|
||||
const end = Date.now() + timeoutMs
|
||||
let last = null
|
||||
while (Date.now() < end) {
|
||||
const lines = (await p.console(id, 'rnpc.status', 6)).filter((l) => l.trim())
|
||||
const at = lines.findIndex((l) => l.startsWith('RunicNPC '))
|
||||
if (at >= 0) {
|
||||
last = lines.slice(at, at + 14)
|
||||
if (last.some((l) => l.includes('navmesh=ready'))) return last
|
||||
}
|
||||
await sleep(30000)
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
async function ensureKit(p, id) {
|
||||
const file = '/oxide/data/Kits/kits_data.json'
|
||||
let data = null
|
||||
for (let i = 0; i < 12 && !data; i++) {
|
||||
data = readJsonText(await p.readOr(id, file, null))
|
||||
if (!data) await sleep(5000)
|
||||
}
|
||||
if (!data) throw new Error(`Kits wrote no ${file}`)
|
||||
data._kits = data._kits || {}
|
||||
if (data._kits.rnhrevolver) return
|
||||
const item = (Shortname, Amount, Position, Condition = 0) => ({ Shortname, Skin: 0, Amount, Condition, MaxCondition: Condition, Ammo: 0, Position, Frequency: -1 })
|
||||
data._kits.rnhrevolver = {
|
||||
Name: 'rnhrevolver',
|
||||
Description: 'RunicNPC test harness: revolver + burlap (staging drill)',
|
||||
RequiredPermission: '',
|
||||
MaximumUses: 0,
|
||||
RequiredAuth: 0,
|
||||
Cooldown: 0,
|
||||
Cost: 0,
|
||||
IsHidden: true,
|
||||
CopyPasteFile: '',
|
||||
KitImage: '',
|
||||
MainItems: [item('ammo.pistol', 120, 0)],
|
||||
WearItems: [item('burlap.shirt', 1, 0), item('burlap.trousers', 1, 1), item('hat.boonie', 1, 2)],
|
||||
BeltItems: [item('pistol.revolver', 1, 0, 100)],
|
||||
}
|
||||
await p.write(id, file, JSON.stringify(data, null, 2))
|
||||
await p.command(id, 'oxide.reload Kits')
|
||||
await sleep(10000)
|
||||
log('added the rnhrevolver kit and reloaded Kits')
|
||||
}
|
||||
|
||||
function readJsonText(t) {
|
||||
try {
|
||||
return t ? JSON.parse(t) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function waitHarness(p, id, timeoutMs) {
|
||||
const end = Date.now() + timeoutMs
|
||||
let lines = []
|
||||
while (Date.now() < end) {
|
||||
await sleep(30000)
|
||||
lines = readJsonText(await p.readOr(id, '/oxide/data/RunicNpcTest.json', null)) || lines
|
||||
if (lines.some((l) => /^# done all:/.test(l))) break
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function summarise(lines) {
|
||||
const failed = lines.filter((l) => / FAIL( |$)/.test(l))
|
||||
const passed = lines.filter((l) => / PASS( |$)/.test(l))
|
||||
const done = lines.find((l) => /^# done all:/.test(l))
|
||||
return { done: !!done, summary: done || null, pass: passed.length, fail: failed.length, failed }
|
||||
}
|
||||
|
||||
/** RunicNPC's and the harness's error lines from today's Oxide log. */
|
||||
async function oxideErrors(p, id) {
|
||||
const day = now().slice(0, 10)
|
||||
const text = (await p.readOr(id, `/oxide/logs/oxide_${day}.txt`, '')) || ''
|
||||
return text
|
||||
.split('\n')
|
||||
.filter((l) => /RunicNPC|RunicNpcTest/.test(l) && /\[Error\]|Exception|Failed|error CS/.test(l))
|
||||
.slice(-40)
|
||||
}
|
||||
|
||||
/** The console's last lines, without the startup line (it carries the RCON password). */
|
||||
function safeTail(text, n) {
|
||||
return (text || '')
|
||||
.split('\n')
|
||||
.filter((l) => !/rcon\.password|\+rcon/i.test(l))
|
||||
.slice(-n)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- report: one issue per build
|
||||
|
||||
async function report(a) {
|
||||
const work = path.resolve(a.work || 'drill-work')
|
||||
const statePath = a.state
|
||||
const state = readJson(statePath, {})
|
||||
const stat = readJson(path.join(work, 'static.json'), null)
|
||||
const liv = readJson(path.join(work, 'live.json'), null)
|
||||
const branch = (stat && stat.branch) || 'staging'
|
||||
const manifest = (stat && stat.manifest) || ''
|
||||
const buildid = (liv && liv.installed && liv.installed.buildid) || ''
|
||||
// How a build is named: Steam's build id when the rig installed it, and always the manifest's date.
|
||||
const build = `${buildid ? `build ${buildid}, ` : ''}manifest ${manifest || '?'} of ${(stat && stat.manifestDate) || '?'}`
|
||||
const read = (f, n = 80) => {
|
||||
try {
|
||||
return fs.readFileSync(path.join(work, f), 'utf8').replace(/\r/g, '').split('\n').slice(0, n).join('\n').trim()
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
const findings = []
|
||||
const sections = []
|
||||
let outcome
|
||||
let markChecked = true
|
||||
|
||||
if (!stat || stat.managed !== 'ok' || stat.fields === 'error' || stat.compile === 'error') {
|
||||
outcome = 'could not run'
|
||||
markChecked = false
|
||||
findings.push(`The static half could not run: ${(stat && stat.note) || 'no static.json'}`)
|
||||
if (stat && stat.fields === 'error') sections.push(['tools/fieldlist', read('fields.diff')])
|
||||
} else if (stat.oxideBehind !== 0) {
|
||||
// A wait from an earlier drill week (or a run by hand outside one) starts over.
|
||||
const current = state.waiting && state.waiting.manifest === manifest && Date.now() - Date.parse(state.waiting.since) < 7 * 86400000
|
||||
const waiting = current ? state.waiting : { manifest, since: now() }
|
||||
const days = (Date.now() - Date.parse(waiting.since)) / 86400000
|
||||
if (days < WAIT_DAYS) {
|
||||
outcome = `waiting for Oxide (${stat.oxideBehind} behind, day ${Math.floor(days) + 1} of ${WAIT_DAYS})`
|
||||
if (branch === 'staging') {
|
||||
state.waiting = { ...waiting, behind: stat.oxideBehind, lastSeen: now() }
|
||||
markChecked = false
|
||||
}
|
||||
} else {
|
||||
outcome = 'Oxide has not caught up'
|
||||
findings.push(
|
||||
`Oxide's staging build is still ${stat.oxideBehind} member(s) behind Rust's staging build after ${WAIT_DAYS} days, ` +
|
||||
'so an Oxide server cannot boot on it and the live half did not run. The static half ran against Oxide\'s older build.'
|
||||
)
|
||||
sections.push(['What Oxide lacks (first 20)', read('oxide-behind.txt', 21)])
|
||||
}
|
||||
}
|
||||
|
||||
if (!outcome || outcome === 'Oxide has not caught up') {
|
||||
if (stat.fields === 'differs') {
|
||||
findings.push("The swap's field list differs from the one the plugin carries (D232): regenerate it with tools/fieldlist and check what changed.")
|
||||
sections.push(['Field list (carried → staging)', read('fields.diff', 120)])
|
||||
}
|
||||
if (stat.compile === 'errors') {
|
||||
findings.push("RunicNPC does not compile against this build with Oxide's staging build.")
|
||||
sections.push(['Compiler output', read('compile.txt', 60)])
|
||||
}
|
||||
if (!liv && stat.oxideBehind === 0) findings.push('The live half did not run: see the job log (is PTERODACTYL_DRILL_KEY set?).')
|
||||
if (liv && !liv.skipped) {
|
||||
findings.push(...liv.findings)
|
||||
if (liv.consoleTail && liv.consoleTail.length) sections.push(['Console, last lines', liv.consoleTail.join('\n')])
|
||||
if (liv.oxideErrors && liv.oxideErrors.length) sections.push(["Oxide's log (RunicNPC lines)", liv.oxideErrors.join('\n')])
|
||||
if (liv.status) sections.push(['rnpc.status', liv.status.join('\n')])
|
||||
}
|
||||
if (!outcome) outcome = findings.length ? 'fail' : 'pass'
|
||||
if (branch === 'staging' && markChecked) delete state.waiting
|
||||
}
|
||||
|
||||
const installed = liv && liv.installed ? `Rust ${liv.installed.branch} ${liv.installed.buildid}, Oxide build of ${liv.installed.oxideModified}` : ''
|
||||
const lines = [
|
||||
`Static: Oxide behind ${stat ? stat.oxideBehind : '?'}, fields ${stat ? stat.fields : '?'}, compile ${stat ? stat.compile : '?'}.`,
|
||||
liv ? (liv.skipped ? `Live: skipped (${liv.skipped}).` : `Live: ${liv.harness ? `${liv.harness.pass} pass, ${liv.harness.fail} fail` : 'did not reach the harness'}${installed ? `, on ${installed}` : ''}${liv.bootMinutes ? `, booted in ${liv.bootMinutes} min` : ''}.`) : 'Live: did not run.',
|
||||
]
|
||||
log(`${build} on ${branch}: ${outcome}`)
|
||||
lines.forEach((l) => log(l))
|
||||
findings.forEach((f) => log('-', f))
|
||||
if (process.env.GITHUB_STEP_SUMMARY) {
|
||||
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Staging drill: ${build}: ${outcome}\n\n${lines.join('\n\n')}\n\n${findings.map((f) => `- ${f}`).join('\n')}\n`)
|
||||
}
|
||||
|
||||
// A rehearsal on another branch reports here and changes nothing.
|
||||
if (branch !== 'staging') return log('not the staging branch: no issue, no state')
|
||||
|
||||
let issue = null
|
||||
if (findings.length) {
|
||||
const body = [
|
||||
`The staging drill (docs/runicnpc/PLAN.md stage 9c) checked Rust's staging branch (**${build}**) on ${now().slice(0, 10)}: **${outcome}**.`,
|
||||
'',
|
||||
...findings.map((f) => `- ${f}`),
|
||||
'',
|
||||
...lines,
|
||||
...sections.flatMap(([title, text]) => (text ? ['', `### ${title}`, '', '```', text, '```'] : [])),
|
||||
'',
|
||||
process.env.GITHUB_RUN_NUMBER ? `Run: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_NUMBER}` : '',
|
||||
].join('\n')
|
||||
issue = await fileIssue(`Staging drill: Rust staging manifest ${manifest} (${(stat && stat.manifestDate) || '?'})`, body)
|
||||
}
|
||||
|
||||
if (markChecked) state.checked = { manifest, buildid, at: now(), outcome, issue, oxideBehind: stat ? stat.oxideBehind : null }
|
||||
state.lastRun = { at: now(), manifest, buildid, outcome, issue }
|
||||
fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n')
|
||||
output('outcome', outcome)
|
||||
}
|
||||
|
||||
async function fileIssue(title, body) {
|
||||
const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}`
|
||||
const headers = { Authorization: 'token ' + process.env.GITEA_TOKEN, 'Content-Type': 'application/json', Accept: 'application/json' }
|
||||
const call = async (method, p, data) => {
|
||||
const r = await fetch(base + p, { method, headers, body: data ? JSON.stringify(data) : undefined })
|
||||
if (!r.ok) throw new Error(`${method} ${p}: ${r.status} ${(await r.text()).slice(0, 200)}`)
|
||||
return r.json()
|
||||
}
|
||||
|
||||
const open = await call('GET', `/issues?state=open&type=issues&labels=staging-drill&limit=50`)
|
||||
const same = open.find((i) => i.title === title)
|
||||
if (same) {
|
||||
await call('POST', `/issues/${same.number}/comments`, { body })
|
||||
log(`commented on #${same.number}`)
|
||||
return same.number
|
||||
}
|
||||
const labels = await call('GET', '/labels?limit=100')
|
||||
let label = labels.find((l) => l.name === 'staging-drill')
|
||||
if (!label) label = await call('POST', '/labels', { name: 'staging-drill', color: '#d93f0b', description: "Rust's staging branch breaks RunicNPC (the daily staging drill)" })
|
||||
const created = await call('POST', '/issues', { title, body, labels: [label.id] })
|
||||
log(`opened #${created.number}`)
|
||||
return created.number
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- main
|
||||
|
||||
const a = args()
|
||||
const commands = { window: window_, gate, live, report }
|
||||
const command = commands[a._[0]]
|
||||
if (!command) {
|
||||
console.error('usage: node tools/drill/drill.js window|gate|live|report [--state f] [--work dir] [--branch b] [--force]')
|
||||
process.exit(2)
|
||||
}
|
||||
command(a).catch((e) => {
|
||||
console.error(e.message || e)
|
||||
process.exit(1)
|
||||
})
|
||||
68
tools/drill/rig-start.sh
Normal file
68
tools/drill/rig-start.sh
Normal file
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
# The staging drill rig's startup (docs/runicnpc/PLAN.md D307). Never shipped.
|
||||
#
|
||||
# rust-staging runs the published egg in its vanilla mode. That image's entrypoint always runs
|
||||
# `app_update 258550` before the startup, and the egg cannot be told otherwise, so the server's
|
||||
# startup is this script in front of the egg's own command line:
|
||||
#
|
||||
# bash ./staging-drill.sh ./RustDedicated -batchmode …
|
||||
#
|
||||
# It moves the install to a Steam branch, lays Oxide's build for that branch over it, writes down what
|
||||
# it installed (staging-drill.json), and becomes the server. The branch is `staging` unless the file
|
||||
# staging-drill.branch names another (`public` rehearses the drill on a pair known to match).
|
||||
# tools/drill/drill.js writes this file to the rig as /staging-drill.sh before every start, so the
|
||||
# copy in git is the one that runs.
|
||||
#
|
||||
# Steam keeps the branch: the entrypoint's own app_update, which names none, then answers "already
|
||||
# up to date" and leaves a staging install on staging (seen 2026-10-06).
|
||||
|
||||
cd /home/container || exit 1
|
||||
|
||||
branch=staging
|
||||
if [ -s staging-drill.branch ]; then branch=$(tr -dc 'a-z0-9-' < staging-drill.branch); fi
|
||||
if [ "$branch" = public ]; then
|
||||
OXIDE_URL=https://github.com/OxideMod/Oxide.Rust/releases/latest/download/Oxide.Rust-linux.zip
|
||||
else
|
||||
OXIDE_URL=https://downloads.oxidemod.com/artifacts/Oxide.Rust/$branch/Oxide.Rust-linux.zip
|
||||
fi
|
||||
|
||||
# The branch Steam has INSTALLED: MountedConfig's key (UserConfig's is only the one asked for). The
|
||||
# public branch has no key.
|
||||
mounted() {
|
||||
sed -n '/"MountedConfig"/,/}/s/^[[:space:]]*"BetaKey"[[:space:]]*"\([^"]*\)".*/\1/p' steamapps/appmanifest_258550.acf | head -1
|
||||
}
|
||||
|
||||
# A branch switch downloads the whole server (5.4 GB), and one can fail ("UpdateResult" 13 on
|
||||
# 2026-10-06, the next attempt fine). Rust on one branch with Oxide's build for another dies at boot,
|
||||
# so the server is not started on a pair that does not match.
|
||||
want=$branch
|
||||
[ "$want" = public ] && want=""
|
||||
for attempt in 1 2; do
|
||||
echo "[staging-drill] Rust: app_update 258550 -beta $branch (attempt $attempt)"
|
||||
./steamcmd/steamcmd.sh +force_install_dir /home/container +login anonymous +app_update 258550 -beta "$branch" +quit
|
||||
installed=$(mounted)
|
||||
[ "${installed:-public}" = "${want:-public}" ] && break
|
||||
done
|
||||
if [ "${installed:-public}" != "${want:-public}" ]; then
|
||||
echo "[staging-drill] Steam installed '${installed:-public}', not '$branch': not starting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Oxide's build carries its own patched Assembly-CSharp.dll, so it goes on AFTER Rust.
|
||||
echo "[staging-drill] Oxide: $OXIDE_URL"
|
||||
oxide_modified=$(curl -sSIL "$OXIDE_URL" | tr -d '\r' | sed -n 's/^[Ll]ast-[Mm]odified: //p' | tail -1)
|
||||
if curl -sSL --fail -o oxide-drill.zip "$OXIDE_URL" && unzip -o -q oxide-drill.zip; then
|
||||
oxide_ok=true
|
||||
else
|
||||
oxide_ok=false
|
||||
echo "[staging-drill] Oxide's $branch build could not be installed"
|
||||
fi
|
||||
rm -f oxide-drill.zip
|
||||
|
||||
buildid=$(sed -n 's/^[[:space:]]*"buildid"[[:space:]]*"\([0-9]*\)".*/\1/p' steamapps/appmanifest_258550.acf | head -1)
|
||||
printf '{"buildid":"%s","branch":"%s","oxideModified":"%s","oxideInstalled":%s,"at":"%s"}\n' \
|
||||
"$buildid" "${installed:-public}" "$oxide_modified" "$oxide_ok" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > staging-drill.json
|
||||
echo "[staging-drill] $(cat staging-drill.json)"
|
||||
if [ "$oxide_ok" != true ]; then exit 1; fi
|
||||
|
||||
exec "$@"
|
||||
145
tools/drill/static.sh
Normal file
145
tools/drill/static.sh
Normal file
@@ -0,0 +1,145 @@
|
||||
#!/bin/bash
|
||||
# The staging drill's static half (docs/runicnpc/PLAN.md D308). Never shipped.
|
||||
#
|
||||
# tools/drill/static.sh (WORK=drill-work BRANCH=staging by default)
|
||||
#
|
||||
# Needs no server. It downloads only RustDedicated_Data/Managed/*.dll from a Steam branch, and
|
||||
# Oxide's build for that branch, and:
|
||||
#
|
||||
# 1. asks whether Oxide's build has caught up with Rust's (tools/fieldlist --oxide-behind). Until it
|
||||
# has, an Oxide server on that branch dies at boot, so the drill waits for it (D317);
|
||||
# 2. regenerates the swap's field list (tools/fieldlist, D232) from Rust's own Assembly-CSharp.dll
|
||||
# and compares it with the one plugin/RunicNPC.cs carries;
|
||||
# 3. compiles plugin/RunicNPC.cs against those assemblies with Oxide's laid over them, as an Oxide
|
||||
# server would. Oxide's patched Assembly-CSharp.dll is needed: the plugin uses members Rust
|
||||
# ships non-public (IAISenses, for one).
|
||||
#
|
||||
# It writes $WORK/static.json and, beside it, oxide-behind.txt, fields.diff and compile.txt. In
|
||||
# static.json, oxideBehind is a count (-1: not measured), fields "same"/"differs", compile
|
||||
# "ok"/"errors", and either is "error" when its step could not run. The script exits 0 whenever it
|
||||
# ran; what it found is in static.json. DEPOTDOWNLOADER and DOTNET point at other copies (on
|
||||
# Windows, DEPOTDOWNLOADER=…/DepotDownloader.exe from Git Bash).
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
WORK=${WORK:-$ROOT/drill-work}
|
||||
BRANCH=${BRANCH:-staging}
|
||||
if [ "$BRANCH" = public ]; then
|
||||
OXIDE_URL=${OXIDE_URL:-https://github.com/OxideMod/Oxide.Rust/releases/latest/download/Oxide.Rust-linux.zip}
|
||||
else
|
||||
OXIDE_URL=${OXIDE_URL:-https://downloads.oxidemod.com/artifacts/Oxide.Rust/$BRANCH/Oxide.Rust-linux.zip}
|
||||
fi
|
||||
DD_URL=https://github.com/SteamRE/DepotDownloader/releases/download/DepotDownloader_3.4.0/DepotDownloader-linux-x64.zip
|
||||
DOTNET=${DOTNET:-dotnet}
|
||||
|
||||
mkdir -p "$WORK"
|
||||
rm -rf "$WORK/depot" "$WORK/oxide" "$WORK/refs" "$WORK/fields"
|
||||
rm -f "$WORK"/static.json "$WORK"/oxide-behind.txt "$WORK"/fields.diff "$WORK"/compile.txt
|
||||
|
||||
managed=error
|
||||
behind=-1
|
||||
fields=error
|
||||
compile=error
|
||||
note=""
|
||||
oxide_modified=""
|
||||
manifest=""
|
||||
manifest_date=""
|
||||
|
||||
finish() {
|
||||
printf '{"branch":"%s","manifest":"%s","manifestDate":"%s","managed":"%s","oxideBehind":%s,"fields":"%s","compile":"%s","oxideModified":"%s","note":"%s"}\n' \
|
||||
"$BRANCH" "$manifest" "$manifest_date" "$managed" "$behind" "$fields" "$compile" "$oxide_modified" "$note" > "$WORK/static.json"
|
||||
cat "$WORK/static.json"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Git Bash does not translate paths inside a response file; elsewhere this is the path as it is.
|
||||
native() { if command -v cygpath > /dev/null; then cygpath -m "$1"; else printf '%s' "$1"; fi; }
|
||||
|
||||
# ---- Rust's managed assemblies, from Steam (no login: the dedicated server is anonymous) ----
|
||||
|
||||
DD=${DEPOTDOWNLOADER:-}
|
||||
if [ -z "$DD" ]; then
|
||||
curl -sSL --fail -o "$WORK/dd.zip" "$DD_URL" && unzip -oq "$WORK/dd.zip" -d "$WORK/dd" && chmod +x "$WORK/dd/DepotDownloader" \
|
||||
|| { note="DepotDownloader could not be fetched"; finish; }
|
||||
DD=$WORK/dd/DepotDownloader
|
||||
fi
|
||||
|
||||
printf 'regex:^RustDedicated_Data/Managed/[^/]+\\.dll$\n' > "$WORK/files.txt"
|
||||
"$DD" -app 258550 -depot 258552 -branch "$BRANCH" -filelist "$WORK/files.txt" -dir "$WORK/depot" -os linux \
|
||||
> "$WORK/depot.txt" 2>&1
|
||||
MANAGED=$WORK/depot/RustDedicated_Data/Managed
|
||||
# Which build this is, from Steam itself: the Linux depot's manifest and its date. (api.steamcmd.net
|
||||
# answered a build id a day stale on 2026-10-06, so the drill does not ask it.)
|
||||
manifest=$(tr -d '\r' < "$WORK/depot.txt" | sed -n 's/^Manifest \([0-9]*\) (\(.*\))$/\1/p' | head -1)
|
||||
manifest_date=$(tr -d '\r' < "$WORK/depot.txt" | sed -n 's/^Manifest \([0-9]*\) (\(.*\))$/\2/p' | head -1)
|
||||
if [ ! -f "$MANAGED/Assembly-CSharp.dll" ]; then
|
||||
note="no Assembly-CSharp.dll from Steam's $BRANCH branch: $(tail -3 "$WORK/depot.txt" | tr '\n"' ' ')"
|
||||
finish
|
||||
fi
|
||||
managed=ok
|
||||
|
||||
# ---- Oxide's build for the branch ----
|
||||
|
||||
oxide_modified=$(curl -sSIL "$OXIDE_URL" | tr -d '\r' | sed -n 's/^[Ll]ast-[Mm]odified: //p' | tail -1)
|
||||
if ! { curl -sSL --fail -o "$WORK/oxide.zip" "$OXIDE_URL" && unzip -oq "$WORK/oxide.zip" -d "$WORK/oxide"; }; then
|
||||
note="Oxide's $BRANCH build could not be downloaded"
|
||||
finish
|
||||
fi
|
||||
OXIDE_MANAGED=$WORK/oxide/RustDedicated_Data/Managed
|
||||
|
||||
# The tool is built once, here, so a build failure is not mistaken for anything it reports.
|
||||
if ! "$DOTNET" build "$ROOT/tools/fieldlist" -c Release -o "$WORK/fieldlist" > "$WORK/fieldlist-build.txt" 2>&1; then
|
||||
note="tools/fieldlist did not build: $(grep -m1 'error' "$WORK/fieldlist-build.txt" | tr '"' ' ')"
|
||||
finish
|
||||
fi
|
||||
FIELDLIST=("$DOTNET" "$WORK/fieldlist/FieldList.dll")
|
||||
|
||||
# ---- 1. Has Oxide caught up? ----
|
||||
|
||||
if "${FIELDLIST[@]}" --oxide-behind "$MANAGED" "$OXIDE_MANAGED/Assembly-CSharp.dll" > "$WORK/oxide-behind.txt" 2>&1; then
|
||||
behind=$(sed -n 's/^behind \([0-9]*\)$/\1/p' "$WORK/oxide-behind.txt" | head -1)
|
||||
behind=${behind:--1}
|
||||
fi
|
||||
|
||||
# ---- 2. The swap's field list (D232) ----
|
||||
|
||||
mkdir -p "$WORK/fields"
|
||||
cp "$ROOT/plugin/RunicNPC.cs" "$WORK/fields/RunicNPC.cs"
|
||||
block() { sed -n '/\/\/ <fieldlist>/,/\/\/ <\/fieldlist>/p' "$1" | grep -v "Generated by tools/fieldlist" | tr -d '\r'; }
|
||||
if "${FIELDLIST[@]}" "$MANAGED" "$WORK/fields/RunicNPC.cs" > "$WORK/fields/run.txt" 2>&1; then
|
||||
if diff -u --label carried <(block "$ROOT/plugin/RunicNPC.cs") --label "$BRANCH" <(block "$WORK/fields/RunicNPC.cs") > "$WORK/fields.diff"; then
|
||||
fields=same
|
||||
else
|
||||
fields=differs
|
||||
fi
|
||||
else
|
||||
cp "$WORK/fields/run.txt" "$WORK/fields.diff"
|
||||
fi
|
||||
|
||||
# ---- 3. Compile against the branch with Oxide's build laid over ----
|
||||
|
||||
mkdir -p "$WORK/refs"
|
||||
cp "$MANAGED"/*.dll "$WORK/refs/"
|
||||
cp "$OXIDE_MANAGED"/*.dll "$WORK/refs/"
|
||||
# Oxide.References.dll bundles its own older copy of Newtonsoft.Json, which Oxide's compiler never
|
||||
# references; with it, every JObject is ambiguous.
|
||||
rm -f "$WORK/refs/Oxide.References.dll"
|
||||
|
||||
DOTNET_DIR=$(dirname "$(readlink -f "$(command -v "$DOTNET")")")
|
||||
CSC=$(ls "$DOTNET_DIR"/sdk/*/Roslyn/bincore/csc.dll 2>/dev/null | tail -1)
|
||||
if [ -z "$CSC" ]; then
|
||||
note="no csc.dll under $DOTNET_DIR/sdk"
|
||||
finish
|
||||
fi
|
||||
|
||||
# A response file: some 250 references outgrow a Windows command line.
|
||||
for f in "$WORK/refs"/*.dll; do printf '"-r:%s"\n' "$(native "$f")"; done > "$WORK/refs.rsp"
|
||||
if "$DOTNET" "$CSC" -nologo -noconfig -nostdlib -target:library -nowarn:1701,1702 -warn:0 \
|
||||
-out:"$WORK/RunicNPC.dll" "@$WORK/refs.rsp" "$ROOT/plugin/RunicNPC.cs" > "$WORK/compile.txt" 2>&1; then
|
||||
compile=ok
|
||||
else
|
||||
compile=errors
|
||||
fi
|
||||
|
||||
finish
|
||||
@@ -12,12 +12,28 @@
|
||||
// the unmodified assembly, it picks the same fields on both frameworks. The result replaces the
|
||||
// block between the two marker lines in the plugin, and the plugin resolves each name at load.
|
||||
|
||||
//
|
||||
// The staging drill (D308) also asks it whether Oxide's build has caught up with Rust's:
|
||||
//
|
||||
// dotnet run --project tools/fieldlist -- --oxide-behind <managed dir> <Oxide's Assembly-CSharp.dll>
|
||||
//
|
||||
// Oxide ships its own patched Assembly-CSharp.dll, which replaces Rust's. Built from an older Rust,
|
||||
// it lacks what Rust has added since, and the server dies at boot ("The referenced script
|
||||
// (ItemModFoodVisual) on this Behaviour is missing!"). Oxide's patcher only ever adds, so every
|
||||
// type and member Rust's own assembly declares, compiler-generated ones aside, must be in Oxide's.
|
||||
// It prints "behind <n>" and the first of them; 0 means caught up. Measured 2026-10-06: public Rust
|
||||
// and public Oxide, 0; staging Rust 25766353 and Oxide's staging build of 2026-10-05, 482.
|
||||
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
|
||||
if (args.Length == 3 && args[0] == "--oxide-behind")
|
||||
return OxideBehind(args[1], args[2]);
|
||||
|
||||
if (args.Length < 1)
|
||||
{
|
||||
Console.Error.WriteLine("usage: dotnet run --project tools/fieldlist -- <managed dir> [plugin/RunicNPC.cs]");
|
||||
Console.Error.WriteLine(" dotnet run --project tools/fieldlist -- --oxide-behind <managed dir> <Oxide's Assembly-CSharp.dll>");
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -96,3 +112,28 @@ string replacement = block.ToString().Replace("\r\n", "\n").Replace("\n", nl);
|
||||
File.WriteAllText(plugin, text[..start] + replacement + text[(end + Close.Length)..]);
|
||||
Console.WriteLine($"{plugin}: {npc.Count} NPC fields, {brain.Count} brain fields (Assembly-CSharp module {mvid}).");
|
||||
return 0;
|
||||
|
||||
static int OxideBehind(string managed, string oxideAssembly)
|
||||
{
|
||||
HashSet<string> Declared(string assembly)
|
||||
{
|
||||
IEnumerable<string> others = Directory.GetFiles(managed, "*.dll").Where(f => Path.GetFileName(f) != "Assembly-CSharp.dll");
|
||||
using var context = new MetadataLoadContext(new PathAssemblyResolver(others.Append(assembly)), "mscorlib");
|
||||
var names = new HashSet<string>();
|
||||
foreach (Type t in context.LoadFromAssemblyPath(assembly).GetTypes())
|
||||
{
|
||||
names.Add(t.FullName!);
|
||||
foreach (MemberInfo m in t.GetMembers(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly))
|
||||
names.Add($"{t.FullName}::{m.Name}");
|
||||
}
|
||||
// Compiler-generated names (lambdas, closures, iterators) are renumbered by every build.
|
||||
names.RemoveWhere(n => n.Contains('<'));
|
||||
return names;
|
||||
}
|
||||
|
||||
List<string> missing = Declared(Path.Combine(managed, "Assembly-CSharp.dll")).Except(Declared(oxideAssembly)).OrderBy(n => n).ToList();
|
||||
Console.WriteLine($"behind {missing.Count}");
|
||||
foreach (string n in missing.Take(20))
|
||||
Console.WriteLine(" " + n);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"tokenFile": "/path/to/pterodactyl_api_token",
|
||||
"rigs": {
|
||||
"oxide": "00000000",
|
||||
"carbon": "00000000"
|
||||
"carbon": "00000000",
|
||||
"staging": "00000000"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user