Module-Rust/server/boot.js

// ── The lifecycle hooks ───────────────────────────────────────────────────
//
// `register()` may not touch the database (MODULE_API.md §2.2). This file is
// where everything it could not do goes.
//
//   core schema  →  this module's schema fragment  →  onBoot(ctx)  →  the listener binds
//
// So by the time `onBoot` runs the tables exist, core's settings are seeded, and
// nothing is serving traffic yet.
//
// **`onBoot` has no timeout.** Shutdown races the process being killed; boot does
// not. A slow `onBoot` delays the listener, which is the promise above rather
// than a problem to be timed out.
//
// **If `onBoot` throws, the module is `startup_failed` and the site still comes
// up.** Its routes stay mounted but answer 503, because a module that failed to
// warm up serving half-initialised data is worse than one that says it is down.
// There is then NO `onShutdown` — being handed a half-built world to tear down is
// worse than not closing cleanly. Which is why the poll below catches everything:
// a sidecar that is not there yet is the ordinary state of a fresh install, and
// letting that fail the boot would make installing the module before installing
// the bridge impossible.
//
// ── Four timers, and they answer four different questions ─────────────────
//
//   refresh   (30s)  what is each server, and who is on it — the BOARDS
//   ingest     (5s)  what has happened since we last looked — the CURSOR
//   sweep      (1m)  which login attempts were never let in (PLAN.md §25)
//   prune       (1h)  forgetting the detail we promised not to keep for ever
//
// The boards poll and the ingest are deliberately separate rather than one loop
// reading both. They fail differently and they matter differently: a board that
// is 30 seconds stale shows a player count slightly behind, and an ingest that
// is 30 seconds behind shows a killfeed that feels broken. Splitting them lets
// the cheap one run often and the expensive one run rarely, and it means a
// sidecar that answers one and not the other degrades in exactly one place.
//
// The poll was never a placeholder for a socket: a sidecar's store-backed reads
// are what answer while a game server is off, which is most of what this module
// renders. See `ingest.js` for why the live feed is a cursor and not a
// WebSocket.

const core = require('./core')

const db = require('./model/servers/servers.db')
const engagement = require('./engagement/emit')
const eventsDb = require('./model/events/events.db')
const eventWorld = require('./eventWorld')
const ingest = require('./ingest')
const permSync = require('./permSync')
const servers = require('./model/servers/servers.model')
const sidecar = require('./sidecarClient')

const log = core.logger('boot')

let refreshTimer = null
let ingestTimer = null
let pruneTimer = null
let sweepTimer = null

const REFRESH_MS = 30 * 1000
const INGEST_MS = 5 * 1000
const PRUNE_MS = 60 * 60 * 1000
const SWEEP_MS = 60 * 1000

/**
 * How long this module keeps raw events.
 *
 * Longer than the sidecar's 14 days, because this is the richer store and the
 * one a page reads — and because the sidecar lives on somebody's game host while
 * this lives on the website's own database. What is NOT bounded by it is the
 * record: `rust_player_wipe_stats` and `rust_gather_totals` are permanent, which
 * is the whole of R12's "a wipe does not erase a player's history".
 */
const EVENT_RETENTION_DAYS = 30

/**
 * Ask every configured sidecar how its server is doing, and store what it said.
 *
 * **Every server is polled independently and one failure never stops the
 * others.** `Promise.allSettled`, not `Promise.all`: six servers behind one
 * unreachable host would otherwise mean the whole fleet stops updating because
 * one of them does, and the site would report five healthy servers offline.
 */
async function refresh() {
  let rows
  try {
    rows = await servers.listForPolling()
  } catch (err) {
    log.warn('could not read the server list', { error: err.message })
    return
  }

  await Promise.allSettled(rows.map(refreshOne))
}

async function refreshOne(server) {
  try {
    // One call for both boards. `/server` would answer the same question about
    // the server itself, but presence would then be a second round trip to the
    // same process for a fact it already had in hand.
    //
    // **And one for `/health`, because the boards cannot say whether the game is
    // there NOW** (D68, PLAN.md §25.1). The sidecar keeps its last `server.hello`
    // after the plugin disconnects — that is what lets a page render a server
    // that is off — so a game that hung, or whose bridge was unloaded, while the
    // sidecar stayed up read as online here from phase 4 until phase 10. Only
    // `/health`'s `plugin_connected` answers the question, and the two are asked
    // together so they describe the same moment.
    const [board, health] = await Promise.all([sidecar.boards(server), sidecar.health(server)])

    // Three outcomes, and collapsing any two of them loses something an operator
    // needs:
    //
    //   • the sidecar answered with a frame   → the server has connected at least once
    //   • the sidecar answered 204 (`empty`)  → the sidecar is up and the game never connected
    //   • the sidecar did not answer          → the bridge is unreachable
    //
    // The middle case is the one that is easy to lose. It is a fresh install
    // whose plugin is not loaded yet, and reporting it as unreachable sends the
    // operator to look at the network instead of at the game server.
    if (!board.ok) {
      // `markUnreachable`, not `putState`: nothing answered, so the only new fact
      // is that nothing answered. Writing the whole row from that one fact would
      // blank the hostname, the map, the seed and the wipe — the last thing this
      // server said, which is exactly what the pages exist to render while it is
      // off.
      await db.markUnreachable(server.id, false)
      engagement.serverObserved(server, false)
      return
    }

    const boards = (board.data && board.data.boards) || {}
    const frame = boards['server.hello']

    if (!frame) {
      // The sidecar is up and has never heard from the game. Presence is emptied
      // rather than left alone: a stale list of players on a server nobody can
      // reach is worse than an empty one, because it looks current.
      await db.markUnreachable(server.id, true)
      await ingest.applyBoards(server.id, {})
      engagement.serverObserved(server, false)
      return
    }

    // Unknown is not connected. A `/health` that did not answer while `/boards`
    // did is odd enough to be worth a line, and reporting the server up on the
    // strength of a board the game may have left behind hours ago is the defect
    // this call exists to remove.
    const connected = Boolean(health.ok && health.data && health.data.plugin_connected === true)
    if (!health.ok) log.warn('the sidecar answered /boards but not /health', { server: server.id })

    // The presence board is the plugin's last word too. While the game is not
    // connected it names people as online who may have left hours ago, which is
    // both wrong and — under §23's rule — a claim about named people nobody made.
    await ingest.applyBoards(
      server.id,
      connected ? boards : { ...boards, 'players.online': { players: [] } },
    )

    await db.putState({
      serverId: server.id,
      reachable: true,
      // The plugin is connected NOW (D68). A stored `server.hello` only says it
      // connected once; the model still applies its own freshness on top.
      online: connected,
      // A board the game left behind is a description, not a sighting.
      seen: connected,
      players: connected ? Number(frame.players) || 0 : 0,
      maxPlayers: Number(frame.maxPlayers) || 0,
      hostname: frame.hostname || null,
      level: frame.level || null,
      seed: frame.seed === undefined ? null : Number(frame.seed),
      worldSize: frame.worldSize === undefined ? null : Number(frame.worldSize),
      bootId: frame.bootId || null,
      saveCreatedAt: frame.saveCreatedAt || null,
      wipeId: frame.wipeId || null,
      protocol: frame.protocol === undefined ? null : Number(frame.protocol),
      raw: frame,
    })

    // After the write, so a transition announced is one a page already shows.
    engagement.serverObserved(server, connected)

    // A restart or a wipe under a running event is the moment core must be told
    // to ask what the world still holds (§11.1). Only a CONNECTED plugin's hello
    // counts: a board the game left behind says nothing about now.
    if (connected) {
      eventWorld.observeServer(server.id, { bootId: frame.bootId, wipeId: frame.wipeId, worldReady: frame.worldReady })
    }
  } catch (err) {
    // A failure here is one server's, and it must not reach `Promise.allSettled`
    // as a rejection that hides which one. Log with the id and carry on.
    log.warn('could not refresh a server', { server: server.id, error: err.message })
  }
}

/**
 * Runs once, after the schema and before the listener binds.
 *
 * Receives the same frozen `ctx` `register()` was given — not a second object
 * built to look like it — so a module that only needs core at boot time can skip
 * `core.init` entirely and use this argument.
 */
/** Runs the cursor for every configured server, independently. */
async function ingestAll() {
  let rows

  try {
    rows = await servers.listForPolling()
  } catch (err) {
    log.warn('could not read the server list', { error: err.message })
    return
  }

  // `allSettled`, for the same reason the board poll uses it: six servers behind
  // one unreachable host must not stop the other five being ingested.
  await Promise.allSettled(rows.map((server) => ingest.ingestServer(server)))
}

async function prune() {
  try {
    const gone = await eventsDb.pruneEvents(EVENT_RETENTION_DAYS)
    if (gone > 0) log.info('pruned old events', { events: gone, days: EVENT_RETENTION_DAYS })
  } catch (err) {
    log.warn('could not prune events', { error: err.message })
  }
}

/**
 * Login attempts that were never approved (D64, PLAN.md §25).
 *
 * On its own minute timer rather than the prune's hour: an attempt waits a
 * minute for its approval, and a staff alert an hour late is not an alert. A
 * query over stored rows, so it needs nothing kept in memory and a restart loses
 * nothing; the dedupe key makes a second pass over the same attempt a no-op.
 */
async function sweep() {
  let rows
  try {
    rows = await servers.listForPolling()
  } catch (err) {
    log.warn('could not read the server list', { error: err.message })
    return
  }
  const sent = await engagement.sweepLoginDenied(rows)
  if (sent > 0) log.info('unapproved logins reported', { attempts: sent })
}

async function onBoot() {
  await refresh()
  // The permission mirror owns its own loop and its own cadence (see
  // `permSync.js`). It is started rather than run here: a first pass would write
  // to every configured game server before the website had finished booting, and
  // nothing about R2 is urgent enough to delay a listener for.
  permSync.start()
  refreshTimer = setInterval(refresh, REFRESH_MS)
  ingestTimer = setInterval(ingestAll, INGEST_MS)
  pruneTimer = setInterval(prune, PRUNE_MS)
  sweepTimer = setInterval(sweep, SWEEP_MS)
  // Node keeps the process alive for a pending timer. Core's own intervals are
  // unref'd for exactly this reason: a module that forgets turns `Ctrl-C` into a
  // thirty-second wait, and on a host it turns a `systemctl stop` into a SIGKILL.
  for (const timer of [refreshTimer, ingestTimer, pruneTimer, sweepTimer]) {
    if (timer && typeof timer.unref === 'function') timer.unref()
  }

  log.info('booted', { refreshMs: REFRESH_MS, ingestMs: INGEST_MS, permSyncMs: permSync.TICK_MS })
}

/**
 * Runs on SIGINT/SIGTERM, before core closes anything of its own.
 *
 * The database pool, the push dispatcher and the SSE fan-out are all still open,
 * because flushing through them is the only thing this hook is for. There is a
 * five-second budget per module, after which the hook is abandoned — abandoned
 * rather than cancelled, since nothing can stop a promise that is still running.
 */
async function onShutdown() {
  permSync.stop()

  for (const timer of [refreshTimer, ingestTimer, pruneTimer, sweepTimer]) {
    if (timer) clearInterval(timer)
  }

  refreshTimer = null
  ingestTimer = null
  pruneTimer = null
  sweepTimer = null

  log.info('shut down')
}

module.exports = {
  onBoot,
  onShutdown,
  refresh,
  refreshOne,
  ingestAll,
  prune,
  REFRESH_MS,
  INGEST_MS,
  EVENT_RETENTION_DAYS,
}