Module-Rust/server/permSync.js

// ── Keeping a game's permission store equal to what the site authored ─────
//
// R2's whole mechanism, and it is chapter 4's board pointed the other way: the
// site is the single producer of a set, it re-sends the whole thing rather than
// a stream of edits, and the receiver reconciles. What is new is the direction —
// the module telling the game what the site knows, where every earlier phase
// asked the game what it knew.
//
// ── One verb (D32) ────────────────────────────────────────────────────────
//
// A sync sends the whole desired set and the plugin diffs it against the live
// store. The website never holds a copy of the game's permissions, which is the
// point: a second source of truth is stale the moment it lands, and the store is
// the bigger of the two sets.
//
// The delta the site DOES compute is the one the game cannot: what this site put
// there and has since withdrawn (`retirements`). A name in the store that is not
// in the desired set is either that, or a hand edit — and only the pushed ledger
// can tell them apart (D31).
//
// ── When it runs ──────────────────────────────────────────────────────────
//
// Every tick asks a cheap question — does the digest of the desired set still
// equal what this server last confirmed — and does nothing when the answer is
// yes. A sync therefore happens when:
//
//   • an operator changed something (the dirty flag, and the digest behind it)
//   • the game restarted or wiped (a new boot id or wipe id: the store may have
//     been emptied, and R2's promise is that a wipe is not a data-loss event)
//   • a permission hook fired in the game that we did not cause (`ingest.js`
//     marks the server dirty; the authoritative answer is this sync's report)
//   • the audit interval elapsed — the backstop that finds drift on a quiet
//     server nobody has touched
//   • the last attempt failed, after a backoff
//
// ── What it never does ────────────────────────────────────────────────────
//
// It does not remove a grant it did not make (D31), it does not invent a
// permission the server has not registered (D33), and it does not treat a
// silent sidecar as a reason to forget anything. A server that is unreachable
// keeps its retirements and its revocations until it comes back.

const core = require('./core')

const db = require('./model/permissions/permissions.db')
const model = require('./model/permissions/permissions.model')
const servers = require('./model/servers/servers.model')
const serversDb = require('./model/servers/servers.db')
const sidecar = require('./sidecarClient')

const log = core.logger('permissions')

/** How often the loop asks whether anything needs pushing. */
const TICK_MS = 30 * 1000

/**
 * How long a server may go without a full reconciliation, however quiet it is.
 *
 * The digest comparison is what keeps the loop cheap, and on its own it would
 * also mean a server whose store somebody edited by hand is never asked about
 * again. This is the interval at which the question gets asked anyway.
 */
const AUDIT_MS = 15 * 60 * 1000

/** How long to leave a failing server alone before trying again. */
const FAIL_BACKOFF_MS = 2 * 60 * 1000

/**
 * The most rows one sync may carry.
 *
 * Below the sidecar's line cap and below the plugin's operation ceiling, so the
 * refusal happens here — where it can name the server and reach an operator —
 * rather than as a `413` or a `too-large` from two processes away.
 */
const MAX_ROWS = 15000

let timer = null

function start() {
  if (timer) return

  timer = setInterval(() => {
    tick().catch((err) => log.error('permission sync tick failed', { error: err.message }))
  }, TICK_MS)

  if (timer.unref) timer.unref()
}

function stop() {
  if (!timer) return

  clearInterval(timer)
  timer = null
}

/**
 * One pass over every enabled server.
 *
 * The authored set is read ONCE and handed to each server's build: six servers
 * are six different answers derived from the same four tables, and re-reading
 * them per server is six times the queries for identical rows.
 */
async function tick({ force = null } = {}) {
  await db.ensureSyncRows()

  const [rows, state, sync, authored] = await Promise.all([
    servers.listForPolling(),
    serversDb.listState(),
    db.listSync(),
    model.readAuthored(),
  ])

  const syncById = new Map(sync.map((row) => [row.serverId, row]))
  const stateById = new Map(state.map((row) => [row.serverId, row]))

  // `allSettled`, for the same reason the board poll uses it: one unreachable
  // host must not stop the other five being reconciled.
  await Promise.allSettled(
    rows
      .filter((server) => force === null || force === server.id)
      .map((server) =>
        syncOne(server, {
          authored,
          sync: syncById.get(server.id) || null,
          state: stateById.get(server.id) || null,
          force: force !== null,
        }),
      ),
  )
}

/**
 * Whether this server needs a push right now.
 *
 * Returns a reason rather than a boolean, because the reason is worth logging:
 * "why did the website just write to my game server" is a question an operator
 * asks, and `wipe` and `drift` are very different answers.
 */
function reasonToSync({ desiredHash, sync, state, force }) {
  if (force) return 'requested'
  // Not while the world is loading (PLAN_FIXES F7). The plugin connects before
  // the save loads, and the first walk's restart sync went 35 s before "Server
  // startup complete", timed out behind the busy main thread, and was retried
  // 2.5 minutes later as "0 applied" — so nothing said what the restart had
  // restored. The plugin says `worldReady: true` in the hello it sends the moment
  // the world is up, and the sync goes on the next tick. A human's "sync now" is
  // not held: they asked, and a failure then is theirs to read.
  if (state && state.worldReady === false) return null
  // Nor while the game is not connected (F7, the step-2 walk). The stored hello is
  // the LAST one: between a restart and the poll that reads the new boot's hello,
  // it still says the old world is ready. On Carbon a due audit went out in that
  // gap, 80 s before "Server startup complete". The poll marks the server offline
  // the moment it goes away, so offline holds until a fresh hello says otherwise.
  // Unknown (`online` absent) is not held — a state row always carries it.
  if (state && state.online !== undefined && state.online !== null && !Number(state.online)) return null
  if (!sync) return 'first'
  if (sync.state !== 'ok' && sync.lastAttemptAt && age(sync.lastAttemptAt) < FAIL_BACKOFF_MS && !sync.dirty) {
    return null
  }
  if (sync.state !== 'ok') return 'retry'
  if (desiredHash !== sync.syncedHash) return 'changed'
  if (sync.dirty) return 'dirty'

  const bootId = state && state.bootId ? state.bootId : null
  const wipeId = state && state.wipeId ? state.wipeId : null

  // A restart or a wipe is the case R2 exists for: the game may have forgotten
  // everything, and the site has not.
  if (bootId && bootId !== sync.bootId) return 'restart'
  if (wipeId && wipeId !== sync.wipeId) return 'wipe'

  if (!sync.lastAttemptAt || age(sync.lastAttemptAt) >= AUDIT_MS) return 'audit'

  return null
}

function age(value) {
  const at = value instanceof Date ? value.getTime() : new Date(value).getTime()
  return Number.isFinite(at) ? Date.now() - at : Number.MAX_SAFE_INTEGER
}

async function syncOne(server, { authored, sync, state, force }) {
  const desired = model.buildDesired(server.id, authored)
  const reason = reasonToSync({ desiredHash: desired.hash, sync, state, force })

  if (!reason) return null

  const [pushed, revocations] = await Promise.all([
    db.listPushed(server.id),
    db.listRevocations(server.id),
  ])

  const retirements = model.retirements(pushed, desired.rows)
  const { styleRetired, sent: styleRetire } = styleRetirements(retirements)
  const retire = [
    ...retirements
      .filter((row) => row.kind !== 'chat-field')
      .map((row) => ({ kind: row.kind, subject: row.subject, object: row.object })),
    ...styleRetire,
    ...revocations.map((row) => ({ kind: row.kind, subject: row.subject, object: row.object })),
  ]

  const bootId = state && state.bootId ? state.bootId : null
  const wipeId = state && state.wipeId ? state.wipeId : null

  if (desired.rows.length + retire.length > MAX_ROWS) {
    // Refused here rather than sent: the sidecar would answer `413` and the
    // plugin would answer `too-large`, and neither of those messages reaches the
    // person who has to make the set smaller.
    const error = `the permission set is too large to push (${desired.rows.length + retire.length} rows, limit ${MAX_ROWS})`
    log.error('permission sync refused', { server: server.id, rows: desired.rows.length })
    await db.putSyncResult(server.id, {
      state: 'failed',
      desiredHash: desired.hash,
      syncedHash: sync ? sync.syncedHash : null,
      bootId,
      wipeId,
      report: null,
      error,
    })

    return 'too-large'
  }

  log.info('syncing permissions', {
    server: server.id,
    reason,
    rows: desired.rows.length,
    retire: retire.length,
  })

  const result = await sidecar.permSync(server, {
    setId: desired.hash,
    groups: withExpect(desired.payload.groups, pushed),
    grants: desired.payload.grants,
    managed: desired.payload.managed,
    credits: desired.payload.credits,
    retire,
  })

  if (!result.ok) {
    // Said in the log as well as on the row (F7): the first walk's restart sync
    // timed out with nothing in the log at all, while the titles push that failed
    // beside it did log.
    log.warn('permission sync failed', { server: server.id, reason, status: result.status })
    await db.putSyncResult(server.id, {
      state: 'failed',
      desiredHash: desired.hash,
      syncedHash: sync ? sync.syncedHash : null,
      bootId,
      wipeId,
      report: null,
      error: result.status,
    })

    return result.status
  }

  const report = result.data || {}

  // The plugin refuses a whole sync with `perm.error` — `busy` while an earlier
  // one is still draining, `too-large` past its own ceiling. Both are answers
  // rather than transport failures, exactly like a refused link code, so they
  // arrive as a 200 and are told apart by `kind`.
  if (report.kind === 'perm.error') {
    log.warn('permission sync refused by the game', { server: server.id, reason, refused: report.reason || 'unknown' })
    await db.putSyncResult(server.id, {
      state: 'failed',
      desiredHash: desired.hash,
      syncedHash: sync ? sync.syncedHash : null,
      bootId,
      wipeId,
      report: null,
      error: `the game refused the sync: ${report.reason || 'unknown'}`,
    })

    return report.reason || 'refused'
  }

  await applyReport(server, { desired, retire, styleRetired, report, bootId, wipeId })

  return 'ok'
}

/**
 * The groups as the wire carries them: each style field with the value this
 * site last pushed to THIS server (`expect`), or null for one it never has.
 * The plugin writes a field only when the game still holds that, so a field
 * somebody changed by hand is reported instead of overwritten (D138).
 */
function withExpect(groups, pushed) {
  const expect = new Map(
    pushed.filter((row) => row.kind === 'chat-field').map((row) => [`${row.subject} ${row.object}`, row.value]),
  )

  return groups.map((group) => {
    if (!group.chat) return group

    const chat = {}
    for (const [field, value] of Object.entries(group.chat)) {
      const last = expect.get(`${group.name} ${field}`)
      chat[field] = { value, expect: last === undefined ? null : last }
    }

    return { ...group, chat }
  })
}

/**
 * Style fields the site pushed and no longer wants, as the wire says it: ONE
 * `chat-group` retirement per group, because a style is all twelve fields or
 * none, and the only way to take one out of BetterChat is to remove its group
 * (D139).
 *
 * **`default` is never removed.** It is BetterChat's fallback, which it warns
 * about on every line when it is missing; a style withdrawn from the site's
 * `default` group stops being pushed and is left as it stands (§33.5).
 */
function styleRetirements(retirements) {
  const styleRetired = retirements.filter((row) => row.kind === 'chat-field')
  const groups = [...new Set(styleRetired.map((row) => row.subject))].filter((name) => name !== 'default')

  return { styleRetired, sent: groups.map((name) => ({ kind: 'chat-group', subject: name, object: '' })) }
}

/**
 * Record what the game said it did.
 *
 * Three writes, and the order matters only in that all three are safe to repeat:
 * a sync that crashes here is re-run next tick and reaches the same place, which
 * is the property that lets this loop be the only writer.
 */
async function applyReport(server, { desired, retire, styleRetired = [], report, bootId, wipeId }) {
  const unresolved = new Set((report.unresolved || []).map(model.normaliseName))
  const pending = new Set(report.pending || [])
  // Grants the plugin made and then did not find in the store when it read it
  // back (D85). Before protocol 9 there was no such read-back, and on Oxide every
  // grant of another plugin's permission landed nowhere while this site recorded
  // it as pushed (PLAN.md §27.6).
  const notLanded = new Set((report.notLanded || []).map((entry) => String(entry).toLowerCase()))

  // A grant naming a permission this server has not registered did NOT land —
  // `GrantUserPermission` no-ops silently for an unregistered name, which is
  // why the plugin pre-checks and says so. Recording it as pushed would make the
  // site believe it had given a privilege it had not.
  //
  // The same for a member the store could not place: the membership is waiting
  // on their first connection, and it is not in the game yet.
  // Protocol 12. A style field landed when BetterChat was there to take it and
  // the report names it neither drift nor failed. With BetterChat absent none
  // did, and each is sent again with the same `expect` next time (§33.2).
  const chat = report.chat && typeof report.chat === 'object' ? report.chat : null
  const chatLoaded = Boolean(chat && chat.loaded === true)
  const fieldKey = (group, field) => `${group} ${String(field || '').toLowerCase()}`
  const chatHeld = new Set([
    ...((chatLoaded && chat.drift) || []).map((row) => fieldKey(row.group, row.field)),
    ...((chatLoaded && chat.failed) || []).filter((row) => row.field).map((row) => fieldKey(row.group, row.field)),
  ])
  const chatGroupFailed = new Set(((chatLoaded && chat.failed) || []).filter((row) => !row.field).map((row) => row.group))

  const landed = desired.rows.filter((row) => {
    if (row.kind === 'chat-field') {
      return chatLoaded && !chatGroupFailed.has(row.subject) && !chatHeld.has(fieldKey(row.subject, row.object))
    }
    if (row.kind === 'grant' || row.kind === 'group-permission') {
      return !unresolved.has(row.object) && !notLanded.has(`${row.subject}:${row.object}`.toLowerCase())
    }
    if (row.kind === 'member') return !pending.has(`${row.subject}:${row.object}`)
    return true
  })

  await db.addPushed(server.id, landed)

  // Everything retired is gone from the game whether the plugin removed it or
  // found it already absent, so it stops being something this site put there.
  // A `chat-group` is not a ledger row; its fields are, below.
  await db.removePushed(server.id, retire.filter((row) => row.kind !== 'chat-group'))

  // A style's fields leave the ledger only once BetterChat has removed the group
  // — or for `default`, which is never removed — so a style withdrawn while
  // BetterChat was absent is retired by the first sync that can (D139).
  const removed = new Set((chatLoaded && chat.removed) || [])
  await db.removePushed(
    server.id,
    styleRetired.filter((row) => row.subject === 'default' || removed.has(row.subject)),
  )

  const revocations = await db.listRevocations(server.id)
  await db.deleteRevocations(revocations.map((row) => row.id))

  await db.replaceDrift(server.id, [
    ...(report.foreign || []).map((row) => ({
      kind: String(row.kind || ''),
      subject: String(row.subject || ''),
      object: String(row.object || ''),
    })),
    // A style field somebody changed by hand, with what it holds now (D138).
    ...((chatLoaded && chat.drift) || []).map((row) => ({
      kind: 'chat-field',
      subject: String(row.group || ''),
      object: String(row.field || ''),
      detail: row.game === undefined || row.game === null ? null : String(row.game).slice(0, 255),
    })),
  ])

  await db.putSyncResult(server.id, {
    state: 'ok',
    desiredHash: desired.hash,
    syncedHash: desired.hash,
    bootId,
    wipeId,
    report: JSON.stringify(report),
    error: null,
  })

  // The option source, refreshed from the same server that just answered. It is
  // a second round trip and it is worth it: the form must not offer a name that
  // stopped being registered when somebody uninstalled a plugin, because a grant
  // against one is a privilege nobody ever gets and nothing ever reports.
  const catalogue = await sidecar.permCatalogue(server)

  if (catalogue.ok && catalogue.data && Array.isArray(catalogue.data.permissions)) {
    await db.putCatalogue(
      server.id,
      catalogue.data.permissions.map(model.normaliseName).filter(Boolean),
    )
  }

  log.info('permissions synced', {
    server: server.id,
    applied: report.applied,
    unresolved: (report.unresolved || []).length,
    foreign: (report.foreign || []).length,
    pending: (report.pending || []).length,
    notLanded: (report.notLanded || []).length,
    ...(chat
      ? {
          betterChat: chatLoaded,
          ...(chatLoaded
            ? { styleApplied: chat.applied, styleSaved: chat.saved, styleDrift: (chat.drift || []).length, styleFailed: (chat.failed || []).length }
            : {}),
        }
      : {}),
    ...(report.creditsApplied !== undefined
      ? { creditsApplied: report.creditsApplied, creditsWithdrawn: report.creditsWithdrawn }
      : {}),
  })
}

module.exports = {
  TICK_MS,
  AUDIT_MS,
  FAIL_BACKOFF_MS,
  MAX_ROWS,
  start,
  stop,
  tick,
  syncOne,
  reasonToSync,
  applyReport,
  withExpect,
  styleRetirements,
}