Module-Rust/server/permSync.js

// ── Keeping a game's permission store equal to what the site authored ─────
//
// R2's whole mechanism, and it is chapter 4's board pointed the other way: the
// site is the single producer of a set, it re-sends the whole thing rather than
// a stream of edits, and the receiver reconciles. What is new is the direction —
// the module telling the game what the site knows, where every earlier phase
// asked the game what it knew.
//
// ── One verb (D32) ────────────────────────────────────────────────────────
//
// A sync sends the whole desired set and the plugin diffs it against the live
// store. The website never holds a copy of the game's permissions, which is the
// point: a second source of truth is stale the moment it lands, and the store is
// the bigger of the two sets.
//
// The delta the site DOES compute is the one the game cannot: what this site put
// there and has since withdrawn (`retirements`). A name in the store that is not
// in the desired set is either that, or a hand edit — and only the pushed ledger
// can tell them apart (D31).
//
// ── When it runs ──────────────────────────────────────────────────────────
//
// Every tick asks a cheap question — does the digest of the desired set still
// equal what this server last confirmed — and does nothing when the answer is
// yes. A sync therefore happens when:
//
//   • an operator changed something (the dirty flag, and the digest behind it)
//   • the game restarted or wiped (a new boot id or wipe id: the store may have
//     been emptied, and R2's promise is that a wipe is not a data-loss event)
//   • a permission hook fired in the game that we did not cause (`ingest.js`
//     marks the server dirty; the authoritative answer is this sync's report)
//   • the audit interval elapsed — the backstop that finds drift on a quiet
//     server nobody has touched
//   • the last attempt failed, after a backoff
//
// ── What it never does ────────────────────────────────────────────────────
//
// It does not remove a grant it did not make (D31), it does not invent a
// permission the server has not registered (D33), and it does not treat a
// silent sidecar as a reason to forget anything. A server that is unreachable
// keeps its retirements and its revocations until it comes back.

const core = require('./core')

const db = require('./model/permissions/permissions.db')
const model = require('./model/permissions/permissions.model')
const servers = require('./model/servers/servers.model')
const serversDb = require('./model/servers/servers.db')
const sidecar = require('./sidecarClient')

const log = core.logger('permissions')

/** How often the loop asks whether anything needs pushing. */
const TICK_MS = 30 * 1000

/**
 * How long a server may go without a full reconciliation, however quiet it is.
 *
 * The digest comparison is what keeps the loop cheap, and on its own it would
 * also mean a server whose store somebody edited by hand is never asked about
 * again. This is the interval at which the question gets asked anyway.
 */
const AUDIT_MS = 15 * 60 * 1000

/** How long to leave a failing server alone before trying again. */
const FAIL_BACKOFF_MS = 2 * 60 * 1000

/**
 * The most rows one sync may carry.
 *
 * Below the sidecar's line cap and below the plugin's operation ceiling, so the
 * refusal happens here — where it can name the server and reach an operator —
 * rather than as a `413` or a `too-large` from two processes away.
 */
const MAX_ROWS = 15000

let timer = null

function start() {
  if (timer) return

  timer = setInterval(() => {
    tick().catch((err) => log.error('permission sync tick failed', { error: err.message }))
  }, TICK_MS)

  if (timer.unref) timer.unref()
}

function stop() {
  if (!timer) return

  clearInterval(timer)
  timer = null
}

/**
 * One pass over every enabled server.
 *
 * The authored set is read ONCE and handed to each server's build: six servers
 * are six different answers derived from the same four tables, and re-reading
 * them per server is six times the queries for identical rows.
 */
async function tick({ force = null } = {}) {
  await db.ensureSyncRows()

  const [rows, state, sync, authored] = await Promise.all([
    servers.listForPolling(),
    serversDb.listState(),
    db.listSync(),
    model.readAuthored(),
  ])

  const syncById = new Map(sync.map((row) => [row.serverId, row]))
  const stateById = new Map(state.map((row) => [row.serverId, row]))

  // `allSettled`, for the same reason the board poll uses it: one unreachable
  // host must not stop the other five being reconciled.
  await Promise.allSettled(
    rows
      .filter((server) => force === null || force === server.id)
      .map((server) =>
        syncOne(server, {
          authored,
          sync: syncById.get(server.id) || null,
          state: stateById.get(server.id) || null,
          force: force !== null,
        }),
      ),
  )
}

/**
 * Whether this server needs a push right now.
 *
 * Returns a reason rather than a boolean, because the reason is worth logging:
 * "why did the website just write to my game server" is a question an operator
 * asks, and `wipe` and `drift` are very different answers.
 */
function reasonToSync({ desiredHash, sync, state, force }) {
  if (force) return 'requested'
  if (!sync) return 'first'
  if (sync.state !== 'ok' && sync.lastAttemptAt && age(sync.lastAttemptAt) < FAIL_BACKOFF_MS && !sync.dirty) {
    return null
  }
  if (sync.state !== 'ok') return 'retry'
  if (desiredHash !== sync.syncedHash) return 'changed'
  if (sync.dirty) return 'dirty'

  const bootId = state && state.bootId ? state.bootId : null
  const wipeId = state && state.wipeId ? state.wipeId : null

  // A restart or a wipe is the case R2 exists for: the game may have forgotten
  // everything, and the site has not.
  if (bootId && bootId !== sync.bootId) return 'restart'
  if (wipeId && wipeId !== sync.wipeId) return 'wipe'

  if (!sync.lastAttemptAt || age(sync.lastAttemptAt) >= AUDIT_MS) return 'audit'

  return null
}

function age(value) {
  const at = value instanceof Date ? value.getTime() : new Date(value).getTime()
  return Number.isFinite(at) ? Date.now() - at : Number.MAX_SAFE_INTEGER
}

async function syncOne(server, { authored, sync, state, force }) {
  const desired = model.buildDesired(server.id, authored)
  const reason = reasonToSync({ desiredHash: desired.hash, sync, state, force })

  if (!reason) return null

  const [pushed, revocations] = await Promise.all([
    db.listPushed(server.id),
    db.listRevocations(server.id),
  ])

  const retirements = model.retirements(pushed, desired.rows)
  const retire = [
    ...retirements.map((row) => ({ kind: row.kind, subject: row.subject, object: row.object })),
    ...revocations.map((row) => ({ kind: row.kind, subject: row.subject, object: row.object })),
  ]

  const bootId = state && state.bootId ? state.bootId : null
  const wipeId = state && state.wipeId ? state.wipeId : null

  if (desired.rows.length + retire.length > MAX_ROWS) {
    // Refused here rather than sent: the sidecar would answer `413` and the
    // plugin would answer `too-large`, and neither of those messages reaches the
    // person who has to make the set smaller.
    const error = `the permission set is too large to push (${desired.rows.length + retire.length} rows, limit ${MAX_ROWS})`
    log.error('permission sync refused', { server: server.id, rows: desired.rows.length })
    await db.putSyncResult(server.id, {
      state: 'failed',
      desiredHash: desired.hash,
      syncedHash: sync ? sync.syncedHash : null,
      bootId,
      wipeId,
      report: null,
      error,
    })

    return 'too-large'
  }

  log.info('syncing permissions', {
    server: server.id,
    reason,
    rows: desired.rows.length,
    retire: retire.length,
  })

  const result = await sidecar.permSync(server, {
    setId: desired.hash,
    groups: desired.payload.groups,
    grants: desired.payload.grants,
    managed: desired.payload.managed,
    retire,
  })

  if (!result.ok) {
    await db.putSyncResult(server.id, {
      state: 'failed',
      desiredHash: desired.hash,
      syncedHash: sync ? sync.syncedHash : null,
      bootId,
      wipeId,
      report: null,
      error: result.status,
    })

    return result.status
  }

  const report = result.data || {}

  // The plugin refuses a whole sync with `perm.error` — `busy` while an earlier
  // one is still draining, `too-large` past its own ceiling. Both are answers
  // rather than transport failures, exactly like a refused link code, so they
  // arrive as a 200 and are told apart by `kind`.
  if (report.kind === 'perm.error') {
    await db.putSyncResult(server.id, {
      state: 'failed',
      desiredHash: desired.hash,
      syncedHash: sync ? sync.syncedHash : null,
      bootId,
      wipeId,
      report: null,
      error: `the game refused the sync: ${report.reason || 'unknown'}`,
    })

    return report.reason || 'refused'
  }

  await applyReport(server, { desired, retire, report, bootId, wipeId })

  return 'ok'
}

/**
 * Record what the game said it did.
 *
 * Three writes, and the order matters only in that all three are safe to repeat:
 * a sync that crashes here is re-run next tick and reaches the same place, which
 * is the property that lets this loop be the only writer.
 */
async function applyReport(server, { desired, retire, report, bootId, wipeId }) {
  const unresolved = new Set((report.unresolved || []).map(model.normaliseName))
  const pending = new Set(report.pending || [])

  // A grant naming a permission this server has not registered did NOT land —
  // `GrantUserPermission` no-ops silently for an unregistered name, which is
  // why the plugin pre-checks and says so. Recording it as pushed would make the
  // site believe it had given a privilege it had not.
  //
  // The same for a member the store could not place: the membership is waiting
  // on their first connection, and it is not in the game yet.
  const landed = desired.rows.filter((row) => {
    if (row.kind === 'grant' || row.kind === 'group-permission') return !unresolved.has(row.object)
    if (row.kind === 'member') return !pending.has(`${row.subject}:${row.object}`)
    return true
  })

  await db.addPushed(server.id, landed)

  // Everything retired is gone from the game whether the plugin removed it or
  // found it already absent, so it stops being something this site put there.
  await db.removePushed(server.id, retire)

  const revocations = await db.listRevocations(server.id)
  await db.deleteRevocations(revocations.map((row) => row.id))

  await db.replaceDrift(server.id, (report.foreign || []).map((row) => ({
    kind: String(row.kind || ''),
    subject: String(row.subject || ''),
    object: String(row.object || ''),
  })))

  await db.putSyncResult(server.id, {
    state: 'ok',
    desiredHash: desired.hash,
    syncedHash: desired.hash,
    bootId,
    wipeId,
    report: JSON.stringify(report),
    error: null,
  })

  // The option source, refreshed from the same server that just answered. It is
  // a second round trip and it is worth it: the form must not offer a name that
  // stopped being registered when somebody uninstalled a plugin, because a grant
  // against one is a privilege nobody ever gets and nothing ever reports.
  const catalogue = await sidecar.permCatalogue(server)

  if (catalogue.ok && catalogue.data && Array.isArray(catalogue.data.permissions)) {
    await db.putCatalogue(
      server.id,
      catalogue.data.permissions.map(model.normaliseName).filter(Boolean),
    )
  }

  log.info('permissions synced', {
    server: server.id,
    applied: report.applied,
    unresolved: (report.unresolved || []).length,
    foreign: (report.foreign || []).length,
    pending: (report.pending || []).length,
  })
}

module.exports = {
  TICK_MS,
  AUDIT_MS,
  FAIL_BACKOFF_MS,
  MAX_ROWS,
  start,
  stop,
  tick,
  syncOne,
  reasonToSync,
  applyReport,
}