website/server/src/modules/install.js

// ── Installing and removing a module bundle ────────────────────────────────
//
// Phase 4, slice 1 of docs/website/MODULE_SYSTEM.md §2.7.2 — the consumer half
// of a release the module's own CI already publishes. Nothing here touches the
// database or the loader: this file moves bytes onto the volume and off it, and
// the caller (router/v1/admin/modules.controller.js) writes down what happened.
//
// The shape of an install, and why it is this shape:
//
//   1. The operator pastes the URL of a release's install manifest — a small
//      JSON document naming the artifact, its sha256 and its size (decision 2).
//      There is no catalog, because a catalog would make core's release cadence
//      decide which modules can exist.
//   2. Every URL fetched — the manifest, the artifact, and every redirect hop —
//      is checked against the admin-managed host allowlist (decision 6). That is
//      what keeps a pasted URL from also being an SSRF primitive.
//   3. The artifact is streamed to a temporary file under a byte cap, hashed as
//      it arrives, and compared against the manifest's `sha256`. The hash is the
//      trust anchor; releases are unsigned and say so.
//   4. modules/archive.js inspects the archive in full before a single byte is
//      unpacked, and only then unpacks it — into a TEMPORARY directory.
//   5. The unpacked tree's own `module.json` must agree with the manifest about
//      what it is. A manifest that promises `uo` and delivers something else is
//      refused rather than installed under the name it was promised.
//   6. Only then is anything moved into `modules/<id>/`, and the directory it
//      replaces is kept aside until the move has succeeded.
//
// **Nothing is written into the modules directory until every check has passed**,
// and that is not belt-and-braces. Verified against node-tar 7.5.22 while writing
// this: extracting an archive whose fourth member escapes upward throws — and
// leaves the first three members on disk. A loader scans that directory at
// require time on the next boot; a half-unpacked module is a module.
//
// One thing this file deliberately does NOT do: mount anything. Installing puts
// a directory on the volume, and §1.12 means the volume is read at require time,
// so the module appears when the process restarts. The admin screen offers that
// restart (decision 1); it is not implied here.

const crypto = require('crypto')
const fs = require('fs')
const fsp = require('fs/promises')
const path = require('path')
const { Readable, Transform } = require('stream')
const { pipeline } = require('stream/promises')

const archive = require('./archive')
const loader = require('./loader')

const log = require('../utils/logger')('modules')

// An install manifest is a small JSON document. A megabyte of it is not a
// manifest, and reading it into memory unbounded is the one place this file
// would otherwise trust a remote length.
const MAX_MANIFEST_BYTES = 256 * 1024
// The artifact cap is the archive's own unpacked cap — a compressed bundle
// larger than what it is allowed to unpack to has nothing to offer.
const MAX_ARTIFACT_BYTES = archive.MAX_BYTES
const FETCH_TIMEOUT_MS = 30000
// Gitea serves a release asset through at least one redirect. Following them is
// necessary; following them blindly is how an allowlist gets bypassed, so each
// hop is re-checked and the chain is bounded.
const MAX_REDIRECTS = 5

// Same id rule the loader enforces when scanning, restated rather than imported
// so an install cannot put a directory on the volume that the loader would then
// refuse to look at.
const ID = /^[a-z][a-z0-9-]{1,31}$/
const SHA256 = /^[0-9a-f]{64}$/

class InstallError extends Error {
  constructor(message, { status = 400 } = {}) {
    super(message)
    this.name = 'InstallError'
    // Carried so the controller can answer 400 for "your URL is wrong" and 502
    // for "the host you named misbehaved" without re-deriving it from the text.
    this.status = status
  }
}

// ── The allowlist ──────────────────────────────────────────────────────────

// The settings row the allowlist lives in (decision 6): seeded from
// MODULE_SOURCE_HOSTS on a fresh install and admin-managed from then on. The KEY
// lives here rather than in the admin controller because it is now read from two
// places — the controller, and the boot-time resolution of the declared module
// set (modules/declared.js), which has no route and no request.
const HOSTS_SETTING = 'module_source_hosts'

/**
 * Parse the stored allowlist setting into hostnames.
 *
 * Comma or whitespace separated, case-insensitive, empty entries dropped. An
 * empty list means nothing may be installed from anywhere — a refusal, never a
 * wildcard. That direction matters: a setting an admin accidentally blanks
 * should stop installs, not permit every host on the internet.
 */
function parseHosts(value) {
  return String(value || '')
    .split(/[,\s]+/)
    .map((h) => h.trim().toLowerCase())
    .filter(Boolean)
}

/**
 * Check one URL against the allowlist, and return it parsed.
 *
 * `https` only. A plaintext fetch of code this process is going to execute is
 * not something an operator should be able to opt into by typing a URL, and the
 * sha256 does not help — whoever can rewrite the artifact in flight can rewrite
 * the manifest that declares its hash.
 */
function checkUrl(raw, hosts) {
  let url
  try {
    url = new URL(String(raw))
  } catch {
    throw new InstallError(`"${raw}" is not a valid URL`)
  }
  if (url.protocol !== 'https:') {
    throw new InstallError(`only https URLs may be installed from (got "${url.protocol}")`)
  }
  if (!hosts.length) {
    throw new InstallError(
      'no module source hosts are allowed — set one in Admin → Modules before installing',
    )
  }
  if (!hosts.includes(url.hostname.toLowerCase())) {
    throw new InstallError(
      `"${url.hostname}" is not an allowed module source host (allowed: ${hosts.join(', ')})`,
    )
  }
  return url
}

// ── Fetching ───────────────────────────────────────────────────────────────

/**
 * GET a URL, following redirects by hand so every hop is re-checked.
 *
 * `fetch`'s own redirect following would take the first hop off the allowlist
 * and the rest wherever it was pointed, which is exactly the hole the allowlist
 * exists to close.
 *
 * `fetchImpl` is the same injection seam `replayFragments({query})` and
 * `lifecycle.boot({model})` use, and it exists for the same reason: the rules
 * this file enforces — https only, allowlisted host, allowlisted REDIRECT host,
 * bounded body, matching hash — are all decisions about a response, and testing
 * them against a real TLS server would mean testing Node's certificate handling
 * instead. Production never passes it.
 *
 * @returns {Promise<Response>} a response whose body has not been read.
 */
async function get(rawUrl, hosts, fetchImpl = fetch) {
  let url = checkUrl(rawUrl, hosts)

  for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
    let res
    try {
      res = await fetchImpl(url, {
        redirect: 'manual',
        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
        headers: { accept: '*/*' },
      })
    } catch (err) {
      throw new InstallError(`could not reach ${url.hostname}: ${err.message}`, { status: 502 })
    }

    if (res.status >= 300 && res.status < 400) {
      const location = res.headers.get('location')
      if (!location) throw new InstallError(`${url.hostname} redirected without a location`, { status: 502 })
      // Resolved against the current URL, then re-checked — a relative redirect
      // is normal and a cross-host one is the thing being guarded against.
      url = checkUrl(new URL(location, url).toString(), hosts)
      continue
    }

    if (!res.ok) {
      throw new InstallError(`${url.href} returned ${res.status}`, { status: 502 })
    }
    return res
  }

  throw new InstallError(`too many redirects (more than ${MAX_REDIRECTS})`, { status: 502 })
}

/** Read a bounded response body as text. */
async function readText(res, cap, what) {
  const declared = Number(res.headers.get('content-length'))
  if (Number.isFinite(declared) && declared > cap) {
    throw new InstallError(`${what} is larger than ${cap} bytes`)
  }
  const buf = Buffer.from(await res.arrayBuffer())
  // Checked again after reading: content-length is the server's claim, not a
  // limit it is obliged to honour.
  if (buf.length > cap) throw new InstallError(`${what} is larger than ${cap} bytes`)
  return buf.toString('utf8')
}

/**
 * Stream a response body to a file, hashing as it goes and stopping at the cap.
 *
 * The hash is computed from the bytes that were written rather than by re-reading
 * the file, so there is no window in which the file could differ from what was
 * verified.
 */
async function download(res, dest, cap) {
  const hash = crypto.createHash('sha256')
  let bytes = 0

  const body = Readable.fromWeb(res.body)
  const counter = new Transform({
    transform(chunk, _enc, cb) {
      bytes += chunk.length
      if (bytes > cap) {
        cb(new InstallError(`the artifact is larger than ${Math.round(cap / 1024 / 1024)} MB`))
        return
      }
      hash.update(chunk)
      cb(null, chunk)
    },
  })

  await pipeline(body, counter, fs.createWriteStream(dest))
  return { sha256: hash.digest('hex'), bytes }
}

// ── The install manifest ───────────────────────────────────────────────────

/**
 * Fetch and validate a release's install manifest.
 *
 * The shape is the one module-uo's release workflow already writes:
 * `{schema, id, name, version, coreApi, artifact, url, sha256, size}`. Only the
 * fields this side needs are required — a module publisher may carry more.
 */
async function fetchManifest(manifestUrl, hosts, fetchImpl = fetch) {
  const res = await get(manifestUrl, hosts, fetchImpl)
  const text = await readText(res, MAX_MANIFEST_BYTES, 'the install manifest')

  let manifest
  try {
    manifest = JSON.parse(text)
  } catch (err) {
    throw new InstallError(`the install manifest is not valid JSON: ${err.message}`)
  }
  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
    throw new InstallError('the install manifest is not a JSON object')
  }

  const { id, name, version, sha256 } = manifest
  if (!ID.test(String(id || ''))) {
    throw new InstallError(`the install manifest has an invalid module id: ${JSON.stringify(id)}`)
  }
  if (!name || !version) throw new InstallError('the install manifest is missing name or version')
  if (!SHA256.test(String(sha256 || '').toLowerCase())) {
    throw new InstallError('the install manifest has no valid sha256 for its artifact')
  }

  // `url` is the absolute artifact URL the publisher wrote; `artifact` is its
  // filename. Prefer the URL, fall back to resolving the filename beside the
  // manifest — which is where a release's assets sit — so a manifest that
  // travelled without its absolute URL still installs.
  const artifactUrl = new URL(manifest.url || manifest.artifact || '', manifestUrl).toString()

  return {
    id: String(id),
    name: String(name),
    version: String(version),
    sha256: String(sha256).toLowerCase(),
    size: Number(manifest.size) || null,
    artifactUrl,
    coreApi: manifest.coreApi ? String(manifest.coreApi) : null,
  }
}

// ── Paths on the volume ────────────────────────────────────────────────────

/** Where a module with this id lives. Rejects an id that is not one. */
function moduleDir(id) {
  if (!ID.test(String(id || ''))) throw new InstallError(`invalid module id: ${JSON.stringify(id)}`)
  return path.join(loader.dir(), String(id))
}

/** Is there a directory for this module on the volume right now? */
function isInstalled(id) {
  try {
    return fs.statSync(moduleDir(id)).isDirectory()
  } catch {
    return false
  }
}

/**
 * The absolute path of a module's `purge.sql`, or null.
 *
 * Read from the module's own `module.json` on disk rather than from the loader,
 * because purge has to work for a module that never loaded — a `startup_failed`
 * one is exactly when an operator wants its tables gone.
 */
function purgeFile(id) {
  try {
    const manifest = JSON.parse(fs.readFileSync(path.join(moduleDir(id), 'module.json'), 'utf8'))
    if (!manifest.purge) return null
    const file = path.resolve(moduleDir(id), manifest.purge)
    // The same containment check the loader applies to `client.entry`: a
    // manifest may not point core at a file outside the module.
    if (!file.startsWith(moduleDir(id) + path.sep)) return null
    return fs.existsSync(file) ? file : null
  } catch {
    return null
  }
}

// ── Install ────────────────────────────────────────────────────────────────

/**
 * Download, verify, unpack and install a module from an install-manifest URL.
 *
 * Never leaves a partial module on the volume: everything happens under a
 * scratch directory that is removed on any failure, and the move into place is
 * the last step.
 *
 * `expect` is what the CALLER was promised, as opposed to what the manifest
 * promises about itself — the declared module set (modules/declared.js) pins an
 * id and a version in the environment, and a URL that resolves to something else
 * has to be refused rather than installed. Checked against the manifest, before
 * a byte is downloaded: catching it after the unpack would mean the undeclared
 * module is already on the volume when the objection is raised. The admin panel
 * passes nothing, because there a URL is the whole of what was asked for.
 *
 * @param {object} args
 * @param {string} args.url    the install manifest URL the admin pasted
 * @param {string[]} args.hosts the allowlist, already parsed
 * @param {{id?: string, version?: string}} [args.expect] what the caller pinned
 * @returns {Promise<{id,name,version,sha256,source,bytes,replaced}>}
 */
async function install({ url, hosts, expect = null, fetchImpl = fetch }) {
  const manifest = await fetchManifest(url, hosts, fetchImpl)

  if (expect && expect.id && manifest.id !== expect.id) {
    throw new InstallError(
      `that URL installs the module "${manifest.id}", but "${expect.id}" was asked for`,
    )
  }
  if (expect && expect.version && manifest.version !== expect.version) {
    throw new InstallError(
      `that URL installs ${manifest.id} v${manifest.version}, but v${expect.version} was asked for`,
    )
  }

  const target = moduleDir(manifest.id)
  const scratch = await fsp.mkdtemp(path.join(loader.dir(), `.install-${manifest.id}-`))
  const tarball = path.join(scratch, 'bundle.tar.gz')
  const unpacked = path.join(scratch, 'unpacked')

  try {
    const res = await get(manifest.artifactUrl, hosts, fetchImpl)
    const { sha256, bytes } = await download(res, tarball, MAX_ARTIFACT_BYTES)

    if (sha256 !== manifest.sha256) {
      // The whole trust model in one comparison. Deliberately does not name the
      // computed hash in a way that reads like a value to copy into the
      // manifest — a mismatch means stop, not reconcile.
      throw new InstallError(
        `the downloaded artifact does not match the sha256 in the install manifest — refusing to install`,
      )
    }
    if (manifest.size && bytes !== manifest.size) {
      throw new InstallError(`the artifact is ${bytes} bytes but the manifest declares ${manifest.size}`)
    }

    await archive.unpack(tarball, unpacked)

    // What the bundle says it is, checked against what the manifest promised.
    let inner
    try {
      inner = JSON.parse(await fsp.readFile(path.join(unpacked, 'module.json'), 'utf8'))
    } catch {
      throw new InstallError('the bundle has no readable module.json at its root')
    }
    if (inner.id !== manifest.id) {
      throw new InstallError(
        `the bundle declares module id "${inner.id}" but the install manifest promised "${manifest.id}"`,
      )
    }
    if (inner.version !== manifest.version) {
      throw new InstallError(
        `the bundle declares version "${inner.version}" but the install manifest promised "${manifest.version}"`,
      )
    }

    // The swap. The outgoing directory is moved aside rather than deleted first,
    // so a failed rename leaves the previous version recoverable instead of
    // leaving no module at all — the same reasoning the installer repo applies
    // to a ServUO tree.
    const replaced = fs.existsSync(target)
    const aside = `${target}.replaced-${Date.now()}`
    if (replaced) await fsp.rename(target, aside)
    try {
      await fsp.rename(unpacked, target)
    } catch (err) {
      if (replaced) await fsp.rename(aside, target).catch(() => {})
      throw err
    }
    if (replaced) await fsp.rm(aside, { recursive: true, force: true })

    log.info(`installed module "${manifest.id}" v${manifest.version}`, {
      source: url,
      sha256: manifest.sha256,
      bytes,
      replaced,
    })

    return { ...manifest, source: url, bytes, replaced }
  } finally {
    await fsp.rm(scratch, { recursive: true, force: true }).catch(() => {})
  }
}

/**
 * Remove a module's directory from the volume.
 *
 * Returns whether there was one. Does not touch the database, does not run
 * purge.sql, and does not stop the running module — the controller sequences
 * all three, because the order matters and only it knows what the operator
 * asked for.
 */
async function removeDir(id) {
  const dir = moduleDir(id)
  if (!fs.existsSync(dir)) return false
  await fsp.rm(dir, { recursive: true, force: true })
  log.info(`removed module directory for "${id}"`)
  return true
}

module.exports = {
  InstallError,
  HOSTS_SETTING,
  parseHosts,
  checkUrl,
  get,
  fetchManifest,
  install,
  removeDir,
  moduleDir,
  isInstalled,
  purgeFile,
  MAX_MANIFEST_BYTES,
  MAX_ARTIFACT_BYTES,
}