website/server/test/moduleInstall.test.js

// modules/install.js — fetching, verifying and placing a module bundle.
//
// Phase 4, slice 1 of MODULE_SYSTEM.md §2.7.2. The rules under test are all
// refusals, and each one is a step an attacker would otherwise walk through:
// a non-https URL, a host that is not allowed, a REDIRECT to a host that is not
// allowed, a body larger than declared, a hash that does not match, an archive
// that does not agree with the manifest about what it is.
//
// `fetchImpl` is injected rather than a TLS server being stood up, for the same
// reason `replayFragments` takes a `query`: what is being tested is what this
// file decides about a response, and a real server would mostly test Node's
// certificate handling. The responses below are real `Response` objects with
// real bodies, so the streaming, the hashing and the byte cap are exercised for
// real — only the transport is stubbed.
process.env.DB_HOST = '127.0.0.1'
process.env.DB_PORT = '59999'

const crypto = require('crypto')
const fs = require('fs')
const os = require('os')
const path = require('path')
const zlib = require('zlib')

const { test, beforeEach, after } = require('node:test')
const assert = require('node:assert/strict')

const db = require('../src/utils/db')

after(() => db.close())

const HOSTS = ['releases.example.com']
const MANIFEST_URL = 'https://releases.example.com/mod/uo-1.0.0.json'

let tmpRoot
let install

/**
 * A fresh install.js bound to a fresh modules directory.
 *
 * install.js reads `loader.dir()`, which is resolved once at require time from
 * MODULES_DIR — so both have to be re-required per test, exactly as
 * moduleLifecycle.test.js does.
 */
function freshInstall(dir) {
  process.env.MODULES_DIR = dir
  delete require.cache[require.resolve('../src/modules/loader')]
  delete require.cache[require.resolve('../src/modules/install')]
  // eslint-disable-next-line global-require
  return require('../src/modules/install')
}

// ── Building a bundle to serve ─────────────────────────────────────────────

const BLOCK = 512

function octal(value, width) {
  return Number(value).toString(8).padStart(width - 1, '0') + '\0'
}

function member({ name, type = '0', body = '' }) {
  const header = Buffer.alloc(BLOCK, 0)
  const data = Buffer.from(body, 'utf8')
  header.write(name, 0, 100, 'utf8')
  header.write(octal(0o644, 8), 100, 8, 'ascii')
  header.write(octal(0, 8), 108, 8, 'ascii')
  header.write(octal(0, 8), 116, 8, 'ascii')
  header.write(octal(data.length, 12), 124, 12, 'ascii')
  header.write(octal(0, 12), 136, 12, 'ascii')
  header.write('        ', 148, 8, 'ascii')
  header.write(type, 156, 1, 'ascii')
  header.write('ustar\0', 257, 6, 'ascii')
  header.write('00', 263, 2, 'ascii')
  let sum = 0
  for (const byte of header) sum += byte
  header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'ascii')
  const padding = Buffer.alloc((BLOCK - (data.length % BLOCK)) % BLOCK, 0)
  return Buffer.concat([header, data, padding])
}

/** A bundle tarball whose root directory is named the way a release names it. */
function bundle({ id = 'uo', version = '1.0.0', extra = [] } = {}) {
  const root = `module-${id}-${version}`
  return zlib.gzipSync(Buffer.concat([
    member({ name: `${root}/`, type: '5' }),
    member({
      name: `${root}/module.json`,
      body: JSON.stringify({ id, name: 'Ultima Online', version, coreApi: '^1.0.0', server: 'server/index.js' }),
    }),
    member({ name: `${root}/server/`, type: '5' }),
    member({ name: `${root}/server/index.js`, body: 'module.exports = () => {}\n' }),
    ...extra.map(member),
    Buffer.alloc(BLOCK * 2, 0),
  ]))
}

const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex')

/**
 * A fake transport serving one manifest and one artifact.
 *
 * `routes` maps an absolute URL to either a Buffer/string body or
 * `{ status, location }` for a redirect, so a test can describe exactly what the
 * remote host does without describing how it does it.
 */
function fakeFetch(routes) {
  const seen = []
  const impl = async (url) => {
    const href = String(url)
    seen.push(href)
    const route = routes[href]
    if (route === undefined) return new Response('not found', { status: 404 })
    if (route && route.status) {
      return new Response(null, {
        status: route.status,
        headers: route.location ? { location: route.location } : {},
      })
    }
    return new Response(route, { status: 200 })
  }
  impl.seen = seen
  return impl
}

/** The manifest a release publishes, with whatever a test wants to change. */
function manifestFor(tarball, overrides = {}) {
  return JSON.stringify({
    schema: 1,
    id: 'uo',
    name: 'Ultima Online',
    version: '1.0.0',
    coreApi: '^1.0.0',
    artifact: 'uo-1.0.0.tar.gz',
    url: 'https://releases.example.com/mod/uo-1.0.0.tar.gz',
    sha256: sha256(tarball),
    size: tarball.length,
    ...overrides,
  })
}

/** The happy-path pair: a valid manifest and the artifact it describes. */
function goodRoutes(options = {}) {
  const tarball = bundle(options)
  return {
    tarball,
    routes: {
      [MANIFEST_URL]: manifestFor(tarball, options.manifest || {}),
      'https://releases.example.com/mod/uo-1.0.0.tar.gz': tarball,
    },
  }
}

beforeEach(() => {
  tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-install-'))
  install = freshInstall(tmpRoot)
})

// ── The allowlist ──────────────────────────────────────────────────────────

test('parseHosts accepts comma and whitespace separation, and lower-cases', () => {
  assert.deepEqual(install.parseHosts('a.com, B.com\nc.com'), ['a.com', 'b.com', 'c.com'])
  assert.deepEqual(install.parseHosts(''), [])
  assert.deepEqual(install.parseHosts(null), [])
})

test('an empty allowlist forbids everything rather than allowing everything', () => {
  // The direction matters: a setting someone blanks by accident must stop
  // installs, not open the door to every host on the internet.
  assert.throws(() => install.checkUrl('https://releases.example.com/x.json', []), /no module source hosts are allowed/)
})

test('only https may be installed from', () => {
  // The sha256 is no help against a plaintext fetch: whoever can rewrite the
  // artifact in flight can rewrite the manifest that declares its hash.
  assert.throws(() => install.checkUrl('http://releases.example.com/x.json', HOSTS), /only https/)
  assert.throws(() => install.checkUrl('file:///etc/passwd', HOSTS), /only https/)
})

test('a host that is not on the allowlist is refused, and the message says which are', () => {
  assert.throws(
    () => install.checkUrl('https://evil.example.net/x.json', HOSTS),
    /"evil.example.net" is not an allowed module source host \(allowed: releases.example.com\)/,
  )
})

test('the allowlist is matched on the host, not on a substring of the URL', () => {
  // `https://evil.com/?x=releases.example.com` must not pass, and neither must a
  // subdomain nobody listed.
  assert.throws(() => install.checkUrl('https://evil.com/?releases.example.com', HOSTS), /not an allowed/)
  assert.throws(() => install.checkUrl('https://sub.releases.example.com/x', HOSTS), /not an allowed/)
  assert.throws(() => install.checkUrl('https://releases.example.com.evil.net/x', HOSTS), /not an allowed/)
})

// ── Redirects ──────────────────────────────────────────────────────────────

test('a redirect is followed, and re-checked against the allowlist', async () => {
  const { tarball } = goodRoutes()
  const impl = fakeFetch({
    [MANIFEST_URL]: { status: 302, location: 'https://releases.example.com/cdn/uo-1.0.0.json' },
    'https://releases.example.com/cdn/uo-1.0.0.json': manifestFor(tarball),
  })

  const manifest = await install.fetchManifest(MANIFEST_URL, HOSTS, impl)

  assert.equal(manifest.id, 'uo')
  assert.deepEqual(impl.seen, [MANIFEST_URL, 'https://releases.example.com/cdn/uo-1.0.0.json'])
})

test('a redirect to a host that is not allowed is refused', async () => {
  // The hole the allowlist exists to close. `fetch`'s own redirect following
  // would check the first hop and then go wherever it was pointed — which is
  // why get() follows them by hand.
  const impl = fakeFetch({
    [MANIFEST_URL]: { status: 302, location: 'http://169.254.169.254/latest/meta-data/' },
  })

  await assert.rejects(() => install.fetchManifest(MANIFEST_URL, HOSTS, impl), /only https/)
})

test('a redirect loop is bounded rather than followed forever', async () => {
  const impl = fakeFetch({
    [MANIFEST_URL]: { status: 302, location: MANIFEST_URL },
  })

  await assert.rejects(() => install.fetchManifest(MANIFEST_URL, HOSTS, impl), /too many redirects/)
})

// ── The install manifest ───────────────────────────────────────────────────

test('a manifest that is not JSON is refused with a readable reason', async () => {
  const impl = fakeFetch({ [MANIFEST_URL]: 'this is not json' })
  await assert.rejects(() => install.fetchManifest(MANIFEST_URL, HOSTS, impl), /not valid JSON/)
})

test('a manifest with an invalid module id is refused', async () => {
  // The loader would refuse to scan a directory called `../etc`, but the point
  // is that one is never created: the id becomes a path segment.
  for (const id of ['../etc', 'UO', '', 'x', 'a'.repeat(40)]) {
    const impl = fakeFetch({ [MANIFEST_URL]: JSON.stringify({ id, name: 'n', version: '1', sha256: 'a'.repeat(64) }) })
    // eslint-disable-next-line no-await-in-loop
    await assert.rejects(() => install.fetchManifest(MANIFEST_URL, HOSTS, impl), /invalid module id/, `id ${JSON.stringify(id)}`)
  }
})

test('a manifest with no usable sha256 is refused', async () => {
  const impl = fakeFetch({
    [MANIFEST_URL]: JSON.stringify({ id: 'uo', name: 'n', version: '1.0.0', sha256: 'not-a-hash' }),
  })
  await assert.rejects(() => install.fetchManifest(MANIFEST_URL, HOSTS, impl), /no valid sha256/)
})

test('the artifact URL is resolved relative to the manifest when it carries no absolute one', async () => {
  const { tarball } = goodRoutes()
  const impl = fakeFetch({ [MANIFEST_URL]: manifestFor(tarball, { url: undefined }) })

  const manifest = await install.fetchManifest(MANIFEST_URL, HOSTS, impl)

  // Release assets sit beside their manifest, so a manifest that travelled
  // without its absolute URL still resolves to the right place.
  assert.equal(manifest.artifactUrl, 'https://releases.example.com/mod/uo-1.0.0.tar.gz')
})

// ── The install ────────────────────────────────────────────────────────────

test('a good bundle installs into modules/<id>, stripped of its wrapper', async () => {
  const { routes, tarball } = goodRoutes()

  const result = await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) })

  assert.equal(result.id, 'uo')
  assert.equal(result.version, '1.0.0')
  assert.equal(result.sha256, sha256(tarball))
  assert.equal(result.source, MANIFEST_URL)
  assert.equal(result.replaced, false)

  // The directory is named for the module id, not for the archive's root — the
  // loader scans for the former and the publisher chose the latter.
  const dir = path.join(tmpRoot, 'uo')
  assert.ok(fs.existsSync(path.join(dir, 'module.json')))
  assert.ok(fs.existsSync(path.join(dir, 'server', 'index.js')))
  assert.ok(!fs.existsSync(path.join(tmpRoot, 'module-uo-1.0.0')))
})

test('a hash that does not match refuses the install and writes nothing', async () => {
  const { routes } = goodRoutes()
  // The bytes are fine; the manifest lies about them. Which is the same thing an
  // artifact swapped after publication looks like.
  routes[MANIFEST_URL] = manifestFor(Buffer.from('different'), {})

  await assert.rejects(
    () => install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) }),
    /does not match the sha256/,
  )
  assert.deepEqual(fs.readdirSync(tmpRoot), [], 'nothing may be left on the volume')
})

test('a bundle whose module.json disagrees with the manifest is refused', async () => {
  // A manifest promising `uo` and delivering something else would otherwise be
  // installed into `modules/uo/` under a name it is not.
  const tarball = bundle({ id: 'rust', version: '1.0.0' })
  const routes = {
    [MANIFEST_URL]: manifestFor(tarball),
    'https://releases.example.com/mod/uo-1.0.0.tar.gz': tarball,
  }

  await assert.rejects(
    () => install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) }),
    /declares module id "rust" but the install manifest promised "uo"/,
  )
  assert.deepEqual(fs.readdirSync(tmpRoot), [])
})

test('a bundle whose version disagrees with the manifest is refused', async () => {
  const tarball = bundle({ id: 'uo', version: '9.9.9' })
  const routes = {
    [MANIFEST_URL]: manifestFor(tarball),
    'https://releases.example.com/mod/uo-1.0.0.tar.gz': tarball,
  }

  await assert.rejects(
    () => install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) }),
    /declares version "9.9.9"/,
  )
})

test('an artifact whose length differs from the declared size is refused', async () => {
  const tarball = bundle()
  const routes = {
    [MANIFEST_URL]: manifestFor(tarball, { size: tarball.length + 10 }),
    'https://releases.example.com/mod/uo-1.0.0.tar.gz': tarball,
  }

  await assert.rejects(
    () => install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) }),
    /but the manifest declares/,
  )
})

test('a hostile archive is refused, and nothing of it reaches the volume', async () => {
  // The case node-tar alone does NOT cover: it throws on the escaping member,
  // but only once it reaches it — the members before it are already on disk.
  // archive.inspect() decides before extract() runs, and the unpack happens in a
  // scratch directory that is removed either way.
  const tarball = bundle({
    extra: [
      { name: 'module-uo-1.0.0/../../ESCAPED.txt', body: 'escaped' },
    ],
  })
  const routes = {
    [MANIFEST_URL]: manifestFor(tarball),
    'https://releases.example.com/mod/uo-1.0.0.tar.gz': tarball,
  }

  await assert.rejects(
    () => install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) }),
    /escapes upward/,
  )
  assert.deepEqual(fs.readdirSync(tmpRoot), [], 'no scratch directory, no partial module, no escape')
})

test('an upgrade replaces the directory and reports that it did', async () => {
  const first = goodRoutes()
  await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(first.routes) })
  fs.writeFileSync(path.join(tmpRoot, 'uo', 'STALE.txt'), 'from the old version')

  const second = goodRoutes({ version: '2.0.0', manifest: { version: '2.0.0' } })
  second.routes[MANIFEST_URL] = manifestFor(second.tarball, { version: '2.0.0' })
  const result = await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(second.routes) })

  assert.equal(result.replaced, true)
  assert.equal(result.version, '2.0.0')
  // A replace, not a merge: a file the previous version left behind must not
  // survive into the new one, or an upgrade quietly keeps dead code loadable.
  assert.ok(!fs.existsSync(path.join(tmpRoot, 'uo', 'STALE.txt')))
  assert.deepEqual(fs.readdirSync(tmpRoot), ['uo'], 'the aside copy is cleaned up')
})

test('a failed upgrade leaves the previous version in place', async () => {
  const first = goodRoutes()
  await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(first.routes) })

  const badTarball = bundle({ id: 'uo', version: '2.0.0', extra: [{ name: 'evil/', type: '5' }] })
  await assert.rejects(() => install.install({
    url: MANIFEST_URL,
    hosts: HOSTS,
    fetchImpl: fakeFetch({
      [MANIFEST_URL]: manifestFor(badTarball, { version: '2.0.0' }),
      'https://releases.example.com/mod/uo-1.0.0.tar.gz': badTarball,
    }),
  }))

  // The installed module is untouched: the swap is the last step, so a bundle
  // rejected before it never got near the live directory.
  assert.ok(fs.existsSync(path.join(tmpRoot, 'uo', 'module.json')))
  assert.equal(JSON.parse(fs.readFileSync(path.join(tmpRoot, 'uo', 'module.json'), 'utf8')).version, '1.0.0')
  assert.deepEqual(fs.readdirSync(tmpRoot), ['uo'])
})

// ── One module per site ────────────────────────────────────────────────────

/** A second, different module's manifest and artifact, served beside the first. */
function otherModuleRoutes(id = 'rust') {
  const tarball = bundle({ id })
  const artifact = `https://releases.example.com/mod/${id}-1.0.0.tar.gz`
  const url = `https://releases.example.com/mod/${id}-1.0.0.json`
  return {
    url,
    routes: {
      [url]: manifestFor(tarball, { id, name: id, artifact: `${id}-1.0.0.tar.gz`, url: artifact }),
      [artifact]: tarball,
    },
    artifact,
  }
}

test('a second, different module is refused before anything is downloaded', async () => {
  const first = goodRoutes()
  await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(first.routes) })

  const other = otherModuleRoutes('rust')
  const fetchImpl = fakeFetch(other.routes)
  await assert.rejects(
    () => install.install({ url: other.url, hosts: HOSTS, fetchImpl }),
    (err) => {
      assert.equal(err.name, 'InstallError')
      // 409: nothing is wrong with the URL; the SITE is not in a state to take it.
      assert.equal(err.status, 409)
      assert.match(err.message, /already runs the module "uo"/)
      assert.match(err.message, /remove it before installing "rust"/)
      return true
    },
  )

  // Refused on the manifest alone: the artifact was never fetched, and the
  // volume holds exactly what it held before.
  assert.ok(!fetchImpl.seen.includes(other.artifact), 'the artifact was not downloaded')
  assert.deepEqual(fs.readdirSync(tmpRoot), ['uo'])
})

test('the same module is still an upgrade, and removing it frees the site for another', async () => {
  const first = goodRoutes()
  await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(first.routes) })

  // An upgrade of what is installed is exactly what the rule allows.
  const second = goodRoutes({ version: '2.0.0', manifest: { version: '2.0.0' } })
  second.routes[MANIFEST_URL] = manifestFor(second.tarball, { version: '2.0.0' })
  const upgraded = await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(second.routes) })
  assert.equal(upgraded.replaced, true)

  // And once it is gone, the site takes a different one.
  await install.removeDir('uo')
  const other = otherModuleRoutes('rust')
  const result = await install.install({ url: other.url, hosts: HOSTS, fetchImpl: fakeFetch(other.routes) })
  assert.equal(result.id, 'rust')
  assert.deepEqual(install.installedIds(), ['rust'])
})

test('what counts as installed is what the loader would scan', () => {
  // A real module, an install's scratch directory, a swap's aside copy and a
  // directory with no module.json. Only the first is a module.
  fs.mkdirSync(path.join(tmpRoot, 'uo'))
  fs.writeFileSync(path.join(tmpRoot, 'uo', 'module.json'), '{}')
  fs.mkdirSync(path.join(tmpRoot, '.install-rust-abc'))
  fs.writeFileSync(path.join(tmpRoot, '.install-rust-abc', 'module.json'), '{}')
  fs.mkdirSync(path.join(tmpRoot, 'uo.replaced-123'))
  fs.writeFileSync(path.join(tmpRoot, 'uo.replaced-123', 'module.json'), '{}')
  fs.mkdirSync(path.join(tmpRoot, 'notes'))

  assert.deepEqual(install.installedIds(), ['uo'])
})

// ── The volume ─────────────────────────────────────────────────────────────

test('moduleDir refuses an id that is not one', () => {
  for (const id of ['../escape', 'a/b', '', 'UO', '.']) {
    assert.throws(() => install.moduleDir(id), /invalid module id/, `id ${JSON.stringify(id)}`)
  }
  assert.equal(install.moduleDir('uo'), path.join(tmpRoot, 'uo'))
})

test('isInstalled and removeDir report what they did', async () => {
  const { routes } = goodRoutes()
  assert.equal(install.isInstalled('uo'), false)

  await install.install({ url: MANIFEST_URL, hosts: HOSTS, fetchImpl: fakeFetch(routes) })
  assert.equal(install.isInstalled('uo'), true)

  assert.equal(await install.removeDir('uo'), true)
  assert.equal(install.isInstalled('uo'), false)
  // Removing what is not there is not an error — an uninstall of a module whose
  // directory was already deleted by hand should still tidy up the row.
  assert.equal(await install.removeDir('uo'), false)
})

test('purgeFile resolves the manifest declaration, and refuses one that escapes', async () => {
  const dir = path.join(tmpRoot, 'uo')
  fs.mkdirSync(path.join(dir, 'server', 'db'), { recursive: true })
  fs.writeFileSync(path.join(dir, 'server', 'db', 'purge.sql'), 'DROP TABLE IF EXISTS x;')

  const write = (purge) => fs.writeFileSync(
    path.join(dir, 'module.json'),
    JSON.stringify({ id: 'uo', name: 'UO', version: '1.0.0', purge }),
  )

  write('server/db/purge.sql')
  assert.equal(install.purgeFile('uo'), path.join(dir, 'server', 'db', 'purge.sql'))

  // The same containment rule the loader applies to client.entry: a manifest may
  // not point core at a file outside the module it belongs to.
  write('../../../../etc/passwd')
  assert.equal(install.purgeFile('uo'), null)

  // Declared but absent, and not declared at all, are both "nothing to run".
  write('server/db/missing.sql')
  assert.equal(install.purgeFile('uo'), null)
  write(undefined)
  assert.equal(install.purgeFile('uo'), null)
})

test('purgeFile is null for a module that is not on the volume', () => {
  assert.equal(install.purgeFile('nothere'), null)
})