From 9b16f39a528a0bd8b20d78cb7e8f40f4233a2045 Mon Sep 17 00:00:00 2001 From: wtclaude Date: Wed, 12 Aug 2026 07:57:08 -0500 Subject: [PATCH] feat(modules): the declarative Docker path (phase 4, slice 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MODULES declares the module set a deployment runs, one entry per module as `@=`, and the container arrives at it by itself (MODULE_SYSTEM.md §2.7.2 decision 4). A module already unpacked at the declared version is a no-op that makes NO network call, so a restart with the network down comes up unchanged; anything else goes through install.js — same allowlist, same sha256, same inspect-then-extract — and install() now takes an `expect: {id, version}` so a URL resolving to another module or version is refused while it is still only a manifest. Resolution runs inside start(), between the seed and the require of app.js: the seed is where the host allowlist setting comes from, and the require is what scans the volume. That buys it the database, so a compose-installed module gets the same provenance columns an admin install writes. A failure is logged and carried, never fatal — an unreachable release host must not take the site down. The declaration owns what is on the volume; the row owns whether a module runs, so uninstalling a declared module returns its files at the next start and leaves it disabled. The admin list gains that as a fourth source (declared / declaredVersion / declaredError), because a declared module that failed to resolve has no row, no directory and nothing mounted. Deferring the app require moved core's schema ahead of the volume scan, and the module schema-fragment replay was wired to core's schema — so every installed module silently got no tables. Invisible to the suite (each one stubs the loader or the pool) and to a smoke on a database that already had the tables; found by booting against an empty one. ensureSchema() now takes `replayModules: false` for the one caller that scans later, server.js replays them itself after the require, and a bootOrder test pins the five steps in the only order they work in. 741 server tests (+18), 187 client (+5); manifest unchanged at 166 public + 2 internal, OpenAPI byte-identical. Co-Authored-By: Claude --- .env.example | 14 + README.md | 28 ++ client/src/lib/moduleAdmin.js | 59 ++- .../src/routes/admin/views/ModulesAdmin.jsx | 30 +- client/test/moduleAdmin.test.js | 57 ++- docker-compose.yml | 26 +- server/.env.example | 19 + server/src/modules/declared.js | 220 ++++++++++ server/src/modules/install.js | 31 +- .../src/router/v1/admin/modules.controller.js | 53 ++- server/src/server.js | 45 +- server/src/utils/db.js | 11 +- server/test/adminModules.test.js | 52 +++ server/test/bootOrder.test.js | 82 ++++ server/test/moduleDeclared.test.js | 391 ++++++++++++++++++ 15 files changed, 1090 insertions(+), 28 deletions(-) create mode 100644 server/src/modules/declared.js create mode 100644 server/test/bootOrder.test.js create mode 100644 server/test/moduleDeclared.test.js diff --git a/.env.example b/.env.example index 0739b00..520c30f 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,20 @@ BOT_INTERNAL_KEY=change-me-to-a-long-random-string # knows nothing about any of them. A module may read its own env vars, and they # belong here because Compose passes this file to the container. # +# MODULES declares the set this deployment runs, and the container arrives at it +# on its own — no admin panel, no `tar -xf` on the host. One entry per module, +# `@=`, whitespace- or comma-separated: +# +# MODULES=uo@0.3.0=https://gitea.whitlocktech.com/RunicGateway/Module-uo/releases/download/v0.3.0/module-uo-0.3.0.json +# +# A module already unpacked at the declared version is a no-op that never touches +# the network, so a restart with the internet down brings the site up exactly as +# it was; only a missing or different version is fetched, verified against the +# sha256 its manifest declares, and unpacked. A failure is logged and shown in +# Admin → Modules, and the site starts anyway. The variable owns what is on the +# volume, not what runs — a module disabled from the admin panel stays disabled. +# Leave it unset to install from the admin panel instead. +# # RunicGateway/Module-uo, for example, reads UOLINK_BASE_URL / UOLINK_WS_URL / # UOLINK_PROTOCOL as the defaults for its connection to a uo-link sidecar, and # TOWNCRIER_DURATION_SEC for its news leg. Its README documents them; they are diff --git a/README.md b/README.md index 0d21c09..5b15305 100644 --- a/README.md +++ b/README.md @@ -489,6 +489,32 @@ modules/ supported install. The directory is tracked in git (via its README) on purpose: Docker recreates a *missing* bind-mount source as `root:root`, and the container is uid 1000. +### Three ways in, and none of them is a build + +| | How | Where it fits | +|---|---|---| +| **Admin panel** | Admin → Modules, paste the URL of a release's install manifest | The click path. Installs, upgrades, disables, uninstalls and purges, with a restart button — no shell on the box | +| **`MODULES`** | Declare the set in the environment; the container resolves it at every start | The compose-managed host. The running set is a line in a file you version-control, not the residue of past clicks | +| **By hand** | `tar -xf module-uo-0.3.0.tar.gz -C ./modules && mv modules/module-uo-0.3.0 modules/uo`, then restart | Development, and any host where the other two do not fit | + +`MODULES` takes one entry per module, whitespace- or comma-separated: + +``` +MODULES=uo@0.3.0=https://gitea.whitlocktech.com/RunicGateway/Module-uo/releases/download/v0.3.0/module-uo-0.3.0.json +``` + +The id and the version are written out rather than discovered inside the manifest so that **the +no-op case needs no network**: a module already unpacked at the declared version is answered by +reading its own `module.json`, so a restart with the internet down brings the site up exactly as it +was. Only a missing or different version is fetched, and it goes through the same +verify-and-unpack path — allowlisted `https` host, sha256 from the manifest, whole-archive +inspection before anything is written — that the admin panel uses. A version that cannot be +resolved is logged and shown on the admin screen; **it never stops the site from starting**. + +The declaration owns what is *on the volume*, never what runs. A module disabled from the admin +panel gets its files back at the next start and stays disabled, because the row and the variable are +answering different questions. + ### What a module gets, and what it may not do At boot, `app.js` scans the volume synchronously, validates each `module.json`, and calls the @@ -534,6 +560,8 @@ Copy `.env.example` (Compose) or `server/.env.example` (local) and fill in. **`. | `PORT` | `3000` | server listens on `0.0.0.0:PORT` | | `UPLOAD_DIR` | `/uploads` | where post images are written (`/app/uploads`, volume-mounted, in Compose) | | `MODULES_DIR` | `/modules` | where installed modules are scanned from (`/app/modules`, bind-mounted, in Compose) | +| `MODULES` | — | the module set this deployment runs, resolved at every start: `@=`, whitespace/comma separated. Already at the declared version = no network. A failure is logged and shown in Admin → Modules, never fatal. See [Modules](#modules) | +| `MODULE_SOURCE_HOSTS` | `gitea.whitlocktech.com` | **bootstrap only** — seeds the `module_source_hosts` setting on first boot; after that the setting is authoritative and is edited in Admin → Modules | | `DB_HOST` / `DB_PORT` | `db` / `3306` | `db` in Compose; `127.0.0.1` for local dev | | `DB_NAME` / `DB_USER` / `DB_PASSWORD` | `runic_gateway` / `runic` / — | app database credentials | | `DB_ROOT_PASSWORD` | — | MariaDB root (Compose only) | diff --git a/client/src/lib/moduleAdmin.js b/client/src/lib/moduleAdmin.js index 38cbf19..d3e09ab 100644 --- a/client/src/lib/moduleAdmin.js +++ b/client/src/lib/moduleAdmin.js @@ -6,13 +6,15 @@ // runner here has no DOM, and this is the part of the Modules screen that is // actually worth testing. // -// **The screen has three sources of truth and they are allowed to disagree** -// (MODULE_SYSTEM.md §2.4): +// **The screen has four sources of truth and they are allowed to disagree** +// (MODULE_SYSTEM.md §2.4, and slice 3 for the fourth): // // state what the DATABASE row records — what the operator decided, and // what the last boot ended up doing // liveState what the LOADER has mounted in this process and is answering with // onVolume whether there is still a directory there at all +// declared what this container's MODULES variable asks for — the only one of +// the four that no button on this screen can change // // Picking one and rendering it would be simpler and would lie. The case that // makes this concrete is the one decision 3 creates on purpose: an operator @@ -34,6 +36,22 @@ * @returns {{ label: string, tone: 'ok'|'warn'|'bad'|'idle', pending: boolean, detail: string }} */ export function statusOf(m) { + // Declared by the environment and not there at all: no row, no directory, + // nothing mounted. Every other branch below reads one of those three, so + // without this the screen would describe a module it has never had as though + // a row had gone stale — and the one thing the operator needs, the reason + // resolution failed, would be nowhere. + if (m.declared && !m.onVolume && m.state === null) { + return { + label: 'Declared, not installed', + tone: 'bad', + pending: false, + detail: m.declaredError + ? `MODULES asks for v${m.declaredVersion}; the last start could not install it: ${m.declaredError}` + : `MODULES asks for v${m.declaredVersion}. It will be installed when the server next starts.`, + } + } + // Gone from the volume, but still known. Either a hand-deleted directory (the // boot reconcile marks that `startup_failed`) or an uninstall waiting for its // restart. Both are "there is nothing to run here". @@ -125,6 +143,43 @@ export function statusOf(m) { } } +/** + * What the environment's declaration means for this module, as one sentence — or + * null if nothing declares it. + * + * Kept out of `statusOf` on purpose. A module can be running perfectly while its + * declared upgrade is failing, and collapsing both into one label would have to + * pick which of the two is "the" status. This is a second line, beside the first. + * + * The sentence an operator most needs is the uninstall one: MODULES owns what is + * on the volume and the row owns whether it runs, so uninstalling a declared + * module puts its files back at the next start and leaves it switched off. Files + * reappearing unexplained is exactly the kind of thing that gets debugged for an + * afternoon. + * + * @param {object} m a row from GET /admin/modules + * @returns {{ text: string, tone: 'warn'|'idle' }|null} + */ +export function declarationNoteFor(m) { + if (!m.declared) return null + + if (m.declaredError) { + return { + text: `MODULES asks for v${m.declaredVersion} and the last start could not install it: ${m.declaredError}`, + tone: 'warn', + } + } + if (!m.onVolume) { + return { + text: + `MODULES declares v${m.declaredVersion}, so its files come back when the server next starts` + + (m.state === 'disabled' ? ' — switched off, until you enable it.' : '.'), + tone: 'warn', + } + } + return { text: `Declared by this deployment's MODULES variable at v${m.declaredVersion}.`, tone: 'idle' } +} + /** * Which actions are offered for a module, and why the others are not. * diff --git a/client/src/routes/admin/views/ModulesAdmin.jsx b/client/src/routes/admin/views/ModulesAdmin.jsx index 77865c5..ce59623 100644 --- a/client/src/routes/admin/views/ModulesAdmin.jsx +++ b/client/src/routes/admin/views/ModulesAdmin.jsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useState } from 'react' import { Loading, ErrorState } from '../../../components/PageState.jsx' import { dateTime } from '../../../lib/format.js' -import { statusOf, actionsFor, needsRestart, parseHosts } from '../../../lib/moduleAdmin.js' +import { statusOf, actionsFor, declarationNoteFor, needsRestart, parseHosts } from '../../../lib/moduleAdmin.js' import { api } from '../../../api/client.js' // Installed modules: install from a release URL, enable, disable, uninstall, @@ -226,6 +226,7 @@ function ModuleRow({ m, onChanged, onError }) { const [busy, setBusy] = useState('') const status = statusOf(m) const actions = actionsFor(m) + const note = declarationNoteFor(m) async function run(name, fn) { setBusy(name) @@ -272,7 +273,9 @@ function ModuleRow({ m, onChanged, onError }) {
{m.name}
- {m.id} · v{m.version} + {/* A declared module that has never installed has no version to show — + only the one MODULES asks for, which the status column carries. */} + {m.id}{m.version ? ` · v${m.version}` : ''}
{m.capabilities?.length > 0 && (
{m.capabilities.join(' · ')}
@@ -282,18 +285,37 @@ function ModuleRow({ m, onChanged, onError }) { {status.label}
{status.detail}
+ {/* The environment's declaration, on its own line: a module can be + running fine while its declared upgrade is failing, and the status + above can only be one of those two things. */} + {note && ( +
+ {note.text} +
+ )} {/* Provenance. Null for a directory placed on the volume by hand, which - stays a supported install — so it is shown as that, not as missing. */} + stays a supported install — so it is shown as that, not as missing. + A declared module can also reach a boot with no provenance: the + no-op path never fetches, so it has no sha256 to record and no + reason to write a row. Saying "by hand" there would be the one + wrong answer. */} {m.source ? ( <>
{m.source}
{m.sha256 &&
sha256 {m.sha256.slice(0, 12)}…
} ) : ( - Placed on the volume by hand + {m.declared ? 'From the declared module set' : 'Placed on the volume by hand'} )} {m.installedAt &&
{dateTime(m.installedAt)}
} diff --git a/client/test/moduleAdmin.test.js b/client/test/moduleAdmin.test.js index e6e667d..0f85064 100644 --- a/client/test/moduleAdmin.test.js +++ b/client/test/moduleAdmin.test.js @@ -1,6 +1,6 @@ import { test } from 'node:test' import assert from 'node:assert/strict' -import { statusOf, actionsFor, needsRestart, parseHosts } from '../src/lib/moduleAdmin.js' +import { statusOf, actionsFor, declarationNoteFor, needsRestart, parseHosts } from '../src/lib/moduleAdmin.js' // lib/moduleAdmin.js — what the Modules screen says about a module and what it // lets you do to it. Phase 4, slice 2 of MODULE_SYSTEM.md §2.7.2. @@ -28,6 +28,9 @@ const mod = (over = {}) => ({ capabilities: [], onVolume: true, canPurge: true, + declared: false, + declaredVersion: null, + declaredError: null, ...over, }) @@ -224,3 +227,55 @@ test('parseHosts previews exactly what the server will store', () => { assert.deepEqual(parseHosts(' '), []) assert.deepEqual(parseHosts(undefined), []) }) + +// ── the declaration (slice 3) ────────────────────────────────────────────── + +test('a declared module that has never installed says so, with the reason', () => { + // No row, no directory, nothing mounted — invisible to the other three + // sources, so without this branch the screen would describe a module it has + // never had as a row gone stale. + const s = statusOf(mod({ + state: null, + liveState: null, + liveVersion: null, + version: null, + onVolume: false, + declared: true, + declaredVersion: '0.3.0', + declaredError: 'could not reach releases.example.com', + })) + assert.equal(s.label, 'Declared, not installed') + assert.equal(s.tone, 'bad') + assert.equal(s.pending, false, 'a restart will not fix an unreachable host') + assert.match(s.detail, /could not reach releases.example.com/) +}) + +test('a declared module waiting for its first resolution is not reported as failed', () => { + const s = statusOf(mod({ state: null, liveState: null, version: null, onVolume: false, declared: true, declaredVersion: '0.3.0' })) + assert.match(s.detail, /installed when the server next starts/) +}) + +test('a running module whose declared upgrade is failing is still Running', () => { + // Both facts are true at once. The status is one label, so the declaration + // gets its own line rather than overwriting it. + const m = mod({ declared: true, declaredVersion: '2.0.0', declaredError: 'sha256 did not match' }) + assert.equal(statusOf(m).label, 'Running') + const note = declarationNoteFor(m) + assert.equal(note.tone, 'warn') + assert.match(note.text, /sha256 did not match/) +}) + +test('uninstalling a declared module is told that its files come back', () => { + // The sentence that saves an afternoon: MODULES owns what is on the volume, + // the row owns whether it runs. + const note = declarationNoteFor(mod({ state: 'disabled', liveState: null, onVolume: false, declared: true, declaredVersion: '1.0.0' })) + assert.match(note.text, /come back when the server next starts/) + assert.match(note.text, /switched off/) +}) + +test('an ordinary declared module gets a quiet note, and an undeclared one none', () => { + assert.equal(declarationNoteFor(mod()), null) + const note = declarationNoteFor(mod({ declared: true, declaredVersion: '1.0.0' })) + assert.equal(note.tone, 'idle') + assert.match(note.text, /MODULES/) +}) diff --git a/docker-compose.yml b/docker-compose.yml index 43c6189..4641c43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,24 @@ services: # defaults to (/modules, and the repo is /app in the image), set # explicitly because the bind mount below is what makes it meaningful. MODULES_DIR: /app/modules + # WHICH modules this deployment runs (MODULE_SYSTEM.md §2.7.2 decision 4). + # One entry per module, `@=`, whitespace- + # or comma-separated. The container resolves this set for itself at every + # start: a module already unpacked at the declared version is left alone + # without a single network call — so a restart with the internet down comes + # up unchanged — and only a missing or different version is fetched, + # verified against the sha256 its release manifest declares, and unpacked. + # A failure is logged and surfaced in Admin → Modules; it never stops the + # site from starting. + # + # Uncomment to declare a set here, in the file this host version-controls, + # or leave it out and set MODULES in .env (env_file above) — or leave it + # unset entirely and install from the admin panel. What it declares is what + # is ON the volume, never whether a module runs: a module disabled from the + # admin panel gets its files back and stays disabled. + # + # MODULES: >- + # uo@0.3.0=https://gitea.whitlocktech.com/RunicGateway/Module-uo/releases/download/v0.3.0/module-uo-0.3.0.json depends_on: db: condition: service_healthy @@ -56,10 +74,12 @@ services: # pull-only deployment without building anything. A bind mount rather than # a named volume because placing a module directory by hand is a supported # install — `tar -xf uo-1.0.0.tgz -C ./modules` then restart — and that has - # to be doable from the host, not through `docker cp`. + # to be doable from the host, not through `docker cp`. It is no longer the + # usual way in: declare MODULES above, or install from Admin → Modules. # - # Read-WRITE: the admin panel's install/uninstall unpacks and removes - # directories here from inside the container. + # Read-WRITE: MODULES resolution at start, and the admin panel's + # install/uninstall, both unpack and remove directories here from inside + # the container. # # `modules/` is tracked (it ships a README) so the directory exists in the # checkout with the operator's own ownership. Do not delete it — Docker diff --git a/server/.env.example b/server/.env.example index d78b055..988a7d8 100644 --- a/server/.env.example +++ b/server/.env.example @@ -143,5 +143,24 @@ ANNOUNCE_POLL_MS=15000 # deployment does nothing, deliberately, so a redeploy cannot # silently undo an operator's choice. Installs are https-only # and an empty list forbids all of them. +# MODULES The module set this deployment RUNS, resolved at every +# start (§2.7.2 decision 4). One entry per module, separated +# by whitespace or commas: +# +# @= +# +# A module already unpacked at the declared version is left +# alone WITHOUT touching the network, so a restart with no +# route to the internet comes up unchanged; only a missing or +# different version is fetched, through the same verify-and- +# unpack path (and the same host allowlist) the admin panel +# uses. A version that cannot be fetched is logged and shown +# in Admin → Modules — it never stops the site from starting. +# +# This variable owns what is ON the volume, not what runs: a +# module disabled from the admin panel stays disabled even +# though its files are put back. Leave it unset to manage +# modules entirely from the admin panel. # MODULES_DIR=/app/modules # MODULE_SOURCE_HOSTS=gitea.whitlocktech.com +# MODULES=uo@0.3.0=https://gitea.whitlocktech.com/RunicGateway/Module-uo/releases/download/v0.3.0/module-uo-0.3.0.json diff --git a/server/src/modules/declared.js b/server/src/modules/declared.js new file mode 100644 index 0000000..09bca4b --- /dev/null +++ b/server/src/modules/declared.js @@ -0,0 +1,220 @@ +// ── The declared module set ──────────────────────────────────────────────── +// +// Phase 4, slice 3 of docs/website/MODULE_SYSTEM.md §2.7.2 — decision 4. A +// compose-managed host is not driven by clicking: it declares which modules it +// runs, in the file it already edits and version-controls, and the container +// arrives at that set by itself. +// +// MODULES: uo@0.3.0=https:///…/module-uo-0.3.0.json +// +// One entry per module, `@=`, separated by +// whitespace or commas. The id and the version are written out rather than left +// to be discovered inside the manifest for one reason: **the no-op case must not +// need the network.** A module already unpacked at the declared version is +// answered by reading its own `module.json` off the volume, so a restart with +// the network down brings the site up exactly as it was. Only a module that is +// missing, or unpacked at some other version, reaches out — and it reaches out +// through modules/install.js, the same fetch-verify-unpack path the admin panel +// uses, under the same host allowlist. +// +// Three things this file deliberately does not do: +// +// - **It does not decide whether a module RUNS.** Resolution owns what is on +// the volume; `installed_modules` owns whether a mounted module answers. An +// admin who uninstalls a declared module gets its directory back at the next +// boot with the row still `disabled`, so it stays off until they enable it. +// The two never fight because they are not answering the same question. +// - **It does not fail a boot.** A module publisher's host being unreachable +// must not take a shard's website down with it; core is built to serve with +// a module absent (§1.6). Every failure is logged loudly and kept for the +// admin screen, and the site comes up. +// - **It does not mount anything.** §1.12 makes the volume the mounting source +// of truth, read once at require time — which is why this runs before +// `require('./app')` in server.js and not from inside it. +// +// It runs on every boot, not only in Docker: a bare `npm start` with MODULES set +// resolves the same way. The Docker path is the reason it exists, not a special +// case in it. + +const fs = require('fs') +const path = require('path') + +const install = require('./install') + +const log = require('../utils/logger')('modules') + +// The variable an operator sets. Named next to MODULES_DIR, which is the other +// half of the same story: one says where modules live, the other says which. +const VAR = 'MODULES' + +// Same id rule the loader enforces when scanning and install.js enforces when +// placing, restated here so a declaration cannot name something neither would +// accept. +const ID = /^[a-z][a-z0-9-]{1,31}$/ + +// The outcome of the last resolution, in memory, for the admin screen. Not a +// database row: a declaration is a fact about this process's environment, and +// writing it down would put it in front of the boot reconcile, which resets +// every non-disabled row (§2.4). The screen merges it as a fourth source +// alongside the row, the loader and the volume. +let results = [] + +/** + * Parse the declaration into entries. + * + * Malformed entries are collected rather than thrown: one operator typo should + * cost that module, not every module on the host. A duplicate id keeps the + * first — there is no sensible way to run two versions of one module, and + * silently preferring the last would make the outcome depend on the order of a + * list nobody reads as ordered. + * + * @param {string} value the raw variable + * @returns {{entries: Array<{id,version,url}>, errors: string[]}} + */ +function parse(value) { + const entries = [] + const errors = [] + const seen = new Set() + + for (const token of String(value || '').split(/[,\s]+/).filter(Boolean)) { + const at = token.indexOf('@') + const eq = token.indexOf('=') + if (at < 1 || eq < at + 2) { + errors.push(`"${token}" is not @=`) + continue + } + const id = token.slice(0, at) + const version = token.slice(at + 1, eq) + const url = token.slice(eq + 1) + + if (!ID.test(id)) { + errors.push(`"${token}" names an invalid module id "${id}"`) + continue + } + if (!url) { + errors.push(`"${token}" has no install manifest URL`) + continue + } + if (seen.has(id)) { + errors.push(`"${id}" is declared more than once — keeping the first`) + continue + } + seen.add(id) + entries.push({ id, version, url }) + } + + return { entries, errors } +} + +/** + * The version currently unpacked on the volume, or null. + * + * Read straight out of the module's own `module.json`, which is the same file + * the loader trusts for the same fact — and never from `installed_modules`, + * because the row records what was installed and this has to answer what is + * actually there. An unreadable manifest counts as absent: whatever is in that + * directory, it is not a module at the declared version. + */ +function installedVersion(id) { + try { + const manifest = JSON.parse( + fs.readFileSync(path.join(install.moduleDir(id), 'module.json'), 'utf8'), + ) + return manifest && manifest.version ? String(manifest.version) : null + } catch { + return null + } +} + +/** + * Bring the volume in line with the declaration. + * + * Never throws and never rejects. Returns one outcome per declared entry, and + * remembers them for `state()`. + * + * @param {object} args + * @param {string} [args.value] the raw variable (defaults to the environment) + * @param {string[]} args.hosts the install allowlist, already parsed + * @param {object} args.model modules.model, for recording provenance + * @param {object} [args.installImpl] injection seam, as everywhere else here + * @returns {Promise>} + */ +async function resolve({ value = process.env[VAR], hosts = [], model, installImpl = install } = {}) { + const { entries, errors } = parse(value) + results = [] + + for (const message of errors) log.error(`${VAR}: ${message}`) + + if (!entries.length) return results + + log.info(`${VAR} declares ${entries.length} module(s)`, { + modules: entries.map((e) => `${e.id}@${e.version}`).join(' '), + }) + + for (const entry of entries) { + const present = installedVersion(entry.id) + if (present === entry.version) { + // The offline path, and the common one: nothing is fetched, nothing is + // written, and a host with no route to the internet boots unchanged. + log.info(`module "${entry.id}" is already at the declared version ${entry.version}`) + results.push({ ...entry, action: 'noop', message: null }) + continue + } + + try { + // `expect` is the declaration itself, handed down so install.js can refuse + // a URL that resolves to another module or another version while it is + // still only a manifest — a check made after the unpack would be made with + // the undeclared module already on the volume. + const result = await installImpl.install({ + url: entry.url, + hosts, + expect: { id: entry.id, version: entry.version }, + }) + + // Provenance, written exactly as the admin route writes it — the whole + // point of resolving in-process rather than from a script that cannot + // reach the database. A module installed by the compose file and one + // installed by an admin are then indistinguishable on the screen, which + // is what makes this one feature and not two. + if (model) { + await model.recordInstalled({ + id: result.id, + name: result.name, + version: result.version, + source: entry.url, + sha256: result.sha256, + }) + } + + log.warn(`installed declared module "${entry.id}" v${entry.version}`, { + from: present || 'nothing', + source: entry.url, + sha256: result.sha256, + }) + results.push({ ...entry, action: 'installed', message: null }) + } catch (err) { + // Loud, and then onward. The site serves; this module does not, or serves + // the version that was already there. + log.error( + `could not resolve declared module "${entry.id}@${entry.version}": ${err.message}` + + (present ? ` — leaving version ${present} in place` : ''), + ) + results.push({ ...entry, action: 'failed', message: err.message }) + } + } + + return results +} + +/** What the last resolution decided, for the admin screen. */ +function state() { + return results.map((r) => ({ ...r })) +} + +/** Test seam: forget the last resolution. */ +function reset() { + results = [] +} + +module.exports = { VAR, parse, installedVersion, resolve, state, reset } diff --git a/server/src/modules/install.js b/server/src/modules/install.js index 973b05a..100e60a 100644 --- a/server/src/modules/install.js +++ b/server/src/modules/install.js @@ -79,6 +79,13 @@ class InstallError extends Error { // ── The allowlist ────────────────────────────────────────────────────────── +// The settings row the allowlist lives in (decision 6): seeded from +// MODULE_SOURCE_HOSTS on a fresh install and admin-managed from then on. The KEY +// lives here rather than in the admin controller because it is now read from two +// places — the controller, and the boot-time resolution of the declared module +// set (modules/declared.js), which has no route and no request. +const HOSTS_SETTING = 'module_source_hosts' + /** * Parse the stored allowlist setting into hostnames. * @@ -313,13 +320,34 @@ function purgeFile(id) { * scratch directory that is removed on any failure, and the move into place is * the last step. * + * `expect` is what the CALLER was promised, as opposed to what the manifest + * promises about itself — the declared module set (modules/declared.js) pins an + * id and a version in the environment, and a URL that resolves to something else + * has to be refused rather than installed. Checked against the manifest, before + * a byte is downloaded: catching it after the unpack would mean the undeclared + * module is already on the volume when the objection is raised. The admin panel + * passes nothing, because there a URL is the whole of what was asked for. + * * @param {object} args * @param {string} args.url the install manifest URL the admin pasted * @param {string[]} args.hosts the allowlist, already parsed + * @param {{id?: string, version?: string}} [args.expect] what the caller pinned * @returns {Promise<{id,name,version,sha256,source,bytes,replaced}>} */ -async function install({ url, hosts, fetchImpl = fetch }) { +async function install({ url, hosts, expect = null, fetchImpl = fetch }) { const manifest = await fetchManifest(url, hosts, fetchImpl) + + if (expect && expect.id && manifest.id !== expect.id) { + throw new InstallError( + `that URL installs the module "${manifest.id}", but "${expect.id}" was asked for`, + ) + } + if (expect && expect.version && manifest.version !== expect.version) { + throw new InstallError( + `that URL installs ${manifest.id} v${manifest.version}, but v${expect.version} was asked for`, + ) + } + const target = moduleDir(manifest.id) const scratch = await fsp.mkdtemp(path.join(loader.dir(), `.install-${manifest.id}-`)) const tarball = path.join(scratch, 'bundle.tar.gz') @@ -407,6 +435,7 @@ async function removeDir(id) { module.exports = { InstallError, + HOSTS_SETTING, parseHosts, checkUrl, get, diff --git a/server/src/router/v1/admin/modules.controller.js b/server/src/router/v1/admin/modules.controller.js index 2c79dc5..2d018c0 100644 --- a/server/src/router/v1/admin/modules.controller.js +++ b/server/src/router/v1/admin/modules.controller.js @@ -30,6 +30,7 @@ const settings = require('../../../model/settings/settings.model') const loader = require('../../../modules/loader') const lifecycle = require('../../../modules/lifecycle') const install = require('../../../modules/install') +const declared = require('../../../modules/declared') // A namespace import, like every other require in this file, and not // `const { runPurge } = …`: destructuring at require time captures the function // rather than the module, which makes it the one dependency here that cannot be @@ -42,8 +43,9 @@ const log = require('../../../utils/logger')('admin-modules') // The allowlist setting. Seeded from MODULE_SOURCE_HOSTS on first boot and // admin-managed from then on (decision 6) — db/seed.js writes it once and never // overwrites it, so changing the variable later does not silently reach in and -// undo an operator's choice. -const HOSTS_KEY = 'module_source_hosts' +// undo an operator's choice. The key itself lives in install.js, which is now +// read by boot-time declared-set resolution as well as by this controller. +const HOSTS_KEY = install.HOSTS_SETTING // A hostname, not a URL: no scheme, no path, no port, no wildcard. Deliberately // strict — every character allowed here is a character that can appear in the @@ -57,24 +59,29 @@ async function allowedHosts() { /** * One module, as the admin screen needs it. * - * Three sources have to be reconciled, and which one answers which question is + * Four sources have to be reconciled, and which one answers which question is * the whole of §2.4: * * - the ROW says what the operator decided and what the last boot recorded; * - the LOADER says what is mounted and answering right now; - * - the VOLUME says whether there is still a directory there at all. + * - the VOLUME says whether there is still a directory there at all; + * - the DECLARATION (slice 3) says what this container's environment asks for, + * which is the only one of the four an admin cannot change from this screen. * * They can legitimately disagree, and the screen has to show that rather than * pick a winner. A row `enabled` with a loader state of `disabled` is a module * the operator has just switched back on and which is waiting for a restart — * exactly the case decision 3 creates, and it would be a lie to render it as - * either "running" or "off". + * either "running" or "off". A declared module with no row and no directory is + * the newest of those disagreements: MODULES asked for it and resolution could + * not get it, so the screen carries the reason rather than showing nothing. */ -function present(row, live, onVolume) { +function present(row, live, onVolume, declaration = null) { + const id = row ? row.id : live ? live.id : declaration.id return { - id: row ? row.id : live.id, - name: row ? row.name : live.name, - version: row ? row.version : live.version, + id, + name: row ? row.name : live ? live.name : id, + version: row ? row.version : live ? live.version : null, // What the database records. state: row ? row.state : null, failureStage: row ? row.failureStage : (live && live.stage) || null, @@ -93,7 +100,16 @@ function present(row, live, onVolume) { capabilities: live ? live.capabilities : [], // What is on the volume. onVolume, - canPurge: onVolume && Boolean(install.purgeFile(row ? row.id : live.id)), + canPurge: onVolume && Boolean(install.purgeFile(id)), + // What the environment declares. `declaredVersion` is what MODULES pins, not + // what is installed — they differ exactly while a resolution is failing, and + // `declaredError` says why. Uninstalling a declared module from this screen + // removes its directory and disables its row; the next boot puts the + // directory back and leaves the row disabled, so the screen says so rather + // than letting the files reappear unexplained. + declared: Boolean(declaration), + declaredVersion: declaration ? declaration.version : null, + declaredError: declaration && declaration.action === 'failed' ? declaration.message : null, } } @@ -107,18 +123,31 @@ async function list(req, res) { // `npm run seed` never gets here, but a test harness might. const live = loader.isLoaded() ? loader.list() : [] const byId = new Map(live.map((m) => [m.id, m])) + const declaredById = new Map(declared.state().map((d) => [d.id, d])) const seen = new Set() const out = [] for (const row of rows) { seen.add(row.id) - out.push(present(row, byId.get(row.id) || null, install.isInstalled(row.id))) + out.push( + present(row, byId.get(row.id) || null, install.isInstalled(row.id), declaredById.get(row.id)), + ) } // A directory on the volume that has no row yet — a hand-placed install // before its first boot. It has to be listed, or the screen would show // nothing for a module whose routes are already being served. for (const m of live) { - if (!seen.has(m.id)) out.push(present(null, m, true)) + if (!seen.has(m.id)) { + seen.add(m.id) + out.push(present(null, m, true, declaredById.get(m.id))) + } + } + // A module MODULES declares that has neither. Resolution failed and left + // nothing behind — the case an operator most needs told, because from the + // screen's other three sources it is indistinguishable from never having + // asked for it. + for (const d of declaredById.values()) { + if (!seen.has(d.id)) out.push(present(null, null, install.isInstalled(d.id), d)) } return res.json({ modules: out, sourceHosts: await allowedHosts() }) diff --git a/server/src/server.js b/server/src/server.js index 1fc3951..9de900a 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -1,8 +1,12 @@ require('dotenv').config() const http = require('http') -const app = require('./app') -const internalApp = require('./internalApp') +// NOTE: `./app` and `./internalApp` are deliberately NOT required here. Requiring +// app.js runs `modules.load()`, which scans the volume and mounts whatever is on +// it (MODULE_API.md §4.1) — so the declared module set has to be resolved before +// that require, not before the listener. They are required inside start(), after +// resolveDeclaredModules(); everything else this file needs is safe to pull in +// now because none of it reaches the loader's scan. const botScore = require('./middleware/botScore') const announceWorker = require('./utils/announceWorker') const { ensureSchema, close } = require('./utils/db') @@ -11,6 +15,9 @@ const settings = require('./model/settings/settings.model') const revokedSessions = require('./model/revokedSessions/revokedSessions.model') const mobileAuthBridge = require('./model/mobileAuthBridge/mobileAuthBridge.model') const moduleLifecycle = require('./modules/lifecycle') +const declaredModules = require('./modules/declared') +const moduleInstall = require('./modules/install') +const moduleModel = require('./model/modules/modules.model') const createLogger = require('./utils/logger') const { evaluateBotInternalKey } = require('./utils/botInternalKey') const brand = require('./config/brand') @@ -51,7 +58,9 @@ async function start() { } log.info('ensuring database schema...') - await ensureSchema() + // Core's schema only. Each installed module's fragment is replayed further + // down, after the volume has been scanned — see the require of ./app below. + await ensureSchema({ replayModules: false }) log.info('seeding defaults...') await seedDefaults() await createInitialAdminFromEnv() @@ -77,6 +86,36 @@ async function start() { const mode = await settings.get('site_mode') log.info(`site mode: ${String(mode || 'live').toUpperCase()}`) + // Bring the modules volume in line with what MODULES declares (§2.7.2 + // decision 4), and only then require the app — the loader scans and mounts at + // require time, so this is the last moment at which a module can be put on the + // volume and still be part of this process. + // + // After the schema and the seed, because the host allowlist it installs under + // is a settings row that the seed creates on a fresh instance. Never throws: + // an unreachable release host leaves the site serving without that module + // rather than taking the site down with it. + await declaredModules.resolve({ + hosts: moduleInstall.parseHosts(await settings.get(moduleInstall.HOSTS_SETTING)), + model: moduleModel, + }) + + // Requiring app.js is what scans the volume and mounts what is on it. Every + // line above this one runs against a core that has no modules in it yet. + // eslint-disable-next-line global-require + const app = require('./app') + // eslint-disable-next-line global-require + const internalApp = require('./internalApp') + + // Now that the scan has happened, replay each module's schema fragment + // (MODULE_API.md §2.6). This used to ride inside ensureSchema() and could, + // because app.js was required at the top of this file; resolving the declared + // set first moved the scan after it, and a booting server quietly getting no + // module tables is precisely what §7.6 warns about. Caught by the browser + // smoke rather than by a test: every suite here stubs one side or the other. + // eslint-disable-next-line global-require + await require('./modules/schema').replayFragments() + // Reconcile installed_modules with what the loader found on the volume at // require time, then run each module's onBoot (MODULE_API.md §2.5). // diff --git a/server/src/utils/db.js b/server/src/utils/db.js index 86f63c9..755aba3 100644 --- a/server/src/utils/db.js +++ b/server/src/utils/db.js @@ -55,11 +55,18 @@ const SCHEMA_PATH = path.join(__dirname, '..', '..', 'db', 'schema.sql') * below — the discovery, splitting and per-module failure handling all live in * modules/schema.js, required lazily so that requiring the pool never drags the * loader in with it. + * + * `replayModules: false` is for a caller that has not scanned the volume YET and + * intends to. server.js is the one: since slice 3 it resolves the declared + * module set before requiring app.js, which puts core's schema *before* the scan + * — so it replays the fragments itself, in the one place that knows the scan has + * happened. Left true everywhere else, so the ordinary caller cannot get module + * tables by accident and lose them by refactor. */ -async function ensureSchema({ retries = 10, delayMs = 2000 } = {}) { +async function ensureSchema({ retries = 10, delayMs = 2000, replayModules = true } = {}) { await ensureCoreSchema({ retries, delayMs }) // eslint-disable-next-line global-require - await require('../modules/schema').replayFragments() + if (replayModules) await require('../modules/schema').replayFragments() } /** Core's own schema.sql, with the wait-for-the-database retry. */ diff --git a/server/test/adminModules.test.js b/server/test/adminModules.test.js index 46d3e00..e14d175 100644 --- a/server/test/adminModules.test.js +++ b/server/test/adminModules.test.js @@ -29,6 +29,7 @@ const settings = require('../src/model/settings/settings.model') const loader = require('../src/modules/loader') const lifecycle = require('../src/modules/lifecycle') const install = require('../src/modules/install') +const declared = require('../src/modules/declared') const schema = require('../src/modules/schema') const db = require('../src/utils/db') @@ -67,6 +68,7 @@ const originals = { removeDir: install.removeDir, }, schema: { runPurge: schema.runPurge }, + declared: { state: declared.state }, } let logged @@ -79,6 +81,7 @@ beforeEach(() => { Object.assign(lifecycle, originals.lifecycle) Object.assign(install, originals.install) Object.assign(schema, originals.schema) + Object.assign(declared, originals.declared) logged = [] activity.log = async (entry) => { logged.push(entry) } @@ -133,6 +136,55 @@ test('list includes a module on the volume that has no row yet', async () => { assert.equal(res.body.modules[0].liveState, 'started') }) +test('list carries what MODULES declares, including a module it could not install', async () => { + // Slice 3's fourth source. A declared module that failed to resolve has no + // row, no directory and nothing mounted — so it is invisible to the other + // three, and the reason it is missing is the one thing the operator needs. + modules.list = async () => [{ + id: 'uo', name: 'UO', version: '0.2.0', state: 'enabled', + failureStage: null, failureReason: null, source: null, sha256: null, + installedAt: null, startedAt: null, + }] + loader.list = () => [{ id: 'uo', name: 'UO', version: '0.2.0', state: 'started', stage: null, reason: null, capabilities: [] }] + install.isInstalled = (id) => id === 'uo' + install.purgeFile = () => null + declared.state = () => [ + { id: 'uo', version: '0.3.0', url: 'https://x/uo.json', action: 'failed', message: 'the host is down' }, + { id: 'market', version: '1.0.0', url: 'https://x/market.json', action: 'failed', message: 'not found' }, + ] + + const res = mockRes() + await ctrl.list(req(), res) + + const byId = Object.fromEntries(res.body.modules.map((m) => [m.id, m])) + // Running fine at 0.2.0 while the declared upgrade to 0.3.0 is failing: both + // facts survive, because collapsing them would have to discard one. + assert.equal(byId.uo.liveState, 'started') + assert.equal(byId.uo.declared, true) + assert.equal(byId.uo.declaredVersion, '0.3.0') + assert.equal(byId.uo.declaredError, 'the host is down') + // Declared and nowhere: listed anyway, with no version invented for it. + assert.equal(byId.market.declared, true) + assert.equal(byId.market.version, null) + assert.equal(byId.market.state, null) + assert.equal(byId.market.declaredError, 'not found') +}) + +test('a successfully resolved module is marked declared, with no error', async () => { + modules.list = async () => [] + loader.list = () => [{ id: 'uo', name: 'UO', version: '0.3.0', state: 'started', stage: null, reason: null, capabilities: [] }] + install.isInstalled = () => true + install.purgeFile = () => null + declared.state = () => [{ id: 'uo', version: '0.3.0', url: 'https://x/uo.json', action: 'noop', message: null }] + + const res = mockRes() + await ctrl.list(req(), res) + + assert.equal(res.body.modules.length, 1, 'declared and present is ONE module, not two') + assert.equal(res.body.modules[0].declared, true) + assert.equal(res.body.modules[0].declaredError, null) +}) + test('list survives a process where the loader never scanned', async () => { loader.isLoaded = () => false loader.list = () => { throw new Error('modules.list() before modules.load()') } diff --git a/server/test/bootOrder.test.js b/server/test/bootOrder.test.js new file mode 100644 index 0000000..a1de893 --- /dev/null +++ b/server/test/bootOrder.test.js @@ -0,0 +1,82 @@ +// server.js's boot ORDER, which is a contract and not a style choice. +// +// Phase 4, slice 3 of MODULE_SYSTEM.md §2.7.2. Five steps have to happen in one +// order, and each arrow is a dependency that is invisible at the call site: +// +// core schema → the declared set (§2.7.2 decision 4) needs the settings row +// its seed writes, to know which hosts it may install from +// declared set → requiring app.js SCANS the volume (§1.12), so anything put +// there afterwards is not in this process +// require app → module schema fragments (§2.6) can only be replayed once the +// loader knows which modules there are +// fragments → onBoot runs against tables that exist +// +// This is a source-structure test, and it is worth being plain about what that +// does and does not prove: it cannot tell you the server boots, only that nobody +// has quietly moved one of these five lines past another. It exists because the +// defect it guards against has already happened once and was invisible to every +// other kind of test here. Deferring the app require — which slice 3 had to do — +// moved core's schema ahead of the scan, and `ensureSchema` then skipped the +// module fragments entirely. On this machine's dev database the tables already +// existed, so the module started; on a FRESH database it would have started +// against no tables at all. Every suite in this directory stubs either the +// loader or the pool, so none of them could see it. The browser smoke did, from +// one log line. +process.env.DB_HOST = '127.0.0.1' +process.env.DB_PORT = '59999' + +const fs = require('fs') +const path = require('path') + +const { test } = require('node:test') +const assert = require('node:assert/strict') + +const SERVER = fs.readFileSync(path.join(__dirname, '..', 'src', 'server.js'), 'utf8') + +/** Where a marker appears, asserted to appear exactly once. */ +function at(marker) { + const first = SERVER.indexOf(marker) + assert.notEqual(first, -1, `server.js no longer contains ${JSON.stringify(marker)}`) + assert.equal( + SERVER.indexOf(marker, first + 1), + -1, + `${JSON.stringify(marker)} appears more than once in server.js — this test cannot tell which is the boot step`, + ) + return first +} + +test('boot runs core schema, then the declared set, then the scan, then fragments, then onBoot', () => { + const steps = [ + ['core schema', at('await ensureSchema({ replayModules: false })')], + ['declared module set', at('await declaredModules.resolve(')], + ['the volume scan', at("require('./app')")], + ['module schema fragments', at('replayFragments()')], + ['module onBoot', at('await moduleLifecycle.boot()')], + ] + + for (let i = 1; i < steps.length; i += 1) { + assert.ok( + steps[i][1] > steps[i - 1][1], + `${steps[i][0]} must come after ${steps[i - 1][0]} in server.js`, + ) + } +}) + +test('app.js is required inside start(), never at the top of the file', () => { + // The whole mechanism depends on this. A top-level require runs when server.js + // is loaded — before a single line of start() — and the scan would then happen + // before the declared set had a chance to put anything on the volume, silently + // and with no error anywhere. + assert.ok( + at("require('./app')") > at('async function start()'), + 'requiring ./app at the top of server.js scans the volume before the declared set is resolved', + ) +}) + +test('ensureSchema is asked NOT to replay fragments, and something else does', () => { + // Both halves matter. Passing the flag without replaying elsewhere is the + // original defect with an explicit spelling; replaying without the flag runs + // the fragments twice, the second time being the one that matters. + assert.match(SERVER, /ensureSchema\(\{ replayModules: false \}\)/) + assert.match(SERVER, /modules\/schema'\)\.replayFragments\(\)/) +}) diff --git a/server/test/moduleDeclared.test.js b/server/test/moduleDeclared.test.js new file mode 100644 index 0000000..13ab0e2 --- /dev/null +++ b/server/test/moduleDeclared.test.js @@ -0,0 +1,391 @@ +// modules/declared.js — resolving the module set an environment declares. +// +// Phase 4, slice 3 of MODULE_SYSTEM.md §2.7.2, decision 4. Two claims are worth +// more than the rest and both are about what does NOT happen: +// +// - a module already unpacked at the declared version does not touch the +// network, because that is what makes a restart with the internet down come +// up unchanged; +// - a module that cannot be resolved does not fail the boot, and does not stop +// the next declaration from resolving. +// +// The last test in the file runs the whole path through the real install.js +// against a fake transport — a real gzipped tar, really hashed, really unpacked +// — because everything above it stubs `installImpl` and would keep passing if +// the two files stopped agreeing about what an install returns. +process.env.DB_HOST = '127.0.0.1' +process.env.DB_PORT = '59999' + +const crypto = require('crypto') +const fs = require('fs') +const os = require('os') +const path = require('path') +const zlib = require('zlib') + +const { test, beforeEach, after } = require('node:test') +const assert = require('node:assert/strict') + +const db = require('../src/utils/db') + +after(() => db.close()) + +const HOSTS = ['releases.example.com'] +const URL_030 = 'https://releases.example.com/mod/uo-0.3.0.json' + +let tmpRoot +let declared + +/** + * A fresh declared.js bound to a fresh modules directory. + * + * loader.js resolves MODULES_DIR once at require time and install.js reads + * `loader.dir()`, so all three have to come back together — the same dance + * moduleInstall.test.js and moduleLifecycle.test.js do. + */ +function fresh(dir) { + process.env.MODULES_DIR = dir + for (const m of ['loader', 'install', 'declared']) { + delete require.cache[require.resolve(`../src/modules/${m}`)] + } + // eslint-disable-next-line global-require + return require('../src/modules/declared') +} + +/** Put a module directory on the volume, as an unpack would leave it. */ +function place(id, version) { + const dir = path.join(tmpRoot, id) + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync( + path.join(dir, 'module.json'), + JSON.stringify({ id, name: 'Ultima Online', version, coreApi: '^1.0.0' }), + ) + return dir +} + +/** + * A stub install service. + * + * Records every call, so "did this reach the network at all" is a question the + * tests can ask directly rather than inferring from an outcome. + */ +function fakeInstall({ version = '0.3.0', throws = null } = {}) { + const calls = [] + return { + calls, + InstallError: Error, + async install(args) { + calls.push(args) + if (throws) throw new Error(throws) + return { + id: 'uo', + name: 'Ultima Online', + version, + sha256: 'a'.repeat(64), + source: args.url, + bytes: 1024, + replaced: false, + } + }, + } +} + +/** A modules.model stand-in that remembers what provenance it was given. */ +function fakeModel() { + const recorded = [] + return { recorded, async recordInstalled(row) { recorded.push(row); return row } } +} + +beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'rg-declared-')) + declared = fresh(tmpRoot) +}) + +// ── Parsing ──────────────────────────────────────────────────────────────── + +test('parses id@version=url entries separated by whitespace or commas', () => { + const { entries, errors } = declared.parse( + ` uo@0.3.0=https://a.example/uo.json,\n market@1.2.3=https://a.example/market.json `, + ) + assert.deepEqual(errors, []) + assert.deepEqual(entries, [ + { id: 'uo', version: '0.3.0', url: 'https://a.example/uo.json' }, + { id: 'market', version: '1.2.3', url: 'https://a.example/market.json' }, + ]) +}) + +test('an empty or unset declaration parses to nothing, without complaint', () => { + for (const value of [undefined, '', ' ']) { + const { entries, errors } = declared.parse(value) + assert.deepEqual(entries, []) + assert.deepEqual(errors, []) + } +}) + +test('a malformed entry is refused on its own, leaving the others', () => { + // A bare URL, a missing version, and an id that is not one — each of which an + // operator can plausibly type, and none of which should cost the module that + // was written correctly. + const { entries, errors } = declared.parse( + 'https://a.example/uo.json uo@=https://a.example/uo.json Uo@1=https://a.example/uo.json' + + ' good@1.0.0=https://a.example/good.json', + ) + assert.equal(entries.length, 1) + assert.equal(entries[0].id, 'good') + assert.equal(errors.length, 3) +}) + +test('a duplicate id keeps the first and says so', () => { + const { entries, errors } = declared.parse( + 'uo@1.0.0=https://a.example/one.json uo@2.0.0=https://a.example/two.json', + ) + assert.equal(entries.length, 1) + assert.equal(entries[0].version, '1.0.0') + assert.match(errors[0], /declared more than once/) +}) + +// ── Resolution ───────────────────────────────────────────────────────────── + +test('a module already at the declared version is a no-op that never fetches', async () => { + place('uo', '0.3.0') + const installer = fakeInstall() + const model = fakeModel() + + const results = await declared.resolve({ + value: `uo@0.3.0=${URL_030}`, + hosts: HOSTS, + model, + installImpl: installer, + }) + + assert.deepEqual(results.map((r) => r.action), ['noop']) + // The offline guarantee, stated as an assertion: nothing was fetched and + // nothing was written. A restart with no route to the internet is this case. + assert.equal(installer.calls.length, 0) + assert.equal(model.recorded.length, 0) +}) + +test('a missing module is installed, with its provenance recorded', async () => { + const installer = fakeInstall() + const model = fakeModel() + + const results = await declared.resolve({ + value: `uo@0.3.0=${URL_030}`, + hosts: HOSTS, + model, + installImpl: installer, + }) + + assert.deepEqual(results.map((r) => r.action), ['installed']) + assert.equal(installer.calls[0].url, URL_030) + assert.deepEqual(installer.calls[0].hosts, HOSTS) + // The declaration is handed down so install.js can refuse a URL that turns out + // to be another module or another version BEFORE it downloads it. + assert.deepEqual(installer.calls[0].expect, { id: 'uo', version: '0.3.0' }) + // Written exactly as the admin route writes it — the reason this runs in the + // server process rather than in a script that cannot reach the database. + assert.deepEqual(model.recorded, [ + { id: 'uo', name: 'Ultima Online', version: '0.3.0', source: URL_030, sha256: 'a'.repeat(64) }, + ]) +}) + +test('a module at a different version is re-resolved', async () => { + place('uo', '0.2.0') + const installer = fakeInstall() + + const results = await declared.resolve({ + value: `uo@0.3.0=${URL_030}`, + hosts: HOSTS, + model: fakeModel(), + installImpl: installer, + }) + + assert.deepEqual(results.map((r) => r.action), ['installed']) + assert.equal(installer.calls.length, 1) +}) + +test('a directory with no readable module.json counts as absent', async () => { + fs.mkdirSync(path.join(tmpRoot, 'uo'), { recursive: true }) + fs.writeFileSync(path.join(tmpRoot, 'uo', 'module.json'), '{ this is not json') + const installer = fakeInstall() + + await declared.resolve({ + value: `uo@0.3.0=${URL_030}`, + hosts: HOSTS, + model: fakeModel(), + installImpl: installer, + }) + + // Whatever is in that directory, it is not the declared module — so the + // declared module is fetched rather than assumed to be there. + assert.equal(installer.calls.length, 1) +}) + +test('a failure is carried, not thrown, and does not stop the next module', async () => { + const model = fakeModel() + const installer = { + calls: [], + async install(args) { + installer.calls.push(args) + if (args.expect.id === 'uo') throw new Error('could not reach releases.example.com') + return { + id: args.expect.id, + name: 'Market', + version: args.expect.version, + sha256: 'b'.repeat(64), + source: args.url, + bytes: 10, + replaced: false, + } + }, + } + + const results = await declared.resolve({ + value: `uo@0.3.0=${URL_030} market@1.0.0=https://releases.example.com/market.json`, + hosts: HOSTS, + model, + installImpl: installer, + }) + + assert.deepEqual(results.map((r) => r.action), ['failed', 'installed']) + assert.match(results[0].message, /could not reach/) + // The second module still installed: one unreachable release host costs that + // module, not the deployment. + assert.equal(model.recorded.length, 1) + assert.equal(model.recorded[0].id, 'market') +}) + +test('a failed resolution leaves whatever was already on the volume', async () => { + place('uo', '0.2.0') + const installer = fakeInstall({ throws: 'the host is down' }) + + const results = await declared.resolve({ + value: `uo@0.3.0=${URL_030}`, + hosts: HOSTS, + model: fakeModel(), + installImpl: installer, + }) + + assert.equal(results[0].action, 'failed') + // The site comes up serving the version it already had rather than not at all. + assert.equal(declared.installedVersion('uo'), '0.2.0') +}) + +test('resolution without a model records nothing and still installs', async () => { + // `npm run seed` and the test harness both reach code paths with no model to + // hand; the volume half must not depend on the database half. + const installer = fakeInstall() + const results = await declared.resolve({ value: `uo@0.3.0=${URL_030}`, hosts: HOSTS, installImpl: installer }) + assert.deepEqual(results.map((r) => r.action), ['installed']) +}) + +test('state() reports the last resolution and is replaced by the next', async () => { + const installer = fakeInstall() + await declared.resolve({ value: `uo@0.3.0=${URL_030}`, hosts: HOSTS, installImpl: installer }) + assert.equal(declared.state().length, 1) + assert.equal(declared.state()[0].id, 'uo') + + await declared.resolve({ value: '', hosts: HOSTS, installImpl: installer }) + assert.deepEqual(declared.state(), []) +}) + +// ── The whole path, once, for real ───────────────────────────────────────── + +const BLOCK = 512 + +function octal(value, width) { + return Number(value).toString(8).padStart(width - 1, '0') + '\0' +} + +function member({ name, type = '0', body = '' }) { + const header = Buffer.alloc(BLOCK, 0) + const data = Buffer.from(body, 'utf8') + header.write(name, 0, 100, 'utf8') + header.write(octal(0o644, 8), 100, 8, 'ascii') + header.write(octal(0, 8), 108, 8, 'ascii') + header.write(octal(0, 8), 116, 8, 'ascii') + header.write(octal(data.length, 12), 124, 12, 'ascii') + header.write(octal(0, 12), 136, 12, 'ascii') + header.write(' ', 148, 8, 'ascii') + header.write(type, 156, 1, 'ascii') + header.write('ustar\0', 257, 6, 'ascii') + header.write('00', 263, 2, 'ascii') + let sum = 0 + for (const byte of header) sum += byte + header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'ascii') + const padding = Buffer.alloc((BLOCK - (data.length % BLOCK)) % BLOCK, 0) + return Buffer.concat([header, data, padding]) +} + +/** A bundle tarball, named the way module-uo's release workflow names one. */ +function bundle(version) { + const root = `module-uo-${version}` + return zlib.gzipSync(Buffer.concat([ + member({ name: `${root}/`, type: '5' }), + member({ + name: `${root}/module.json`, + body: JSON.stringify({ id: 'uo', name: 'Ultima Online', version, coreApi: '^1.0.0', server: 'server/index.js' }), + }), + member({ name: `${root}/server/`, type: '5' }), + member({ name: `${root}/server/index.js`, body: 'module.exports = () => {}\n' }), + Buffer.alloc(BLOCK * 2, 0), + ])) +} + +test('end to end: a declaration installs a real bundle through the real install path', async (t) => { + const tarball = bundle('0.3.0') + const artifactUrl = 'https://releases.example.com/mod/uo-0.3.0.tar.gz' + const manifest = JSON.stringify({ + schema: 1, + id: 'uo', + name: 'Ultima Online', + version: '0.3.0', + coreApi: '^1.0.0', + artifact: 'uo-0.3.0.tar.gz', + url: artifactUrl, + sha256: crypto.createHash('sha256').update(tarball).digest('hex'), + size: tarball.length, + }) + const routes = { [URL_030]: manifest, [artifactUrl]: tarball } + const fetched = [] + const fetchImpl = async (url) => { + fetched.push(String(url)) + const body = routes[String(url)] + return body === undefined ? new Response('nope', { status: 404 }) : new Response(body, { status: 200 }) + } + + // The real install.js, with only the transport replaced — same seam the + // install tests use, and the same reason: what is under test is the decisions, + // not Node's TLS. + // eslint-disable-next-line global-require + const install = require('../src/modules/install') + const installImpl = { install: (args) => install.install({ ...args, fetchImpl }) } + const model = fakeModel() + + const first = await declared.resolve({ + value: `uo@0.3.0=${URL_030}`, + hosts: HOSTS, + model, + installImpl, + }) + assert.deepEqual(first.map((r) => r.action), ['installed']) + // Unpacked, with the release's top-level directory stripped, under the id. + assert.equal( + JSON.parse(fs.readFileSync(path.join(tmpRoot, 'uo', 'module.json'), 'utf8')).version, + '0.3.0', + ) + assert.equal(model.recorded[0].sha256, JSON.parse(manifest).sha256) + + // And again, which is what every restart after the first one is. + const before = fetched.length + const second = await declared.resolve({ value: `uo@0.3.0=${URL_030}`, hosts: HOSTS, model, installImpl }) + assert.deepEqual(second.map((r) => r.action), ['noop']) + assert.equal(fetched.length, before, 'the second resolution must not fetch anything') + + // A declaration whose URL resolves to a different version is refused BEFORE + // the artifact is downloaded — the module on the volume is left alone. + const wrong = await declared.resolve({ value: `uo@0.4.0=${URL_030}`, hosts: HOSTS, model, installImpl }) + assert.equal(wrong[0].action, 'failed') + assert.match(wrong[0].message, /v0\.4\.0 was asked for/) + assert.equal(declared.installedVersion('uo'), '0.3.0') + t.diagnostic(`fetched ${fetched.length} URL(s) across three resolutions`) +})