feat(modules): the three de-entanglement registries, with core as the registrant

Phase 2 PR 4 of docs/website/MODULE_SYSTEM.md §2.7. Adds server/src/modules/registries.js
and moves core's own notification streams, announce leg and users-detail routes
behind it, so the three seams §1.8 and §1.9 named are exercised on every boot
before any module depends on them.

Registering is validate-then-commit per registrant: the loader stages what a
module claims and the second pass commits it, so a module that throws halfway
through register() — or fails checkDeclared after it — leaves nothing behind.
That is the registry-side twin of PR 2's second-pass mount rule.

Four decisions, all the recommended option:

- announce legs became a child table. `announce_job_legs` replaces the
  towncrier_*/discord_* column groups, so the leg set is data: core registers
  `discord`, module-uo will register `towncrier`, and a module cannot ALTER a
  core table to add its own. Backfill is guarded on information_schema (a
  SELECT of a dropped column is a parse error, not a runtime one) and the
  columns go with DROP COLUMN IF EXISTS. Verified against the live dev DB:
  three legacy jobs migrated faithfully, three replays, no duplicates.
- `mapEvent` dropped from registerNotificationStreams. §1.8 already inverts the
  push path so a module owns fromShardEvent and calls core's publish() with a
  stream id it resolved; a second mapping mechanism was a leftover. The public
  safety filter, the kinds it reads and the streams it protects now live in one
  file and move together.
- core registers through the same staging area a module uses, via an explicit
  registries.registerCore() in app.js before modules.load().
- core's six /admin/users/:id/shard/* paths now go through the
  `admin.users.detail` slot, and getUser moved back to admin.controller.js.

Found on the way, and the reason two build tools changed:

- scripts/routeManifest.js could not decode a parameterised mount. Its
  unwinder expected `(?:([^\/]+?))`; express 4.22 emits `(?:\/([^/]+?))` with
  the separator inside the group. The branch had never run. It threw rather
  than guessing, which is what it is for.
- swagger-autogen cannot follow a route into an extension slot — the slot's
  router is created by declareSlot() and filled later, so there is no literal
  mount for a static parse. Regenerating deleted 407 lines and printed
  `Swagger-autogen: Success`, the spike's exact failure (MODULE_API.md §7.4).
  swagger/slotSpecs.js generates a fragment per filled slot and re-roots it at
  the prefix the router actually hangs at in the live app — read from the
  express stack via routeManifest's own mountPath, so the manifest and the spec
  cannot disagree. swagger/mergeSpec.js is the merge helper core owes for
  module fragments anyway (§6.1a), proved here against core's own slot first.

884 tests pass (856 before). routes.manifest.json is unchanged at 229 routes.
The OpenAPI spec diff is two lines of intent: the retry endpoint's summary, and
its `leg` no longer being a fixed enum.

Co-Authored-By: Claude <noreply@anthropic.com>

server/src/model/announceJobs/announceJobs.db.js

@@ -1,26 +1,68 @@
+// ── Announcement pipeline: SQL ─────────────────────────────────────────────
+//
+// Two tables since PR 4 (MODULE_SYSTEM.md §1.8): `announce_jobs` is one row per
+// publish event, `announce_job_legs` one row per delivery leg of that job. The
+// leg set is registered rather than fixed, so a leg is a stored VALUE now instead
+// of a group of leg-prefixed columns — which is what lets a module bring its own
+// leg without altering a core table.
+//
+// Every read returns the job with a `legs` array attached, so a caller never has
+// to remember to fetch the second table.
+
 const { query } = require('../../utils/db')
 
-const COLS =
-  'id, post_id, status, ' +
-  'towncrier_status, towncrier_attempts, towncrier_last_error, towncrier_next_attempt_at, ' +
-  'discord_status, discord_attempts, discord_last_error, discord_next_attempt_at, ' +
-  'created_at, updated_at'
+const COLS = 'id, post_id, status, created_at, updated_at'
+const LEG_COLS = 'job_id, leg, status, attempts, last_error, next_attempt_at'
 
-// Whitelist so a `leg` value can be interpolated into a column name safely — it
-// never comes from raw user input, but keep the guard explicit.
-const LEGS = ['towncrier', 'discord']
-function assertLeg(leg) {
-  if (!LEGS.includes(leg)) throw new Error(`unknown announce leg: ${leg}`)
+async function legsFor(jobIds) {
+  if (jobIds.length === 0) return new Map()
+  const marks = jobIds.map(() => '?').join(', ')
+  const rows = await query(
+    `SELECT ${LEG_COLS} FROM announce_job_legs WHERE job_id IN (${marks}) ORDER BY job_id, leg`,
+    jobIds,
+  )
+  const byJob = new Map(jobIds.map((id) => [id, []]))
+  for (const row of rows) byJob.get(row.job_id).push(row)
+  return byJob
 }
 
-async function create(postId) {
+async function attachLegs(jobs) {
+  const byJob = await legsFor(jobs.map((j) => j.id))
+  for (const job of jobs) job.legs = byJob.get(job.id) || []
+  return jobs
+}
+
+// Create a job and its leg rows in one go. `legs` is the registered leg id list —
+// an empty list is legal and yields a job with nothing to deliver.
+async function create(postId, legs = []) {
   const res = await query('INSERT INTO announce_jobs (post_id) VALUES (?)', [postId])
-  return res.insertId
+  const jobId = Number(res.insertId)
+  if (legs.length > 0) {
+    const values = legs.map(() => '(?, ?)').join(', ')
+    await query(
+      `INSERT INTO announce_job_legs (job_id, leg) VALUES ${values}`,
+      legs.flatMap((leg) => [jobId, leg]),
+    )
+  }
+  return jobId
+}
+
+// Add any registered legs this job is missing. A job enqueued before a module was
+// installed has no row for that module's leg, and without this it could never
+// deliver one — the worker only ever sees rows that exist.
+async function ensureLegs(jobId, legs = []) {
+  if (legs.length === 0) return
+  const values = legs.map(() => '(?, ?)').join(', ')
+  await query(
+    `INSERT IGNORE INTO announce_job_legs (job_id, leg) VALUES ${values}`,
+    legs.flatMap((leg) => [jobId, leg]),
+  )
 }
 
 async function findById(id) {
   const rows = await query(`SELECT ${COLS} FROM announce_jobs WHERE id = ? LIMIT 1`, [id])
-  return rows[0] || null
+  if (rows.length === 0) return null
+  return (await attachLegs(rows))[0]
 }
 
 async function findByPostId(postId) {
@@ -28,36 +70,38 @@ async function findByPostId(postId) {
     `SELECT ${COLS} FROM announce_jobs WHERE post_id = ? ORDER BY id DESC LIMIT 1`,
     [postId],
   )
-  return rows[0] || null
+  if (rows.length === 0) return null
+  return (await attachLegs(rows))[0]
 }
 
 // Jobs with at least one leg that is due now: pending and either never scheduled
-// (next_attempt_at IS NULL — a fresh enqueue) or past its backoff time.
+// (next_attempt_at IS NULL — a fresh enqueue) or past its backoff time. Returns
+// whole jobs with every leg attached; the worker decides which legs to run, so
+// this stays one query regardless of how many legs are registered.
 async function findDue(now = new Date(), limit = 25) {
-  return query(
-    `SELECT ${COLS} FROM announce_jobs
-      WHERE (towncrier_status = 'pending'
-             AND (towncrier_next_attempt_at IS NULL OR towncrier_next_attempt_at <= ?))
-         OR (discord_status = 'pending'
-             AND (discord_next_attempt_at IS NULL OR discord_next_attempt_at <= ?))
-      ORDER BY id ASC
+  const rows = await query(
+    `SELECT ${COLS} FROM announce_jobs j
+      WHERE EXISTS (
+        SELECT 1 FROM announce_job_legs l
+         WHERE l.job_id = j.id
+           AND l.status = 'pending'
+           AND (l.next_attempt_at IS NULL OR l.next_attempt_at <= ?))
+      ORDER BY j.id ASC
       LIMIT ?`,
-    [now, now, limit],
+    [now, limit],
   )
+  return attachLegs(rows)
 }
 
-// Update one leg's columns. `fields` uses leg-agnostic keys (status, attempts,
-// lastError, nextAttemptAt); we map them onto the leg-prefixed columns.
-async function updateLeg(id, leg, { status, attempts, lastError, nextAttemptAt }) {
-  assertLeg(leg)
+// Update one leg's row. `leg` is a bound VALUE, not an interpolated column name —
+// the reason the old leg allowlist that guarded that interpolation is gone. A
+// module's leg id could not have passed it anyway.
+async function updateLeg(jobId, leg, { status, attempts, lastError, nextAttemptAt }) {
   await query(
-    `UPDATE announce_jobs SET
-       ${leg}_status = ?,
-       ${leg}_attempts = ?,
-       ${leg}_last_error = ?,
-       ${leg}_next_attempt_at = ?
-     WHERE id = ?`,
-    [status, attempts, lastError ?? null, nextAttemptAt ?? null, id],
+    `UPDATE announce_job_legs SET
+       status = ?, attempts = ?, last_error = ?, next_attempt_at = ?
+     WHERE job_id = ? AND leg = ?`,
+    [status, attempts, lastError ?? null, nextAttemptAt ?? null, jobId, leg],
   )
 }
 
@@ -65,4 +109,4 @@ async function setStatus(id, status) {
   await query('UPDATE announce_jobs SET status = ? WHERE id = ?', [status, id])
 }
 
-module.exports = { LEGS, create, findById, findByPostId, findDue, updateLeg, setStatus }
+module.exports = { create, ensureLegs, findById, findByPostId, findDue, updateLeg, setStatus }

server/src/model/announceJobs/announceJobs.logic.js

@@ -1,87 +1,38 @@
 // ── Announcement pipeline: pure logic ──────────────────────────────────────
 //
-// No DB, no network — just the decisions the worker and model make, kept here so
-// they are unit-testable in isolation (server/test/announceJobs.test.js):
-//   • buildTownCrierText — turn a post into sidecar-safe town-crier lines
-//   • classifyTownCrier / classifyDiscord — map a dispatch result to done / retry
-//     / terminal, so a data problem fails fast and a transient outage retries
+// No DB, no network — just the LEG-AGNOSTIC decisions the worker and model make,
+// kept here so they are unit-testable in isolation (server/test/announceJobs.test.js):
 //   • scheduleAfter — exponential backoff schedule + the attempt cap
-//   • rollupStatus — derive the parent job status from the two legs
-
-const { deriveExcerpt } = require('../../utils/sanitizeHtml')
-
-// Sidecar town-crier caps, mirrored from the admin route validation
-// (admin/uoLink.router.js: lines isArray({ max: 8 }), lines.* isLength({ max: 200 })).
-// We pre-truncate to these so a published post never bounces with towncrier.error.
-const MAX_LINES = 8
-const MAX_LINE_LEN = 200
+//   • rollupStatus  — derive the parent job status from its legs
+//   • legError      — squeeze a client result into one error line
+//   • baseUrl / articleUrl — the public link an announcement carries
+//
+// What used to be here and is not any more: `buildTownCrierText`,
+// `classifyTownCrier` and `classifyDiscord`. A leg's own text-building and result
+// classification belong to the leg, and a leg is a registration now
+// (MODULE_SYSTEM.md §1.8) — they live in utils/shardAnnounce.js and
+// utils/discordAnnounce.js. This file is what every leg shares.
 
 // Backoff between retries, indexed by attempts-so-far. Six attempts spread over
 // ~a couple of hours; after the last one a leg is marked failed and surfaced in
-// the post's admin panel. Shared by both legs.
+// the post's admin panel. Shared by every leg.
 const BACKOFF_MS = [30_000, 120_000, 600_000, 1_800_000, 3_600_000, 7_200_000]
 const MAX_ATTEMPTS = BACKOFF_MS.length
 
-// Trim to a hard length, appending an ellipsis only when something was cut.
-function clamp(value, max) {
-  const s = String(value == null ? '' : value)
-    .replace(/\s+/g, ' ')
-    .trim()
-  if (s.length <= max) return s
-  return `${s.slice(0, max - 1).trimEnd()}…`
+// The site's public base, used to build the link an announcement carries.
+function baseUrl() {
+  return (process.env.APP_BASE_URL || 'http://localhost:5173').replace(/\/+$/, '')
 }
 
 // The public link that goes in the announcement. News has no per-post route
 // (App.jsx only has the /site/news list), so we link the list — matches the
 // pre-pipeline Discord announce behavior.
-function articleUrl(baseUrl) {
-  return `${String(baseUrl || '').replace(/\/+$/, '')}/site/news`
-}
-
-// Build the town-crier lines: title, a one-line excerpt, then the URL. Each line
-// is clamped to the sidecar's per-line cap and the whole thing to the line-count
-// cap. Falls back to a stripped body excerpt when the post has no excerpt.
-function buildTownCrierText(post, { baseUrl } = {}) {
-  const title = clamp(post.title, MAX_LINE_LEN)
-  const excerptSource = post.excerpt || deriveExcerpt(post.body, MAX_LINE_LEN) || ''
-  const lines = [title]
-  const excerpt = clamp(excerptSource, MAX_LINE_LEN)
-  if (excerpt) lines.push(excerpt)
-  const url = clamp(articleUrl(baseUrl), MAX_LINE_LEN)
-  if (url) lines.push(url)
-  return lines.filter(Boolean).slice(0, MAX_LINES)
-}
-
-// ── Result classification ──────────────────────────────────────────────────
-// Both clients return { ok, status, error }. Map that to one of:
-//   done     — delivered, mark the leg done
-//   retry    — transient (shard restarting, bot down, network); back off + retry
-//   terminal — will never succeed as-is (over caps, bad auth/config); fail now
-
-function classifyTownCrier(result) {
-  if (result && result.ok) return { outcome: 'done' }
-  const status = result ? result.status : 0
-  // 400 = over the line/duration caps (a data problem — do NOT retry).
-  // 401 = token mismatch, 409 = protocol mismatch (both config problems).
-  if (status === 400 || status === 401 || status === 409) {
-    return { outcome: 'terminal', error: legError(result) }
-  }
-  // 503 (shard not connected), 504 (shard timeout), 0 (network/timeout / not
-  // configured yet), and any other 5xx are transient — retry.
-  return { outcome: 'retry', error: legError(result) }
-}
-
-function classifyDiscord(result) {
-  if (result && result.ok) return { outcome: 'done' }
-  // The bot's /internal/announce collapses failures (503 = not connected,
-  // 400 = no news channel configured) without surfacing Discord's own
-  // retry_after, so there is no reliable terminal signal to key on here. Retry
-  // every failure on the shared backoff; a genuine config problem simply
-  // exhausts its attempts and lands as `failed` in the admin panel, where the
-  // per-leg retry button re-runs it after the channel is set.
-  return { outcome: 'retry', error: legError(result) }
+function articleUrl(base) {
+  return `${String(base || '').replace(/\/+$/, '')}/site/news`
 }
 
+// Every leg's client returns { ok, status, data, error }. Squeeze a failure into
+// the one line stored in announce_job_legs.last_error and shown in the panel.
 function legError(result) {
   if (!result) return 'no response'
   if (result.status) {
@@ -99,29 +50,32 @@ function scheduleAfter(attempts) {
   return BACKOFF_MS[Math.min(attempts - 1, BACKOFF_MS.length - 1)]
 }
 
-// Parent job status derived from the two leg statuses:
-//   done    — both legs delivered
-//   failed  — both legs gave up
-//   partial — at least one leg reached a terminal state while the other has not
-//             matched it (still pending/retrying, or the opposite terminal state)
-//   pending — neither leg is terminal yet
-function rollupStatus(towncrierStatus, discordStatus) {
-  if (towncrierStatus === 'done' && discordStatus === 'done') return 'done'
-  if (towncrierStatus === 'failed' && discordStatus === 'failed') return 'failed'
+// Parent job status derived from its leg statuses:
+//   done    — every leg delivered
+//   failed  — every leg gave up
+//   partial — at least one leg reached a terminal state without all of them
+//             agreeing (some still pending/retrying, or a mix of done and failed)
+//   pending — no leg is terminal yet
+//
+// Takes the list of leg statuses rather than two named arguments, because the leg
+// set is registered rather than fixed (MODULE_SYSTEM.md §1.8). No legs at all
+// rolls up `done`: with nothing registered there is nothing left to deliver, and
+// leaving such jobs `pending` would pile up rows the worker never touches.
+function rollupStatus(statuses) {
+  const list = Array.isArray(statuses) ? statuses : []
   const terminal = (s) => s === 'done' || s === 'failed'
-  if (terminal(towncrierStatus) || terminal(discordStatus)) return 'partial'
+  if (list.every((s) => s === 'done')) return 'done'
+  if (list.every((s) => s === 'failed')) return 'failed'
+  if (list.some(terminal)) return 'partial'
   return 'pending'
 }
 
 module.exports = {
-  MAX_LINES,
-  MAX_LINE_LEN,
   MAX_ATTEMPTS,
   BACKOFF_MS,
-  buildTownCrierText,
+  baseUrl,
   articleUrl,
-  classifyTownCrier,
-  classifyDiscord,
+  legError,
   scheduleAfter,
   rollupStatus,
 }

server/src/model/announceJobs/announceJobs.model.js

@@ -2,19 +2,21 @@
 //
 // Sits between the DB rows and the worker: creates jobs on publish, records each
 // leg's outcome, keeps the parent `status` rollup in sync, stamps the post's
-// announced_at when both legs land, and resets a leg for the admin retry button.
-// The pure decisions (backoff, rollup, classification) live in .logic.js.
+// announced_at when every leg lands, and resets a leg for the admin retry button.
+// The pure decisions (backoff, rollup) live in .logic.js; which legs exist at all
+// is modules/registries.js's answer, not this file's (MODULE_SYSTEM.md §1.8).
 
 const db = require('./announceJobs.db')
 const logic = require('./announceJobs.logic')
+const registries = require('../../modules/registries')
 const posts = require('../posts/posts.model')
 const log = require('../../utils/logger')('announce')
 
-// Enqueue an announcement for a freshly-published news post: one job row (both
-// legs pending, due immediately) plus a back-pointer on the post so the admin
-// panel can find it. Returns the new job id.
+// Enqueue an announcement for a freshly-published news post: one job row, one leg
+// row per registered leg (all pending, due immediately), plus a back-pointer on
+// the post so the admin panel can find it. Returns the new job id.
 async function enqueue(postId) {
-  const jobId = await db.create(postId)
+  const jobId = await db.create(postId, registries.announceLegIds())
   await posts.linkAnnounceJob(postId, jobId)
   log.info('announce job enqueued', { jobId, postId })
   return jobId
@@ -43,12 +45,13 @@ async function enqueueIfNeeded(post, transition) {
   }
 }
 
-// Record a leg's dispatch outcome and refresh the rollup. `outcome` is one of
-// logic.classify*'s results: 'done' | 'retry' | 'terminal'. For 'retry' we bump
-// the attempt count and schedule the next run (or fail the leg once the cap is
-// hit). Returns the updated job row.
+// Record a leg's dispatch outcome and refresh the rollup. `outcome` is one of a
+// leg's classify() results: 'done' | 'retry' | 'terminal'. For 'retry' we bump the
+// attempt count and schedule the next run (or fail the leg once the cap is hit).
+// Returns the updated job row.
 async function recordOutcome(job, leg, { outcome, error }) {
-  const attempts = Number(job[`${leg}_attempts`]) || 0
+  const row = (job.legs || []).find((l) => l.leg === leg)
+  const attempts = Number(row && row.attempts) || 0
 
   if (outcome === 'done') {
     await db.updateLeg(job.id, leg, { status: 'done', attempts, lastError: null, nextAttemptAt: null })
@@ -71,12 +74,12 @@ async function recordOutcome(job, leg, { outcome, error }) {
   return refreshStatus(job.id)
 }
 
-// Recompute and persist the parent status from the two legs; stamp the post's
-// announced_at the moment both legs have delivered.
+// Recompute and persist the parent status from the legs; stamp the post's
+// announced_at the moment every leg has delivered.
 async function refreshStatus(jobId) {
   const job = await db.findById(jobId)
   if (!job) return null
-  const status = logic.rollupStatus(job.towncrier_status, job.discord_status)
+  const status = logic.rollupStatus(job.legs.map((l) => l.status))
   if (status !== job.status) await db.setStatus(jobId, status)
   job.status = status
   if (status === 'done') {
@@ -93,16 +96,33 @@ async function refreshStatus(jobId) {
 // the worker pick it up on the next tick. Resets the attempt count so a retry
 // after a config fix gets a full budget again.
 async function resetLeg(postId, leg) {
-  if (!db.LEGS.includes(leg)) throw new Error(`unknown announce leg: ${leg}`)
+  if (!registries.announceLeg(leg)) throw new Error(`unknown announce leg: ${leg}`)
   const job = await db.findByPostId(postId)
   if (!job) return null
+  // A job enqueued before this leg was registered has no row for it; create it so
+  // the retry button works on an existing post after a module is installed.
+  await db.ensureLegs(job.id, [leg])
   await db.updateLeg(job.id, leg, { status: 'pending', attempts: 0, lastError: null, nextAttemptAt: null })
   log.info('announce leg reset for retry', { jobId: job.id, postId, leg })
-  return refreshStatus(job.id)
+  // Labelled, because this is the response body the admin panel re-renders from.
+  return withLabels(await refreshStatus(job.id))
+}
+
+// Decorate a job's legs with the label their registration carries, so the admin
+// panel renders a module's leg with a real name and no client change
+// (MODULE_SYSTEM.md §1.8). An unregistered leg — a stale row from a module that
+// was since removed — keeps its id as the label rather than disappearing.
+function withLabels(job) {
+  if (!job) return job
+  job.legs = (job.legs || []).map((l) => {
+    const registered = registries.announceLeg(l.leg)
+    return { ...l, label: registered ? registered.label : l.leg }
+  })
+  return job
 }
 
 async function getByPostId(postId) {
-  return db.findByPostId(postId)
+  return withLabels(await db.findByPostId(postId))
 }
 
 module.exports = {
@@ -113,4 +133,5 @@ module.exports = {
   refreshStatus,
   resetLeg,
   getByPostId,
+  withLabels,
 }

server/src/model/notificationSubs/notificationSubs.model.js

@@ -1,8 +1,10 @@
 // Per-user push-notification subscriptions (which streams a user opted into;
-// applied to every device they register). The catalog is config/notificationStreams.
+// applied to every device they register). The catalog is core's plus every
+// installed module's, so it is read back through modules/registries rather than
+// from a config file (MODULE_SYSTEM.md §1.8).
 
 const db = require('./notificationSubs.db')
-const { isValidStream } = require('../../config/notificationStreams')
+const { isValidStream } = require('../../modules/registries')
 
 const getForUser = async (userId) => (await db.listByUser(userId)).map((r) => r.stream_id)