feat(events): a failed step may say when to retry (MODULE_API 1.15.0)

A module's perform() may answer { ok: false, retry: true, retryAfterMs }
and the runner waits that long instead of the flat EVENT_STEP_RETRY_MS
(Rust PLAN_REDESIGNS §14.3.13, D371). Dangerous Treasures' warm-up is the
first user.

- dispatch.classify reads it on the retry shape only, clamped to 60 000 to
  1 800 000 ms; a value that is not a number is dropped and logged, and the
  step retries on the flat delay.
- It is one attempt of EVENT_STEP_MAX_ATTEMPTS and degrades the run like any
  retry. A run paused or cancelled in the wait is not picked up.
- Every step.retry log line now carries retryAt (and retryAfterMs when
  hinted); the run console reads "retrying at 20:10".
- MODULE_API 1.15.0, additions only; the engagement manifest is regenerated.
  Module-uo (^1.10.0) answers none of it and is unaffected.
- Test harness: resourcesDb.finishedRunsWithNothingLeft was never stubbed in
  eventRunner.test.js, so a tick that reached the cleanup sweep with a
  terminal run waited on the dead pool.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E14m6SuuY6i1vASFeGDBeY

server/src/events/dispatch.js

@@ -44,6 +44,38 @@ const MAX_HOLD_SECONDS = 7 * 24 * 60 * 60
 // rendered half of one would be a second bug on top of the first.
 const MAX_DETAIL_BYTES = 4096
 
+// The bounds on a failure's `retryAfterMs`, in milliseconds (MODULE_API 1.15.0,
+// Rust PLAN_REDESIGNS §14.3.13, D371). One minute to thirty. **Clamped, never
+// refused**: it is the module's hint about its own verb — "the treasure is still
+// warming up, try in ten minutes" — not an author's value, so a module that says
+// five seconds or five hours gets the nearest bound rather than a failed step.
+// The floor is the flat delay's own default, so no hint can make a retry come
+// sooner than an unhinted one would on a stock deployment; the ceiling keeps
+// `MAX_ATTEMPTS` tries of a step inside an hour and a half of the moment the
+// event was about.
+const MIN_RETRY_AFTER_MS = 60_000
+const MAX_RETRY_AFTER_MS = 30 * 60_000
+
+/**
+ * A failure's `retryAfterMs`, clamped, or null for "use the flat delay".
+ *
+ * A value that is not a number at all is the hint being absent rather than
+ * wrong: it is dropped and logged, the step retries on the flat delay, and
+ * nothing about the failure changes — the same posture as a bad `detail`.
+ */
+function safeRetryAfter(value, actionId) {
+  if (value === undefined || value === null) return null
+  const n = typeof value === 'number' ? value : Number.NaN
+  if (!Number.isFinite(n)) {
+    log.warn('event action answered a retryAfterMs that is not a number; using the flat delay', {
+      action: actionId,
+      retryAfterMs: String(value).slice(0, 40),
+    })
+    return null
+  }
+  return Math.min(MAX_RETRY_AFTER_MS, Math.max(MIN_RETRY_AFTER_MS, Math.round(n)))
+}
+
 /**
  * A module's own account of what a successful step actually did.
  *
@@ -127,7 +159,7 @@ function withDeadline(fn, ms, actionId) {
 
 /**
  * Turn a raw `perform()` answer into
- * `{ outcome, error?, holdSeconds?, resources?, participants? }`.
+ * `{ outcome, error?, retryAfterMs?, holdSeconds?, resources?, participants? }`.
  *
  * Exported and pure, so the classification rules are testable without a registry,
  * a database or a clock — which matters because they are the rules that decide
@@ -150,10 +182,13 @@ function classify(result, actionId) {
     // says `retry: false`, and an envelope that forgot to say anything gets the
     // benefit of the doubt on the transient question but not on the success one.
     const retry = result.retry !== false
-    return {
-      outcome: retry ? 'retry' : 'terminal',
-      error: result.error ? String(result.error) : `${actionId} refused`,
-    }
+    const error = result.error ? String(result.error) : `${actionId} refused`
+    if (!retry) return { outcome: 'terminal', error }
+    // **When to try again** is the module's to suggest and core's to bound. Read
+    // on the retry shape only: a terminal failure is never attempted again, so a
+    // delay beside `retry: false` describes nothing.
+    const retryAfterMs = safeRetryAfter(result.retryAfterMs, actionId)
+    return retryAfterMs === null ? { outcome: 'retry', error } : { outcome: 'retry', error, retryAfterMs }
   }
 
   // ── The two success shapes that are not "finished" ──
@@ -250,4 +285,15 @@ async function dispatchStep(step, { run, actor = null, verify = false } = {}) {
   return classification
 }
 
-module.exports = { dispatchStep, classify, withDeadline, safeDetail, OUTCOMES, MAX_HOLD_SECONDS, MAX_DETAIL_BYTES }
+module.exports = {
+  dispatchStep,
+  classify,
+  withDeadline,
+  safeDetail,
+  safeRetryAfter,
+  OUTCOMES,
+  MAX_HOLD_SECONDS,
+  MAX_DETAIL_BYTES,
+  MIN_RETRY_AFTER_MS,
+  MAX_RETRY_AFTER_MS,
+}

server/src/modules/version.js

@@ -9,7 +9,16 @@
 // Deliberately separate from PROTOCOL_VERSION (which versions the shard wire and
 // has nothing to say about a website module) and from any module's own version.
 
-// 1.14.0 — an action param may hold several values: `multiple: true`, with an
+// 1.15.0 — a failed step may say when to try again: `{ ok: false, retry: true,
+// retryAfterMs }` (Rust PLAN_REDESIGNS §14.3.13, D371; docs/website/EVENTS.md §F).
+// Clamped to one minute–thirty minutes, never refused; absent, the flat
+// `EVENT_STEP_RETRY_MS` as before. It is one attempt of `EVENT_STEP_MAX_ATTEMPTS`
+// like any retry, and ignored beside `retry: false`. One optional envelope
+// member, so minor. Module-uo declares `^1.10.0` and answers none: an envelope
+// without it is classified exactly as it was. This took 1.15.0 ahead of
+// STEAM_SIGNIN.md, which had claimed it and moves to the next minor.
+//
+// 1.14.0 —an action param may hold several values: `multiple: true`, with an
 // optional `max` (1–100, 20 if left out), on a `string` or `int` param (Rust
 // PLAN_REDESIGNS §12.3, D344; docs/website/EVENTS.md §F). The step stores a JSON
 // array; a bare value is read as a list of one; `required` means at least one;
@@ -198,6 +207,6 @@
 // an admin action a module performs belongs in core's one audit log, the
 // extension slot needs the user its prefix names, and §2.7 forbids a module
 // reading core's `APP_BASE_URL` for itself. Additions only, so minor.
-const MODULE_API_VERSION = '1.14.0'
+const MODULE_API_VERSION = '1.15.0'
 
 module.exports = { MODULE_API_VERSION }

server/src/utils/eventRunner.js

@@ -428,13 +428,28 @@ async function drainStep(run, step, now, carry = {}) {
   }
 
   if (result.outcome === 'retry' && step.attempts + 1 < MAX_ATTEMPTS) {
-    await stepsDb.reschedule(step.id, new Date(now.getTime() + RETRY_MS), result.error)
+    // The module's `retryAfterMs` when it gave one (already clamped by the
+    // dispatcher), else the flat delay. **It is still one attempt** of
+    // MAX_ATTEMPTS: a hint changes when, never how many, so a module that answers
+    // "not yet" for ever fails the step on schedule like any other. A run paused
+    // or cancelled in the wait is not picked up — `findDue` reads only running
+    // runs — and the staff Retry on a paused step is unaffected.
+    const delayMs = result.retryAfterMs || RETRY_MS
+    const retryAt = new Date(now.getTime() + delayMs)
+    await stepsDb.reschedule(step.id, retryAt, result.error)
     await logDb.write({
       runId: run.id,
       stepId: step.id,
       kind: 'step.retry',
       phase: step.phase,
-      detail: { action: step.action_id, attempt: step.attempts + 1, of: MAX_ATTEMPTS, error: result.error },
+      detail: {
+        action: step.action_id,
+        attempt: step.attempts + 1,
+        of: MAX_ATTEMPTS,
+        error: result.error,
+        retryAt: retryAt.toISOString(),
+        ...(result.retryAfterMs ? { retryAfterMs: result.retryAfterMs } : {}),
+      },
     })
     // Degraded from the FIRST retry, not from the eventual failure. An event
     // whose announcements are landing on the second attempt is having trouble