fix(bot): retry boot-time config fetch so bot self-heals on cold start
On `docker compose up`/restart the bot and app start together. The bot's `depends_on: app` uses `condition: service_started`, which only waits for the app container to launch — not for its internal server (3001) to be listening after it reaches the DB and boots Express. bootstrap.js did a single un-retried fetch, lost that race, gave up, and left the bot disconnected while the DB `enabled` flag stayed true — so the admin panel showed "enabled but disconnected" until an admin toggled off/on to force a pushConfig. Retry the boot config fetch with backoff (~1 min, 2s apart) until the app answers: retry on network errors and 5xx, bail on 4xx (a real misconfig, not a startup race). Also wrap discordManager.start in try/catch so a bad-token boot logs and keeps the process alive instead of crashing it via server.js's exit-on-start-failure. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0114TpmrNW4wNXsHq5CR72jQ
This commit is contained in:
56
bot/src/bootstrap.js
vendored
56
bot/src/bootstrap.js
vendored
@@ -4,11 +4,50 @@
|
||||
// container restart (crash, `docker compose restart`, host reboot) self-heals
|
||||
// without any admin-panel interaction. Node 20's built-in fetch is used; no
|
||||
// extra HTTP client dependency needed for a single startup call.
|
||||
//
|
||||
// The fetch RETRIES with backoff: on `docker compose up`, the bot and the app
|
||||
// start together and the bot's `depends_on: app` only waits for the container
|
||||
// to *start*, not for the app's internal server to be listening (it still has
|
||||
// to reach the DB and boot Express). Without retries the very first fetch loses
|
||||
// that race, bootstrap gives up, and the bot sits disconnected while the DB
|
||||
// still says enabled — the exact "enabled but disconnected until I toggle it"
|
||||
// bug. Retrying until the site answers makes a cold whole-stack start heal on
|
||||
// its own.
|
||||
const discordManager = require('./discord/discordManager')
|
||||
const createLogger = require('./utils/logger')
|
||||
|
||||
const log = createLogger('bootstrap')
|
||||
|
||||
const MAX_ATTEMPTS = 30 // ~30 tries * ~2s ≈ 1 min of patience for the app to come up
|
||||
const RETRY_DELAY_MS = 2000
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
// Fetch config from the site, retrying while the site is unreachable or not yet
|
||||
// ready (network error or 5xx). Returns the parsed config, or null if we gave
|
||||
// up after MAX_ATTEMPTS. A 4xx (e.g. bad internal key) is a real misconfig, not
|
||||
// a transient startup race, so we don't retry those.
|
||||
async function fetchConfig(siteUrl, key) {
|
||||
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
const res = await fetch(siteUrl, { headers: { 'X-Internal-Key': key } })
|
||||
if (res.ok) return await res.json()
|
||||
if (res.status >= 400 && res.status < 500) {
|
||||
log.error('boot-time config fetch rejected — not retrying', { status: res.status })
|
||||
return null
|
||||
}
|
||||
log.warn('boot-time config fetch not ready — retrying', { status: res.status, attempt })
|
||||
} catch (err) {
|
||||
log.warn('boot-time config fetch errored — retrying', { message: err.message, attempt })
|
||||
}
|
||||
if (attempt < MAX_ATTEMPTS) await sleep(RETRY_DELAY_MS)
|
||||
}
|
||||
log.error('boot-time config fetch gave up after retries — staying disconnected until the admin panel pushes config', {
|
||||
attempts: MAX_ATTEMPTS,
|
||||
})
|
||||
return null
|
||||
}
|
||||
|
||||
async function bootstrap() {
|
||||
const siteUrl = process.env.SITE_INTERNAL_URL
|
||||
const key = process.env.BOT_INTERNAL_KEY
|
||||
@@ -17,22 +56,19 @@ async function bootstrap() {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch(siteUrl, { headers: { 'X-Internal-Key': key } })
|
||||
if (!res.ok) {
|
||||
log.error('boot-time config fetch failed', { status: res.status })
|
||||
return
|
||||
}
|
||||
const config = await res.json()
|
||||
const config = await fetchConfig(siteUrl, key)
|
||||
if (!config) return
|
||||
|
||||
if (config.enabled) {
|
||||
log.info('boot-time config says enabled — reconnecting', { guildId: config.guildId })
|
||||
try {
|
||||
await discordManager.start({ token: config.token, guildId: config.guildId })
|
||||
} catch (err) {
|
||||
log.error('boot-time reconnect failed', { message: err.message })
|
||||
}
|
||||
} else {
|
||||
log.info('boot-time config says disabled — staying disconnected')
|
||||
}
|
||||
} catch (err) {
|
||||
log.error('boot-time config fetch errored', { message: err.message })
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = bootstrap
|
||||
|
||||
Reference in New Issue
Block a user