diff --git a/server/src/model/teams/teamForumUploads.model.js b/server/src/model/teams/teamForumUploads.model.js new file mode 100644 index 0000000..c006bbf --- /dev/null +++ b/server/src/model/teams/teamForumUploads.model.js @@ -0,0 +1,177 @@ +// ── `uploads` mode, and what had to harden first (TEAMS.md §5.5.4) ───────── +// +// The existing admin upload path (router/v1/admin/imageUpload.js) is already good +// for an admin: an 8 MB cap, a mimetype allowlist, a random filename, an extension +// derived from the MIMETYPE MAP and never from `originalname`, and +// `X-Content-Type-Options: nosniff` forced on serve. All of that is kept and this +// file adds the four things that path never needed, because until now it has never +// had a hostile uploader. +// +// 1. MAGIC-BYTE SNIFFING. `file.mimetype` is the client's own Content-Type +// header. A player can send `image/png` with arbitrary bytes and land +// arbitrary content under a `.png`. Trusted from an admin, not from a player. +// 2. QUOTAS. A per-post attachment cap and a per-account daily byte quota. +// Community uploads with no ceiling is disk exhaustion on the operator's own +// host. (The per-request RATE limit is core's rateLimit middleware, applied +// at the route.) +// 3. ATTRIBUTION. Every accepted file gets a `team_forum_uploads` row. Not +// bookkeeping: the acknowledgement in §5.5.5 is meaningless if "who uploaded +// this" cannot be answered afterwards. +// 4. LIFECYCLE. Deleting a post soft-deletes its uploads; the sweep removes the +// bytes after a retention window, and files with no row at all. The admin +// upload path never deletes anything, which is fine at admin volume and is +// not fine here. + +const fs = require('fs/promises') +const path = require('path') + +const forumDb = require('./teamForum.db') +const { UPLOAD_DIR } = require('../../router/v1/admin/imageUpload') + +// Leading bytes → the type they actually are. Deliberately not a library: five +// signatures, checked exactly, is less surface than a dependency that accepts +// hundreds of formats when the allowlist only wants these. +// +// WebP and AVIF are container formats, so both need a second check past the first +// four bytes — RIFF alone is also .wav, and the `ftyp` box also fronts .mp4. +const SIGNATURES = [ + { mime: 'image/png', test: (b) => b.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) }, + { mime: 'image/jpeg', test: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff }, + { mime: 'image/gif', test: (b) => b.subarray(0, 6).toString('latin1').match(/^GIF8[79]a$/) != null }, + { + mime: 'image/webp', + test: (b) => b.subarray(0, 4).toString('latin1') === 'RIFF' && b.subarray(8, 12).toString('latin1') === 'WEBP', + }, + { + mime: 'image/avif', + test: (b) => b.subarray(4, 8).toString('latin1') === 'ftyp' + && ['avif', 'avis'].includes(b.subarray(8, 12).toString('latin1')), + }, +] + +// Per-post attachment cap and per-account rolling byte quota. +const MAX_ATTACHMENTS_PER_POST = 6 +const DAILY_QUOTA_BYTES = 25 * 1024 * 1024 +const QUOTA_WINDOW_HOURS = 24 + +// Lifecycle windows. A soft-deleted file survives long enough for a mis-click to +// be recoverable; an orphan is one uploaded into a composer that was never +// submitted, which is a normal thing to do and so gets a generous grace. +const RETENTION_DAYS = 30 +const ORPHAN_GRACE_HOURS = 48 + +/** + * What do these bytes actually claim to be? + * + * Returns the sniffed mimetype, or null when nothing matches. Null is a rejection + * and never a "trust the header instead" — an unrecognised file is exactly the + * case this check exists for. + */ +function sniff(buffer) { + if (!Buffer.isBuffer(buffer) || buffer.length < 12) return null + return SIGNATURES.find((s) => s.test(buffer))?.mime || null +} + +/** + * Accept a file multer has already written to disk. + * + * The file is on disk before it can be sniffed — multer streams it there — so the + * rejection path has to REMOVE it. A rejected upload that stays on disk is exactly + * the disk-exhaustion vector the quota exists to close, reached by a different + * route. + */ +async function accept({ team, actor, file }) { + const stored = path.join(UPLOAD_DIR, file.filename) + const discard = async () => { await fs.rm(stored, { force: true }) } + + let head + try { + const handle = await fs.open(stored, 'r') + try { + head = Buffer.alloc(16) + await handle.read(head, 0, 16, 0) + } finally { + await handle.close() + } + } catch { + await discard() + return { ok: false, status: 400, error: 'Could not read the uploaded file' } + } + + const sniffed = sniff(head) + if (!sniffed || sniffed !== file.mimetype) { + await discard() + return { ok: false, status: 400, error: 'That file is not the image type it claims to be' } + } + + const used = await forumDb.bytesUploadedSince(actor.id, QUOTA_WINDOW_HOURS) + if (used + file.size > DAILY_QUOTA_BYTES) { + await discard() + return { ok: false, status: 429, error: 'Daily upload limit reached. Try again tomorrow.' } + } + + const id = await forumDb.insertUpload({ + teamId: team.id, + postId: null, // attached when the post that embeds it is written + uploaderUserId: actor.id, + uploaderUsername: actor.username, + filename: file.filename, + mimetype: sniffed, // the SNIFFED type, never the client's header + byteSize: file.size, + }) + return { ok: true, id, url: `/uploads/${file.filename}`, bytes: file.size } +} + +/** + * Remove an upload. The uploader may, within the edit window; staff may at any + * time. Soft — the bytes go with the sweep, not with the button. + */ +async function remove({ id, actor, isStaff }) { + const row = await forumDb.uploadById(id) + if (!row || row.deleted_at) return { ok: false, status: 404, error: 'No such upload' } + if (!isStaff && row.uploader_user_id !== actor.id) { + return { ok: false, status: 403, error: 'Not your upload' } + } + await forumDb.softDeleteUpload(id, actor.id) + return { ok: true } +} + +/** + * The nightly sweep: bytes for soft-deleted rows past retention, plus files on + * disk with no row at all. + * + * The orphan half deliberately only considers files whose names match the upload + * naming scheme AND appear in no row. UPLOAD_DIR is shared with the admin upload + * path, whose files have no row here and must never be swept — so the sweep works + * from the FORUM's own rows outward and never from the directory listing inward. + */ +async function sweep({ retentionDays = RETENTION_DAYS, orphanGraceHours = ORPHAN_GRACE_HOURS } = {}) { + const expired = await forumDb.sweepableUploads(retentionDays) + const orphans = await forumDb.orphanedUploads(orphanGraceHours) + const doomed = [...expired, ...orphans] + const cleared = [] + for (const row of doomed) { + try { + await fs.rm(path.join(UPLOAD_DIR, row.filename), { force: true }) + cleared.push(row.id) + } catch { + // Leave the ROW as well as the file. A file we could not delete is one the + // next run should try again, and dropping its row would lose the only + // record that the bytes are still there. + } + } + await forumDb.deleteUploadRows(cleared) + return { swept: doomed.length, filesRemoved: cleared.length } +} + +module.exports = { + MAX_ATTACHMENTS_PER_POST, + DAILY_QUOTA_BYTES, + QUOTA_WINDOW_HOURS, + RETENTION_DAYS, + ORPHAN_GRACE_HOURS, + sniff, + accept, + remove, + sweep, +} diff --git a/server/src/server.js b/server/src/server.js index 4474214..9b4dbce 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -10,6 +10,7 @@ const http = require('http') const botScore = require('./middleware/botScore') const announceWorker = require('./utils/announceWorker') const teamActivityPrune = require('./utils/teamActivityPrune') +const teamForumUploadSweep = require('./utils/teamForumUploadSweep') const { ensureSchema, close } = require('./utils/db') const { seedDefaults, createInitialAdminFromEnv } = require('../db/seed') const settings = require('./model/settings/settings.model') @@ -155,6 +156,7 @@ async function start() { // is the obvious unbounded-growth failure, so retention starts with the feed // rather than after someone notices. No-op on a deployment with no Teams. teamActivityPrune.start() + teamForumUploadSweep.start() setupShutdown(server, internalServer) } @@ -174,6 +176,7 @@ function setupShutdown(server, internalServer) { botScore.stopSweeper() // stop the bot-store cleanup interval announceWorker.stop() // stop the news-announcement dispatcher poller teamActivityPrune.stop() // stop the Team activity retention timer + teamForumUploadSweep.stop() // stop the forum upload sweep server.close(() => log.info('http server closed')) if (internalServer) internalServer.close(() => log.info('internal http server closed')) try { diff --git a/server/src/utils/teamForumUploadSweep.js b/server/src/utils/teamForumUploadSweep.js new file mode 100644 index 0000000..5d6e886 --- /dev/null +++ b/server/src/utils/teamForumUploadSweep.js @@ -0,0 +1,66 @@ +// ── Team forum upload sweep ──────────────────────────────────────────────── +// +// TEAMS.md §5.5.4's lifecycle half: soft-deleted uploads lose their bytes after a +// retention window, and files uploaded into a composer that was never submitted +// lose theirs after a grace period. The existing admin upload path never deletes +// anything, which is fine at admin volume and is not fine once a community can +// upload. +// +// Same in-process shape as utils/teamActivityPrune — setInterval + unref + stop(), +// wired into server.js start/shutdown. There is no cron in this stack. +// +// **It runs whether or not `teams_forum_images` is `uploads`, and that is the +// point.** An operator who turns uploads off after a problem has files already on +// disk; a sweep that switched itself off with the setting would strand exactly the +// bytes they were trying to be rid of. The admin help text says the same thing in +// the other direction — disabling uploads stops new files, it does not delete old +// ones — and this is the only thing that eventually does. + +const uploads = require('../model/teams/teamForumUploads.model') +const log = require('./logger')('teams') + +const INTERVAL_MS = Number(process.env.TEAM_FORUM_SWEEP_MS) || 24 * 60 * 60 * 1000 +// Later than the activity prune's five minutes, so two table-walking jobs do not +// land on the same boot at the same moment. +const FIRST_RUN_MS = Number(process.env.TEAM_FORUM_SWEEP_DELAY_MS) || 10 * 60 * 1000 + +let timer = null +let firstRun = null + +/** One sweep. Never throws — it runs on a timer with nobody to catch it. */ +async function tick() { + try { + const result = await uploads.sweep() + if (result.swept) log.info('team forum upload sweep', result) + return result + } catch (err) { + log.error('team forum upload sweep failed', { message: err.message }) + return null + } +} + +function start() { + if (timer || firstRun) return timer + firstRun = setTimeout(() => { + firstRun = null + tick() + timer = setInterval(() => { tick() }, INTERVAL_MS) + if (timer.unref) timer.unref() + }, FIRST_RUN_MS) + if (firstRun.unref) firstRun.unref() + log.info('team forum upload sweep started', { intervalMs: INTERVAL_MS, firstRunMs: FIRST_RUN_MS }) + return timer +} + +function stop() { + if (firstRun) { + clearTimeout(firstRun) + firstRun = null + } + if (timer) { + clearInterval(timer) + timer = null + } +} + +module.exports = { start, stop, tick, INTERVAL_MS, FIRST_RUN_MS }