server/src/model/teams/teamForumUploads.model.js

@@ -0,0 +1,177 @@
+// ── `uploads` mode, and what had to harden first (TEAMS.md §5.5.4) ─────────
+//
+// The existing admin upload path (router/v1/admin/imageUpload.js) is already good
+// for an admin: an 8 MB cap, a mimetype allowlist, a random filename, an extension
+// derived from the MIMETYPE MAP and never from `originalname`, and
+// `X-Content-Type-Options: nosniff` forced on serve. All of that is kept and this
+// file adds the four things that path never needed, because until now it has never
+// had a hostile uploader.
+//
+//   1. MAGIC-BYTE SNIFFING. `file.mimetype` is the client's own Content-Type
+//      header. A player can send `image/png` with arbitrary bytes and land
+//      arbitrary content under a `.png`. Trusted from an admin, not from a player.
+//   2. QUOTAS. A per-post attachment cap and a per-account daily byte quota.
+//      Community uploads with no ceiling is disk exhaustion on the operator's own
+//      host. (The per-request RATE limit is core's rateLimit middleware, applied
+//      at the route.)
+//   3. ATTRIBUTION. Every accepted file gets a `team_forum_uploads` row. Not
+//      bookkeeping: the acknowledgement in §5.5.5 is meaningless if "who uploaded
+//      this" cannot be answered afterwards.
+//   4. LIFECYCLE. Deleting a post soft-deletes its uploads; the sweep removes the
+//      bytes after a retention window, and files with no row at all. The admin
+//      upload path never deletes anything, which is fine at admin volume and is
+//      not fine here.
+
+const fs = require('fs/promises')
+const path = require('path')
+
+const forumDb = require('./teamForum.db')
+const { UPLOAD_DIR } = require('../../router/v1/admin/imageUpload')
+
+// Leading bytes → the type they actually are. Deliberately not a library: five
+// signatures, checked exactly, is less surface than a dependency that accepts
+// hundreds of formats when the allowlist only wants these.
+//
+// WebP and AVIF are container formats, so both need a second check past the first
+// four bytes — RIFF alone is also .wav, and the `ftyp` box also fronts .mp4.
+const SIGNATURES = [
+  { mime: 'image/png', test: (b) => b.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) },
+  { mime: 'image/jpeg', test: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
+  { mime: 'image/gif', test: (b) => b.subarray(0, 6).toString('latin1').match(/^GIF8[79]a$/) != null },
+  {
+    mime: 'image/webp',
+    test: (b) => b.subarray(0, 4).toString('latin1') === 'RIFF' && b.subarray(8, 12).toString('latin1') === 'WEBP',
+  },
+  {
+    mime: 'image/avif',
+    test: (b) => b.subarray(4, 8).toString('latin1') === 'ftyp'
+      && ['avif', 'avis'].includes(b.subarray(8, 12).toString('latin1')),
+  },
+]
+
+// Per-post attachment cap and per-account rolling byte quota.
+const MAX_ATTACHMENTS_PER_POST = 6
+const DAILY_QUOTA_BYTES = 25 * 1024 * 1024
+const QUOTA_WINDOW_HOURS = 24
+
+// Lifecycle windows. A soft-deleted file survives long enough for a mis-click to
+// be recoverable; an orphan is one uploaded into a composer that was never
+// submitted, which is a normal thing to do and so gets a generous grace.
+const RETENTION_DAYS = 30
+const ORPHAN_GRACE_HOURS = 48
+
+/**
+ * What do these bytes actually claim to be?
+ *
+ * Returns the sniffed mimetype, or null when nothing matches. Null is a rejection
+ * and never a "trust the header instead" — an unrecognised file is exactly the
+ * case this check exists for.
+ */
+function sniff(buffer) {
+  if (!Buffer.isBuffer(buffer) || buffer.length < 12) return null
+  return SIGNATURES.find((s) => s.test(buffer))?.mime || null
+}
+
+/**
+ * Accept a file multer has already written to disk.
+ *
+ * The file is on disk before it can be sniffed — multer streams it there — so the
+ * rejection path has to REMOVE it. A rejected upload that stays on disk is exactly
+ * the disk-exhaustion vector the quota exists to close, reached by a different
+ * route.
+ */
+async function accept({ team, actor, file }) {
+  const stored = path.join(UPLOAD_DIR, file.filename)
+  const discard = async () => { await fs.rm(stored, { force: true }) }
+
+  let head
+  try {
+    const handle = await fs.open(stored, 'r')
+    try {
+      head = Buffer.alloc(16)
+      await handle.read(head, 0, 16, 0)
+    } finally {
+      await handle.close()
+    }
+  } catch {
+    await discard()
+    return { ok: false, status: 400, error: 'Could not read the uploaded file' }
+  }
+
+  const sniffed = sniff(head)
+  if (!sniffed || sniffed !== file.mimetype) {
+    await discard()
+    return { ok: false, status: 400, error: 'That file is not the image type it claims to be' }
+  }
+
+  const used = await forumDb.bytesUploadedSince(actor.id, QUOTA_WINDOW_HOURS)
+  if (used + file.size > DAILY_QUOTA_BYTES) {
+    await discard()
+    return { ok: false, status: 429, error: 'Daily upload limit reached. Try again tomorrow.' }
+  }
+
+  const id = await forumDb.insertUpload({
+    teamId: team.id,
+    postId: null,          // attached when the post that embeds it is written
+    uploaderUserId: actor.id,
+    uploaderUsername: actor.username,
+    filename: file.filename,
+    mimetype: sniffed,     // the SNIFFED type, never the client's header
+    byteSize: file.size,
+  })
+  return { ok: true, id, url: `/uploads/${file.filename}`, bytes: file.size }
+}
+
+/**
+ * Remove an upload. The uploader may, within the edit window; staff may at any
+ * time. Soft — the bytes go with the sweep, not with the button.
+ */
+async function remove({ id, actor, isStaff }) {
+  const row = await forumDb.uploadById(id)
+  if (!row || row.deleted_at) return { ok: false, status: 404, error: 'No such upload' }
+  if (!isStaff && row.uploader_user_id !== actor.id) {
+    return { ok: false, status: 403, error: 'Not your upload' }
+  }
+  await forumDb.softDeleteUpload(id, actor.id)
+  return { ok: true }
+}
+
+/**
+ * The nightly sweep: bytes for soft-deleted rows past retention, plus files on
+ * disk with no row at all.
+ *
+ * The orphan half deliberately only considers files whose names match the upload
+ * naming scheme AND appear in no row. UPLOAD_DIR is shared with the admin upload
+ * path, whose files have no row here and must never be swept — so the sweep works
+ * from the FORUM's own rows outward and never from the directory listing inward.
+ */
+async function sweep({ retentionDays = RETENTION_DAYS, orphanGraceHours = ORPHAN_GRACE_HOURS } = {}) {
+  const expired = await forumDb.sweepableUploads(retentionDays)
+  const orphans = await forumDb.orphanedUploads(orphanGraceHours)
+  const doomed = [...expired, ...orphans]
+  const cleared = []
+  for (const row of doomed) {
+    try {
+      await fs.rm(path.join(UPLOAD_DIR, row.filename), { force: true })
+      cleared.push(row.id)
+    } catch {
+      // Leave the ROW as well as the file. A file we could not delete is one the
+      // next run should try again, and dropping its row would lose the only
+      // record that the bytes are still there.
+    }
+  }
+  await forumDb.deleteUploadRows(cleared)
+  return { swept: doomed.length, filesRemoved: cleared.length }
+}
+
+module.exports = {
+  MAX_ATTACHMENTS_PER_POST,
+  DAILY_QUOTA_BYTES,
+  QUOTA_WINDOW_HOURS,
+  RETENTION_DAYS,
+  ORPHAN_GRACE_HOURS,
+  sniff,
+  accept,
+  remove,
+  sweep,
+}

server/src/server.js

@@ -10,6 +10,7 @@ const http = require('http')
 const botScore = require('./middleware/botScore')
 const announceWorker = require('./utils/announceWorker')
 const teamActivityPrune = require('./utils/teamActivityPrune')
+const teamForumUploadSweep = require('./utils/teamForumUploadSweep')
 const { ensureSchema, close } = require('./utils/db')
 const { seedDefaults, createInitialAdminFromEnv } = require('../db/seed')
 const settings = require('./model/settings/settings.model')
@@ -155,6 +156,7 @@ async function start() {
   // is the obvious unbounded-growth failure, so retention starts with the feed
   // rather than after someone notices. No-op on a deployment with no Teams.
   teamActivityPrune.start()
+  teamForumUploadSweep.start()
 
   setupShutdown(server, internalServer)
 }
@@ -174,6 +176,7 @@ function setupShutdown(server, internalServer) {
     botScore.stopSweeper() // stop the bot-store cleanup interval
     announceWorker.stop() // stop the news-announcement dispatcher poller
     teamActivityPrune.stop() // stop the Team activity retention timer
+    teamForumUploadSweep.stop() // stop the forum upload sweep
     server.close(() => log.info('http server closed'))
     if (internalServer) internalServer.close(() => log.info('internal http server closed'))
     try {

server/src/utils/teamForumUploadSweep.js

@@ -0,0 +1,66 @@
+// ── Team forum upload sweep ────────────────────────────────────────────────
+//
+// TEAMS.md §5.5.4's lifecycle half: soft-deleted uploads lose their bytes after a
+// retention window, and files uploaded into a composer that was never submitted
+// lose theirs after a grace period. The existing admin upload path never deletes
+// anything, which is fine at admin volume and is not fine once a community can
+// upload.
+//
+// Same in-process shape as utils/teamActivityPrune — setInterval + unref + stop(),
+// wired into server.js start/shutdown. There is no cron in this stack.
+//
+// **It runs whether or not `teams_forum_images` is `uploads`, and that is the
+// point.** An operator who turns uploads off after a problem has files already on
+// disk; a sweep that switched itself off with the setting would strand exactly the
+// bytes they were trying to be rid of. The admin help text says the same thing in
+// the other direction — disabling uploads stops new files, it does not delete old
+// ones — and this is the only thing that eventually does.
+
+const uploads = require('../model/teams/teamForumUploads.model')
+const log = require('./logger')('teams')
+
+const INTERVAL_MS = Number(process.env.TEAM_FORUM_SWEEP_MS) || 24 * 60 * 60 * 1000
+// Later than the activity prune's five minutes, so two table-walking jobs do not
+// land on the same boot at the same moment.
+const FIRST_RUN_MS = Number(process.env.TEAM_FORUM_SWEEP_DELAY_MS) || 10 * 60 * 1000
+
+let timer = null
+let firstRun = null
+
+/** One sweep. Never throws — it runs on a timer with nobody to catch it. */
+async function tick() {
+  try {
+    const result = await uploads.sweep()
+    if (result.swept) log.info('team forum upload sweep', result)
+    return result
+  } catch (err) {
+    log.error('team forum upload sweep failed', { message: err.message })
+    return null
+  }
+}
+
+function start() {
+  if (timer || firstRun) return timer
+  firstRun = setTimeout(() => {
+    firstRun = null
+    tick()
+    timer = setInterval(() => { tick() }, INTERVAL_MS)
+    if (timer.unref) timer.unref()
+  }, FIRST_RUN_MS)
+  if (firstRun.unref) firstRun.unref()
+  log.info('team forum upload sweep started', { intervalMs: INTERVAL_MS, firstRunMs: FIRST_RUN_MS })
+  return timer
+}
+
+function stop() {
+  if (firstRun) {
+    clearTimeout(firstRun)
+    firstRun = null
+  }
+  if (timer) {
+    clearInterval(timer)
+    timer = null
+  }
+}
+
+module.exports = { start, stop, tick, INTERVAL_MS, FIRST_RUN_MS }