feat(server): port the UO models, utils and schema fragment
The data half of the extraction: 8 model directories, 13 utils, the shard
stream catalog and the 27-table schema fragment with its purge.
server/core.js is what makes the port a one-line import change per file rather
than a signature change per function. Ported code requires its dependencies at
file scope -- `const { query } = require('../../core')` -- which runs before
register() has been called and before any ctx exists. So every member is a
stable function that resolves ctx when CALLED, and nothing may be destructured
off ctx at init either, because core is free to hand over a getter.
Two helpers are vendored rather than taken from ctx, and the line between them
is the point. utils/excerpt.js is core's deriveExcerpt -- nine lines of pure
text handling. Core's sanitiser next to it was NOT copied: a second copy of a
security control diverges silently the moment either is fixed. announceLinks.js
vendors legError and articleUrl the same way, but baseUrl could not be: core's
reads APP_BASE_URL, and §2.7 forbids a module reading core's environment, so it
comes off ctx.site.baseUrl.
The schema fragment is core's 27 shard_*/uo_link_* statements, verbs CREATE,
ALTER and UPDATE only, every CREATE TABLE guarded. Two of its tables carry a
foreign key INTO users, which is allowed and is why the replay order matters --
core's schema is in place before this runs. The reverse never occurs and must
not: it would make core unable to boot without a module installed.
One real port bug caught by the integration run, not by tests: the atlas art
map resolved `../../../db/data`, which pointed at core's tree when this file
lived there and points outside server/ now. A path that happens to resolve is
exactly what survives a green suite, because the absent-file branch returns {}
and looks like the normal case.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
46
server/model/shardEvents/shardEvents.db.js
Normal file
46
server/model/shardEvents/shardEvents.db.js
Normal file
@@ -0,0 +1,46 @@
|
||||
const { query } = require('../../core')
|
||||
|
||||
// INSERT IGNORE on the UNIQUE dedupe_key — a re-ingested event (WS-reconnect
|
||||
// backfill overlap) is silently skipped rather than duplicated. Returns true if
|
||||
// a new row was actually inserted.
|
||||
async function insertIgnore({ kind, t, bootId, payload, dedupeKey }) {
|
||||
const res = await query(
|
||||
`INSERT IGNORE INTO shard_events (kind, t, boot_id, payload, dedupe_key)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
[kind, t, bootId || null, JSON.stringify(payload), dedupeKey],
|
||||
)
|
||||
return res.affectedRows > 0
|
||||
}
|
||||
|
||||
// Recent events, newest first. Filter by a single `kind`, or an allowlist of
|
||||
// `kinds` (IN clause) — the public feed uses the allowlist so it can never leak
|
||||
// staff/sensitive kinds. limit is clamped by the model.
|
||||
async function list({ kind, kinds, limit }) {
|
||||
// An allowlist that resolved to NOTHING means "serve nothing" — never "serve
|
||||
// everything". Falling through to the unfiltered query below would have turned
|
||||
// a fully-gated visibility config into a full dump of the event log, staff
|
||||
// audit and cheat detections included.
|
||||
if (kinds && kinds.length === 0) return []
|
||||
if (kinds && kinds.length) {
|
||||
const placeholders = kinds.map(() => '?').join(', ')
|
||||
return query(
|
||||
`SELECT id, kind, t, boot_id, payload, created_at
|
||||
FROM shard_events WHERE kind IN (${placeholders}) ORDER BY t DESC LIMIT ?`,
|
||||
[...kinds, limit],
|
||||
)
|
||||
}
|
||||
if (kind) {
|
||||
return query(
|
||||
`SELECT id, kind, t, boot_id, payload, created_at
|
||||
FROM shard_events WHERE kind = ? ORDER BY t DESC LIMIT ?`,
|
||||
[kind, limit],
|
||||
)
|
||||
}
|
||||
return query(
|
||||
`SELECT id, kind, t, boot_id, payload, created_at
|
||||
FROM shard_events ORDER BY t DESC LIMIT ?`,
|
||||
[limit],
|
||||
)
|
||||
}
|
||||
|
||||
module.exports = { insertIgnore, list }
|
||||
61
server/model/shardEvents/shardEvents.model.js
Normal file
61
server/model/shardEvents/shardEvents.model.js
Normal file
@@ -0,0 +1,61 @@
|
||||
// Append-only shard event log. The WS ingest dispatcher calls append() for the
|
||||
// notable kinds; the public/admin read endpoints call list(). The DB layer only
|
||||
// sees an already-computed dedupe_key so INSERT IGNORE is idempotent across
|
||||
// WS-reconnect backfill.
|
||||
|
||||
const crypto = require('crypto')
|
||||
const db = require('./shardEvents.db')
|
||||
|
||||
const MAX_LIMIT = 1000
|
||||
const DEFAULT_LIMIT = 100
|
||||
|
||||
// Stable stringify — keys sorted — so the dedupe hash is independent of the
|
||||
// property order the sidecar happened to serialize with.
|
||||
function stableStringify(value) {
|
||||
if (value === null || typeof value !== 'object') return JSON.stringify(value)
|
||||
if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`
|
||||
const keys = Object.keys(value).sort()
|
||||
const entries = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`)
|
||||
return `{${entries.join(',')}}`
|
||||
}
|
||||
|
||||
// dedupe_key = sha256(kind + t + stable-json(payload)), truncated to 40 hex chars.
|
||||
// This is a content fingerprint for idempotent INSERT IGNORE, not a security value,
|
||||
// but we use SHA-256 rather than SHA-1 anyway; the truncation keeps it inside the
|
||||
// CHAR(40) column (160 bits is ample collision resistance for dedupe). Two identical
|
||||
// events (same kind, same timestamp, same body) collapse to one row.
|
||||
function dedupeKey(kind, t, payload) {
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(`${kind}|${t}|${stableStringify(payload)}`)
|
||||
.digest('hex')
|
||||
.slice(0, 40)
|
||||
}
|
||||
|
||||
// Append one event. Returns true if a new row was inserted (false = deduped).
|
||||
async function append({ kind, t, bootId, payload }) {
|
||||
return db.insertIgnore({ kind, t, bootId, payload, dedupeKey: dedupeKey(kind, t, payload) })
|
||||
}
|
||||
|
||||
function normalizeLimit(limit) {
|
||||
const n = Number(limit)
|
||||
if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT
|
||||
return Math.min(Math.floor(n), MAX_LIMIT)
|
||||
}
|
||||
|
||||
// Recent events, newest first. Each row's JSON payload is parsed back to an
|
||||
// object. `kinds` (array) restricts to an allowlist; `kind` filters a single kind.
|
||||
async function list({ kind, kinds, limit } = {}) {
|
||||
const rows = await db.list({ kind, kinds, limit: normalizeLimit(limit) })
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
kind: row.kind,
|
||||
t: row.t,
|
||||
bootId: row.boot_id || null,
|
||||
// mariadb returns JSON columns as strings on some versions; parse defensively.
|
||||
payload: typeof row.payload === 'string' ? JSON.parse(row.payload) : row.payload,
|
||||
createdAt: row.created_at,
|
||||
}))
|
||||
}
|
||||
|
||||
module.exports = { append, list, dedupeKey }
|
||||
Reference in New Issue
Block a user