diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a0e958d1..f3a24258 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -77,7 +77,7 @@ so old imports keep working: `public/js/ui/gameLogic.js` (`public/js/ui/gameLogi | path | what | |---|---| | `server/index.js` | the process entry (`npm start`); `startServer()` wires `server/http/` | -| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | +| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `assetIndex.js` (the content hashes of `public/assets/` and `public/fonts/`, cached in `.cache/asset-hashes.json`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | | `server/net.js` | sessions and reconnect tokens, rate limits, message validation | | `server/lobby.js` | rooms, seats, AI seats, spectators; starts a `Match` | | `server/data.js` | loads `data/*.json` once (frozen) | diff --git a/docs/design/overview.md b/docs/design/overview.md index ed82f088..e871de44 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -50,6 +50,7 @@ server/ buildTag.js build tag of the served browser runtime (/healthz build, public/js/ui/buildGuard.js) routes.js request listener: 414 / 400 / 405 guards, GET /healthz (?traffic=1: the counters), then static files traffic.js traffic counters: HTTP body bytes by category (assets / audio / code / data), WebSocket wire / payload + assetIndex.js content hashes (SHA-256, 16 hex) of public/assets + public/fonts, background build, .cache/asset-hashes.json common.js security headers, URL split, error page, JSON replies, bare 400 for unparseable requests boot.js boot banner (Local / LAN / tunnel URLs), port-in-use hint, graceful SIGINT / SIGTERM net.js session registry, send helpers, per-connection rate limit, message validation (uses shared/protocol.js) diff --git a/server/http/assetIndex.js b/server/http/assetIndex.js new file mode 100644 index 00000000..d2a497d6 --- /dev/null +++ b/server/http/assetIndex.js @@ -0,0 +1,352 @@ +// server/http/assetIndex.js — the content hashes of the served art (host bandwidth, layer 1; docs/ASSETS.md "The served +// manifests"): +// +// * built in the background shortly after the start (ASSET_INDEX_DELAY_MS): every file under public/assets/ and +// public/fonts/ (dot names skipped like the static server does, linked folders followed once), the first 16 hex +// digits of the SHA-256 of its bytes (WebCrypto, off the main thread); +// * remembered in .cache/asset-hashes.json by path + size + mtime: a restart only stats the files and reads none of +// the unchanged ones again; a file whose size or mtime changed is hashed again (a re-extraction, `setup --force`, +// a Docker rebuild then costs the players only the files whose bytes changed); +// * never trusted beyond what it checked: `hashOf(rel, stat)` answers only while the file still has the size and +// mtime it was hashed with, else null — the caller then serves the plain way (1-day cache, size-mtime ETag) and +// the miss asks for a rescan (at most one every RESCAN_GAP_MS); +// * models: lists of files that share one URL hash — a Spine skeleton, its atlas and its pages (pixi-spine finds the +// atlas next to the skeleton and the pages next to the atlas, so all their URLs carry the same segment). A model's +// hash is the SHA-256 of its members' hashes; assetViews.js hands the lists over (`setModels`), read from the +// manifests. `urlHashOk` accepts a file's own hash or the hash of a model it belongs to. +// +// Rel paths are POSIX and relative to public/ ('assets/char/avatar/x.png', 'fonts/bender-regular.woff2'). A rel that +// differs from a file's name only in case finds that file when no other name matches it that way (Windows and macOS +// serve `wah-y.png` from `WAH-Y.png`; on Linux the request is a 404 anyway). + +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { createHash, webcrypto } from 'node:crypto'; +import { ASSET_HASH_HEX } from '../../shared/assetPaths.js'; +import { noopLog } from './config.js'; +import { IMMUTABLE_CACHE } from './files.js'; + +/** Folders of public/ whose files are hashed. */ +export const ASSET_INDEX_ROOTS = Object.freeze(['assets', 'fonts']); +/** The first build waits this long after the start (a server that is only started and stopped again — tests — reads nothing). */ +export const ASSET_INDEX_DELAY_MS = 1000; +/** Shortest time between two rescans a stale or unknown file asks for. */ +export const RESCAN_GAP_MS = 10_000; +/** Version of .cache/asset-hashes.json. */ +export const HASH_CACHE_VERSION = 1; +const STAT_CONCURRENCY = 64; +const HASH_CONCURRENCY = 4; + +const hex = (bytes) => Buffer.from(bytes).toString('hex').slice(0, ASSET_HASH_HEX); + +/** First ASSET_HASH_HEX hex digits of the SHA-256 of `data` (synchronous; model hashes, manifests, tests). */ +export function hash16(data) { + return createHash('sha256').update(data).digest('hex').slice(0, ASSET_HASH_HEX); +} + +/** The same for a file's bytes, computed off the main thread (WebCrypto runs in libuv's thread pool). */ +async function digest16(buf) { + return hex(await webcrypto.subtle.digest('SHA-256', buf)); +} + +/** Run `fn` over `items` with at most `n` at a time; stops early when `stop()` says so. */ +async function eachLimit(items, n, fn, stop) { + let i = 0; + const worker = async () => { + while (i < items.length && !stop()) { + const item = items[i++]; + await fn(item); + } + }; + await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker)); +} + +/** + * The asset index of one server. + * @param {{ publicDir: string, roots?: readonly string[], cacheFile?: string | null, log?: object, delayMs?: number, + * rescanGapMs?: number, now?: () => number }} opts + * cacheFile: where the hashes persist (null: in memory only — a test's temporary public/) + */ +export function createAssetIndex({ publicDir, roots = ASSET_INDEX_ROOTS, cacheFile = null, log = noopLog, delayMs = ASSET_INDEX_DELAY_MS, + rescanGapMs = RESCAN_GAP_MS, now = Date.now }) { + const base = path.resolve(publicDir); + /** @type {Map} */ + let records = new Map(); + /** @type {Map} lower-case rel → the rel (null: two names differ only in case) */ + let lower = new Map(); + /** @type {string[][]} */ + let modelLists = []; + /** @type {Map>} rel → hashes of the models it belongs to */ + let modelOf = new Map(); + let generation = 0; + let ready = false; + let closed = false; + let building = null; // the build in flight (a promise) + let again = false; // another build was asked for while one ran + let startTimer = null; + let rescanTimer = null; + let lastBuildAt = -Infinity; + let readyResolve; + const readyPromise = new Promise((r) => { readyResolve = r; }); + const listeners = new Set(); + const last = { files: 0, bytes: 0, hashed: 0, reused: 0, ms: 0, at: null, error: null }; + + /** The cache file's records ({} when absent, unreadable or another version). */ + function loadCache() { + if (!cacheFile) return new Map(); + try { + const doc = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); + if (!doc || doc.version !== HASH_CACHE_VERSION || typeof doc.files !== 'object' || !doc.files) return new Map(); + const out = new Map(); + for (const [rel, v] of Object.entries(doc.files)) { + if (Array.isArray(v) && Number.isFinite(v[0]) && Number.isFinite(v[1]) && typeof v[2] === 'string' && v[2].length === ASSET_HASH_HEX) { + out.set(rel, { size: v[0], mtimeMs: v[1], hash: v[2] }); + } + } + return out; + } catch { + return new Map(); + } + } + + /** Write the records (temp file + rename: a concurrent reader never sees half a file). Failures only log. */ + async function saveCache(map) { + if (!cacheFile) return; + const files = {}; + for (const [rel, r] of [...map].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))) files[rel] = [r.size, r.mtimeMs, r.hash]; + const tmp = `${cacheFile}.${process.pid}.tmp`; + try { + await fsp.mkdir(path.dirname(cacheFile), { recursive: true }); + await fsp.writeFile(tmp, JSON.stringify({ version: HASH_CACHE_VERSION, algo: 'sha256/16', files })); + await fsp.rename(tmp, cacheFile); + } catch (e) { + log.debug?.('[assets] hash cache not saved', e?.code || e?.message); + fsp.rm(tmp, { force: true }).catch(() => {}); + } + } + + /** Every file under the roots: [rel, abs] (dot names skipped, a folder reached twice — a link loop — walked once). */ + async function listFiles() { + const out = []; + const seen = new Set(); + const walk = async (abs, rel) => { + let real; + try { real = await fsp.realpath(abs); } catch { return; } + if (seen.has(real)) return; + seen.add(real); + let ents; + try { ents = await fsp.readdir(abs, { withFileTypes: true }); } catch { return; } + for (const d of ents) { + if (closed) return; + if (d.name.startsWith('.')) continue; + const childAbs = path.join(abs, d.name); + const childRel = `${rel}/${d.name}`; + if (d.isDirectory()) await walk(childAbs, childRel); + else if (d.isFile()) out.push([childRel, childAbs]); + else if (d.isSymbolicLink()) { + const st = await fsp.stat(childAbs).catch(() => null); + if (st?.isDirectory()) await walk(childAbs, childRel); + else if (st?.isFile()) out.push([childRel, childAbs]); + } + } + }; + for (const root of roots) await walk(path.join(base, root), root); + return out; + } + + /** Model hashes from the current records (a model with a member the index does not know has none). */ + function computeModels() { + const next = new Map(); + for (const list of modelLists) { + const h = modelHashOf(list); + if (!h) continue; + for (const rel of list) { + const c = canonical(rel); + let set = next.get(c); + if (!set) next.set(c, (set = new Set())); + set.add(h); + } + } + modelOf = next; + } + + /** The indexed name of `rel`: itself, or the one file whose name differs from it only in case (else null). */ + function canonical(rel) { + if (typeof rel !== 'string') return null; + if (records.has(rel)) return rel; + return lower.get(rel.toLowerCase()) || null; + } + + function recordOf(rel) { + const c = canonical(rel); + return c ? records.get(c) : null; + } + + function modelHashOf(list) { + if (!Array.isArray(list) || !list.length) return null; + const parts = []; + for (const rel of list) { + const r = recordOf(rel); + if (!r) return null; + parts.push(r.hash); + } + return hash16(parts.join('\n')); + } + + /** One build; true when the records changed (a new generation). */ + async function build() { + const t0 = now(); + const known = records.size ? records : loadCache(); + const next = new Map(); + let hashed = 0; + let reused = 0; + let bytes = 0; + const files = await listFiles(); + const toHash = []; + await eachLimit(files, STAT_CONCURRENCY, async ([rel, abs]) => { + const st = await fsp.stat(abs).catch(() => null); + if (!st || !st.isFile()) return; + bytes += st.size; + const k = known.get(rel); + if (k && k.size === st.size && k.mtimeMs === st.mtimeMs) { next.set(rel, k); reused++; } else toHash.push([rel, abs, st]); + }, () => closed); + await eachLimit(toHash, HASH_CONCURRENCY, async ([rel, abs, st]) => { + try { + const buf = await fsp.readFile(abs); + const after = await fsp.stat(abs); + // changed while it was read: hashed again by the next build + if (after.size !== st.size || after.mtimeMs !== st.mtimeMs || buf.length !== st.size) return; + next.set(rel, { size: st.size, mtimeMs: st.mtimeMs, hash: await digest16(buf) }); + hashed++; + } catch { /* vanished or unreadable: not in the index */ } + }, () => closed); + if (closed) return false; + // commit: records, model hashes and the generation change together (one synchronous step) + let changed = next.size !== records.size; + if (!changed) for (const [rel, r] of next) { const o = records.get(rel); if (!o || o.hash !== r.hash || o.size !== r.size || o.mtimeMs !== r.mtimeMs) { changed = true; break; } } + const cacheChanged = hashed > 0 || next.size !== known.size; + records = next; + lower = new Map(); + for (const rel of next.keys()) { const k = rel.toLowerCase(); lower.set(k, lower.has(k) ? null : rel); } + if (changed) { generation++; computeModels(); } + Object.assign(last, { files: next.size, bytes, hashed, reused, ms: now() - t0, at: new Date().toISOString(), error: null }); + lastBuildAt = now(); + if (cacheChanged) await saveCache(next); + if (hashed) log.info?.(`[assets] content hashes: ${next.size} files, ${hashed} hashed, ${reused} unchanged (${Math.round(last.ms)} ms)`); + return changed; + } + + function markReady() { + if (ready) return; + ready = true; + readyResolve(); + } + + function notify() { + for (const fn of [...listeners]) { try { fn(generation); } catch (e) { log.error?.('[assets] index listener failed', e); } } + } + + /** Build now (or once more after the build in flight). */ + function refresh() { + if (closed) return Promise.resolve(); + if (building) { again = true; return building; } + building = (async () => { + try { + do { + again = false; + const changed = await build(); + if (closed) break; + markReady(); + if (changed) notify(); + } while (again && !closed); + } catch (e) { + last.error = e?.message || String(e); + log.warn?.('[assets] content hash index failed', e?.message || e); + } finally { + building = null; + if (!closed) markReady(); + } + })(); + return building; + } + + return { + /** Start the background build (after `delayMs`); returns at once. Idempotent. */ + start() { + if (closed || startTimer || building || ready) return; + if (!(delayMs > 0)) { refresh(); return; } + startTimer = setTimeout(() => { startTimer = null; refresh(); }, delayMs); + startTimer.unref?.(); + }, + refresh, + /** Resolves once the first build is done (never rejects). */ + whenReady() { return readyPromise; }, + get ready() { return ready; }, + get generation() { return generation; }, + /** Files with a hash. */ + get size() { return records.size; }, + /** + * The file's content hash while it still has the size and mtime it was hashed with (`stat` given), else null. + * Without `stat`: the hash of the last build. + * @param {string} rel @param {{ size: number, mtimeMs: number }} [stat] + */ + hashOf(rel, stat) { + const r = recordOf(rel); + if (!r) return null; + if (stat && (stat.size !== r.size || stat.mtimeMs !== r.mtimeMs)) return null; + return r.hash; + }, + /** Does `urlHash` name exactly these bytes: the file's own hash, or the hash of a model it belongs to? */ + urlHashOk(rel, stat, urlHash) { + if (typeof urlHash !== 'string') return false; + const own = this.hashOf(rel, stat); + if (!own) return false; + return own === urlHash || !!modelOf.get(canonical(rel))?.has(urlHash); + }, + /** The hash of a model (its members' hashes hashed), null while a member is unknown. */ + modelHash: (list) => modelHashOf(list), + /** Replace the model lists (assetViews.js, from the manifests). */ + setModels(lists) { + modelLists = Array.isArray(lists) ? lists.filter((l) => Array.isArray(l) && l.length) : []; + computeModels(); + }, + /** A file was served that the index does not know, or no longer matches: rescan (at most one per rescanGapMs). */ + noteMiss() { + if (closed || !ready || rescanTimer || building) return; + const wait = Math.max(0, lastBuildAt + rescanGapMs - now()); + rescanTimer = setTimeout(() => { rescanTimer = null; refresh(); }, wait); + rescanTimer.unref?.(); + }, + /** `fn(generation)` after a build changed the records; returns the unsubscribe. */ + onChange(fn) { listeners.add(fn); return () => listeners.delete(fn); }, + stats() { + return { ready, building: !!building, generation, files: records.size, models: modelLists.length, last: { ...last } }; + }, + /** Stop: timers cleared, a build in flight abandoned. */ + close() { + closed = true; + if (startTimer) { clearTimeout(startTimer); startTimer = null; } + if (rescanTimer) { clearTimeout(rescanTimer); rescanTimer = null; } + readyResolve(); // a waiter is let go; `ready` stays as it was + }, + }; +} + +/** + * The ETag source and cache policy of a file of the asset index's folders (public/assets, public/fonts), for + * serveFile: its content hash when the index has it for these bytes (a miss asks for a rescan); for a URL with a hash + * segment `immutable` when that hash names these bytes, else `no-cache`. + * @param {ReturnType | null} assetIndex + * @param {string} rel the file under public/ ('assets/…', POSIX) + * @param {import('node:fs').Stats} stat + * @param {string | null} urlHash the URL's hash segment + * @returns {{ contentHash?: string | null, cacheControl?: string }} + */ +export function hashedFileOpts(assetIndex, rel, stat, urlHash) { + if (!assetIndex) return urlHash ? { cacheControl: 'no-cache' } : {}; + const contentHash = assetIndex.hashOf(rel, stat); + if (!contentHash) assetIndex.noteMiss(rel); + if (!urlHash) return { contentHash }; + return { contentHash, cacheControl: contentHash && assetIndex.urlHashOk(rel, stat, urlHash) ? IMMUTABLE_CACHE : 'no-cache' }; +} diff --git a/server/http/config.js b/server/http/config.js index 3d1bd123..113c4441 100644 --- a/server/http/config.js +++ b/server/http/config.js @@ -4,6 +4,8 @@ // * TRUST_PROXY ('auto' default: honour CF-Connecting-IP / X-Real-IP / X-Forwarded-For only from loopback/private // peers such as a local cloudflared; '1' always; '0' never) → net.js trustProxy; // * DEBUG → the console logger's debug level; +// * SP_ASSET_INDEX=0 (or the `assetIndex: false` option) → no content-hash index: plain asset URLs, size-mtime ETags +// (assetIndex.js); the index of the default public/ persists in .cache/asset-hashes.json; // * the served directories (public/, data/, shared/ and the content packs' packs/ of this repository unless the // options name others), and which startServer() options are handed on to net.js Network and lobby.js Lobby. @@ -64,6 +66,25 @@ export function serveDirs(opts) { }; } +/** + * The asset index options out of the startServer() options and the environment, or null for none: `assetIndex: false` + * or SP_ASSET_INDEX=0 / off / false / no switches it off; `assetIndex: { cacheFile, delayMs }` overrides the defaults. + * The hashes persist in .cache/asset-hashes.json only for the repository's own public/ (a test's temporary folder keeps + * them in memory). + * @param {{ assetIndex?: boolean | { cacheFile?: string | null, delayMs?: number } }} opts + * @param {string} publicDir + * @returns {{ cacheFile: string | null, delayMs?: number } | null} + */ +export function assetIndexOptionsFrom(opts, publicDir) { + if (opts.assetIndex === false) return null; + const env = String(process.env.SP_ASSET_INDEX ?? '').trim().toLowerCase(); + if (opts.assetIndex == null && ['0', 'off', 'false', 'no'].includes(env)) return null; + const o = typeof opts.assetIndex === 'object' && opts.assetIndex ? opts.assetIndex : {}; + const own = path.resolve(publicDir) === path.join(ROOT, 'public'); + const cacheFile = o.cacheFile !== undefined ? o.cacheFile : (own ? path.join(ROOT, '.cache', 'asset-hashes.json') : null); + return { cacheFile, ...(Number.isFinite(o.delayMs) ? { delayMs: o.delayMs } : {}) }; +} + /** net.js Network options out of the startServer() options; `trustProxy` falls back to TRUST_PROXY. */ export function netOptionsFrom(opts) { const netOptions = {}; diff --git a/server/http/files.js b/server/http/files.js index c15de0e6..98910651 100644 --- a/server/http/files.js +++ b/server/http/files.js @@ -64,8 +64,8 @@ const GZIP_CACHE_MAX_FILE = 8 << 20; // larger files are gzip-streamed on t const GZIP_CACHE_MAX_TOTAL = 96 << 20; // Asset URLs carry no content hash yet, and tools/fetch-assets.mjs / tools/vendor.mjs can rewrite files in // place (atlas + png + skel must stay consistent), so "long" is one day; revalidation after that is a cheap 304. -const LONG_CACHE = 'public, max-age=86400'; // 1 day -const IMMUTABLE_CACHE = 'public, max-age=31536000, immutable'; +export const LONG_CACHE = 'public, max-age=86400'; // 1 day +export const IMMUTABLE_CACHE = 'public, max-age=31536000, immutable'; const LONG_CACHE_DIRS = ['assets', 'fonts', 'vendor']; // first path segment under public/ const gzipAsync = promisify(zlib.gzip); diff --git a/server/http/routes.js b/server/http/routes.js index 3b785fd5..3bef18da 100644 --- a/server/http/routes.js +++ b/server/http/routes.js @@ -17,12 +17,14 @@ import { setSecurityHeaders, sendError, sendJson, splitUrl } from './common.js'; const MAX_URL_LENGTH = 4096; /** - * The GET /healthz body (`withTraffic`: plus the traffic counters, `?traffic=1`). + * The GET /healthz body (`withTraffic`: plus the traffic counters and the asset index's state, `?traffic=1`). * @param {{ startedAt: number, network: import('../net.js').Network, registry: import('../net.js').SessionRegistry, - * lobby: import('../lobby.js').Lobby, traffic?: ReturnType }} health + * lobby: import('../lobby.js').Lobby, traffic?: ReturnType, + * assetIndex?: ReturnType | null, + * views?: ReturnType | null }} health * @param {boolean} [withTraffic] */ -export function healthReport({ startedAt, network, registry, lobby, traffic }, withTraffic = false) { +export function healthReport({ startedAt, network, registry, lobby, traffic, assetIndex, views }, withTraffic = false) { const report = { ok: true, version: PROTOCOL_VERSION, app: APP_VERSION, uptimeSec: Math.round((Date.now() - startedAt) / 1000), // the runtime the server is serving right now (public/js/ui/buildGuard.js): a page whose own build is @@ -31,6 +33,8 @@ export function healthReport({ startedAt, network, registry, lobby, traffic }, w sockets: network.connectionCount, sessions: registry.size, ...lobby.stats(), }; if (withTraffic && traffic) report.traffic = traffic.snapshot(); + // content-hashed asset URLs (assetIndex.js / assetViews.js): off, building, or ready with how many files and URLs + if (withTraffic) report.assetHashes = assetIndex ? { ...assetIndex.stats(), views: views ? views.stats() : null } : null; return report; } diff --git a/server/index.js b/server/index.js index 80675f4c..108e05f1 100644 --- a/server/index.js +++ b/server/index.js @@ -9,6 +9,8 @@ // the content packs (/packs/index.json, /packs// — the registry is packs.js) // http/media.js /media/bgm/act1 → public/assets/audio/bgm/act1.mp3 (audio addressed without its extension) // http/files.js one file → response: MIME, gzip + memory cache, ETag / Last-Modified / 304, Cache-Control, ranges +// http/assetIndex.js the content hashes of public/assets and public/fonts (built in the background, cached in +// .cache/asset-hashes.json by path + size + mtime; Spine models share one hash) // http/buildTag.js the build tag of the served browser runtime (/healthz `build`, public/js/ui/buildGuard.js) // http/routes.js the request listener: security headers, 414 / 400 / 405, GET /healthz → JSON status, else static // http/traffic.js the traffic counters: bytes sent per category (assets / audio / code / data) and on the @@ -26,7 +28,7 @@ import http from 'node:http'; import { getData, loadData } from './data.js'; -import { ROOT, listenAddress, bindCandidates, serveDirs, makeLogger, parseTrustProxy } from './http/config.js'; +import { ROOT, listenAddress, bindCandidates, serveDirs, makeLogger, parseTrustProxy, assetIndexOptionsFrom } from './http/config.js'; import { WS_MAX_PAYLOAD, createSessionStack, attachWebSocket } from './http/websocket.js'; import { DATA_SHIM_JS, createStaticHandler } from './http/static.js'; import { createPackRegistry } from './packs.js'; @@ -34,6 +36,7 @@ import { MIME, COMPRESSIBLE, acceptsGzip, parseRange } from './http/files.js'; import { BUILD_INPUTS, computeBuildTag, buildTag, resetBuildTag } from './http/buildTag.js'; import { createRequestHandler } from './http/routes.js'; import { createTrafficCounter } from './http/traffic.js'; +import { createAssetIndex } from './http/assetIndex.js'; import { answerClientError } from './http/common.js'; import { lanUrls, displayHost, isProcessEntry, runMain } from './http/boot.js'; @@ -53,11 +56,14 @@ export { * ratePerSec?: number, rateBurst?: number, maxConnections?: number, maxRooms?: number, * maxConnectionsPerAddr?: number, maxRoomsPerAddr?: number, maxMatchesPerAddr?: number, resyncMinGapMs?: number, * heavyPerSec?: number, heavyBurst?: number, trustProxy?: 'auto' | boolean, soloReconnectWindowMs?: number, + * assetIndex?: boolean | { cacheFile?: string | null, delayMs?: number }, * }} [opts] + * assetIndex: the content-hash index (default on; false or SP_ASSET_INDEX=0: off — config.js assetIndexOptionsFrom) * @returns {Promise<{ port: number, host: string, url: string, server: http.Server, wss: import('ws').WebSocketServer, * lobby: import('./lobby.js').Lobby, network: import('./net.js').Network, * registry: import('./net.js').SessionRegistry, packs: ReturnType, - * traffic: ReturnType, close: () => Promise }>} + * traffic: ReturnType, assetIndex: ReturnType | null, + * close: () => Promise }>} */ export async function startServer(opts = {}) { const { port, host } = listenAddress(opts); @@ -70,6 +76,9 @@ export async function startServer(opts = {}) { // content packs (docs/PACKS.md): scanned now — the start log names them — and again whenever their folders change const packs = createPackRegistry({ publicDir, dataDir, packsDir }, { log }); packs.refresh(true); + // the content hashes of the art (built in the background once listening) + const indexOptions = assetIndexOptionsFrom(opts, publicDir); + const assetIndex = indexOptions ? createAssetIndex({ publicDir, log, ...indexOptions }) : null; const serveStatic = createStaticHandler({ publicDir, dataDir, sharedDir, packsDir, packs, log }); const startedAt = Date.now(); // The tag is per process (see buildTag): read the browser runtime once, here, not on every /healthz. @@ -77,7 +86,7 @@ export async function startServer(opts = {}) { buildTag(); const traffic = createTrafficCounter(); - const server = http.createServer(createRequestHandler({ serveStatic, health: { startedAt, network, registry, lobby, traffic }, log })); + const server = http.createServer(createRequestHandler({ serveStatic, health: { startedAt, network, registry, lobby, traffic, assetIndex }, log })); server.on('clientError', answerClientError); const wss = attachWebSocket(server, { network, log, traffic }); @@ -112,9 +121,11 @@ export async function startServer(opts = {}) { boundHost = bound; } catch (e) { network.close(); // stop heartbeat/sweep timers of the half-built server + assetIndex?.close(); throw e; } server.on('error', (e) => log.error('[http] server error', e)); + assetIndex?.start(); const addr = server.address(); const actualPort = typeof addr === 'object' && addr ? addr.port : port; @@ -126,6 +137,7 @@ export async function startServer(opts = {}) { closing = (async () => { try { lobby.shutdown('shutdown'); } catch (e) { log.error('[shutdown] lobby', e); } network.close(); + assetIndex?.close(); await new Promise((resolve) => { server.close(() => resolve()); server.closeIdleConnections?.(); @@ -136,7 +148,7 @@ export async function startServer(opts = {}) { return closing; } - return { port: actualPort, host: boundHost, url, server, wss, lobby, network, registry, packs, traffic, close }; + return { port: actualPort, host: boundHost, url, server, wss, lobby, network, registry, packs, traffic, assetIndex, close }; } // `node server/index.js` / npm start: listen, print the banner, stop on SIGINT / SIGTERM (http/boot.js). diff --git a/test/asset-index.test.js b/test/asset-index.test.js new file mode 100644 index 00000000..0df7b361 --- /dev/null +++ b/test/asset-index.test.js @@ -0,0 +1,124 @@ +// Content-hashed asset URLs (host bandwidth, layer 1; docs/ASSETS.md "The served manifests"): +// * server/http/assetIndex.js — SHA-256 (16 hex) of every file under public/assets and public/fonts, cached by path + +// size + mtime in a JSON file, never trusted for a file whose size or mtime changed; Spine models share one hash. +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createHash, randomBytes } from 'node:crypto'; +import { createAssetIndex, hash16 } from '../server/http/assetIndex.js'; + +const sha16 = (buf) => createHash('sha256').update(buf).digest('hex').slice(0, 16); + +function tree(files) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sp-hash-')); + for (const [rel, content] of Object.entries(files)) { + const p = path.join(dir, rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, content); + } + return dir; +} + +const until = async (fn, ms = 3000) => { for (const t0 = Date.now(); Date.now() - t0 < ms;) { if (await fn()) return true; await new Promise((r) => setTimeout(r, 20)); } return false; }; + +describe('the asset index', () => { + test('hashes every file under assets/ and fonts/ (dot names skipped, a linked folder followed), persists them, rehashes only what changed', async () => { + const a = randomBytes(2000); + const b = Buffer.from('atlas text\n'.repeat(50)); + const dir = tree({ + 'public/assets/char/a.png': a, 'public/assets/spine/x/x.atlas': b, 'public/assets/.hidden': 'h', 'public/assets/.git/obj': 'o', + 'public/fonts/f.woff2': 'font', 'public/js/app.js': 'not indexed', 'elsewhere/linked/l.png': 'linked', + }); + fs.symlinkSync(path.join(dir, 'elsewhere/linked'), path.join(dir, 'public/assets/link'), 'dir'); + const cacheFile = path.join(dir, '.cache/asset-hashes.json'); + try { + const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), cacheFile, delayMs: 0 }); + assert.equal(idx.ready, false); + idx.start(); + await idx.whenReady(); + assert.equal(idx.ready, true); + const st = (rel) => fs.statSync(path.join(dir, 'public', rel)); + assert.equal(idx.hashOf('assets/char/a.png', st('assets/char/a.png')), sha16(a)); + assert.equal(idx.hashOf('assets/spine/x/x.atlas'), sha16(b)); + assert.equal(idx.hashOf('fonts/f.woff2'), sha16(Buffer.from('font'))); + assert.equal(idx.hashOf('assets/link/l.png'), sha16(Buffer.from('linked')), 'a linked folder is followed'); + assert.equal(idx.hashOf('assets/.hidden'), null, 'dot names are never served, never hashed'); + assert.equal(idx.hashOf('assets/.git/obj'), null); + assert.equal(idx.hashOf('js/app.js'), null, 'only the asset folders'); + assert.equal(idx.size, 4); + assert.equal(idx.stats().last.hashed, 4); + assert.equal(idx.hashOf('assets/CHAR/A.PNG'), sha16(a), 'a name that differs only in case finds the file'); + // a stat that no longer matches: no hash (the caller serves the plain way) + assert.equal(idx.hashOf('assets/char/a.png', { size: 1, mtimeMs: st('assets/char/a.png').mtimeMs }), null); + const cached = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); + assert.equal(cached.version, 1); + assert.deepEqual(cached.files['assets/char/a.png'], [a.length, st('assets/char/a.png').mtimeMs, sha16(a)]); + idx.close(); + + // a restart: the cache answers for the unchanged files; a changed file is hashed again; a removed one is dropped + const a2 = randomBytes(2100); + fs.writeFileSync(path.join(dir, 'public/assets/char/a.png'), a2); + fs.rmSync(path.join(dir, 'public/fonts/f.woff2')); + const idx2 = createAssetIndex({ publicDir: path.join(dir, 'public'), cacheFile, delayMs: 0 }); + idx2.start(); + await idx2.whenReady(); + assert.equal(idx2.stats().last.hashed, 1, 'only the changed file was read'); + assert.equal(idx2.stats().last.reused, 2); + assert.equal(idx2.hashOf('assets/char/a.png'), sha16(a2)); + assert.equal(idx2.hashOf('fonts/f.woff2'), null); + assert.equal(JSON.parse(fs.readFileSync(cacheFile, 'utf8')).files['fonts/f.woff2'], undefined); + idx2.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + test('models: one hash for a skeleton, its atlas and pages; urlHashOk accepts the own or the model hash, nothing else', async () => { + const dir = tree({ 'public/assets/s/x.skel': 'skel', 'public/assets/s/x.atlas': 'atlas', 'public/assets/s/x.png': 'png', 'public/assets/o.png': 'other' }); + try { + const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), delayMs: 0 }); + idx.start(); + await idx.whenReady(); + const trio = ['assets/s/x.skel', 'assets/s/x.atlas', 'assets/s/x.png']; + const m = idx.modelHash(trio); + assert.equal(m, hash16(['skel', 'atlas', 'png'].map((s) => sha16(Buffer.from(s))).join('\n'))); + assert.equal(idx.modelHash([...trio, 'assets/missing.png']), null, 'a member the index does not know: no model hash'); + const st = (rel) => fs.statSync(path.join(dir, 'public', rel)); + assert.equal(idx.urlHashOk('assets/s/x.atlas', st('assets/s/x.atlas'), m), false, 'not before the lists are known'); + idx.setModels([trio]); + for (const rel of trio) assert.equal(idx.urlHashOk(rel, st(rel), m), true, rel); + assert.equal(idx.urlHashOk('assets/s/x.png', st('assets/s/x.png'), sha16(Buffer.from('png'))), true, 'the own hash too'); + assert.equal(idx.urlHashOk('assets/o.png', st('assets/o.png'), m), false, 'not a member'); + assert.equal(idx.urlHashOk('assets/s/x.png', st('assets/s/x.png'), '0000000000000000'), false); + assert.equal(idx.urlHashOk('assets/s/x.png', { size: 99, mtimeMs: 1 }, m), false, 'changed since: never immutable'); + idx.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + test('a served file the index no longer matches asks for a rescan (rate-limited); the new hash follows', async () => { + const dir = tree({ 'public/assets/a.png': 'one' }); + try { + let changes = 0; + const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), delayMs: 0, rescanGapMs: 50 }); + idx.onChange(() => changes++); + idx.start(); + await idx.whenReady(); + assert.equal(changes, 1); + const gen = idx.generation; + fs.writeFileSync(path.join(dir, 'public/assets/a.png'), 'two!'); + assert.equal(idx.hashOf('assets/a.png', fs.statSync(path.join(dir, 'public/assets/a.png'))), null); + idx.noteMiss('assets/a.png'); + idx.noteMiss('assets/a.png'); + assert.ok(await until(() => idx.generation > gen), 'rescanned'); + assert.equal(idx.hashOf('assets/a.png'), sha16(Buffer.from('two!'))); + assert.equal(changes, 2, 'one rescan for both misses'); + idx.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +});