// Content-hashed asset URLs (host bandwidth, layer 1; docs/ASSETS.md "The served manifests"): // * server/http/assetIndex.js — SHA-256 (16 hex) of every file under public/assets and public/fonts, cached by path + // size + mtime in a JSON file, never trusted for a file whose size or mtime changed; Spine models share one hash; // * server/http/assetViews.js — the served data/assets.json / data/local-assets.json with every art URL carrying its // hash segment, the committed files untouched; // * the ~hash routes of static.js / media.js — the segment stripped, `immutable` only when it names the file's bytes, // the current file with `no-cache` otherwise; content ETags; the plain URLs as before. import { test, describe, before, after } from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { createHash, randomBytes } from 'node:crypto'; import { createAssetIndex, hash16 } from '../server/http/assetIndex.js'; import { rewriteManifest, relOf } from '../server/http/assetViews.js'; import { startServer } from '../server/index.js'; const sha16 = (buf) => createHash('sha256').update(buf).digest('hex').slice(0, 16); const IMMUTABLE = 'public, max-age=31536000, immutable'; function tree(files) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sp-hash-')); for (const [rel, content] of Object.entries(files)) { const p = path.join(dir, rel); fs.mkdirSync(path.dirname(p), { recursive: true }); fs.writeFileSync(p, content); } return dir; } function get(port, p, headers = {}, method = 'GET') { return new Promise((resolve, reject) => { const req = http.request({ host: '127.0.0.1', port, path: p, headers, method, agent: false }, (res) => { const chunks = []; res.on('data', (c) => chunks.push(c)); res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) })); }); req.on('error', reject); req.end(); }); } const until = async (fn, ms = 3000) => { for (const t0 = Date.now(); Date.now() - t0 < ms;) { if (await fn()) return true; await new Promise((r) => setTimeout(r, 20)); } return false; }; describe('the asset index', () => { test('hashes every file under assets/ and fonts/ (dot names skipped, a linked folder followed), persists them, rehashes only what changed', async () => { const a = randomBytes(2000); const b = Buffer.from('atlas text\n'.repeat(50)); const dir = tree({ 'public/assets/char/a.png': a, 'public/assets/spine/x/x.atlas': b, 'public/assets/.hidden': 'h', 'public/assets/.git/obj': 'o', 'public/fonts/f.woff2': 'font', 'public/js/app.js': 'not indexed', 'elsewhere/linked/l.png': 'linked', }); fs.symlinkSync(path.join(dir, 'elsewhere/linked'), path.join(dir, 'public/assets/link'), 'dir'); const cacheFile = path.join(dir, '.cache/asset-hashes.json'); try { const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), cacheFile, delayMs: 0 }); assert.equal(idx.ready, false); idx.start(); await idx.whenReady(); assert.equal(idx.ready, true); const st = (rel) => fs.statSync(path.join(dir, 'public', rel)); assert.equal(idx.hashOf('assets/char/a.png', st('assets/char/a.png')), sha16(a)); assert.equal(idx.hashOf('assets/spine/x/x.atlas'), sha16(b)); assert.equal(idx.hashOf('fonts/f.woff2'), sha16(Buffer.from('font'))); assert.equal(idx.hashOf('assets/link/l.png'), sha16(Buffer.from('linked')), 'a linked folder is followed'); assert.equal(idx.hashOf('assets/.hidden'), null, 'dot names are never served, never hashed'); assert.equal(idx.hashOf('assets/.git/obj'), null); assert.equal(idx.hashOf('js/app.js'), null, 'only the asset folders'); assert.equal(idx.size, 4); assert.equal(idx.stats().last.hashed, 4); assert.equal(idx.hashOf('assets/CHAR/A.PNG'), sha16(a), 'a name that differs only in case finds the file'); // a stat that no longer matches: no hash (the caller serves the plain way) assert.equal(idx.hashOf('assets/char/a.png', { size: 1, mtimeMs: st('assets/char/a.png').mtimeMs }), null); const cached = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); assert.equal(cached.version, 1); assert.deepEqual(cached.files['assets/char/a.png'], [a.length, st('assets/char/a.png').mtimeMs, sha16(a)]); idx.close(); // a restart: the cache answers for the unchanged files; a changed file is hashed again; a removed one is dropped const a2 = randomBytes(2100); fs.writeFileSync(path.join(dir, 'public/assets/char/a.png'), a2); fs.rmSync(path.join(dir, 'public/fonts/f.woff2')); const idx2 = createAssetIndex({ publicDir: path.join(dir, 'public'), cacheFile, delayMs: 0 }); idx2.start(); await idx2.whenReady(); assert.equal(idx2.stats().last.hashed, 1, 'only the changed file was read'); assert.equal(idx2.stats().last.reused, 2); assert.equal(idx2.hashOf('assets/char/a.png'), sha16(a2)); assert.equal(idx2.hashOf('fonts/f.woff2'), null); assert.equal(JSON.parse(fs.readFileSync(cacheFile, 'utf8')).files['fonts/f.woff2'], undefined); idx2.close(); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); test('models: one hash for a skeleton, its atlas and pages; urlHashOk accepts the own or the model hash, nothing else', async () => { const dir = tree({ 'public/assets/s/x.skel': 'skel', 'public/assets/s/x.atlas': 'atlas', 'public/assets/s/x.png': 'png', 'public/assets/o.png': 'other' }); try { const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), delayMs: 0 }); idx.start(); await idx.whenReady(); const trio = ['assets/s/x.skel', 'assets/s/x.atlas', 'assets/s/x.png']; const m = idx.modelHash(trio); assert.equal(m, hash16(['skel', 'atlas', 'png'].map((s) => sha16(Buffer.from(s))).join('\n'))); assert.equal(idx.modelHash([...trio, 'assets/missing.png']), null, 'a member the index does not know: no model hash'); const st = (rel) => fs.statSync(path.join(dir, 'public', rel)); assert.equal(idx.urlHashOk('assets/s/x.atlas', st('assets/s/x.atlas'), m), false, 'not before the lists are known'); idx.setModels([trio]); for (const rel of trio) assert.equal(idx.urlHashOk(rel, st(rel), m), true, rel); assert.equal(idx.urlHashOk('assets/s/x.png', st('assets/s/x.png'), sha16(Buffer.from('png'))), true, 'the own hash too'); assert.equal(idx.urlHashOk('assets/o.png', st('assets/o.png'), m), false, 'not a member'); assert.equal(idx.urlHashOk('assets/s/x.png', st('assets/s/x.png'), '0000000000000000'), false); assert.equal(idx.urlHashOk('assets/s/x.png', { size: 99, mtimeMs: 1 }, m), false, 'changed since: never immutable'); idx.close(); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); test('a served file the index no longer matches asks for a rescan (rate-limited); the new hash follows', async () => { const dir = tree({ 'public/assets/a.png': 'one' }); try { let changes = 0; const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), delayMs: 0, rescanGapMs: 50 }); idx.onChange(() => changes++); idx.start(); await idx.whenReady(); assert.equal(changes, 1); const gen = idx.generation; fs.writeFileSync(path.join(dir, 'public/assets/a.png'), 'two!'); assert.equal(idx.hashOf('assets/a.png', fs.statSync(path.join(dir, 'public/assets/a.png'))), null); idx.noteMiss('assets/a.png'); idx.noteMiss('assets/a.png'); assert.ok(await until(() => idx.generation > gen), 'rescanned'); assert.equal(idx.hashOf('assets/a.png'), sha16(Buffer.from('two!'))); assert.equal(changes, 2, 'one rescan for both misses'); idx.close(); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); }); describe('the served manifests (rewriteManifest)', () => { const H = { '/assets/char/a.png': 'aaaaaaaaaaaaaaaa', '/assets/audio/bgm/x.mp3': 'bbbbbbbbbbbbbbbb', '/fonts/f.woff2': 'cccccccccccccccc', '/assets/spine/x/x.skel': '1111111111111111', '/assets/spine/x/x.atlas': '2222222222222222', '/assets/spine/x/x.png': '3333333333333333', '/assets/local/spine/token/t/t.skel': '4444444444444444', '/assets/local/spine/token/t/t.atlas': '5555555555555555', '/assets/local/spine/token/t/t.png': '6666666666666666', '/assets/local/ui/battle/b.png': '7777777777777777' }; const index = { hashOf: (rel) => H[`/${rel}`] || null, modelHash: (rels) => (rels.every((r) => H[`/${r}`]) ? hash16(rels.map((r) => H[`/${r}`]).join('\n')) : null), }; test('data/assets.json: own hashes, one model hash per Spine object, unknown files and names left alone, the input untouched', () => { const doc = { version: 1, hash: 'a1b2c3d4e5f6', generator: 'tools/fetch-assets.mjs', chars: { c: { avatar: '/assets/char/a.png', portrait: '/assets/char/missing.png', spine: { front: { skel: '/assets/spine/x/x.skel', atlas: '/assets/spine/x/x.atlas', textures: ['/assets/spine/x/x.png'], pma: false, anims: { idle: 'Idle' } } } } }, enemies: { e: { icon: '/assets/char/a.png', spineLocal: { group: 'spine/enemy/e', skel: 'e.skel', atlas: 'e.atlas', textures: ['e.png'] } } }, audio: { bgm: { lobby: { loop: '/assets/audio/bgm/x.mp3' } } }, fonts: { css: '/fonts/fonts.css', faces: { f: { woff2: '/fonts/f.woff2', family: 'F' } } }, }; const copy = JSON.parse(JSON.stringify(doc)); const r = rewriteManifest(doc, 'assets.json', index); assert.deepEqual(doc, copy, 'the committed manifest object is not changed'); const m = index.modelHash(['assets/spine/x/x.skel', 'assets/spine/x/x.atlas', 'assets/spine/x/x.png']); assert.equal(r.doc.chars.c.avatar, '/assets/~aaaaaaaaaaaaaaaa/char/a.png'); assert.equal(r.doc.chars.c.portrait, '/assets/char/missing.png', 'no hash: the plain URL'); assert.deepEqual(r.doc.chars.c.spine.front, { skel: `/assets/~${m}/spine/x/x.skel`, atlas: `/assets/~${m}/spine/x/x.atlas`, textures: [`/assets/~${m}/spine/x/x.png`], pma: false, anims: { idle: 'Idle' } }); assert.deepEqual(r.doc.enemies.e.spineLocal, copy.enemies.e.spineLocal, 'spineLocal holds names, not URLs'); assert.equal(r.doc.audio.bgm.lobby.loop, '/assets/~bbbbbbbbbbbbbbbb/audio/bgm/x.mp3', 'audio keeps /assets/…/audio/….mp3 (media.js makes it /media/~hash/…)'); assert.equal(r.doc.fonts.faces.f.woff2, '/fonts/~cccccccccccccccc/f.woff2'); assert.equal(r.doc.fonts.css, '/fonts/fonts.css'); assert.equal(r.doc.hash, 'a1b2c3d4e5f6'); assert.deepEqual(r.models, [['assets/spine/x/x.skel', 'assets/spine/x/x.atlas', 'assets/spine/x/x.png']]); assert.equal(r.hashed, 7); assert.equal(r.plain, 2); }); test('data/local-assets.json: a spine/ group is one model, other groups own hashes', () => { const doc = { version: 1, source: 'local-client', count: 4, groups: { 'spine/token/t': { 't.atlas': { path: '/assets/local/spine/token/t/t.atlas', kind: 'TextAsset' }, 't.png': { path: '/assets/local/spine/token/t/t.png', w: 4, h: 4 }, 't.skel': { path: '/assets/local/spine/token/t/t.skel' } }, 'ui/battle': { b: { path: '/assets/local/ui/battle/b.png', w: 2, h: 2, kind: 'Sprite' } }, } }; const r = rewriteManifest(doc, 'local-assets.json', index); const m = index.modelHash(['assets/local/spine/token/t/t.atlas', 'assets/local/spine/token/t/t.png', 'assets/local/spine/token/t/t.skel']); assert.ok(m); for (const [n, e] of Object.entries(r.doc.groups['spine/token/t'])) assert.equal(e.path, doc.groups['spine/token/t'][n].path.replace('/assets/', `/assets/~${m}/`), n); assert.equal(r.doc.groups['spine/token/t']['t.png'].w, 4); assert.deepEqual(r.doc.groups['ui/battle'].b, { path: '/assets/~7777777777777777/local/ui/battle/b.png', w: 2, h: 2, kind: 'Sprite' }); assert.deepEqual([r.doc.version, r.doc.source, r.doc.count], [1, 'local-client', 4]); assert.equal(r.models.length, 1); }); test('relOf: URL path → index path, refusing what the static server refuses', () => { assert.equal(relOf('/assets/a%20b.png'), 'assets/a b.png'); assert.equal(relOf('/fonts/f.woff2'), 'fonts/f.woff2'); for (const u of ['/data/x.json', '/assets/../x', '/assets//x', '/assets/x?v=1', '/assets/%E0%A4%A', 'assets/x', null]) assert.equal(relOf(u), null, String(u)); }); }); describe('content-hashed URLs through a real server', () => { let srv; let dir; const png = randomBytes(3000); const mp3 = randomBytes(6000); const skel = Buffer.from('skel-binary'.repeat(80)); const atlas = Buffer.from('x.png\nsize: 64,64\n'.repeat(30)); const page = randomBytes(1500); const manifest = { version: 1, hash: 'abcdefabcdef', chars: { c: { avatar: '/assets/char/a.png', spine: { front: { skel: '/assets/spine/x/x.skel', atlas: '/assets/spine/x/x.atlas', textures: ['/assets/spine/x/x.png'], anims: { idle: 'Idle' } } } } }, audio: { bgm: { lobby: { loop: '/assets/audio/bgm/lobby.mp3' } } }, fonts: { css: '/fonts/fonts.css', faces: { f: { woff2: '/fonts/f.woff2' } } }, stats: { note: 'a manifest large enough to be compressed '.repeat(40) }, }; const manifestText = JSON.stringify(manifest, null, 2); before(async () => { dir = tree({ 'secret.txt': 'TOP-SECRET-CONTENT', 'public/index.html': '
hi
', 'public/assets/char/a.png': png, 'public/assets/audio/bgm/lobby.mp3': mp3, 'public/assets/spine/x/x.skel': skel, 'public/assets/spine/x/x.atlas': atlas, 'public/assets/spine/x/x.png': page, 'public/assets/spine/x/tiles.json': '{"t":1}', 'public/fonts/f.woff2': randomBytes(500), 'public/fonts/fonts.css': '@font-face{}', 'data/assets.json': manifestText, 'data/config.json': '{}', 'shared/x.js': 'export const y = 2;', }); srv = await startServer({ port: 0, host: '127.0.0.1', quiet: true, publicDir: path.join(dir, 'public'), dataDir: path.join(dir, 'data'), sharedDir: path.join(dir, 'shared'), assetIndex: { delayMs: 0 } }); await srv.assetIndex.whenReady(); }); after(async () => { await srv?.close(); fs.rmSync(dir, { recursive: true, force: true }); }); const view = async () => JSON.parse((await get(srv.port, '/data/assets.json')).body); test('/data/assets.json is served with hashed URLs (no-cache, its own ETag, 304, gzip, HEAD); the committed file stays as it was', async () => { const r = await get(srv.port, '/data/assets.json'); assert.equal(r.status, 200); assert.equal(r.headers['content-type'], 'application/json; charset=utf-8'); assert.equal(r.headers['cache-control'], 'no-cache'); assert.match(r.headers.etag, /^"[0-9a-f]{16}"$/); const v = JSON.parse(r.body); assert.equal(v.chars.c.avatar, `/assets/~${sha16(png)}/char/a.png`); assert.equal(v.audio.bgm.lobby.loop, `/assets/~${sha16(mp3)}/audio/bgm/lobby.mp3`); assert.match(v.fonts.faces.f.woff2, /^\/fonts\/~[0-9a-f]{16}\/f\.woff2$/); const m = hash16([skel, atlas, page].map(sha16).join('\n')); assert.deepEqual([v.chars.c.spine.front.skel, v.chars.c.spine.front.atlas, v.chars.c.spine.front.textures[0]], [`/assets/~${m}/spine/x/x.skel`, `/assets/~${m}/spine/x/x.atlas`, `/assets/~${m}/spine/x/x.png`]); assert.equal((await get(srv.port, '/data/assets.json', { 'if-none-match': r.headers.etag })).status, 304); const gz = await get(srv.port, '/data/assets.json', { 'accept-encoding': 'gzip' }); assert.equal(gz.headers['content-encoding'], 'gzip'); assert.notEqual(gz.headers.etag, r.headers.etag); const head = await get(srv.port, '/data/assets.json', {}, 'HEAD'); assert.equal(head.status, 200); assert.equal(head.body.length, 0); assert.equal(fs.readFileSync(path.join(dir, 'data/assets.json'), 'utf8'), manifestText, 'never written'); assert.equal((await get(srv.port, '/data/local-assets.json')).status, 200, 'an absent local manifest is still the empty stand-in'); assert.deepEqual(JSON.parse((await get(srv.port, '/data/local-assets.json')).body).groups, {}); }); test('a hashed URL: immutable with a content ETag when the hash names the file; the current file with no-cache otherwise; plain URLs as before', async () => { const v = await view(); const ok = await get(srv.port, v.chars.c.avatar); assert.equal(ok.status, 200); assert.deepEqual(ok.body, png); assert.equal(ok.headers['cache-control'], IMMUTABLE); assert.equal(ok.headers.etag, `"${sha16(png)}"`); const old = await get(srv.port, '/assets/~0000000000000000/char/a.png'); assert.deepEqual(old.body, png, 'an old page still gets the file'); assert.equal(old.headers['cache-control'], 'no-cache'); const plain = await get(srv.port, '/assets/char/a.png'); assert.equal(plain.headers['cache-control'], 'public, max-age=86400'); assert.equal(plain.headers.etag, `"${sha16(png)}"`, 'a content ETag on the plain URL too (an mtime change alone is a 304)'); assert.equal((await get(srv.port, '/assets/char/a.png', { 'if-none-match': `"${sha16(png)}"` })).status, 304); const font = await get(srv.port, v.fonts.faces.f.woff2); assert.equal(font.headers['cache-control'], IMMUTABLE); const query = await get(srv.port, '/assets/char/a.png?v=1'); assert.match(query.headers['cache-control'], /immutable/, 'the old ?v= rule is kept'); }); test('a Spine model: skeleton, atlas, page and the atlas derived from the skeleton URL are immutable under the model hash; a file derived beside them is no-cache', async () => { const sp = (await view()).chars.c.spine.front; for (const u of [sp.skel, sp.atlas, sp.textures[0], sp.skel.replace(/\.skel$/, '.atlas')]) { const r = await get(srv.port, u); assert.equal(r.status, 200, u); assert.equal(r.headers['cache-control'], IMMUTABLE, u); } const sk = await get(srv.port, sp.skel); assert.deepEqual(sk.body, skel); assert.equal(sk.headers.etag, `"${sha16(skel)}"`, 'the ETag is the file\'s own content hash'); const tiles = await get(srv.port, sp.skel.replace(/[^/]+$/, 'tiles.json')); assert.equal(tiles.status, 200); assert.equal(tiles.headers['cache-control'], 'no-cache', 'not a member of the model'); }); test('audio: /media/~hash/… resolves like /media/…, immutable when the hash matches, ranges and ETags as before', async () => { const loop = (await view()).audio.bgm.lobby.loop; const media = loop.replace(/^\/assets\/(~[0-9a-f]{16})\/audio\/(.+)\.mp3$/, '/media/$1/$2'); assert.match(media, /^\/media\/~[0-9a-f]{16}\/bgm\/lobby$/); const r = await get(srv.port, media); assert.equal(r.status, 200); assert.equal(r.headers['content-type'], 'audio/mpeg'); assert.equal(r.headers['cache-control'], IMMUTABLE); assert.equal(r.headers.etag, `"${sha16(mp3)}"`); assert.deepEqual(r.body, mp3); const range = await get(srv.port, media, { range: 'bytes=0-9' }); assert.equal(range.status, 206); assert.deepEqual(range.body, mp3.subarray(0, 10)); assert.equal((await get(srv.port, loop)).headers['cache-control'], IMMUTABLE, 'the .mp3 fallback URL too'); assert.equal((await get(srv.port, '/media/~0000000000000000/bgm/lobby')).headers['cache-control'], 'no-cache'); assert.equal((await get(srv.port, '/media/bgm/lobby')).headers['cache-control'], 'public, max-age=86400'); }); test('the segment never opens a way out of the served folders', async () => { const h = sha16(png); for (const p of [`/assets/~${h}/../../secret.txt`, `/assets/~${h}/%2e%2e/%2e%2e/secret.txt`, `/assets/~${h}/..%2f..%2fsecret.txt`, `/media/~${h}/..%2f..%2f..%2fsecret.txt`, `/fonts/~${h}/../../secret.txt`, `/assets/~${h}/.hidden`, `/assets/~${h}`, '/assets/~/char/a.png']) { const r = await get(srv.port, p); assert.ok([301, 400, 403, 404].includes(r.status), `${p} → ${r.status}`); assert.ok(!r.body.toString().includes('TOP-SECRET'), `${p} leaked`); } }); test('a file replaced on disk: its old hashed URL turns no-cache at once, and after the rescan the view names the new bytes', async () => { const before = (await view()).chars.c.avatar; const next = randomBytes(3100); fs.writeFileSync(path.join(dir, 'public/assets/char/a.png'), next); const stale = await get(srv.port, before); assert.deepEqual(stale.body, next, 'the current file'); assert.equal(stale.headers['cache-control'], 'no-cache', 'the index no longer vouches for these bytes'); // the miss asked for a rescan (RESCAN_GAP_MS after the last build) — trigger it now instead of waiting await srv.assetIndex.refresh(); assert.ok(await until(async () => (await view()).chars.c.avatar === `/assets/~${sha16(next)}/char/a.png`), 'the view follows'); assert.equal((await get(srv.port, `/assets/~${sha16(next)}/char/a.png`)).headers['cache-control'], IMMUTABLE); }); }); test('without the index (assetIndex: false) the manifest goes out as committed and a hashed URL is served no-cache', async () => { const dir = tree({ 'public/assets/a.png': 'png', 'data/assets.json': JSON.stringify({ chars: { c: { avatar: '/assets/a.png' } } }) }); const srv = await startServer({ port: 0, host: '127.0.0.1', quiet: true, publicDir: path.join(dir, 'public'), dataDir: path.join(dir, 'data'), assetIndex: false }); try { assert.equal(srv.assetIndex, null); assert.equal(JSON.parse((await get(srv.port, '/data/assets.json')).body).chars.c.avatar, '/assets/a.png'); const r = await get(srv.port, `/assets/~${sha16(Buffer.from('png'))}/a.png`); assert.equal(r.status, 200); assert.equal(r.headers['cache-control'], 'no-cache'); assert.match(r.headers.etag, /^"[0-9a-f]+-[0-9a-f]+"$/, 'size-mtime ETag'); } finally { await srv.close(); fs.rmSync(dir, { recursive: true, force: true }); } });