From 69011d2b54cd2a9d9d51e62c09602f7bf017aa9c Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 15:45:38 +0800 Subject: [PATCH 01/11] 0.2.6-dev: no BGM download while the music is muted or at BGM volume 0; the wanted track is fetched once it is audible again (host bandwidth layer 0.1); golden: unchanged --- docs/design/client.md | 2 +- public/js/audio.js | 23 +++++++++++++++-- test/ui/audio.test.js | 57 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/design/client.md b/docs/design/client.md index 9f425265..4d2a59ff 100644 --- a/docs/design/client.md +++ b/docs/design/client.md @@ -62,7 +62,7 @@ its board-space position and facing, including on a mirrored boss half. They can from this preview. The inspection affordance does not make the preview a combat unit; the battle sim still creates the selected variant at battle start (§30.17). -Audio: autochess BGM per phase, UI SFX (buy/sell/refresh/level/merge/ready/timer), per-unit attack/hit SFX (throttled; an impact plays for a hostile attack's phys / arts / true damage only, never an operator's skill-mode file for a normal attack, ≤ 2 copies of one battle sound — §18.4; a skill cue that reaches the client before its unit — the sim emits a deploy-tick cast ahead of the unit's `['spawn']` — is held and sounds once when the unit is tracked, `AudioManager.pendingSkill`, GitHub PR #292 by @LimitlessHPPK; entering a field late hands the sound the buffered `spawn` tuples only, `screens/game/early.js audioEarly`, so no old death, deploy or cast replays), boss music in R14/15. +Audio: autochess BGM per phase, UI SFX (buy/sell/refresh/level/merge/ready/timer), per-unit attack/hit SFX (throttled; an impact plays for a hostile attack's phys / arts / true damage only, never an operator's skill-mode file for a normal attack, ≤ 2 copies of one battle sound — §18.4; a skill cue that reaches the client before its unit — the sim emits a deploy-tick cast ahead of the unit's `['spawn']` — is held and sounds once when the unit is tracked, `AudioManager.pendingSkill`, GitHub PR #292 by @LimitlessHPPK; entering a field late hands the sound the buffered `spawn` tuples only, `screens/game/early.js audioEarly`, so no old death, deploy or cast replays), boss music in R14/15. While the music is silent — muted, or the BGM volume at 0 — no BGM is downloaded: the track the screen wants is remembered, a different one still playing stops, and the wanted track is fetched once the music is audible again (`AudioManager.bgmSilent`; about 14 MB a match for a player who turned the music off). --- diff --git a/public/js/audio.js b/public/js/audio.js index 53cb8686..5ab23809 100644 --- a/public/js/audio.js +++ b/public/js/audio.js @@ -13,7 +13,9 @@ // never block or warn; everything requested before that is remembered (BGM) or dropped (SFX). // - Channels: master → { bgm, sfx, voice } gains; volumes from settings (0..1) + mute. Tab hidden ⇒ suspend. // - BGM: `intro` then `loop` (1 s crossfade); switching tracks fades out/in (0.8 s). The same loop URL -// keeps playing across phases (prep and combat share a track). +// keeps playing across phases (prep and combat share a track). While the music is silent — muted, or the BGM volume +// at 0 — no track is downloaded (≈ 14 MB a match): the wanted one is remembered, another one playing stops, and the +// track the screen wants then starts once the music is audible again (`setVolumes`; host bandwidth, layer 0). // - 开战 BGM: `bgm.combatAlts` are the mode's own battle tracks (塞壬唱片 骑士之日 / 无畏者). The track is fixed per // round, not drawn: 无畏者 through rounds 1–7 and 骑士之日 from round 8 on (`combatTrackFor`, the official // schedule), so every client of a match hears the same one, a fight never switches track halfway through and the @@ -662,6 +664,7 @@ export class AudioManager { */ setVolumes(v) { const n = (x, d) => (Number.isFinite(x) ? Math.max(0, Math.min(1, x)) : d); + const wasSilent = this.bgmSilent; this.volumes = { bgm: n(v?.bgm, this.volumes.bgm), sfx: n(v?.sfx, this.volumes.sfx), @@ -670,8 +673,13 @@ export class AudioManager { }; this._applyVolumes(); if (this.volumes.muted || this.volumes.sfx <= 0) this.stopBondCue(); + // the music became audible again: download and start the track the screen wants now (none was fetched meanwhile) + if (wasSilent && !this.bgmSilent && this.ctx && this.wantBgm) { const k = this.wantBgm; this.wantBgm = null; this.playBgm(k); } } + /** Is the music silent — muted, or the BGM volume at 0? Then no BGM is downloaded (playBgm). */ + get bgmSilent() { return this.volumes.muted || this.volumes.bgm <= 0; } + /** * The voice dub (settings 语音语言): 'jp' plays `audio.voiceJp`, anything else `audio.voice` (中文, the default). The * line on air finishes in its own dub; the next one follows the setting. @@ -766,12 +774,23 @@ export class AudioManager { // ---- BGM ------------------------------------------------------------------------------------------------ /** - * Switch BGM (null stops). Same loop URL ⇒ no restart. + * Switch BGM (null stops). Same loop URL ⇒ no restart. Before the first gesture, or while the music is silent + * (`bgmSilent`), the key is only remembered (`wantBgm`) and nothing is downloaded; a different track that is still + * playing stops. `_unlock` / `setVolumes` start the remembered track once it can be heard. * @param {string|null} key see bgmKeyFor */ playBgm(key) { try { if (!this.ctx) { this.wantBgm = key; return; } + if (this.bgmSilent) { + this.wantBgm = key; + const silentTrack = resolveBgm(this.getManifest(), key); + if (this.bgm && silentTrack && this.bgm.loopUrl === silentTrack.loop) { this.bgm.key = key; return; } + ++this.bgmToken; // a start still waiting for its download stays silent too + this._fadeOutBgm(); + return; + } + this.wantBgm = null; const track = resolveBgm(this.getManifest(), key); if (this.bgm && track && this.bgm.loopUrl === track.loop) { this.bgm.key = key; return; } if (!track && !this.bgm) return; diff --git a/test/ui/audio.test.js b/test/ui/audio.test.js index 6e6a5895..d11bb99b 100644 --- a/test/ui/audio.test.js +++ b/test/ui/audio.test.js @@ -577,6 +577,63 @@ describe('AudioManager', () => { globalThis.fetch = origFetch; } }); + test('BGM is not downloaded while the music is muted or at volume 0; the wanted track starts once it is audible', async () => { + const fw = fakeWindow(); + const origFetch = globalThis.fetch; + const urls = []; + globalThis.fetch = async (u) => { urls.push(u); return { ok: true, arrayBuffer: async () => new ArrayBuffer(8) }; }; + const tick = () => new Promise((r) => setTimeout(r, 10)); + const lobby = manifest.audio.bgm.lobby.loop; + const prep = manifest.audio.bgm.prep.loop; + assert.notEqual(lobby, prep, 'premise: two different loops'); + try { + const a = new AudioManager({ win: fw.win, getManifest: () => manifest }); + a.setVolumes({ muted: true }); + a.install(); + a.playBgm('lobby'); + fw.fire('pointerdown'); + await tick(); + assert.equal(a.unlocked, true); + assert.equal(a.bgmSilent, true); + assert.ok(!asked(urls, lobby) && !asked(urls, manifest.audio.bgm.lobby.intro), 'muted: nothing fetched at unlock'); + a.playBgm('prep'); + await tick(); + assert.ok(!asked(urls, prep), 'muted: a phase change fetches nothing'); + assert.equal(a.wantBgm, 'prep', 'the wanted track is remembered'); + a.setVolumes({ muted: false }); + await tick(); + assert.ok(asked(urls, prep), 'unmuted: the track the screen wants now is fetched'); + assert.ok(!asked(urls, lobby), 'never the track wanted before'); + assert.equal(a.bgm?.loopUrl, prep); + + // BGM volume 0 is silent too: another track stops and is not fetched; SFX keep playing + a.setVolumes({ bgm: 0 }); + a.playBgm('lobby'); + await tick(); + assert.ok(!asked(urls, lobby), 'BGM volume 0: no download'); + assert.equal(a.bgm, null, 'the other track stopped'); + a.sfx('buy'); + await tick(); + assert.ok(asked(urls, manifest.audio.sfx.ui.buy), 'SFX are not gated by the BGM volume'); + a.setVolumes({ bgm: 0.5 }); + await tick(); + assert.ok(asked(urls, lobby), 'audible again: fetched'); + assert.equal(a.bgm?.loopUrl, lobby); + + // the loop that already plays keeps playing through a silent phase change, and is not fetched again + a.setVolumes({ muted: true }); + const n = urls.length; + const started = fw.made.started; + a.playBgm('lobby'); + a.setVolumes({ muted: false }); + await tick(); + assert.equal(a.bgm?.loopUrl, lobby); + assert.equal(urls.length, n, 'no new request'); + assert.equal(fw.made.started, started, 'no restart'); + } finally { + globalThis.fetch = origFetch; + } + }); test('fetch failures are swallowed', async () => { const fw = fakeWindow(); const origFetch = globalThis.fetch; From 8edda589f3f8896a2497b11968ed34b6e4bee3c9 Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 15:57:33 +0800 Subject: [PATCH 02/11] =?UTF-8?q?0.2.6-dev:=20WebSocket=20permessage-defla?= =?UTF-8?q?te=20for=20frames=20from=201=20KB=20(no=20server=20context=20ta?= =?UTF-8?q?keover,=202=20KB=20server=20window,=20memLevel=207;=20a=20clien?= =?UTF-8?q?t=20offering=20a=20narrower=20window=20connects=20uncompressed)?= =?UTF-8?q?=20=E2=80=94=20host=20bandwidth=20layer=200.2;=20golden:=20unch?= =?UTF-8?q?anged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/DEPLOY.md | 2 +- docs/design/network.md | 2 +- docs/design/overview.md | 3 +- server/http/websocket.js | 38 +++++++++++++-- server/index.js | 4 +- test/ws-compression.test.js | 94 +++++++++++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 test/ws-compression.test.js diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index da59e7f9..709f6d55 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -9,7 +9,7 @@ |---|---| | 服务器 CPU | 战斗在各玩家浏览器里模拟（DESIGN §14），服务器只负责回合、经济和校验：**每个房间每个作战回合约 1 ms CPU**。AI 队友 / 掉线玩家的战场由服务器模拟：作战开始时 3 个 AI 战场在开发机上约 0.2–0.5 s CPU，小主机上可能要几秒（分成 8 ms 小片执行，不会卡住其他房间）。`SP_VERIFY=all` 会复算每个真人战场，CPU 明显增加，小主机建议保持 `off` 或 `sample`。 | | 服务器内存 | 空闲约 100 MB，每个进行中的对局再增加几 MB。 | -| 网络 | 4 人对局中服务器每回合下行约 0.25 MB（DESIGN §14 实测）。首次进入游戏时浏览器要从主机下载所需的图片 / Spine 模型 / 音频（按需加载，之后走浏览器缓存），公网隧道带宽小时第一次会慢一些。 | +| 网络 | 4 人对局中服务器每回合下行约 0.25 MB（DESIGN §14 实测；0.2.6 起 1 KB 以上的 WebSocket 消息用 permessage-deflate 压缩，实测一局单人对局的服务器消息从 169 KB 降到 63 KB，约为原来的四成）。首次进入游戏时浏览器要从主机下载所需的图片 / Spine 模型 / 音频（按需加载，之后走浏览器缓存），公网隧道带宽小时第一次会慢一些。 | | 磁盘 | 素材约 550 MB（`public/assets`，含中文约 65 MB、日文约 85 MB 两套干员语音）+ 依赖约 125 MB（`node_modules`；整合包只带运行依赖，约 65 MB）；可选的本地提取约 40 MB（`.venv-extract`）+ 70 MB 贴图（见第 6 节）。完整包解压后约 710 MB。 | | 玩家设备 | 支持 WebGL 的现代浏览器（Chrome / Edge / Firefox / Safari 最新版），电脑或手机平板（横屏）。老旧设备可在设置里调低画质或访问 `/?board=2d`。 | diff --git a/docs/design/network.md b/docs/design/network.md index 377ee918..39181960 100644 --- a/docs/design/network.md +++ b/docs/design/network.md @@ -4,7 +4,7 @@ Part of [DESIGN.md](../DESIGN.md) (the index; section numbers are global). ## 8. Network protocol (shared/protocol.js is normative) -Transport: one WebSocket per tab at `/ws`, JSON text frames `{ t: '', ...payload }`. Client requests may carry `rid` (request id); the server's direct reply echoes `rid` (`ok` or `error`). Server pushes have no `rid`. Max frame 64 KB inbound; rate limit 40 msgs/s per connection (excess dropped with `error RATE`). +Transport: one WebSocket per tab at `/ws`, JSON text frames `{ t: '', ...payload }`. Client requests may carry `rid` (request id); the server's direct reply echoes `rid` (`ok` or `error`). Server pushes have no `rid`. Max frame 64 KB inbound (a compressed frame counts its inflated size); rate limit 40 msgs/s per connection (excess dropped with `error RATE`). Frames from 1 KB up are compressed with permessage-deflate (RFC 7692; `server/http/websocket.js WS_DEFLATE`: no server context takeover, a 2 KB server window, zlib memLevel 7 — about 72 KB of zlib state per connection); smaller frames go out plain, and a client that offers no compression, or caps the server window below 2 KB, gets plain frames. A measured solo session (rounds 3–5): 169 KB of server frames → 63 KB on the wire. ### 8.1 Session & lobby C→S: `hello {name, token?, version}` · `room.create {mode:'solo'|'coop', difficulty}` · `room.join {code}` · `room.leave` · `room.ready {ready}` · `room.setDifficulty {difficulty}` (host) · `room.setAiPicksLast {on}` (host, co-op, before the match — `ROOM_STARTED` during it, `BAD_TARGET` in a solo room; un-readies the other humans like a difficulty change; the match takes it as `opts.aiPicksLast`: humans draft before AI seats, off by default, no extra random draw) · `room.addBot` / `room.removeBot {seat}` (host) · `room.kick {seat, playerId}` (host, before the match: another human leaves the room and gets `room.closed {reason:'kicked'}`; may join again; `playerId` = the player the host confirmed, a seat that changed hands meanwhile → `BAD_TARGET`) · `room.start` (host) · `room.loadout {entries, ops?}` (§16: any time in the lobby / room and during INFO_CHECK, then `WRONG_PHASE`; heavy rate-limit bucket; the match exposes it as `m.private.loadout`; `ops` — 0.2.2 — the per-operator 潜能 / 练度 `{ [charId]: { potential?: 1–6, cultivate?: 0–3 } }` of the 干员调配 roster and the 自选 owned pool, ≤ 256, checked strictly with the entries, `m.private.ops`; absent = none set: every operator 潜能 6, 精英2 Lv.60) · `room.ownership {notOwned}` (0.2.0 补位, the 干员持有 tab: the base chess ids marked as not owned — ≤ 160 ids, kept leniently: anything but a droppable NORMAL chess is dropped; stored on the session and the seat, a match takes the seat's list when it starts and never changes it — during a match `ROOM_STARTED` 'stored for the next match'; heavy bucket; the match exposes it as `m.private.standIns`) · `room.diy {picks}` (0.2.0 自选编队: `{ [slotBaseId]: { charId, skillIndex?, uniEquipId? } | null }` for the four DIY slots — ≤ 8 entries, kept leniently: shared/protocol.js `checkDiyPicks` drops an illegal pick (an operator without a kit, another tier's prototype, a prototype off its locked skill, a module of another game mode (集成战略 / 生息演算), an owned operator's second slot, the same operator twice in a tier, an unknown slot), only malformed picks are `BAD_MSG`; stored on the session and the seat like the not-owned list, a match takes the seat's picks when it starts — during a match `ROOM_STARTED` 'stored for the next match'; heavy bucket; the match exposes them as `m.private.diy`) · `ping {c}` diff --git a/docs/design/overview.md b/docs/design/overview.md index 875b7e69..90df7f61 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -42,7 +42,8 @@ server/ index.js process entry: startServer() wires server/http/ (plain node:http + ws); boots when run directly http/ config.js ROOT, served dirs, env (PORT, HOST, TRUST_PROXY, DEBUG), options handed to net.js / lobby.js - websocket.js session wiring (SessionRegistry → Lobby → Network), WebSocket upgrade at /ws (64 KB frames) + websocket.js session wiring (SessionRegistry → Lobby → Network), WebSocket upgrade at /ws (64 KB frames, + permessage-deflate from 1 KB) static.js mounts / → public, /data/, /shared/, /sim/ (.js only), the /data.js browser stand-in, path guards media.js /media/… extension-less audio → public/assets/audio files.js one file → response: MIME, gzip (.skel/.atlas/.json/.js/.css …), ETag / 304, cache policy, ranges diff --git a/server/http/websocket.js b/server/http/websocket.js index 0f207665..f74e0acd 100644 --- a/server/http/websocket.js +++ b/server/http/websocket.js @@ -2,8 +2,8 @@ // // * session wiring: SessionRegistry (reconnect tokens) → Lobby (rooms, server/lobby.js) → Network (the socket // protocol, server/net.js), built from the startServer() options (config.js decides which go where); -// * WebSocket (ws) at /ws, maxPayload 64 KB, no per-message deflate → Network.handleConnection. Refused at the -// upgrade: any other path 404; per-network socket limit for internet clients (maxConnectionsPerAddr, see net.js +// * WebSocket (ws) at /ws, maxPayload 64 KB, permessage-deflate (WS_DEFLATE) → Network.handleConnection. Refused at +// the upgrade: any other path 404; per-network socket limit for internet clients (maxConnectionsPerAddr, see net.js // clientAddress; local/LAN peers are exempt) 429; server full (maxConnections) or shutting down 503. import { WebSocketServer } from 'ws'; @@ -15,6 +15,37 @@ import { netOptionsFrom, lobbyOptionsFrom } from './config.js'; /** Inbound WebSocket frame limit (DESIGN §8). */ export const WS_MAX_PAYLOAD = 64 * 1024; +/** + * permessage-deflate (RFC 7692) for the JSON text frames. A message from 1 KB up (m.public, m.private, b.start, welcome + * …) is compressed on its own — no context is kept from one message to the next (`serverNoContextTakeover`) — with a + * 2 KB window and zlib memLevel 7, so a connection holds about 72 KB of zlib state; smaller frames (pong, ok) go out + * as they are. Measured on a solo session (rounds 3–5, 216 server frames): 169 KB of payload → 63 KB on the wire; a + * larger window gains nothing on these messages. Browsers negotiate it by themselves; a client that offers none gets + * plain frames. The inflated size of a client frame still counts against WS_MAX_PAYLOAD. + */ +export const WS_DEFLATE = Object.freeze({ + threshold: 1024, + serverNoContextTakeover: true, + serverMaxWindowBits: 11, + zlibDeflateOptions: Object.freeze({ memLevel: 7 }), +}); + +/** + * Does every permessage-deflate offer of this `Sec-WebSocket-Extensions` header cap the server window below + * WS_DEFLATE.serverMaxWindowBits? ws cannot accept such an offer and refuses the whole handshake (400), so the upgrade + * drops the header instead and the connection goes uncompressed. No browser sends `server_max_window_bits`. + * @param {string | string[] | undefined} header + */ +export function narrowDeflateOffer(header) { + if (typeof header !== 'string') return false; + const offers = header.split(',').map((s) => s.trim()).filter((s) => /^permessage-deflate\b/i.test(s)); + if (!offers.length) return false; + return offers.every((offer) => { + const m = /(?:^|;)\s*server_max_window_bits\s*=\s*"?(\d+)"?/i.exec(offer); + return !!m && Number(m[1]) < WS_DEFLATE.serverMaxWindowBits; + }); +} + /** * The session stack of one server. * @param {{ MatchClass?: Function, seedFn?: () => number, [option: string]: any }} opts startServer() options @@ -37,7 +68,7 @@ export function createSessionStack(opts, { data, log }) { * @returns {WebSocketServer} */ export function attachWebSocket(server, { network, log }) { - const wss = new WebSocketServer({ noServer: true, maxPayload: WS_MAX_PAYLOAD, perMessageDeflate: false, clientTracking: false }); + const wss = new WebSocketServer({ noServer: true, maxPayload: WS_MAX_PAYLOAD, perMessageDeflate: { ...WS_DEFLATE }, clientTracking: false }); wss.on('connection', (ws, req) => network.handleConnection(ws, req)); wss.on('error', (e) => log.error('[ws] server error', e)); @@ -51,6 +82,7 @@ export function attachWebSocket(server, { network, log }) { const refused = network.admission(req); if (refused === 'per-address') { reject(429, 'Too Many Requests'); return; } if (refused) { reject(503, 'Service Unavailable'); return; } + if (narrowDeflateOffer(req.headers['sec-websocket-extensions'])) delete req.headers['sec-websocket-extensions']; try { wss.handleUpgrade(req, socket, head, (ws) => wss.emit('connection', ws, req)); } catch (e) { diff --git a/server/index.js b/server/index.js index 2543c3de..91178128 100644 --- a/server/index.js +++ b/server/index.js @@ -3,8 +3,8 @@ // // http/config.js ROOT, the served directories, the environment (PORT 3000, HOST '::' dual-stack, TRUST_PROXY auto, DEBUG), // which startServer() options go to net.js / lobby.js, the console logger -// http/websocket.js session wiring (SessionRegistry → Lobby → Network) and the WebSocket at /ws (maxPayload 64 KB; -// refused at upgrade with 404 / 429 per network / 503) +// http/websocket.js session wiring (SessionRegistry → Lobby → Network) and the WebSocket at /ws (maxPayload 64 KB, +// permessage-deflate from 1 KB; refused at upgrade with 404 / 429 per network / 503) // http/static.js the static mounts (/ → public/, /data/, /shared/, /sim/ `.js` only), the /data.js browser stand-in, // the content packs (/packs/index.json, /packs// — the registry is packs.js) // http/media.js /media/bgm/act1 → public/assets/audio/bgm/act1.mp3 (audio addressed without its extension) diff --git a/test/ws-compression.test.js b/test/ws-compression.test.js new file mode 100644 index 00000000..5a02e283 --- /dev/null +++ b/test/ws-compression.test.js @@ -0,0 +1,94 @@ +// WebSocket permessage-deflate (server/http/websocket.js WS_DEFLATE, host bandwidth layer 0): a browser-like client +// negotiates it with no server context takeover and a 2 KB server window, a large JSON frame goes out compressed, a +// frame under the 1 KB threshold and a client without the extension get plain frames, and an offer that caps the server +// window below ours is answered without compression instead of a refused handshake. +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import WebSocket from 'ws'; +import { attachWebSocket, narrowDeflateOffer, WS_DEFLATE } from '../server/http/websocket.js'; +import { startServer } from '../server/index.js'; + +const BIG = JSON.stringify({ t: 'm.public', players: Array.from({ length: 200 }, (_, i) => ({ playerId: `p_${i % 4}`, name: '阿米娅', lp: 30, phase: 'PREP' })) }); +const SMALL = JSON.stringify({ t: 'pong', at: 12345 }); + +/** A bare http server whose /ws sends BIG then SMALL to every socket (a stand-in for net.js Network). */ +async function fakeServer() { + const server = http.createServer((req, res) => { res.writeHead(404); res.end(); }); + const network = { + admission: () => null, + handleConnection: (ws) => { ws.send(BIG); ws.send(SMALL); }, + }; + const wss = attachWebSocket(server, { network, log: { error() {} } }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return { url: `ws://127.0.0.1:${server.address().port}/ws`, close: () => new Promise((r) => { wss.close(); server.close(() => r()); }) }; +} + +/** Connect, collect two messages, report the server's Sec-WebSocket-Extensions answer and the bytes the socket read. */ +function receive(url, opts) { + return new Promise((resolve, reject) => { + const ws = new WebSocket(url, opts); + const msgs = []; + let negotiated = ''; + ws.on('upgrade', (res) => { negotiated = res.headers['sec-websocket-extensions'] || ''; }); + ws.on('error', reject); + ws.on('message', (data) => { + msgs.push(String(data)); + if (msgs.length === 2) { + const read = ws._socket.bytesRead; + ws.close(); + resolve({ msgs, read, extensions: negotiated }); + } + }); + }); +} + +describe('WebSocket permessage-deflate', () => { + let srv; + before(async () => { srv = await fakeServer(); }); + after(async () => { await srv.close(); }); + + test('negotiated with no server context takeover and a small server window; a large frame is compressed', async () => { + const r = await receive(srv.url); + assert.match(r.extensions, /permessage-deflate/); + assert.match(r.extensions, /server_no_context_takeover/); + assert.match(r.extensions, new RegExp(`server_max_window_bits=${WS_DEFLATE.serverMaxWindowBits}`)); + assert.deepEqual(r.msgs, [BIG, SMALL], 'the payloads arrive intact'); + assert.ok(BIG.length > 10000); + assert.ok(r.read < BIG.length / 4, `compressed on the wire: ${r.read} bytes read for ${BIG.length} of payload`); + assert.equal(WS_DEFLATE.threshold, 1024); + }); + + test('a client without the extension gets the same messages uncompressed', async () => { + const r = await receive(srv.url, { perMessageDeflate: false }); + assert.equal(r.extensions, ''); + assert.deepEqual(r.msgs, [BIG, SMALL]); + assert.ok(r.read > BIG.length + SMALL.length, `plain frames: ${r.read}`); + }); + + test('an offer that caps the server window below ours connects uncompressed instead of a refused handshake', async () => { + const r = await receive(srv.url, { perMessageDeflate: { serverMaxWindowBits: 9 } }); + assert.equal(r.extensions, ''); + assert.deepEqual(r.msgs, [BIG, SMALL]); + assert.equal(narrowDeflateOffer('permessage-deflate; server_max_window_bits=9'), true); + assert.equal(narrowDeflateOffer('permessage-deflate; server_max_window_bits=12; client_max_window_bits'), false); + assert.equal(narrowDeflateOffer('permessage-deflate; server_max_window_bits=9, permessage-deflate'), false, 'another offer is acceptable'); + assert.equal(narrowDeflateOffer('permessage-deflate; client_max_window_bits'), false); + assert.equal(narrowDeflateOffer(undefined), false); + }); +}); + +test('the real server negotiates compression at /ws and keeps the 64 KB inflated frame limit', async () => { + const srv = await startServer({ port: 0, host: '127.0.0.1', quiet: true }); + try { + const ws = new WebSocket(`ws://127.0.0.1:${srv.port}/ws`); + await new Promise((resolve, reject) => { ws.once('open', resolve); ws.once('error', reject); }); + assert.match(ws.extensions, /permessage-deflate/); + // a compressed client frame whose inflated size is over the limit closes the socket (1009), like a plain one + const closed = new Promise((resolve) => ws.once('close', (code) => resolve(code))); + ws.send(JSON.stringify({ t: 'hello', name: 'x'.repeat(70 * 1024) })); + assert.equal(await closed, 1009); + } finally { + await srv.close(); + } +}); From 05b0dba8b533e26b4b9d4baeb3aa4a0887cceea4 Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 16:00:08 +0800 Subject: [PATCH 03/11] =?UTF-8?q?0.2.6-dev:=20the=2019=20local=20=E7=8E=A9?= =?UTF-8?q?=E6=B3=95=E8=AF=B4=E6=98=8E=20pages=20get=20a=20lossless=20WebP?= =?UTF-8?q?=20copy=20in=20tools/local-extract=20(25.6=20=E2=86=92=2017.4?= =?UTF-8?q?=20MB;=20lossy=20waits=20for=20the=20owner's=20comparison)=20an?= =?UTF-8?q?d=20the=20guide=20preloads=20only=20the=20next=20page=20(host?= =?UTF-8?q?=20bandwidth=20layer=200.3);=20golden:=20unchanged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/ASSETS.md | 2 +- docs/DEPLOY.md | 2 ++ docs/design/client.md | 2 +- public/js/ui/guide.js | 19 ++++++++++--- test/local-extract.test.js | 41 ++++++++++++++++++++++++++++ test/ui/localArt.test.js | 13 +++++++++ tools/local-extract/extract.py | 49 ++++++++++++++++++++++++++-------- 7 files changed, 112 insertions(+), 16 deletions(-) diff --git a/docs/ASSETS.md b/docs/ASSETS.md index 2467d4dc..32d6729c 100644 --- a/docs/ASSETS.md +++ b/docs/ASSETS.md @@ -211,7 +211,7 @@ The `stem` of a Spine model is the upstream file name. Two examples: `char_107_l - **Fonts:** OTF/TTF files are converted to WOFF2 by a built-in encoder (`tools/assets/woff2.mjs`: Brotli with null transforms). - Its output was verified lossless against Google's reference `woff2` decoder. - `fonts.css` lists WOFF2 first and falls back to the original file. -- Images stay PNG. WebP conversion is not done: it would need a native dependency. The local-client board textures are the exception: `tools/local-extract/extract.py` (Python, where Pillow is already a dependency) writes WebP copies of the 12 textures the 3D board downloads (`WEBP`: colour maps lossy at quality 95 with the alpha and the RGB under transparent texels kept, normal and data maps lossless) and `data/local-assets.json` lists the copies; the PNGs stay beside them for `tools/crop-board-atlas.mjs`. `extract.py --webp` adds the copies to an existing extraction without the client. +- Images stay PNG. WebP conversion is not done: it would need a native dependency. The local-client board textures are the exception: `tools/local-extract/extract.py` (Python, where Pillow is already a dependency) writes WebP copies of the 12 textures the 3D board downloads (`WEBP`: colour maps lossy at quality 95 with the alpha and the RGB under transparent texels kept, normal and data maps lossless) and `data/local-assets.json` lists the copies; the PNGs stay beside them for `tools/crop-board-atlas.mjs`. The 19 local 玩法说明 pages get a WebP copy too (`GUIDE_WEBP_MODE`, lossless: 25.6 MB → 17.4 MB on the CN client's pages; lossy quality 90 would be 4.2 MB and waits for the maintainer's side-by-side look), listed in place of the PNG, which stays beside it; the mirror copies under `ui/guide/` stay PNG. `extract.py --webp` adds the copies to an existing extraction without the client. ## Manifest schema (`data/assets.json`) diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 709f6d55..387d0da5 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -315,6 +315,8 @@ services: **3D 棋盘贴图的下载量**：每位玩家进入对局时都要从开服的电脑下载 3D 棋盘的 12 张贴图。提取时会给这 12 张各写一份 WebP（颜色贴图有损、质量 95，法线和数据贴图无损），清单里列的是 WebP，同名 PNG 留在旁边给裁切工具和 setup 用。这部分下载量从约 6.7 MB 降到约 2 MB，网速慢的远程联机最明显。只有 PNG 的本地素材（例如在这一改动之前提取的）可以用提取时的 Python 环境运行 `tools/local-extract/extract.py --webp` 就地补上，只需要 Pillow，不需要客户端。 +**「玩法说明」教程图的下载量**（0.2.6 起）：本地提取的 19 页教程图每页 1.2–1.5 MB，玩家每翻一页都要从开服的电脑下载。提取时同样给每页写一份无损 WebP，清单列 WebP，PNG 留在旁边；19 页合计从约 25.6 MB 降到约 17.4 MB，画面完全不变。看说明时只预载下一页（以前前后两页都预载）。已经提取过的同样用 `tools/local-extract/extract.py --webp` 补上。 + ## 7. 打包发布（维护者） Releases 的 zip（完整包、精简包，0.2.1 起还有更新包）由 `tools/package.mjs` 生成，在**源码仓库**里运行（整合包里没有这个工具）： diff --git a/docs/design/client.md b/docs/design/client.md index 4d2a59ff..f3a6a7e8 100644 --- a/docs/design/client.md +++ b/docs/design/client.md @@ -73,7 +73,7 @@ Audio: autochess BGM per phase, UI SFX (buy/sell/refresh/level/merge/ready/timer - `scene//` and `scene/tex/` — the official scene of a battlefield (§15 "Official battlefield scenes", DESIGN §30.17): `scene.json` (materials, key light, chunk records), `scene.bin` (the meshes in board space), the lightmap pages `lm`, and the materials' albedo textures shared by the scenes. Read from the scene bundles `scenes/activities//level_/…`, which the client carries only while the season runs: `extract.py --only scene` takes them from the client or from `--scenes ` / `.cache/official-scenes/*` (`tools/local-extract/fetch_scenes.py` downloads them from the official CDN's in-season version). The 0.2.5 full release zip includes the extracted files when the maintainer extracted them before packaging; a lite zip and a source install may fetch and extract them (`node tools/setup.mjs --scenes`). The repository never stores the bundles or the extracted files. - `ui/{common,battle,outer}/*` — every sprite of the three official autochess UI bundles (tier chips I–VI, bond disc rings, frames, banners, HUD pieces, badges, backgrounds, mode art). - `emoticon//.png` — the 36 official battle emotes, 6 themes (`shared/constants.js → EMOTE_THEMES / EMOTES`, `emoteArtPath`; data/emotes.json); also downloaded from the mirror (§22.5). -- `guide/autochess_{home,shop,handbook}_N.png` — the 19 official tutorial pages; stored squashed to 1024² — **display at 16:9** (the mirror copies too, §22.5). +- `guide/autochess_{home,shop,handbook}_N.png` — the 19 official tutorial pages; stored squashed to 1024² — **display at 16:9** (the mirror copies too, §22.5). `extract.py` writes a lossless WebP copy beside each page and the manifest lists the copy (`GUIDE_WEBP_MODE`; 25.6 MB → 17.4 MB); the viewer preloads only the next page (`ui/guide.js guidePreloadUrl`), not the previous one too. - `projectiles/*` — battle projectile sprites. - `spine/enemy//` — enemy models no web dump carries (灼热 / 炽焰源石虫, `extract.py ENEMY_SPINES`): manifest `enemies[id].spineLocal` (metadata from the committed `tools/assets/local-enemy-spines.json`); drawn instead of the tinted web alias when every file is listed (§21.14). - `spine/token//` — token (summon) models no web dump carries (40: most 自选 summons, 凯瑟琳's 爬行号·防护单元, 凛御银灰's 风雪之眼; `extract.py TOKEN_SPINES`, the Front renderer's skeleton of each battle token prefab): manifest `tokens[id].spineLocal` (metadata from the committed `tools/assets/local-token-spines.json`); drawn instead of the avatar diamond when every file is listed (docs/ASSETS.md "Token models from the local client"). diff --git a/public/js/ui/guide.js b/public/js/ui/guide.js index 907abd0e..46cb9626 100644 --- a/public/js/ui/guide.js +++ b/public/js/ui/guide.js @@ -3,7 +3,8 @@ // downloads from the public mirror (data/assets.json → ui['guide/']; GitHub issue #42) — data.js artUrls; a page // whose picture fails to load tries its next URL. The pages are stored squashed to 1024² (both copies) and are // displayed stretched back to 16:9. Three chapters (基础规则 / 调度手册 / 进阶图鉴), ‹ › buttons, ←/→ (A/D) keys, page -// dots, thumbnails; Esc or the backdrop closes. Adjacent pages are preloaded. +// dots, thumbnails; Esc or the backdrop closes. Only the next page is preloaded (a page is 0.9–1.5 MB; the one before +// has usually just been seen, so preloading it too doubled the download — host bandwidth, layer 0). // With no page listed the viewer shows the official loading-screen tips (config.tips) instead, and so does a page none // of whose copies loads (`guideStage`: data/assets.json lists the downloaded pages but the files are not on disk — a // `git pull` and restart without setup, an older asset folder, a failed download the shrink guard kept); the chapter @@ -66,6 +67,19 @@ export function guideStage(page, failed) { return src ? { kind: 'image', src } : { kind: 'tips' }; } +/** + * The one page the viewer preloads while page `i` is shown: the next one (wrapping around), its first URL that has not + * failed — null without pages. The previous page is not preloaded (see the header). + * @param {Array<{ urls?: string[] }>} pages guidePages() + * @param {number} i + * @param {Set} [failed] + */ +export function guidePreloadUrl(pages, i, failed) { + const n = Array.isArray(pages) ? pages.length : 0; + if (!n || n < 2) return null; + return nextArtUrl(pages[(((i + 1) % n) + n) % n]?.urls, failed); +} + /** Open/closed + current page. */ export const guideStore = createStore({ open: false, page: 0 }); @@ -136,8 +150,7 @@ export function GuideHost() { useEffect(() => { if (!open || !n) return; - preload(nextArtUrl(pages[(i + 1) % n]?.urls, failed)); - preload(nextArtUrl(pages[(i + n - 1) % n]?.urls, failed)); + preload(guidePreloadUrl(pages, i, failed)); }, [open, i, n, failed]); if (!open) return null; diff --git a/test/local-extract.test.js b/test/local-extract.test.js index 86577f1c..e22ca151 100644 --- a/test/local-extract.test.js +++ b/test/local-extract.test.js @@ -209,6 +209,47 @@ print(json.dumps(e.prefab_node(go)))`); const raw = spawnSync(PY, ['-c', `import sys; sys.path.insert(0, ${JSON.stringify(TOOL)}); import extract as e, json; print(json.dumps(e.vec(type('V', (), {'x': -0.0, 'y': float('inf'), 'z': 1e-9})())))`], { encoding: 'utf8', env: ENV }); assert.equal(raw.stdout.trim(), '[0.0, 0, 0.0]'); }); + + test('guide pages (host bandwidth layer 0): --webp writes a lossless WebP copy of every page PNG, the manifest lists it, the PNG stays', + { skip: spawnSync(PY, ['-c', "import sys; from PIL import features; sys.exit(0 if features.check('webp') else 1)"], { env: ENV }).status !== 0 && 'no Pillow with WebP' }, () => { + const jobs = JSON.parse(spawnSync(PY, [path.join(TOOL, 'extract.py'), '--print-jobs'], { encoding: 'utf8', env: ENV }).stdout); + assert.deepEqual(jobs.guideWebp.sub, 'guide'); + assert.equal(jobs.guideWebp.mode, 'lossless', 'lossy pages wait for the maintainer\'s comparison'); + assert.ok(!jobs.webp.some((w) => w.sub === 'guide'), 'the board texture table is unchanged'); + const out = py(` +import tempfile, pathlib +from PIL import Image +root = pathlib.Path(tempfile.mkdtemp()) +sub = root / 'guide' +sub.mkdir() +for i, name in enumerate(['autochess_home_1', 'autochess_shop_2']): + img = Image.new('RGBA', (48, 32), (10, 20, 30, 255)) + for x in range(48): + for y in range(32): img.putpixel((x, y), ((x * 5 + i * 40) % 256, (y * 7) % 256, (x * y) % 256, 255 if x % 7 else 254)) + img.save(sub / f'{name}.png') +groups = {'guide': {n: {'path': f'/assets/local/guide/{n}.png', 'w': 48, 'h': 32, 'kind': 'Sprite'} for n in ['autochess_home_1', 'autochess_shop_2']}, + 'ui/battle': {'x': {'path': '/assets/local/ui/battle/x.png', 'w': 2, 'h': 2, 'kind': 'Sprite'}}} +groups['guide']['autochess_handbook_9'] = {'path': '/assets/local/guide/autochess_handbook_9.png', 'kind': 'Sprite'} # no PNG on disk +mf = root / 'local-assets.json' +mf.write_text(json.dumps({'version': 1, 'source': 'local-client', 'count': 4, 'groups': groups})) +code = e.webp_only(root, mf, lambda *a: None) +doc = json.loads(mf.read_text()) +g = doc['groups']['guide'] +px = lambda p: list(Image.open(p).convert('RGBA').getdata()) +print(json.dumps({'code': code, 'paths': {k: v['path'] for k, v in g.items()}, 'modes': {k: v.get('webp') for k, v in g.items()}, + 'other': doc['groups']['ui/battle']['x']['path'], + 'same': all(px(sub / f'{n}.png') == px(sub / f'{n}.webp') for n in ['autochess_home_1', 'autochess_shop_2']), + 'png': all((sub / f'{n}.png').exists() for n in ['autochess_home_1', 'autochess_shop_2']), + 'again': e.run_webp(root, 'guide', doc['groups'], lambda *a: None)}))`); + assert.equal(out.code, 0); + const at = (n, ext) => `/assets/local/guide/${n}.${ext}`; + assert.deepEqual(out.paths, { autochess_home_1: at('autochess_home_1', 'webp'), autochess_shop_2: at('autochess_shop_2', 'webp'), autochess_handbook_9: at('autochess_handbook_9', 'png') }); + assert.deepEqual(out.modes, { autochess_home_1: 'lossless', autochess_shop_2: 'lossless', autochess_handbook_9: null }); + assert.equal(out.other, '/assets/local/ui/battle/x.png', 'other groups keep their PNGs'); + assert.ok(out.same, 'lossless: the same pixels, alpha included'); + assert.ok(out.png, 'the PNG stays beside the copy'); + assert.equal(out.again, 2, 'a second run (another GUIDE_WEBP_MODE) rewrites the copies the manifest already lists'); + }); }); describe('extracted board-scene files (when a local client was extracted)', { skip: !manifest && 'no data/local-assets.json' }, () => { diff --git a/test/ui/localArt.test.js b/test/ui/localArt.test.js index 956f6cba..13ffa2a5 100644 --- a/test/ui/localArt.test.js +++ b/test/ui/localArt.test.js @@ -163,6 +163,19 @@ describe('emotes and 玩法说明 pages: local art first, then the mirror copy ( assert.ok(!src.includes('该页面暂时无法显示'), 'no blank notice left'); }); + test('the guide preloads only the next page (host bandwidth layer 0): wraps around, skips a failed copy, never the previous page', async () => { + const { guidePreloadUrl } = await import('../../public/js/ui/guide.js'); + const page = (k) => ({ urls: [`/assets/local/guide/${k}.webp`, `/assets/ui/guide/${k}.png`] }); + const pages = ['autochess_home_1', 'autochess_home_2', 'autochess_home_3'].map(page); + assert.equal(guidePreloadUrl(pages, 0), '/assets/local/guide/autochess_home_2.webp'); + assert.equal(guidePreloadUrl(pages, 2), '/assets/local/guide/autochess_home_1.webp', 'wraps to the first page'); + assert.equal(guidePreloadUrl(pages, 1, new Set(['/assets/local/guide/autochess_home_3.webp'])), '/assets/ui/guide/autochess_home_3.png', 'the copy that has not failed'); + assert.equal(guidePreloadUrl([page('autochess_home_1')], 0), null, 'a single page needs no preload'); + assert.equal(guidePreloadUrl([], 0), null); + const src = readFileSync(path.join(ROOT, 'public/js/ui/guide.js'), 'utf8'); + assert.deepEqual(src.match(/^\s+preload\(.*$/gm).map((l) => l.trim()), ['preload(guidePreloadUrl(pages, i, failed));'], 'one preload call in the viewer'); + }); + test('tools/assets/plan.mjs GUIDE_PAGES is the viewer\'s page list (GUIDE_CHAPTERS), in reading order', async () => { const { GUIDE_CHAPTERS } = await import('../../public/js/ui/guide.js'); assert.deepEqual([...GUIDE_PAGES], GUIDE_CHAPTERS.flatMap((c) => c.pages.map(([key]) => key))); diff --git a/tools/local-extract/extract.py b/tools/local-extract/extract.py index 129d18e4..2531b645 100644 --- a/tools/local-extract/extract.py +++ b/tools/local-extract/extract.py @@ -32,8 +32,8 @@ aklz4.py registers a decoder for it. This script pulls the art the web sources l DXT5nm in an older bundle: AG = XY, R = 1) and metallic/gloss with the smoothness in A; the board renderer needs an RGB normal map (Z rebuilt) and a roughness map in G (1 − smoothness; metalness B = 0), written next to the source as _rgb.png / _rough.png - - WebP copies of the board textures every player downloads with the 3D board (WEBP), written next to their PNGs; - the manifest lists the copy + - WebP copies of the board textures every player downloads with the 3D board (WEBP) and of the 19 guidebook pages + (GUIDE_WEBP_*), written next to their PNGs; the manifest lists the copy - the official battlefield scenes (scenes.py SCENES: one prebuilt scene per 卫戍协议 battlefield) → scene// scene.json + scene.bin + lightmap pages and scene/tex/ (the materials' albedo textures). The scene bundles ship only while the season runs: they are read from the client when it has them, else from --scenes (default @@ -43,8 +43,9 @@ Usage: python3 -m venv .venv && .venv/bin/pip install -r tools/local-extract/requirements.txt .venv/bin/python tools/local-extract/extract.py [--game ] [--out public/assets/local] [--only ] python3 tools/local-extract/extract.py --print-jobs (the job table as JSON; needs no dependencies) - python3 tools/local-extract/extract.py --webp (only the WebP copies, from the PNGs already extracted — - e.g. the local art copied from a release bundle; needs Pillow) + python3 tools/local-extract/extract.py --webp (only the WebP copies of the board textures and the guide + pages, from the PNGs already extracted — e.g. the local art + copied from a release bundle; needs Pillow) .venv/bin/python tools/local-extract/extract.py --only scene [--scenes ] Writes /**.png|.webp|.skel|.atlas and data/local-assets.json (manifest of what was extracted). With --only, just @@ -188,6 +189,19 @@ WEBP_OPTIONS = { 'lossless': {'lossless': True, 'quality': 100, 'method': 6, 'exact': True}, } +# The 19 玩法说明 pages (output subdir guide: 1024², opaque) get a WebP copy too — every page PNG of the group, each a +# 1.2–1.5 MB download whenever a player opens the guide (public/js/ui/guide.js shows the local page first and preloads +# the next one). Lossless by default: 25.6 MB → 17.4 MB on the CN client's pages. Lossy quality 90 would make them +# 4.2 MB; the maintainer decides on that after a side-by-side comparison (docs/history/0.2.5.md, the 0.2.6-dev note), +# and then only GUIDE_WEBP_MODE changes (`extract.py --webp` rewrites the copies). Method 4: method 6 spends about 8 s +# a page for 0.4 % less. +GUIDE_WEBP_SUB = 'guide' +GUIDE_WEBP_MODE = 'lossless' +GUIDE_WEBP_OPTIONS = { + 'lossless': {'lossless': True, 'quality': 100, 'method': 4, 'exact': True}, + 'lossy': {'quality': 90, 'method': 4, 'alpha_quality': 100}, +} + # Shader bundles loaded beside every job that exports Materials, only so that the materials' shader references # (external CABs) resolve to a name in materials.json (e.g. Torappu/Scene/StandardDirectional); nothing is exported # from them. @@ -454,11 +468,22 @@ def run_derived(out_root, sub, manifest, log): return n +def webp_targets(sub, manifest): + """(name, mode, Pillow options) of every WebP copy output subdir `sub` gets: its WEBP rows and, for the guide, each + page of the group (an image entry: its path ends in .png, or in .webp after an earlier run).""" + out = [(name, mode, WEBP_OPTIONS[mode]) for wsub, name, mode in WEBP if wsub == sub] + if sub == GUIDE_WEBP_SUB: + out += [(name, GUIDE_WEBP_MODE, GUIDE_WEBP_OPTIONS[GUIDE_WEBP_MODE]) + for name, entry in sorted(manifest.get(sub, {}).items()) + if isinstance(entry, dict) and str(entry.get('path', '')).endswith(('.png', '.webp'))] + return out + + def run_webp(out_root, sub, manifest, log): - """Write the WEBP copies of output subdir `sub` from its PNGs and point their manifest entries at them.""" + """Write the WebP copies of output subdir `sub` from its PNGs and point their manifest entries at them.""" n = 0 - for wsub, name, mode in WEBP: - entry = manifest.get(sub, {}).get(name) if wsub == sub else None + for name, mode, options in webp_targets(sub, manifest): + entry = manifest.get(sub, {}).get(name) png = Path(out_root) / sub / f'{name}.png' if not entry or not png.exists(): continue @@ -466,7 +491,7 @@ def run_webp(out_root, sub, manifest, log): try: from PIL import Image with Image.open(png) as img: - img.save(dst, 'WEBP', **WEBP_OPTIONS[mode]) + img.save(dst, 'WEBP', **options) except Exception as e: # no WebP support in this Pillow, or a broken PNG: the entry keeps the PNG log(f' warn webp {name}: {e}') continue @@ -486,7 +511,7 @@ def webp_only(out_root, manifest_path, log): if not isinstance(groups, dict): print(f'{manifest_path} has no groups. Extract the local art first.', file=sys.stderr) return 2 - n = sum(run_webp(out_root, sub, groups, log) for sub in dict.fromkeys(s for s, _, _ in WEBP)) + n = sum(run_webp(out_root, sub, groups, log) for sub in dict.fromkeys([*(s for s, _, _ in WEBP), GUIDE_WEBP_SUB])) Path(manifest_path).write_text(json.dumps(doc, ensure_ascii=False, separators=(',', ':')), encoding='utf-8') log(f'done: {n} WebP copies, manifest {manifest_path}') return 0 if n else 1 @@ -846,8 +871,8 @@ def main(): 'searched after --game and before .cache/official-scenes/*)') ap.add_argument('--print-jobs', action='store_true', help='print the job table as JSON and exit') ap.add_argument('--webp', action='store_true', - help='only write the WebP copies of the board textures from the PNGs already under --out and list ' - 'them in the manifest (needs Pillow, not the client)') + help='only write the WebP copies of the board textures and the guide pages from the PNGs already ' + 'under --out and list them in the manifest (needs Pillow, not the client)') args = ap.parse_args() if args.print_jobs: @@ -857,6 +882,8 @@ def main(): webp = [{'sub': sub, 'name': name, 'mode': mode} for sub, name, mode in WEBP] print(json.dumps({'emoteThemes': [{'themeId': t, 'dir': d} for t, d in EMOTE_THEMES], 'jobs': jobs, 'derived': derived, 'webp': webp, + 'guideWebp': {'sub': GUIDE_WEBP_SUB, 'mode': GUIDE_WEBP_MODE, + 'options': GUIDE_WEBP_OPTIONS[GUIDE_WEBP_MODE]}, 'enemySpines': {'bundles': ENEMY_ART, 'sub': ENEMY_SPINE_SUB, 'ids': ENEMY_SPINES}, 'tokenSpines': {'bundles': TOKEN_PACKS, 'sub': TOKEN_SPINE_SUB, 'ids': TOKEN_SPINES}, 'scenes': {'sub': scenes.SCENE_SUB, 'tex': scenes.TEX_SUB, From be8d382c05aceee0539a96647c3237e8c2af737f Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 16:05:49 +0800 Subject: [PATCH 04/11] =?UTF-8?q?0.2.6-dev:=20traffic=20counters=20?= =?UTF-8?q?=E2=80=94=20HTTP=20body=20bytes=20by=20category=20(assets=20/?= =?UTF-8?q?=20audio=20/=20code=20/=20data)=20and=20WebSocket=20wire=20/=20?= =?UTF-8?q?payload=20bytes=20since=20the=20start=20at=20GET=20/healthz=3Ft?= =?UTF-8?q?raffic=3D1,=20and=20the=20page's=20own=20downloads=20(Resource?= =?UTF-8?q?=20Timing=20transferSize)=20as=20a=20traffic=20line=20in=20the?= =?UTF-8?q?=20=E9=97=AE=E9=A2=98=E5=8F=8D=E9=A6=88=20diagnostics=20(host?= =?UTF-8?q?=20bandwidth=20layer=200.4);=20golden:=20unchanged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/ARCHITECTURE.md | 2 +- docs/DEPLOY.md | 1 + docs/PLAYING.md | 2 +- docs/design/overview.md | 3 +- public/js/diag.js | 86 ++++++++++++++++++++++++++++++-- public/js/main.js | 4 +- server/http/routes.js | 23 ++++++--- server/http/traffic.js | 102 +++++++++++++++++++++++++++++++++++++ server/http/websocket.js | 28 +++++++++-- server/index.js | 12 +++-- shared/assetPaths.js | 78 +++++++++++++++++++++++++++++ test/asset-paths.test.js | 41 +++++++++++++++ test/diag.test.js | 52 ++++++++++++++++++- test/traffic.test.js | 105 +++++++++++++++++++++++++++++++++++++++ 14 files changed, 516 insertions(+), 23 deletions(-) create mode 100644 server/http/traffic.js create mode 100644 shared/assetPaths.js create mode 100644 test/asset-paths.test.js create mode 100644 test/traffic.test.js diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 50efc4c6..a0e958d1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -77,7 +77,7 @@ so old imports keep working: `public/js/ui/gameLogic.js` (`public/js/ui/gameLogi | path | what | |---|---| | `server/index.js` | the process entry (`npm start`); `startServer()` wires `server/http/` | -| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | +| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | | `server/net.js` | sessions and reconnect tokens, rate limits, message validation | | `server/lobby.js` | rooms, seats, AI seats, spectators; starts a `Match` | | `server/data.js` | loads `data/*.json` once (frozen) | diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 387d0da5..4ef91f4f 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -289,6 +289,7 @@ services: | 表情显示成默认图标、「玩法说明」只有文字要点 | 素材没下载完整：重新运行 `node tools/setup.mjs`（表情和教程图随其他素材一起从公开镜像下载，不需要客户端）；缺失明细在 `.cache/assets-report.json` | | 本地提取失败 | 游戏照常运行，只是第 6 节表格里的几样换成替代样式。确认客户端已下载全部资源；Python 版本太新导致依赖安装失败时，安装 Python 3.12 后删除 `.venv-extract` 再运行 `node tools/setup.mjs --local` | | 3D 棋盘没出现 | 需要本地提取的棋盘贴图（`node tools/doctor.mjs` 会显示「3D 棋盘可用」），以及支持 WebGL2 的浏览器。没有客户端的服务器可以从同一版本的整合包复制本地素材（第 6 节） | +| 想知道开服电脑上传了多少流量 | 浏览器打开 `http://localhost:3000/healthz?traffic=1`（换成你的端口）：`traffic.http` 是服务器启动以来各类内容（`assets` 图片和模型、`audio` 音频、`code` 页面和脚本、`data` 游戏数据和素材清单）发出的字节数和请求数（压缩后的实际字节，`notModified` 是只回了「没变」的 304），`traffic.ws` 是 WebSocket 的线上字节（`wireOut` / `wireIn`，压缩后）和压缩前的消息字节（`payloadOut` / `payloadIn`）。数字只在内存里，重启后从 0 开始。玩家那边：设置 →「问题反馈」复制的诊断里有一行 `traffic:`，是这个页面从打开起下载的字节数（`cached` 是浏览器缓存直接给的，不占流量；`external` 是其他网站，例如 Google 字体） | | 断线 | 同盟模拟 10 分钟内、独立模拟 24 小时内（`config.constants.singleReconnectTime`）用同一浏览器重新打开页面，自动回到原座位。同盟掉线期间按原阵容自动作战、到时自动准备（不会代为购买；想让 AI 代打请用「离开模拟 → 暂离（AI 托管）」）；独立模拟不计时，等你回来 | ## 6. 本地客户端素材（可选） diff --git a/docs/PLAYING.md b/docs/PLAYING.md index 36cc34af..aa2e7cbd 100644 --- a/docs/PLAYING.md +++ b/docs/PLAYING.md @@ -184,7 +184,7 @@ 标题画面页脚的「项目仓库」在新标签页打开本项目的 [GitHub 仓库](https://github.com/sganggs/Stronghold-Protocol)。 -遇到报错、卡住或结果不对时，打开「设置」→「问题反馈」，点「复制诊断信息」，再到 [GitHub issue](https://github.com/sganggs/Stronghold-Protocol/issues) 里描述发生了什么并粘贴进去。诊断信息包括游戏版本、浏览器和设备、当前所在的阶段与回合，以及打开页面以来记录到的错误（最多保留最近 40 条）。战斗中复制时，默认附上这场战斗的数据，开发者可以用它重现这场战斗；不想附上可以关掉「附上本场战斗」。诊断信息只在你的浏览器里生成，不会自动上传；你和其他玩家的名字、房间号会被替换成 ``、``、`` 这样的占位符。剪贴板不可用时（部分 App 内置浏览器），内容会显示在下方的文本框里，手动全选复制即可。 +遇到报错、卡住或结果不对时，打开「设置」→「问题反馈」，点「复制诊断信息」，再到 [GitHub issue](https://github.com/sganggs/Stronghold-Protocol/issues) 里描述发生了什么并粘贴进去。诊断信息包括游戏版本、浏览器和设备、当前所在的阶段与回合、打开页面以来按类别统计的下载量（图片和模型、音频、代码、数据），以及打开页面以来记录到的错误（最多保留最近 40 条）。战斗中复制时，默认附上这场战斗的数据，开发者可以用它重现这场战斗；不想附上可以关掉「附上本场战斗」。诊断信息只在你的浏览器里生成，不会自动上传；你和其他玩家的名字、房间号会被替换成 ``、``、`` 这样的占位符。剪贴板不可用时（部分 App 内置浏览器），内容会显示在下方的文本框里，手动全选复制即可。 ## 13. 统计（本机） diff --git a/docs/design/overview.md b/docs/design/overview.md index 90df7f61..ed82f088 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -48,7 +48,8 @@ server/ media.js /media/… extension-less audio → public/assets/audio files.js one file → response: MIME, gzip (.skel/.atlas/.json/.js/.css …), ETag / 304, cache policy, ranges buildTag.js build tag of the served browser runtime (/healthz build, public/js/ui/buildGuard.js) - routes.js request listener: 414 / 400 / 405 guards, GET /healthz, then static files + routes.js request listener: 414 / 400 / 405 guards, GET /healthz (?traffic=1: the counters), then static files + traffic.js traffic counters: HTTP body bytes by category (assets / audio / code / data), WebSocket wire / payload common.js security headers, URL split, error page, JSON replies, bare 400 for unparseable requests boot.js boot banner (Local / LAN / tunnel URLs), port-in-use hint, graceful SIGINT / SIGTERM net.js session registry, send helpers, per-connection rate limit, message validation (uses shared/protocol.js) diff --git a/public/js/diag.js b/public/js/diag.js index 959ae1e2..4132fca7 100644 --- a/public/js/diag.js +++ b/public/js/diag.js @@ -5,19 +5,23 @@ // the room code and the reconnect token replaced and the page's host cut from every URL. A secret is a whole token in // the prose and an exact string value in the attached battle JSON, so a name never rewrites a longer id. The battle // on screen can be attached as its BattleSpec (the b.start a browser receives: player ids, no names), which the dev -// tools replay. +// tools replay. The report's `traffic:` line sums what this page downloaded (Resource Timing `transferSize`: bytes +// on the wire, 0 for a response from the browser cache) by the categories of shared/assetPaths.js trafficCategory, +// other sites apart as `external` (host bandwidth, layer 0; the server's own counters: /healthz?traffic=1). // // recordError(source, err, detail?) source: 'error' (uncaught) | 'rejection' (unhandled promise) | 'sim' (a content // error the battle isolated) | 'runner' (the local battle runner) | 'request' // (a failed request or error push the player was shown) // errorLog() / errorCount() / clearErrorLog() // setBattleSource(fn) the battle runner registers how to read the battle on screen +// installTrafficMeter() / recordResourceEntry(entry) / trafficSummary() / clearTraffic() the page's download totals // buildDiagnostics(input) the report text (pure: the state, the environment and the battle are passed in) // diagnosticsText({ state, settings, attachBattle }) the report of this page (this browser, the battle on screen) // // Report keys are English on purpose: the text is read by developers and searched in issues, whatever the UI language. import { APP_VERSION } from '../../shared/constants.js'; +import { TRAFFIC_CATEGORIES, trafficCategory } from '../../shared/assetPaths.js'; export const LOG_MAX = 40; export const STACK_LINES = 6; @@ -87,6 +91,78 @@ export const errorLog = () => log.map((e) => ({ ...e })); export const errorCount = () => total; export function clearErrorLog() { log.length = 0; total = 0; dropped = 0; } +// ---- traffic: what this page downloaded --------------------------------------------------------------------- + +/** @type {Map} category → totals */ +const traffic = new Map(); +let trafficMeter = null; + +/** + * Add one Resource Timing entry (a PerformanceResourceTiming, or the page's own navigation entry) to the totals: bytes + * on the wire (`transferSize`; headers included, 0 when the browser cache answered — counted as `cached`). A same-origin + * URL counts under trafficCategory(path), another site's under `external`; data: / blob: URLs are not downloads. + * @param {{ name?: string, transferSize?: number, decodedBodySize?: number }} entry + * @param {string} [origin] the page's origin + */ +export function recordResourceEntry(entry, origin = globalThis.location?.origin) { + if (!entry || typeof entry.name !== 'string') return; + let u; + try { u = new URL(entry.name, origin || 'http://localhost'); } catch { return; } + if (u.protocol !== 'http:' && u.protocol !== 'https:') return; + const cat = origin && u.origin !== origin ? 'external' : trafficCategory(u.pathname); + let t = traffic.get(cat); + if (!t) traffic.set(cat, (t = { requests: 0, bytes: 0, cached: 0 })); + const bytes = Number(entry.transferSize) || 0; + t.requests++; + t.bytes += bytes; + if (bytes === 0 && (Number(entry.decodedBodySize) || 0) > 0) t.cached++; +} + +/** + * Start summing this page's downloads (main.js at boot): the entries already in the Resource Timing buffer, then + * every new one (a PerformanceObserver also gets the entries a full buffer drops). False where unsupported. + * @param {any} [win] + */ +export function installTrafficMeter(win = globalThis) { + if (trafficMeter) return true; + const PO = win?.PerformanceObserver; + if (typeof PO !== 'function') return false; + try { + const origin = win.location?.origin; + for (const e of win.performance?.getEntriesByType?.('navigation') || []) recordResourceEntry(e, origin); + trafficMeter = new PO((list) => { for (const e of list.getEntries()) recordResourceEntry(e, origin); }); + trafficMeter.observe({ type: 'resource', buffered: true }); + return true; + } catch { + trafficMeter = null; + return false; + } +} + +/** The totals so far: `{ [category]: { requests, bytes, cached } }` in report order, plus `total`. */ +export function trafficSummary() { + const out = {}; + const total = { requests: 0, bytes: 0, cached: 0 }; + for (const cat of [...TRAFFIC_CATEGORIES, 'external']) { + const t = traffic.get(cat); + if (!t) continue; + out[cat] = { ...t }; + if (cat !== 'external') { total.requests += t.requests; total.bytes += t.bytes; total.cached += t.cached; } + } + out.total = total; + return out; +} +export function clearTraffic() { traffic.clear(); } + +const sizeOf = (n) => (n >= 1e6 ? `${(n / 1e6).toFixed(1)} MB` : n >= 1e3 ? `${Math.round(n / 1e3)} KB` : `${n} B`); +/** `assets 9.7 MB (254, 41 cached) | … | host total 21.4 MB (869)` — the report's traffic line; null when nothing was recorded. */ +export function trafficLine(summary) { + if (!summary || !summary.total || !(summary.total.requests > 0)) return null; + const parts = Object.entries(summary).filter(([k]) => k !== 'total') + .map(([k, t]) => `${k} ${sizeOf(t.bytes)} (${t.requests}${t.cached ? `, ${t.cached} cached` : ''})`); + return `${parts.join(' | ')} | host total ${sizeOf(summary.total.bytes)} (${summary.total.requests})`; +} + let battleSource = null; /** @param {() => ({ battleId: string, fieldId?: string, kind?: string, spec: object, time?: number }|null)} fn */ export function setBattleSource(fn) { battleSource = typeof fn === 'function' ? fn : null; } @@ -177,10 +253,10 @@ const join = (...parts) => parts.filter((p) => p != null && p !== '' && p !== fa * error log newest first) and, with `battle`, a ```json block of its spec. Secrets of `state` are whole tokens in the * prose and exact string values in the JSON. * @param {{ state: any, env?: any, settings?: any, battle?: any, entries?: any[], dropped?: number, total?: number, - * now?: number, pageStart?: number, version?: string }} input + * now?: number, pageStart?: number, version?: string, traffic?: ReturnType | null }} input */ export function buildDiagnostics(input) { - const { state = {}, env = {}, settings = null, battle = null } = input; + const { state = {}, env = {}, settings = null, battle = null, traffic: trafficTotals = null } = input; const entries = input.entries ?? errorLog(); const now = input.now ?? Date.now(); const lines = []; @@ -208,6 +284,8 @@ export function buildDiagnostics(input) { if (settings) { lines.push(`settings: ${join(`quality ${settings.quality}`, settings.damageNumbers ? 'damage numbers' : null, settings.officialDamageNumbers ? 'official damage numbers' : null, settings.muted ? 'muted' : null)}`); } + const tl = trafficLine(trafficTotals); + if (tl) lines.push(`traffic: ${tl}`); const shown = entries.slice().reverse(); const tot = input.total ?? total; const drop = input.dropped ?? dropped; @@ -265,5 +343,5 @@ export function browserEnv() { * @param {{ state: any, settings?: any, attachBattle?: boolean }} opts */ export function diagnosticsText({ state, settings = null, attachBattle = false }) { - return buildDiagnostics({ state, settings, env: browserEnv(), battle: attachBattle ? currentBattle() : null }); + return buildDiagnostics({ state, settings, env: browserEnv(), battle: attachBattle ? currentBattle() : null, traffic: trafficSummary() }); } diff --git a/public/js/main.js b/public/js/main.js index f40b6545..52782875 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -55,7 +55,7 @@ import { installLoadoutSync, installOwnershipSync, installDiySync } from './ui/l import { startBuildGuard } from './ui/buildGuard.js'; import { initLang, useLang, tickerText } from './ui/lang.js'; import { t, N_, translateWire } from '../../shared/i18n.js'; -import { recordError } from './diag.js'; +import { recordError, installTrafficMeter } from './diag.js'; const RESTORE_GRACE_MS = 1500; const JOIN_DELAY_MS = 350; @@ -346,6 +346,8 @@ function installGlobalErrorHandlers() { async function boot() { installGlobalErrorHandlers(); + // what this page downloads, by category, for the 问题反馈 report's traffic line (diag.js; nothing is sent anywhere) + installTrafficMeter(); // touch / hover / fullscreen classes, zoom-gesture blocking, rotation re-layout (ui/device.js, css/devices.css) installDeviceSupport(); // A page restored from the back/forward cache has a dead socket and a stale token choice: start over. diff --git a/server/http/routes.js b/server/http/routes.js index ab4a12ff..3b785fd5 100644 --- a/server/http/routes.js +++ b/server/http/routes.js @@ -4,9 +4,11 @@ // * a URL longer than 4096 characters → 414; one that does not parse → 400; // * any method but GET / HEAD → 405 with `Allow: GET, HEAD`; // * GET /healthz → JSON status (protocol `version`, release `app`, uptime, the served `build`, sockets, sessions, -// rooms, matches), never cached; +// rooms, matches), never cached; `?traffic=1` adds the traffic counters (traffic.js: bytes by category since the +// start, WebSocket wire and payload bytes); // * everything else → the static files (static.js). -// A route that throws is logged and answers 500. +// Every response's body bytes count in the traffic counters (traffic.js meter). A route that throws is logged and +// answers 500. import { PROTOCOL_VERSION, APP_VERSION } from '../../shared/constants.js'; import { buildTag } from './buildTag.js'; @@ -15,20 +17,26 @@ import { setSecurityHeaders, sendError, sendJson, splitUrl } from './common.js'; const MAX_URL_LENGTH = 4096; /** - * The GET /healthz body. + * The GET /healthz body (`withTraffic`: plus the traffic counters, `?traffic=1`). * @param {{ startedAt: number, network: import('../net.js').Network, registry: import('../net.js').SessionRegistry, - * lobby: import('../lobby.js').Lobby }} health + * lobby: import('../lobby.js').Lobby, traffic?: ReturnType }} health + * @param {boolean} [withTraffic] */ -export function healthReport({ startedAt, network, registry, lobby }) { - return { +export function healthReport({ startedAt, network, registry, lobby, traffic }, withTraffic = false) { + const report = { ok: true, version: PROTOCOL_VERSION, app: APP_VERSION, uptimeSec: Math.round((Date.now() - startedAt) / 1000), // the runtime the server is serving right now (public/js/ui/buildGuard.js): a page whose own build is // older than this reloads itself, so a deploy reaches clients that never reload build: buildTag(), sockets: network.connectionCount, sessions: registry.size, ...lobby.stats(), }; + if (withTraffic && traffic) report.traffic = traffic.snapshot(); + return report; } +/** Does the query ask for the traffic counters (`traffic`, `traffic=1`, `traffic=true`)? */ +const wantsTraffic = (query) => /(?:^|&)traffic(?:=(?:1|true|yes))?(?:&|$)/i.test(query || ''); + /** * The request listener for `http.createServer`. * @param {{ serveStatic: (req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse, @@ -48,7 +56,7 @@ export function createRequestHandler({ serveStatic, health, log }) { return; } if (parts.rawPath === '/healthz') { - sendJson(req, res, 200, healthReport(health)); + sendJson(req, res, 200, healthReport(health, wantsTraffic(parts.query))); return; } await serveStatic(req, res, parts.rawPath, parts.query); @@ -56,6 +64,7 @@ export function createRequestHandler({ serveStatic, health, log }) { return (req, res) => { setSecurityHeaders(res); + if (health.traffic) health.traffic.meter(res, splitUrl(req.url || '/')?.rawPath || '/'); handleRequest(req, res).catch((e) => { log.error('[http] request failed', e); sendError(req, res, 500, '服务器内部错误 · Internal error'); diff --git a/server/http/traffic.js b/server/http/traffic.js new file mode 100644 index 00000000..66818bbe --- /dev/null +++ b/server/http/traffic.js @@ -0,0 +1,102 @@ +// server/http/traffic.js — what this server has sent and received since it started (host bandwidth, layer 0): +// +// * HTTP: per category of shared/assetPaths.js trafficCategory (assets / audio / code / data / other) the responses +// and the body bytes actually written (after gzip / brotli; headers not counted), and how many were 304s; +// * WebSocket: the connections, the bytes on the wire both ways (the sockets' own counters, after permessage-deflate, +// frame headers included, the HTTP upgrade not) and the message payloads before compression — their ratio is what +// the compression saves. +// +// routes.js meters every response (`meter`), websocket.js every socket (`wsOpen` / `wsClose` / `wsSent` / `wsReceived`); +// `snapshot()` is the `traffic` object of GET /healthz?traffic=1 (docs/DEPLOY.md §5). Nothing is stored or sent +// anywhere else; the counters start at zero with the process. + +import { TRAFFIC_CATEGORIES, trafficCategory } from '../../shared/assetPaths.js'; + +/** Byte length of a chunk handed to res.write / res.end (a string in `encoding`, utf8 by default). */ +function chunkBytes(chunk, encoding) { + if (chunk == null || typeof chunk === 'function') return 0; + if (typeof chunk === 'string') return Buffer.byteLength(chunk, typeof encoding === 'string' && Buffer.isEncoding(encoding) ? encoding : 'utf8'); + return typeof chunk.byteLength === 'number' ? chunk.byteLength : 0; +} + +/** Length of a ws 'message' payload (a Buffer, an ArrayBuffer or, fragmented, a Buffer list). */ +function payloadBytes(data) { + if (Array.isArray(data)) return data.reduce((n, b) => n + payloadBytes(b), 0); + return data && typeof data.byteLength === 'number' ? data.byteLength : typeof data === 'string' ? Buffer.byteLength(data) : 0; +} + +/** + * A traffic counter for one server. + * @param {{ now?: () => number }} [opts] + */ +export function createTrafficCounter({ now = Date.now } = {}) { + const startedAt = now(); + const http = Object.fromEntries(TRAFFIC_CATEGORIES.map((c) => [c, { requests: 0, bytes: 0, notModified: 0 }])); + const ws = { connections: 0, wireOut: 0, wireIn: 0, payloadOut: 0, payloadIn: 0, messagesOut: 0, messagesIn: 0 }; + /** @type {Map} live sockets and their counters at the upgrade */ + const live = new Map(); + + /** One finished HTTP response. */ + function record(path, status, bytes) { + const c = http[trafficCategory(path)] || http.other; + c.requests++; + c.bytes += bytes; + if (status === 304) c.notModified++; + } + + return { + /** + * Count the body bytes of `res` (every res.write / res.end) under the category of `path`, once the response is + * done or its connection gone. + * @param {import('node:http').ServerResponse} res + * @param {string} path the request's URL path + */ + meter(res, path) { + let bytes = 0; + const { write, end } = res; + res.write = function meteredWrite(chunk, encoding, cb) { + bytes += chunkBytes(chunk, encoding); + return write.call(this, chunk, encoding, cb); + }; + res.end = function meteredEnd(chunk, encoding, cb) { + bytes += chunkBytes(chunk, encoding); + return end.call(this, chunk, encoding, cb); + }; + res.once('close', () => record(path, res.statusCode, bytes)); + }, + /** A WebSocket connection opened on `socket` (after its upgrade answer was written). */ + wsOpen(socket) { + ws.connections++; + if (socket && !live.has(socket)) live.set(socket, { out: socket.bytesWritten || 0, in: socket.bytesRead || 0 }); + }, + /** …and closed: its wire bytes move to the totals. */ + wsClose(socket) { + const base = live.get(socket); + if (!base) return; + live.delete(socket); + ws.wireOut += Math.max(0, (socket.bytesWritten || 0) - base.out); + ws.wireIn += Math.max(0, (socket.bytesRead || 0) - base.in); + }, + /** A message handed to ws.send (its payload before compression). */ + wsSent(data) { ws.messagesOut++; ws.payloadOut += payloadBytes(data); }, + /** A message received (its payload after inflating). */ + wsReceived(data) { ws.messagesIn++; ws.payloadIn += payloadBytes(data); }, + /** The counters now (plain JSON). */ + snapshot() { + let wireOut = ws.wireOut; + let wireIn = ws.wireIn; + for (const [socket, base] of live) { + wireOut += Math.max(0, (socket.bytesWritten || 0) - base.out); + wireIn += Math.max(0, (socket.bytesRead || 0) - base.in); + } + const byCat = Object.fromEntries(Object.entries(http).map(([k, v]) => [k, { ...v }])); + const total = Object.values(http).reduce((t, v) => ({ requests: t.requests + v.requests, bytes: t.bytes + v.bytes }), { requests: 0, bytes: 0 }); + return { + since: new Date(startedAt).toISOString(), + seconds: Math.max(0, Math.round((now() - startedAt) / 1000)), + http: { ...byCat, total }, + ws: { ...ws, open: live.size, wireOut, wireIn }, + }; + }, + }; +} diff --git a/server/http/websocket.js b/server/http/websocket.js index f74e0acd..303b0834 100644 --- a/server/http/websocket.js +++ b/server/http/websocket.js @@ -61,15 +61,37 @@ export function createSessionStack(opts, { data, log }) { return { registry, lobby, network }; } +/** + * Count a socket in the traffic counters (traffic.js): its wire bytes from now until it closes, the payload of every + * message it sends (ws.send wrapped) and receives. + * @param {import('ws').WebSocket} ws + * @param {import('node:http').IncomingMessage} req + * @param {ReturnType} traffic + */ +function meterSocket(ws, req, traffic) { + const socket = req.socket; + traffic.wsOpen(socket); + const send = ws.send; + ws.send = function meteredSend(data, ...rest) { + traffic.wsSent(data); + return send.call(this, data, ...rest); + }; + ws.on('message', (data) => traffic.wsReceived(data)); + ws.once('close', () => traffic.wsClose(socket)); +} + /** * Serve the WebSocket endpoint /ws on `server` (its 'upgrade' event). * @param {import('node:http').Server} server - * @param {{ network: Network, log: object }} deps + * @param {{ network: Network, log: object, traffic?: ReturnType }} deps * @returns {WebSocketServer} */ -export function attachWebSocket(server, { network, log }) { +export function attachWebSocket(server, { network, log, traffic = null }) { const wss = new WebSocketServer({ noServer: true, maxPayload: WS_MAX_PAYLOAD, perMessageDeflate: { ...WS_DEFLATE }, clientTracking: false }); - wss.on('connection', (ws, req) => network.handleConnection(ws, req)); + wss.on('connection', (ws, req) => { + if (traffic) meterSocket(ws, req, traffic); + network.handleConnection(ws, req); + }); wss.on('error', (e) => log.error('[ws] server error', e)); server.on('upgrade', (req, socket, head) => { diff --git a/server/index.js b/server/index.js index 91178128..80675f4c 100644 --- a/server/index.js +++ b/server/index.js @@ -11,6 +11,8 @@ // http/files.js one file → response: MIME, gzip + memory cache, ETag / Last-Modified / 304, Cache-Control, ranges // http/buildTag.js the build tag of the served browser runtime (/healthz `build`, public/js/ui/buildGuard.js) // http/routes.js the request listener: security headers, 414 / 400 / 405, GET /healthz → JSON status, else static +// http/traffic.js the traffic counters: bytes sent per category (assets / audio / code / data) and on the +// WebSocket, since the start (GET /healthz?traffic=1) // http/common.js what every answer shares: security headers, URL split, error page, JSON replies, bare 400 // http/boot.js a pending update package first (update.js: old files deleted, the install verified against // MANIFEST.json), banner (Local / LAN / tunnel URLs), port-in-use hint, graceful shutdown on SIGINT / @@ -31,6 +33,7 @@ import { createPackRegistry } from './packs.js'; import { MIME, COMPRESSIBLE, acceptsGzip, parseRange } from './http/files.js'; import { BUILD_INPUTS, computeBuildTag, buildTag, resetBuildTag } from './http/buildTag.js'; import { createRequestHandler } from './http/routes.js'; +import { createTrafficCounter } from './http/traffic.js'; import { answerClientError } from './http/common.js'; import { lanUrls, displayHost, isProcessEntry, runMain } from './http/boot.js'; @@ -54,7 +57,7 @@ export { * @returns {Promise<{ port: number, host: string, url: string, server: http.Server, wss: import('ws').WebSocketServer, * lobby: import('./lobby.js').Lobby, network: import('./net.js').Network, * registry: import('./net.js').SessionRegistry, packs: ReturnType, - * close: () => Promise }>} + * traffic: ReturnType, close: () => Promise }>} */ export async function startServer(opts = {}) { const { port, host } = listenAddress(opts); @@ -73,9 +76,10 @@ export async function startServer(opts = {}) { resetBuildTag(); buildTag(); - const server = http.createServer(createRequestHandler({ serveStatic, health: { startedAt, network, registry, lobby }, log })); + const traffic = createTrafficCounter(); + const server = http.createServer(createRequestHandler({ serveStatic, health: { startedAt, network, registry, lobby, traffic }, log })); server.on('clientError', answerClientError); - const wss = attachWebSocket(server, { network, log }); + const wss = attachWebSocket(server, { network, log, traffic }); // The address actually bound. The default may fall back to IPv4; the returned host and url follow that. let boundHost; @@ -132,7 +136,7 @@ export async function startServer(opts = {}) { return closing; } - return { port: actualPort, host: boundHost, url, server, wss, lobby, network, registry, packs, close }; + return { port: actualPort, host: boundHost, url, server, wss, lobby, network, registry, packs, traffic, close }; } // `node server/index.js` / npm start: listen, print the banner, stop on SIGINT / SIGTERM (http/boot.js). diff --git a/shared/assetPaths.js b/shared/assetPaths.js new file mode 100644 index 00000000..24bb68a8 --- /dev/null +++ b/shared/assetPaths.js @@ -0,0 +1,78 @@ +// Asset URL paths shared by the server and the browser (host bandwidth, docs/ASSETS.md "The served manifests"): +// +// * the content-hash segment. The server hands out data/assets.json and data/local-assets.json with every file URL +// carrying the first 16 hex digits of the file's SHA-256 (of the model's files for a Spine model) as its first +// segment — `/assets/char/avatar/x.png` → `/assets/~0123456789abcdef/char/avatar/x.png`, audio at +// `/media/~/…` (shared/media.js) — so a browser may keep it for a year. The server strips the segment and +// serves the file at the plain path (server/http/static.js, media.js). A path segment, never `?v=`: several +// loaders look at the URL's ending (`.skel`, `.mp3`) and pixi-spine finds the atlas and its pages next to the +// skeleton, which a query string would not survive. +// * trafficCategory: the category a request path counts under in the traffic counters (server/http/traffic.js, +// /healthz?traffic=1; public/js/diag.js for the browser's own totals). + +/** Hex digits of a content hash in a URL. */ +export const ASSET_HASH_HEX = 16; + +/** The content-hash segment: `~` + ASSET_HASH_HEX lower-case hex digits. */ +export const HASH_SEGMENT_RE = /^~([0-9a-f]{16})$/; + +/** URL roots whose first segment may be a content hash. */ +export const HASHED_ROOTS = Object.freeze(['/assets/', '/fonts/', '/media/']); + +/** + * Split the content-hash segment off a URL path. + * '/assets/~0123456789abcdef/x.png' → { hash: '0123456789abcdef', path: '/assets/x.png' }; any other path (no segment, + * a malformed one, another root) → { hash: null, path } unchanged. + * @param {string} path a URL path (no query) + * @returns {{ hash: string | null, path: string }} + */ +export function splitHashedPath(path) { + const p = String(path ?? ''); + for (const root of HASHED_ROOTS) { + if (!p.startsWith(root)) continue; + const slash = p.indexOf('/', root.length); + if (slash < 0) break; + const m = HASH_SEGMENT_RE.exec(p.slice(root.length, slash)); + return m ? { hash: m[1], path: root + p.slice(slash + 1) } : { hash: null, path: p }; + } + return { hash: null, path: p }; +} + +/** + * The URL path with the content-hash segment after its root ('/assets/x.png', 'abc…' → '/assets/~abc…/x.png'). A + * path outside HASHED_ROOTS, one that already has a segment, or no valid hash → the path unchanged. + * @param {string} path + * @param {string | null | undefined} hash + */ +export function hashedPath(path, hash) { + const p = String(path ?? ''); + if (typeof hash !== 'string' || !HASH_SEGMENT_RE.test(`~${hash}`)) return p; + const root = HASHED_ROOTS.find((r) => p.startsWith(r)); + if (!root || p.length === root.length || splitHashedPath(p).hash) return p; + return `${root}~${hash}/${p.slice(root.length)}`; +} + +/** The URL with its content-hash segment removed (a URL derived from a hashed one names another file). */ +export const unhashedUrl = (url) => (typeof url === 'string' ? splitHashedPath(url).path : url); + +/** Traffic categories, in report order. */ +export const TRAFFIC_CATEGORIES = Object.freeze(['assets', 'audio', 'code', 'data', 'other']); + +/** + * The traffic category of a request path (the content-hash segment does not matter): `audio` (/media/, + * /assets/audio/), `assets` (/assets/, /fonts/, /icons/, the art of a content pack), `data` (/data/*.json, the UI + * language packs /i18n/, the pack index, other JSON), `code` (the page, scripts, styles: /, *.js, *.css, *.html, + * /vendor/, /sim/, /shared/), `other` (/healthz, error pages for anything else). + * @param {string} path a URL path (no query) + * @returns {'assets'|'audio'|'code'|'data'|'other'} + */ +export function trafficCategory(path) { + const p = splitHashedPath(path).path; + if (p.startsWith('/media/') || p.startsWith('/assets/audio/')) return 'audio'; + if (p.startsWith('/assets/') || p.startsWith('/fonts/') || p.startsWith('/icons/')) return 'assets'; + if (p.startsWith('/packs/')) return /\.json$/i.test(p) ? 'data' : 'assets'; + if (p.startsWith('/data/') || p.startsWith('/i18n/') || /\.json$/i.test(p)) return 'data'; + if (p === '/' || p.endsWith('/') || /\.(?:m?js|css|html?|map|webmanifest)$/i.test(p) + || p.startsWith('/vendor/') || p.startsWith('/sim/') || p.startsWith('/shared/')) return 'code'; + return 'other'; +} diff --git a/test/asset-paths.test.js b/test/asset-paths.test.js new file mode 100644 index 00000000..3f58a94b --- /dev/null +++ b/test/asset-paths.test.js @@ -0,0 +1,41 @@ +// shared/assetPaths.js: the content-hash segment of asset URLs (split / insert / strip) and the traffic categories the +// server's counters and the browser's diagnostics share (host bandwidth, layers 0 and 1). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { splitHashedPath, hashedPath, unhashedUrl, trafficCategory, TRAFFIC_CATEGORIES, ASSET_HASH_HEX, HASH_SEGMENT_RE } from '../shared/assetPaths.js'; + +const H = '0123456789abcdef'; + +test('the hash segment: split off, inserted after the root, removed again', () => { + assert.equal(ASSET_HASH_HEX, 16); + assert.ok(HASH_SEGMENT_RE.test(`~${H}`)); + assert.deepEqual(splitHashedPath(`/assets/~${H}/char/avatar/a.png`), { hash: H, path: '/assets/char/avatar/a.png' }); + assert.deepEqual(splitHashedPath(`/fonts/~${H}/f.woff2`), { hash: H, path: '/fonts/f.woff2' }); + assert.deepEqual(splitHashedPath(`/media/~${H}/bgm/a`), { hash: H, path: '/media/bgm/a' }); + for (const p of ['/assets/char/a.png', `/assets/~${H.toUpperCase()}/a.png`, `/assets/~${H}0/a.png`, `/assets/~${H}`, `/data/~${H}/x.json`, + `/assets/x/~${H}/a.png`, '/assets/~zz/a.png', '']) { + assert.deepEqual(splitHashedPath(p), { hash: null, path: p }, p); + } + assert.equal(hashedPath('/assets/char/a.png', H), `/assets/~${H}/char/a.png`); + assert.equal(hashedPath('/media/bgm/a', H), `/media/~${H}/bgm/a`); + assert.equal(hashedPath('/fonts/f.woff2', H), `/fonts/~${H}/f.woff2`); + assert.equal(hashedPath('/data/assets.json', H), '/data/assets.json', 'not a hashed root'); + assert.equal(hashedPath('/assets/a.png', null), '/assets/a.png'); + assert.equal(hashedPath('/assets/a.png', 'nothex'), '/assets/a.png'); + assert.equal(hashedPath(`/assets/~${H}/a.png`, 'fedcba9876543210'), `/assets/~${H}/a.png`, 'never two segments'); + assert.equal(hashedPath('/assets/', H), '/assets/'); + assert.equal(unhashedUrl(`/assets/~${H}/audio/sfx/b_char_dead.mp3`), '/assets/audio/sfx/b_char_dead.mp3'); + assert.equal(unhashedUrl(null), null); +}); + +test('traffic categories: audio, assets, code, data, other — the hash segment does not matter', () => { + assert.deepEqual([...TRAFFIC_CATEGORIES], ['assets', 'audio', 'code', 'data', 'other']); + const cases = { + '/media/bgm/act1': 'audio', [`/media/~${H}/bgm/act1`]: 'audio', '/assets/audio/bgm/a.mp3': 'audio', [`/assets/~${H}/audio/sfx/a.mp3`]: 'audio', + '/assets/char/avatar/a.png': 'assets', [`/assets/~${H}/spine/op/x/front/x.skel`]: 'assets', '/fonts/f.woff2': 'assets', '/icons/app.svg': 'assets', + '/packs/foo/art/a.png': 'assets', '/packs/index.json': 'data', '/data/assets.json': 'data', '/data/i18n/en.json': 'data', '/i18n/en.json': 'data', + '/manifest.json': 'data', '/': 'code', '/index.html': 'code', '/js/main.js': 'code', '/css/theme.css': 'code', '/vendor/pixi.min.js': 'code', + '/sim/Battle.js': 'code', '/shared/protocol.js': 'code', '/data.js': 'code', '/healthz': 'other', '/favicon.ico': 'other', + }; + for (const [p, cat] of Object.entries(cases)) assert.equal(trafficCategory(p), cat, p); +}); diff --git a/test/diag.test.js b/test/diag.test.js index 0a16ebbf..3f951991 100644 --- a/test/diag.test.js +++ b/test/diag.test.js @@ -5,7 +5,7 @@ import { test, describe, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; import { recordError, errorLog, errorCount, clearErrorLog, messageOf, stackOf, stripHosts, redact, redactValues, secretsOf, buildDiagnostics, - setBattleSource, currentBattle, LOG_MAX, STACK_LINES, + setBattleSource, currentBattle, LOG_MAX, STACK_LINES, recordResourceEntry, trafficSummary, trafficLine, clearTraffic, installTrafficMeter, } from '../public/js/diag.js'; import { APP_VERSION } from '../shared/constants.js'; @@ -184,3 +184,53 @@ describe('the report', () => { setBattleSource(null); }); }); + +describe('the page\'s traffic (host bandwidth layer 0)', () => { + beforeEach(() => clearTraffic()); + const ORIGIN = 'http://192.168.1.5:3000'; + + test('Resource Timing entries sum by category: bytes on the wire, a cache hit counted as cached, other sites apart', () => { + recordResourceEntry({ name: `${ORIGIN}/assets/~0123456789abcdef/spine/op/x/front/x.skel`, transferSize: 120000, decodedBodySize: 400000 }, ORIGIN); + recordResourceEntry({ name: `${ORIGIN}/assets/char/avatar/a.png`, transferSize: 0, decodedBodySize: 50000 }, ORIGIN); + recordResourceEntry({ name: `${ORIGIN}/media/~0123456789abcdef/bgm/m_bat`, transferSize: 2000000, decodedBodySize: 2000000 }, ORIGIN); + recordResourceEntry({ name: `${ORIGIN}/js/main.js`, transferSize: 9000, decodedBodySize: 30000 }, ORIGIN); + recordResourceEntry({ name: `${ORIGIN}/data/assets.json`, transferSize: 300, decodedBodySize: 1300000 }, ORIGIN); + recordResourceEntry({ name: 'https://fonts.gstatic.com/s/x.woff2', transferSize: 50000 }, ORIGIN); + recordResourceEntry({ name: 'data:image/png;base64,AAAA', transferSize: 0 }, ORIGIN); + recordResourceEntry({ name: 'blob:http://192.168.1.5:3000/1234', transferSize: 0 }, ORIGIN); + recordResourceEntry(null, ORIGIN); + const t = trafficSummary(); + assert.deepEqual(t.assets, { requests: 2, bytes: 120000, cached: 1 }); + assert.deepEqual(t.audio, { requests: 1, bytes: 2000000, cached: 0 }); + assert.deepEqual(t.code, { requests: 1, bytes: 9000, cached: 0 }); + assert.deepEqual(t.data, { requests: 1, bytes: 300, cached: 0 }, 'a 304 costs its headers only'); + assert.deepEqual(t.external, { requests: 1, bytes: 50000, cached: 0 }); + assert.deepEqual(t.total, { requests: 5, bytes: 2129300, cached: 1 }, 'the host total leaves other sites out'); + assert.deepEqual(Object.keys(t), ['assets', 'audio', 'code', 'data', 'external', 'total']); + assert.equal(trafficLine(t), 'assets 120 KB (2, 1 cached) | audio 2.0 MB (1) | code 9 KB (1) | data 300 B (1) | external 50 KB (1) | host total 2.1 MB (5)'); + }); + + test('the report carries the traffic line; nothing recorded → no line', () => { + recordResourceEntry({ name: `${ORIGIN}/js/main.js`, transferSize: 1500 }, ORIGIN); + const text = buildDiagnostics({ state: {}, now: 0, pageStart: 0, traffic: trafficSummary() }); + assert.match(text, /\ntraffic: code 2 KB \(1\) \| host total 2 KB \(1\)\n/); + clearTraffic(); + assert.ok(!/traffic:/.test(buildDiagnostics({ state: {}, now: 0, pageStart: 0, traffic: trafficSummary() }))); + assert.ok(!/traffic:/.test(buildDiagnostics({ state: {}, now: 0, pageStart: 0 }))); + }); + + test('installTrafficMeter reads the navigation entry and observes the resources (buffered), once; false without the API', () => { + let observed = null; + class PO { + constructor(fn) { this.fn = fn; } + observe(o) { observed = o; this.fn({ getEntries: () => [{ name: `${ORIGIN}/vendor/pixi.min.js`, transferSize: 140000 }] }); } + } + const win = { PerformanceObserver: PO, location: { origin: ORIGIN }, performance: { getEntriesByType: (t) => (t === 'navigation' ? [{ name: `${ORIGIN}/`, transferSize: 7000 }] : []) } }; + assert.equal(installTrafficMeter({}), false); + assert.equal(installTrafficMeter(win), true); + assert.deepEqual(observed, { type: 'resource', buffered: true }); + assert.deepEqual(trafficSummary().code, { requests: 2, bytes: 147000, cached: 0 }); + assert.equal(installTrafficMeter(win), true, 'already installed'); + assert.equal(trafficSummary().code.requests, 2, 'not twice'); + }); +}); diff --git a/test/traffic.test.js b/test/traffic.test.js new file mode 100644 index 00000000..183978d2 --- /dev/null +++ b/test/traffic.test.js @@ -0,0 +1,105 @@ +// The server's traffic counters (server/http/traffic.js, GET /healthz?traffic=1; host bandwidth layer 0): the body +// bytes every HTTP response wrote, by category, 304s apart, and the WebSocket's wire and payload bytes both ways. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomBytes } from 'node:crypto'; +import { startServer } from '../server/index.js'; +import { createTrafficCounter } from '../server/http/traffic.js'; +import { TestClient } from './helpers/wsClient.js'; + +function get(port, p, headers = {}) { + return new Promise((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port, path: p, headers, agent: false }, (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) })); + }); + req.on('error', reject); + req.end(); + }); +} + +test('HTTP: body bytes by category as written (compressed), 304s counted, /healthz reports them only when asked', async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sp-traffic-')); + const w = (rel, content) => { fs.mkdirSync(path.dirname(path.join(tmp, rel)), { recursive: true }); fs.writeFileSync(path.join(tmp, rel), content); }; + const js = `// app\n${'export const x = 1;\n'.repeat(400)}`; + const png = randomBytes(3000); + const mp3 = randomBytes(5000); + w('public/index.html', '

hi

'); + w('public/js/app.js', js); + w('public/assets/img/a.png', png); + w('public/assets/audio/bgm/a.mp3', mp3); + w('data/config.json', JSON.stringify({ a: 1 })); + w('shared/x.js', 'export const y = 2;'); + const srv = await startServer({ port: 0, host: '127.0.0.1', quiet: true, publicDir: path.join(tmp, 'public'), dataDir: path.join(tmp, 'data'), sharedDir: path.join(tmp, 'shared') }); + try { + const plain = await get(srv.port, '/health'); + assert.equal(plain.status, 404); + const gz = await get(srv.port, '/js/app.js', { 'accept-encoding': 'gzip' }); + assert.ok(gz.body.length < js.length, 'compressed'); + await get(srv.port, '/js/app.js', { 'if-none-match': (await get(srv.port, '/js/app.js')).headers.etag }); + await get(srv.port, '/assets/img/a.png'); + await get(srv.port, '/media/bgm/a'); + await get(srv.port, '/data/config.json'); + const before = await get(srv.port, '/healthz'); + assert.equal(JSON.parse(before.body).traffic, undefined, 'plain /healthz stays as it was'); + const t = JSON.parse((await get(srv.port, '/healthz?traffic=1')).body).traffic; + assert.equal(t.http.code.requests, 3); + assert.equal(t.http.code.bytes, gz.body.length + js.length, 'what was written: the gzip body, the plain body, nothing for the 304'); + assert.equal(t.http.code.notModified, 1); + assert.deepEqual(t.http.assets, { requests: 1, bytes: png.length, notModified: 0 }); + assert.deepEqual(t.http.audio, { requests: 1, bytes: mp3.length, notModified: 0 }); + assert.equal(t.http.data.requests, 1); + assert.equal(t.http.other.requests, 2, 'the 404 page and the first /healthz'); + assert.ok(t.http.other.bytes > 0); + assert.equal(t.http.total.requests, 8); + assert.equal(typeof t.since, 'string'); + assert.equal(srv.traffic.snapshot().http.total.requests, 9, 'the traffic request itself once it has finished'); + } finally { + await srv.close(); + fs.rmSync(tmp, { recursive: true, force: true }); + } +}); + +test('WebSocket: wire bytes both ways (after permessage-deflate) and the payloads, live and after the close', async () => { + const srv = await startServer({ port: 0, host: '127.0.0.1', quiet: true }); + try { + const c = await TestClient.connect(`ws://127.0.0.1:${srv.port}/ws`); + await c.hello('流量计数'); + const live = srv.traffic.snapshot().ws; + assert.equal(live.connections, 1); + assert.equal(live.open, 1); + assert.ok(live.messagesOut >= 1 && live.payloadOut > 0, JSON.stringify(live)); + assert.ok(live.messagesIn >= 1 && live.payloadIn > 0); + assert.ok(live.wireOut > 0 && live.wireIn > 0); + await c.close(); + for (let i = 0; i < 50 && srv.traffic.snapshot().ws.open; i++) await new Promise((r) => setTimeout(r, 20)); + const done = srv.traffic.snapshot().ws; + assert.equal(done.open, 0); + assert.ok(done.wireOut >= live.wireOut, 'the closed socket\'s bytes stay in the totals'); + } finally { + await srv.close(); + } +}); + +test('the counter on its own: a string body in another encoding, an aborted response, the payload of a fragmented message', () => { + const tc = createTrafficCounter({ now: () => 1000 }); + const listeners = {}; + const res = { statusCode: 200, write() { return true; }, end() {}, once(ev, fn) { listeners[ev] = fn; } }; + tc.meter(res, '/data/x.json'); + res.write('aGk=', 'base64'); + res.end('é'); + listeners.close(); + const s = tc.snapshot(); + assert.deepEqual(s.http.data, { requests: 1, bytes: 2 + 2, notModified: 0 }); + tc.wsSent('{"t":"pong"}'); + tc.wsReceived([Buffer.alloc(3), Buffer.alloc(4)]); + const ws = tc.snapshot().ws; + assert.equal(ws.payloadOut, 12); + assert.equal(ws.payloadIn, 7); + assert.equal(tc.snapshot().seconds, 0); +}); From fb5f4493f2c7d7553d8a96725caddc698b4ec42e Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 16:06:35 +0800 Subject: [PATCH 05/11] =?UTF-8?q?0.2.6-dev:=20DEPLOY=20=E2=80=94=20a=20sta?= =?UTF-8?q?ble=20address=20keeps=20every=20player's=20cache=20(quick=20tun?= =?UTF-8?q?nels=20change=20domain),=20HTTP/2=20through=20the=20proxy,=20br?= =?UTF-8?q?otli=20only=20on=20HTTPS=20pages,=20a=20Cloudflare=20cache=20ru?= =?UTF-8?q?le=20for=20/assets/=20/fonts/=20/media/=20(host=20bandwidth=20l?= =?UTF-8?q?ayer=200.5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/DEPLOY.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 4ef91f4f..aa68f551 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -170,6 +170,8 @@ cloudflared tunnel --url http://localhost:3000 把输出的 `https://xxxx.trycloudflare.com` 发给朋友。页面是 https 时客户端自动改用 `wss://`，不需要任何配置；服务器会通过隧道转发的 `CF-Connecting-IP` 识别真实来源（`TRUST_PROXY=auto`）。临时隧道每次启动地址都不同，且没有可用性保证；需要固定地址请使用 Cloudflare 账号 + 自己域名的「命名隧道」。 +**固定访问地址能省很多流量**：浏览器的缓存、设置、统计和断线令牌都按地址（域名或 IP + 端口）分开保存。地址一换，对玩家来说就是一个新网站：每个人进游戏都要把图片、模型和音频重新下载一遍（第一局约 20–50 MB），设置和本机统计也看不到了。临时隧道每次重启都换域名，所以固定朋友圈子长期开服时，建议用命名隧道、Tailscale（2.1）或固定的公网 IP / 局域网 IP（1.3），让大家一直用同一个地址。 + ### 2.3 路由器端口转发 仅当你有**公网 IPv4**（很多宽带是运营商级 NAT，没有公网 IP，此时请用 2.1 / 2.2）： @@ -219,7 +221,11 @@ server { } ``` -**HTTP/2**：客户端由数百个小脚本模块组成（0.2.0 起进入对局时约 340 个），隔着公网时建议让代理以 HTTP/2 提供页面：所有模块走同一条连接，远距离玩家首次进入对局明显更快。Caddy 默认就是 HTTP/2；Nginx 见上面的 `http2`。服务器本身只说 HTTP/1.1，局域网或本机游玩不受影响。 +**HTTP/2**：客户端由数百个小脚本模块组成（0.2.0 起进入对局时约 340 个），隔着公网时建议让代理以 HTTP/2 提供页面：所有模块走同一条连接，远距离玩家首次进入对局明显更快。Caddy 默认就是 HTTP/2；Nginx 见上面的 `http2`。服务器本身只说 HTTP/1.1，局域网或本机游玩不受影响。HTTP/2 省的是往返等待，不减少字节数。 + +**压缩**：服务器自己压缩脚本、样式、数据和 Spine 的 `.skel` / `.atlas`：浏览器支持 brotli 时用 brotli（0.2.6 起，比 gzip 更小，`.skel` 小得最多），否则用 gzip；图片和音频本来就是压缩格式，原样发送。浏览器只在 HTTPS 页面（以及 `localhost`）上声明支持 brotli，所以通过代理或隧道的 https 地址访问时用 brotli，局域网里的 `http://192.168.x.x:3000` 这类地址用 gzip。代理不需要另开压缩（Caddy 的 `encode`、Nginx 的 `gzip` 遇到已经压缩的响应会跳过）。 + +**Cloudflare 缓存**（命名隧道或自己的域名经过 Cloudflare 时）：Cloudflare 默认只缓存常见扩展名，`.skel`、`.atlas` 和没有扩展名的音频地址 `/media/…` 都不在其中，每个请求都会回到开服的电脑。在 Cloudflare 控制台的「缓存规则」里加一条：主机名是你的游戏域名、且 URI 路径以 `/assets/`、`/fonts/` 或 `/media/` 开头时「符合缓存条件」，边缘 TTL 选「使用源站的 Cache-Control」。0.2.6 起素材清单里的地址带内容哈希（`/assets/~<哈希>/…`），服务器给这类地址一年的 `immutable`，换了内容的文件会换地址，所以长期缓存是安全的；`/data/`、页面和脚本不要加进规则（它们要随服务器更新）。注意：边缘公开缓存官方素材会增加被投诉的风险，大量媒体文件也要留意 Cloudflare 的服务条款；只给自己人玩时不加这条规则也可以。 https / wss 说明：页面通过 https 打开时客户端自动连接 `wss://同一域名/ws`；http 时用 `ws://`。服务器本身只提供 http，证书由代理 / 隧道负责。代理与服务器在同一台机器或内网时，`TRUST_PROXY=auto` 会信任它的 `X-Forwarded-For` / `X-Real-IP`；代理在公网另一台机器上时设 `TRUST_PROXY=1`（同时确保游戏端口只对代理开放）。 From 521dd984e76d698c8200b533c299cc52898f130b Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 16:20:17 +0800 Subject: [PATCH 06/11] =?UTF-8?q?0.2.6-dev:=20a=20background=20content-has?= =?UTF-8?q?h=20index=20of=20public/assets=20and=20public/fonts=20(SHA-256,?= =?UTF-8?q?=2016=20hex;=20cached=20by=20path=20+=20size=20+=20mtime=20in?= =?UTF-8?q?=20.cache/asset-hashes.json;=20Spine=20models=20share=20one=20h?= =?UTF-8?q?ash;=20SP=5FASSET=5FINDEX=3D0=20turns=20it=20off)=20=E2=80=94?= =?UTF-8?q?=20host=20bandwidth=20layer=201.1;=20golden:=20unchanged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/ARCHITECTURE.md | 2 +- docs/design/overview.md | 1 + server/http/assetIndex.js | 352 ++++++++++++++++++++++++++++++++++++++ server/http/config.js | 21 +++ server/http/files.js | 4 +- server/http/routes.js | 10 +- server/index.js | 20 ++- test/asset-index.test.js | 124 ++++++++++++++ 8 files changed, 524 insertions(+), 10 deletions(-) create mode 100644 server/http/assetIndex.js create mode 100644 test/asset-index.test.js diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a0e958d1..f3a24258 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -77,7 +77,7 @@ so old imports keep working: `public/js/ui/gameLogic.js` (`public/js/ui/gameLogi | path | what | |---|---| | `server/index.js` | the process entry (`npm start`); `startServer()` wires `server/http/` | -| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | +| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `assetIndex.js` (the content hashes of `public/assets/` and `public/fonts/`, cached in `.cache/asset-hashes.json`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | | `server/net.js` | sessions and reconnect tokens, rate limits, message validation | | `server/lobby.js` | rooms, seats, AI seats, spectators; starts a `Match` | | `server/data.js` | loads `data/*.json` once (frozen) | diff --git a/docs/design/overview.md b/docs/design/overview.md index ed82f088..e871de44 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -50,6 +50,7 @@ server/ buildTag.js build tag of the served browser runtime (/healthz build, public/js/ui/buildGuard.js) routes.js request listener: 414 / 400 / 405 guards, GET /healthz (?traffic=1: the counters), then static files traffic.js traffic counters: HTTP body bytes by category (assets / audio / code / data), WebSocket wire / payload + assetIndex.js content hashes (SHA-256, 16 hex) of public/assets + public/fonts, background build, .cache/asset-hashes.json common.js security headers, URL split, error page, JSON replies, bare 400 for unparseable requests boot.js boot banner (Local / LAN / tunnel URLs), port-in-use hint, graceful SIGINT / SIGTERM net.js session registry, send helpers, per-connection rate limit, message validation (uses shared/protocol.js) diff --git a/server/http/assetIndex.js b/server/http/assetIndex.js new file mode 100644 index 00000000..d2a497d6 --- /dev/null +++ b/server/http/assetIndex.js @@ -0,0 +1,352 @@ +// server/http/assetIndex.js — the content hashes of the served art (host bandwidth, layer 1; docs/ASSETS.md "The served +// manifests"): +// +// * built in the background shortly after the start (ASSET_INDEX_DELAY_MS): every file under public/assets/ and +// public/fonts/ (dot names skipped like the static server does, linked folders followed once), the first 16 hex +// digits of the SHA-256 of its bytes (WebCrypto, off the main thread); +// * remembered in .cache/asset-hashes.json by path + size + mtime: a restart only stats the files and reads none of +// the unchanged ones again; a file whose size or mtime changed is hashed again (a re-extraction, `setup --force`, +// a Docker rebuild then costs the players only the files whose bytes changed); +// * never trusted beyond what it checked: `hashOf(rel, stat)` answers only while the file still has the size and +// mtime it was hashed with, else null — the caller then serves the plain way (1-day cache, size-mtime ETag) and +// the miss asks for a rescan (at most one every RESCAN_GAP_MS); +// * models: lists of files that share one URL hash — a Spine skeleton, its atlas and its pages (pixi-spine finds the +// atlas next to the skeleton and the pages next to the atlas, so all their URLs carry the same segment). A model's +// hash is the SHA-256 of its members' hashes; assetViews.js hands the lists over (`setModels`), read from the +// manifests. `urlHashOk` accepts a file's own hash or the hash of a model it belongs to. +// +// Rel paths are POSIX and relative to public/ ('assets/char/avatar/x.png', 'fonts/bender-regular.woff2'). A rel that +// differs from a file's name only in case finds that file when no other name matches it that way (Windows and macOS +// serve `wah-y.png` from `WAH-Y.png`; on Linux the request is a 404 anyway). + +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { createHash, webcrypto } from 'node:crypto'; +import { ASSET_HASH_HEX } from '../../shared/assetPaths.js'; +import { noopLog } from './config.js'; +import { IMMUTABLE_CACHE } from './files.js'; + +/** Folders of public/ whose files are hashed. */ +export const ASSET_INDEX_ROOTS = Object.freeze(['assets', 'fonts']); +/** The first build waits this long after the start (a server that is only started and stopped again — tests — reads nothing). */ +export const ASSET_INDEX_DELAY_MS = 1000; +/** Shortest time between two rescans a stale or unknown file asks for. */ +export const RESCAN_GAP_MS = 10_000; +/** Version of .cache/asset-hashes.json. */ +export const HASH_CACHE_VERSION = 1; +const STAT_CONCURRENCY = 64; +const HASH_CONCURRENCY = 4; + +const hex = (bytes) => Buffer.from(bytes).toString('hex').slice(0, ASSET_HASH_HEX); + +/** First ASSET_HASH_HEX hex digits of the SHA-256 of `data` (synchronous; model hashes, manifests, tests). */ +export function hash16(data) { + return createHash('sha256').update(data).digest('hex').slice(0, ASSET_HASH_HEX); +} + +/** The same for a file's bytes, computed off the main thread (WebCrypto runs in libuv's thread pool). */ +async function digest16(buf) { + return hex(await webcrypto.subtle.digest('SHA-256', buf)); +} + +/** Run `fn` over `items` with at most `n` at a time; stops early when `stop()` says so. */ +async function eachLimit(items, n, fn, stop) { + let i = 0; + const worker = async () => { + while (i < items.length && !stop()) { + const item = items[i++]; + await fn(item); + } + }; + await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker)); +} + +/** + * The asset index of one server. + * @param {{ publicDir: string, roots?: readonly string[], cacheFile?: string | null, log?: object, delayMs?: number, + * rescanGapMs?: number, now?: () => number }} opts + * cacheFile: where the hashes persist (null: in memory only — a test's temporary public/) + */ +export function createAssetIndex({ publicDir, roots = ASSET_INDEX_ROOTS, cacheFile = null, log = noopLog, delayMs = ASSET_INDEX_DELAY_MS, + rescanGapMs = RESCAN_GAP_MS, now = Date.now }) { + const base = path.resolve(publicDir); + /** @type {Map} */ + let records = new Map(); + /** @type {Map} lower-case rel → the rel (null: two names differ only in case) */ + let lower = new Map(); + /** @type {string[][]} */ + let modelLists = []; + /** @type {Map>} rel → hashes of the models it belongs to */ + let modelOf = new Map(); + let generation = 0; + let ready = false; + let closed = false; + let building = null; // the build in flight (a promise) + let again = false; // another build was asked for while one ran + let startTimer = null; + let rescanTimer = null; + let lastBuildAt = -Infinity; + let readyResolve; + const readyPromise = new Promise((r) => { readyResolve = r; }); + const listeners = new Set(); + const last = { files: 0, bytes: 0, hashed: 0, reused: 0, ms: 0, at: null, error: null }; + + /** The cache file's records ({} when absent, unreadable or another version). */ + function loadCache() { + if (!cacheFile) return new Map(); + try { + const doc = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); + if (!doc || doc.version !== HASH_CACHE_VERSION || typeof doc.files !== 'object' || !doc.files) return new Map(); + const out = new Map(); + for (const [rel, v] of Object.entries(doc.files)) { + if (Array.isArray(v) && Number.isFinite(v[0]) && Number.isFinite(v[1]) && typeof v[2] === 'string' && v[2].length === ASSET_HASH_HEX) { + out.set(rel, { size: v[0], mtimeMs: v[1], hash: v[2] }); + } + } + return out; + } catch { + return new Map(); + } + } + + /** Write the records (temp file + rename: a concurrent reader never sees half a file). Failures only log. */ + async function saveCache(map) { + if (!cacheFile) return; + const files = {}; + for (const [rel, r] of [...map].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))) files[rel] = [r.size, r.mtimeMs, r.hash]; + const tmp = `${cacheFile}.${process.pid}.tmp`; + try { + await fsp.mkdir(path.dirname(cacheFile), { recursive: true }); + await fsp.writeFile(tmp, JSON.stringify({ version: HASH_CACHE_VERSION, algo: 'sha256/16', files })); + await fsp.rename(tmp, cacheFile); + } catch (e) { + log.debug?.('[assets] hash cache not saved', e?.code || e?.message); + fsp.rm(tmp, { force: true }).catch(() => {}); + } + } + + /** Every file under the roots: [rel, abs] (dot names skipped, a folder reached twice — a link loop — walked once). */ + async function listFiles() { + const out = []; + const seen = new Set(); + const walk = async (abs, rel) => { + let real; + try { real = await fsp.realpath(abs); } catch { return; } + if (seen.has(real)) return; + seen.add(real); + let ents; + try { ents = await fsp.readdir(abs, { withFileTypes: true }); } catch { return; } + for (const d of ents) { + if (closed) return; + if (d.name.startsWith('.')) continue; + const childAbs = path.join(abs, d.name); + const childRel = `${rel}/${d.name}`; + if (d.isDirectory()) await walk(childAbs, childRel); + else if (d.isFile()) out.push([childRel, childAbs]); + else if (d.isSymbolicLink()) { + const st = await fsp.stat(childAbs).catch(() => null); + if (st?.isDirectory()) await walk(childAbs, childRel); + else if (st?.isFile()) out.push([childRel, childAbs]); + } + } + }; + for (const root of roots) await walk(path.join(base, root), root); + return out; + } + + /** Model hashes from the current records (a model with a member the index does not know has none). */ + function computeModels() { + const next = new Map(); + for (const list of modelLists) { + const h = modelHashOf(list); + if (!h) continue; + for (const rel of list) { + const c = canonical(rel); + let set = next.get(c); + if (!set) next.set(c, (set = new Set())); + set.add(h); + } + } + modelOf = next; + } + + /** The indexed name of `rel`: itself, or the one file whose name differs from it only in case (else null). */ + function canonical(rel) { + if (typeof rel !== 'string') return null; + if (records.has(rel)) return rel; + return lower.get(rel.toLowerCase()) || null; + } + + function recordOf(rel) { + const c = canonical(rel); + return c ? records.get(c) : null; + } + + function modelHashOf(list) { + if (!Array.isArray(list) || !list.length) return null; + const parts = []; + for (const rel of list) { + const r = recordOf(rel); + if (!r) return null; + parts.push(r.hash); + } + return hash16(parts.join('\n')); + } + + /** One build; true when the records changed (a new generation). */ + async function build() { + const t0 = now(); + const known = records.size ? records : loadCache(); + const next = new Map(); + let hashed = 0; + let reused = 0; + let bytes = 0; + const files = await listFiles(); + const toHash = []; + await eachLimit(files, STAT_CONCURRENCY, async ([rel, abs]) => { + const st = await fsp.stat(abs).catch(() => null); + if (!st || !st.isFile()) return; + bytes += st.size; + const k = known.get(rel); + if (k && k.size === st.size && k.mtimeMs === st.mtimeMs) { next.set(rel, k); reused++; } else toHash.push([rel, abs, st]); + }, () => closed); + await eachLimit(toHash, HASH_CONCURRENCY, async ([rel, abs, st]) => { + try { + const buf = await fsp.readFile(abs); + const after = await fsp.stat(abs); + // changed while it was read: hashed again by the next build + if (after.size !== st.size || after.mtimeMs !== st.mtimeMs || buf.length !== st.size) return; + next.set(rel, { size: st.size, mtimeMs: st.mtimeMs, hash: await digest16(buf) }); + hashed++; + } catch { /* vanished or unreadable: not in the index */ } + }, () => closed); + if (closed) return false; + // commit: records, model hashes and the generation change together (one synchronous step) + let changed = next.size !== records.size; + if (!changed) for (const [rel, r] of next) { const o = records.get(rel); if (!o || o.hash !== r.hash || o.size !== r.size || o.mtimeMs !== r.mtimeMs) { changed = true; break; } } + const cacheChanged = hashed > 0 || next.size !== known.size; + records = next; + lower = new Map(); + for (const rel of next.keys()) { const k = rel.toLowerCase(); lower.set(k, lower.has(k) ? null : rel); } + if (changed) { generation++; computeModels(); } + Object.assign(last, { files: next.size, bytes, hashed, reused, ms: now() - t0, at: new Date().toISOString(), error: null }); + lastBuildAt = now(); + if (cacheChanged) await saveCache(next); + if (hashed) log.info?.(`[assets] content hashes: ${next.size} files, ${hashed} hashed, ${reused} unchanged (${Math.round(last.ms)} ms)`); + return changed; + } + + function markReady() { + if (ready) return; + ready = true; + readyResolve(); + } + + function notify() { + for (const fn of [...listeners]) { try { fn(generation); } catch (e) { log.error?.('[assets] index listener failed', e); } } + } + + /** Build now (or once more after the build in flight). */ + function refresh() { + if (closed) return Promise.resolve(); + if (building) { again = true; return building; } + building = (async () => { + try { + do { + again = false; + const changed = await build(); + if (closed) break; + markReady(); + if (changed) notify(); + } while (again && !closed); + } catch (e) { + last.error = e?.message || String(e); + log.warn?.('[assets] content hash index failed', e?.message || e); + } finally { + building = null; + if (!closed) markReady(); + } + })(); + return building; + } + + return { + /** Start the background build (after `delayMs`); returns at once. Idempotent. */ + start() { + if (closed || startTimer || building || ready) return; + if (!(delayMs > 0)) { refresh(); return; } + startTimer = setTimeout(() => { startTimer = null; refresh(); }, delayMs); + startTimer.unref?.(); + }, + refresh, + /** Resolves once the first build is done (never rejects). */ + whenReady() { return readyPromise; }, + get ready() { return ready; }, + get generation() { return generation; }, + /** Files with a hash. */ + get size() { return records.size; }, + /** + * The file's content hash while it still has the size and mtime it was hashed with (`stat` given), else null. + * Without `stat`: the hash of the last build. + * @param {string} rel @param {{ size: number, mtimeMs: number }} [stat] + */ + hashOf(rel, stat) { + const r = recordOf(rel); + if (!r) return null; + if (stat && (stat.size !== r.size || stat.mtimeMs !== r.mtimeMs)) return null; + return r.hash; + }, + /** Does `urlHash` name exactly these bytes: the file's own hash, or the hash of a model it belongs to? */ + urlHashOk(rel, stat, urlHash) { + if (typeof urlHash !== 'string') return false; + const own = this.hashOf(rel, stat); + if (!own) return false; + return own === urlHash || !!modelOf.get(canonical(rel))?.has(urlHash); + }, + /** The hash of a model (its members' hashes hashed), null while a member is unknown. */ + modelHash: (list) => modelHashOf(list), + /** Replace the model lists (assetViews.js, from the manifests). */ + setModels(lists) { + modelLists = Array.isArray(lists) ? lists.filter((l) => Array.isArray(l) && l.length) : []; + computeModels(); + }, + /** A file was served that the index does not know, or no longer matches: rescan (at most one per rescanGapMs). */ + noteMiss() { + if (closed || !ready || rescanTimer || building) return; + const wait = Math.max(0, lastBuildAt + rescanGapMs - now()); + rescanTimer = setTimeout(() => { rescanTimer = null; refresh(); }, wait); + rescanTimer.unref?.(); + }, + /** `fn(generation)` after a build changed the records; returns the unsubscribe. */ + onChange(fn) { listeners.add(fn); return () => listeners.delete(fn); }, + stats() { + return { ready, building: !!building, generation, files: records.size, models: modelLists.length, last: { ...last } }; + }, + /** Stop: timers cleared, a build in flight abandoned. */ + close() { + closed = true; + if (startTimer) { clearTimeout(startTimer); startTimer = null; } + if (rescanTimer) { clearTimeout(rescanTimer); rescanTimer = null; } + readyResolve(); // a waiter is let go; `ready` stays as it was + }, + }; +} + +/** + * The ETag source and cache policy of a file of the asset index's folders (public/assets, public/fonts), for + * serveFile: its content hash when the index has it for these bytes (a miss asks for a rescan); for a URL with a hash + * segment `immutable` when that hash names these bytes, else `no-cache`. + * @param {ReturnType | null} assetIndex + * @param {string} rel the file under public/ ('assets/…', POSIX) + * @param {import('node:fs').Stats} stat + * @param {string | null} urlHash the URL's hash segment + * @returns {{ contentHash?: string | null, cacheControl?: string }} + */ +export function hashedFileOpts(assetIndex, rel, stat, urlHash) { + if (!assetIndex) return urlHash ? { cacheControl: 'no-cache' } : {}; + const contentHash = assetIndex.hashOf(rel, stat); + if (!contentHash) assetIndex.noteMiss(rel); + if (!urlHash) return { contentHash }; + return { contentHash, cacheControl: contentHash && assetIndex.urlHashOk(rel, stat, urlHash) ? IMMUTABLE_CACHE : 'no-cache' }; +} diff --git a/server/http/config.js b/server/http/config.js index 3d1bd123..113c4441 100644 --- a/server/http/config.js +++ b/server/http/config.js @@ -4,6 +4,8 @@ // * TRUST_PROXY ('auto' default: honour CF-Connecting-IP / X-Real-IP / X-Forwarded-For only from loopback/private // peers such as a local cloudflared; '1' always; '0' never) → net.js trustProxy; // * DEBUG → the console logger's debug level; +// * SP_ASSET_INDEX=0 (or the `assetIndex: false` option) → no content-hash index: plain asset URLs, size-mtime ETags +// (assetIndex.js); the index of the default public/ persists in .cache/asset-hashes.json; // * the served directories (public/, data/, shared/ and the content packs' packs/ of this repository unless the // options name others), and which startServer() options are handed on to net.js Network and lobby.js Lobby. @@ -64,6 +66,25 @@ export function serveDirs(opts) { }; } +/** + * The asset index options out of the startServer() options and the environment, or null for none: `assetIndex: false` + * or SP_ASSET_INDEX=0 / off / false / no switches it off; `assetIndex: { cacheFile, delayMs }` overrides the defaults. + * The hashes persist in .cache/asset-hashes.json only for the repository's own public/ (a test's temporary folder keeps + * them in memory). + * @param {{ assetIndex?: boolean | { cacheFile?: string | null, delayMs?: number } }} opts + * @param {string} publicDir + * @returns {{ cacheFile: string | null, delayMs?: number } | null} + */ +export function assetIndexOptionsFrom(opts, publicDir) { + if (opts.assetIndex === false) return null; + const env = String(process.env.SP_ASSET_INDEX ?? '').trim().toLowerCase(); + if (opts.assetIndex == null && ['0', 'off', 'false', 'no'].includes(env)) return null; + const o = typeof opts.assetIndex === 'object' && opts.assetIndex ? opts.assetIndex : {}; + const own = path.resolve(publicDir) === path.join(ROOT, 'public'); + const cacheFile = o.cacheFile !== undefined ? o.cacheFile : (own ? path.join(ROOT, '.cache', 'asset-hashes.json') : null); + return { cacheFile, ...(Number.isFinite(o.delayMs) ? { delayMs: o.delayMs } : {}) }; +} + /** net.js Network options out of the startServer() options; `trustProxy` falls back to TRUST_PROXY. */ export function netOptionsFrom(opts) { const netOptions = {}; diff --git a/server/http/files.js b/server/http/files.js index c15de0e6..98910651 100644 --- a/server/http/files.js +++ b/server/http/files.js @@ -64,8 +64,8 @@ const GZIP_CACHE_MAX_FILE = 8 << 20; // larger files are gzip-streamed on t const GZIP_CACHE_MAX_TOTAL = 96 << 20; // Asset URLs carry no content hash yet, and tools/fetch-assets.mjs / tools/vendor.mjs can rewrite files in // place (atlas + png + skel must stay consistent), so "long" is one day; revalidation after that is a cheap 304. -const LONG_CACHE = 'public, max-age=86400'; // 1 day -const IMMUTABLE_CACHE = 'public, max-age=31536000, immutable'; +export const LONG_CACHE = 'public, max-age=86400'; // 1 day +export const IMMUTABLE_CACHE = 'public, max-age=31536000, immutable'; const LONG_CACHE_DIRS = ['assets', 'fonts', 'vendor']; // first path segment under public/ const gzipAsync = promisify(zlib.gzip); diff --git a/server/http/routes.js b/server/http/routes.js index 3b785fd5..3bef18da 100644 --- a/server/http/routes.js +++ b/server/http/routes.js @@ -17,12 +17,14 @@ import { setSecurityHeaders, sendError, sendJson, splitUrl } from './common.js'; const MAX_URL_LENGTH = 4096; /** - * The GET /healthz body (`withTraffic`: plus the traffic counters, `?traffic=1`). + * The GET /healthz body (`withTraffic`: plus the traffic counters and the asset index's state, `?traffic=1`). * @param {{ startedAt: number, network: import('../net.js').Network, registry: import('../net.js').SessionRegistry, - * lobby: import('../lobby.js').Lobby, traffic?: ReturnType }} health + * lobby: import('../lobby.js').Lobby, traffic?: ReturnType, + * assetIndex?: ReturnType | null, + * views?: ReturnType | null }} health * @param {boolean} [withTraffic] */ -export function healthReport({ startedAt, network, registry, lobby, traffic }, withTraffic = false) { +export function healthReport({ startedAt, network, registry, lobby, traffic, assetIndex, views }, withTraffic = false) { const report = { ok: true, version: PROTOCOL_VERSION, app: APP_VERSION, uptimeSec: Math.round((Date.now() - startedAt) / 1000), // the runtime the server is serving right now (public/js/ui/buildGuard.js): a page whose own build is @@ -31,6 +33,8 @@ export function healthReport({ startedAt, network, registry, lobby, traffic }, w sockets: network.connectionCount, sessions: registry.size, ...lobby.stats(), }; if (withTraffic && traffic) report.traffic = traffic.snapshot(); + // content-hashed asset URLs (assetIndex.js / assetViews.js): off, building, or ready with how many files and URLs + if (withTraffic) report.assetHashes = assetIndex ? { ...assetIndex.stats(), views: views ? views.stats() : null } : null; return report; } diff --git a/server/index.js b/server/index.js index 80675f4c..108e05f1 100644 --- a/server/index.js +++ b/server/index.js @@ -9,6 +9,8 @@ // the content packs (/packs/index.json, /packs// — the registry is packs.js) // http/media.js /media/bgm/act1 → public/assets/audio/bgm/act1.mp3 (audio addressed without its extension) // http/files.js one file → response: MIME, gzip + memory cache, ETag / Last-Modified / 304, Cache-Control, ranges +// http/assetIndex.js the content hashes of public/assets and public/fonts (built in the background, cached in +// .cache/asset-hashes.json by path + size + mtime; Spine models share one hash) // http/buildTag.js the build tag of the served browser runtime (/healthz `build`, public/js/ui/buildGuard.js) // http/routes.js the request listener: security headers, 414 / 400 / 405, GET /healthz → JSON status, else static // http/traffic.js the traffic counters: bytes sent per category (assets / audio / code / data) and on the @@ -26,7 +28,7 @@ import http from 'node:http'; import { getData, loadData } from './data.js'; -import { ROOT, listenAddress, bindCandidates, serveDirs, makeLogger, parseTrustProxy } from './http/config.js'; +import { ROOT, listenAddress, bindCandidates, serveDirs, makeLogger, parseTrustProxy, assetIndexOptionsFrom } from './http/config.js'; import { WS_MAX_PAYLOAD, createSessionStack, attachWebSocket } from './http/websocket.js'; import { DATA_SHIM_JS, createStaticHandler } from './http/static.js'; import { createPackRegistry } from './packs.js'; @@ -34,6 +36,7 @@ import { MIME, COMPRESSIBLE, acceptsGzip, parseRange } from './http/files.js'; import { BUILD_INPUTS, computeBuildTag, buildTag, resetBuildTag } from './http/buildTag.js'; import { createRequestHandler } from './http/routes.js'; import { createTrafficCounter } from './http/traffic.js'; +import { createAssetIndex } from './http/assetIndex.js'; import { answerClientError } from './http/common.js'; import { lanUrls, displayHost, isProcessEntry, runMain } from './http/boot.js'; @@ -53,11 +56,14 @@ export { * ratePerSec?: number, rateBurst?: number, maxConnections?: number, maxRooms?: number, * maxConnectionsPerAddr?: number, maxRoomsPerAddr?: number, maxMatchesPerAddr?: number, resyncMinGapMs?: number, * heavyPerSec?: number, heavyBurst?: number, trustProxy?: 'auto' | boolean, soloReconnectWindowMs?: number, + * assetIndex?: boolean | { cacheFile?: string | null, delayMs?: number }, * }} [opts] + * assetIndex: the content-hash index (default on; false or SP_ASSET_INDEX=0: off — config.js assetIndexOptionsFrom) * @returns {Promise<{ port: number, host: string, url: string, server: http.Server, wss: import('ws').WebSocketServer, * lobby: import('./lobby.js').Lobby, network: import('./net.js').Network, * registry: import('./net.js').SessionRegistry, packs: ReturnType, - * traffic: ReturnType, close: () => Promise }>} + * traffic: ReturnType, assetIndex: ReturnType | null, + * close: () => Promise }>} */ export async function startServer(opts = {}) { const { port, host } = listenAddress(opts); @@ -70,6 +76,9 @@ export async function startServer(opts = {}) { // content packs (docs/PACKS.md): scanned now — the start log names them — and again whenever their folders change const packs = createPackRegistry({ publicDir, dataDir, packsDir }, { log }); packs.refresh(true); + // the content hashes of the art (built in the background once listening) + const indexOptions = assetIndexOptionsFrom(opts, publicDir); + const assetIndex = indexOptions ? createAssetIndex({ publicDir, log, ...indexOptions }) : null; const serveStatic = createStaticHandler({ publicDir, dataDir, sharedDir, packsDir, packs, log }); const startedAt = Date.now(); // The tag is per process (see buildTag): read the browser runtime once, here, not on every /healthz. @@ -77,7 +86,7 @@ export async function startServer(opts = {}) { buildTag(); const traffic = createTrafficCounter(); - const server = http.createServer(createRequestHandler({ serveStatic, health: { startedAt, network, registry, lobby, traffic }, log })); + const server = http.createServer(createRequestHandler({ serveStatic, health: { startedAt, network, registry, lobby, traffic, assetIndex }, log })); server.on('clientError', answerClientError); const wss = attachWebSocket(server, { network, log, traffic }); @@ -112,9 +121,11 @@ export async function startServer(opts = {}) { boundHost = bound; } catch (e) { network.close(); // stop heartbeat/sweep timers of the half-built server + assetIndex?.close(); throw e; } server.on('error', (e) => log.error('[http] server error', e)); + assetIndex?.start(); const addr = server.address(); const actualPort = typeof addr === 'object' && addr ? addr.port : port; @@ -126,6 +137,7 @@ export async function startServer(opts = {}) { closing = (async () => { try { lobby.shutdown('shutdown'); } catch (e) { log.error('[shutdown] lobby', e); } network.close(); + assetIndex?.close(); await new Promise((resolve) => { server.close(() => resolve()); server.closeIdleConnections?.(); @@ -136,7 +148,7 @@ export async function startServer(opts = {}) { return closing; } - return { port: actualPort, host: boundHost, url, server, wss, lobby, network, registry, packs, traffic, close }; + return { port: actualPort, host: boundHost, url, server, wss, lobby, network, registry, packs, traffic, assetIndex, close }; } // `node server/index.js` / npm start: listen, print the banner, stop on SIGINT / SIGTERM (http/boot.js). diff --git a/test/asset-index.test.js b/test/asset-index.test.js new file mode 100644 index 00000000..0df7b361 --- /dev/null +++ b/test/asset-index.test.js @@ -0,0 +1,124 @@ +// Content-hashed asset URLs (host bandwidth, layer 1; docs/ASSETS.md "The served manifests"): +// * server/http/assetIndex.js — SHA-256 (16 hex) of every file under public/assets and public/fonts, cached by path + +// size + mtime in a JSON file, never trusted for a file whose size or mtime changed; Spine models share one hash. +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createHash, randomBytes } from 'node:crypto'; +import { createAssetIndex, hash16 } from '../server/http/assetIndex.js'; + +const sha16 = (buf) => createHash('sha256').update(buf).digest('hex').slice(0, 16); + +function tree(files) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sp-hash-')); + for (const [rel, content] of Object.entries(files)) { + const p = path.join(dir, rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, content); + } + return dir; +} + +const until = async (fn, ms = 3000) => { for (const t0 = Date.now(); Date.now() - t0 < ms;) { if (await fn()) return true; await new Promise((r) => setTimeout(r, 20)); } return false; }; + +describe('the asset index', () => { + test('hashes every file under assets/ and fonts/ (dot names skipped, a linked folder followed), persists them, rehashes only what changed', async () => { + const a = randomBytes(2000); + const b = Buffer.from('atlas text\n'.repeat(50)); + const dir = tree({ + 'public/assets/char/a.png': a, 'public/assets/spine/x/x.atlas': b, 'public/assets/.hidden': 'h', 'public/assets/.git/obj': 'o', + 'public/fonts/f.woff2': 'font', 'public/js/app.js': 'not indexed', 'elsewhere/linked/l.png': 'linked', + }); + fs.symlinkSync(path.join(dir, 'elsewhere/linked'), path.join(dir, 'public/assets/link'), 'dir'); + const cacheFile = path.join(dir, '.cache/asset-hashes.json'); + try { + const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), cacheFile, delayMs: 0 }); + assert.equal(idx.ready, false); + idx.start(); + await idx.whenReady(); + assert.equal(idx.ready, true); + const st = (rel) => fs.statSync(path.join(dir, 'public', rel)); + assert.equal(idx.hashOf('assets/char/a.png', st('assets/char/a.png')), sha16(a)); + assert.equal(idx.hashOf('assets/spine/x/x.atlas'), sha16(b)); + assert.equal(idx.hashOf('fonts/f.woff2'), sha16(Buffer.from('font'))); + assert.equal(idx.hashOf('assets/link/l.png'), sha16(Buffer.from('linked')), 'a linked folder is followed'); + assert.equal(idx.hashOf('assets/.hidden'), null, 'dot names are never served, never hashed'); + assert.equal(idx.hashOf('assets/.git/obj'), null); + assert.equal(idx.hashOf('js/app.js'), null, 'only the asset folders'); + assert.equal(idx.size, 4); + assert.equal(idx.stats().last.hashed, 4); + assert.equal(idx.hashOf('assets/CHAR/A.PNG'), sha16(a), 'a name that differs only in case finds the file'); + // a stat that no longer matches: no hash (the caller serves the plain way) + assert.equal(idx.hashOf('assets/char/a.png', { size: 1, mtimeMs: st('assets/char/a.png').mtimeMs }), null); + const cached = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); + assert.equal(cached.version, 1); + assert.deepEqual(cached.files['assets/char/a.png'], [a.length, st('assets/char/a.png').mtimeMs, sha16(a)]); + idx.close(); + + // a restart: the cache answers for the unchanged files; a changed file is hashed again; a removed one is dropped + const a2 = randomBytes(2100); + fs.writeFileSync(path.join(dir, 'public/assets/char/a.png'), a2); + fs.rmSync(path.join(dir, 'public/fonts/f.woff2')); + const idx2 = createAssetIndex({ publicDir: path.join(dir, 'public'), cacheFile, delayMs: 0 }); + idx2.start(); + await idx2.whenReady(); + assert.equal(idx2.stats().last.hashed, 1, 'only the changed file was read'); + assert.equal(idx2.stats().last.reused, 2); + assert.equal(idx2.hashOf('assets/char/a.png'), sha16(a2)); + assert.equal(idx2.hashOf('fonts/f.woff2'), null); + assert.equal(JSON.parse(fs.readFileSync(cacheFile, 'utf8')).files['fonts/f.woff2'], undefined); + idx2.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + test('models: one hash for a skeleton, its atlas and pages; urlHashOk accepts the own or the model hash, nothing else', async () => { + const dir = tree({ 'public/assets/s/x.skel': 'skel', 'public/assets/s/x.atlas': 'atlas', 'public/assets/s/x.png': 'png', 'public/assets/o.png': 'other' }); + try { + const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), delayMs: 0 }); + idx.start(); + await idx.whenReady(); + const trio = ['assets/s/x.skel', 'assets/s/x.atlas', 'assets/s/x.png']; + const m = idx.modelHash(trio); + assert.equal(m, hash16(['skel', 'atlas', 'png'].map((s) => sha16(Buffer.from(s))).join('\n'))); + assert.equal(idx.modelHash([...trio, 'assets/missing.png']), null, 'a member the index does not know: no model hash'); + const st = (rel) => fs.statSync(path.join(dir, 'public', rel)); + assert.equal(idx.urlHashOk('assets/s/x.atlas', st('assets/s/x.atlas'), m), false, 'not before the lists are known'); + idx.setModels([trio]); + for (const rel of trio) assert.equal(idx.urlHashOk(rel, st(rel), m), true, rel); + assert.equal(idx.urlHashOk('assets/s/x.png', st('assets/s/x.png'), sha16(Buffer.from('png'))), true, 'the own hash too'); + assert.equal(idx.urlHashOk('assets/o.png', st('assets/o.png'), m), false, 'not a member'); + assert.equal(idx.urlHashOk('assets/s/x.png', st('assets/s/x.png'), '0000000000000000'), false); + assert.equal(idx.urlHashOk('assets/s/x.png', { size: 99, mtimeMs: 1 }, m), false, 'changed since: never immutable'); + idx.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + test('a served file the index no longer matches asks for a rescan (rate-limited); the new hash follows', async () => { + const dir = tree({ 'public/assets/a.png': 'one' }); + try { + let changes = 0; + const idx = createAssetIndex({ publicDir: path.join(dir, 'public'), delayMs: 0, rescanGapMs: 50 }); + idx.onChange(() => changes++); + idx.start(); + await idx.whenReady(); + assert.equal(changes, 1); + const gen = idx.generation; + fs.writeFileSync(path.join(dir, 'public/assets/a.png'), 'two!'); + assert.equal(idx.hashOf('assets/a.png', fs.statSync(path.join(dir, 'public/assets/a.png'))), null); + idx.noteMiss('assets/a.png'); + idx.noteMiss('assets/a.png'); + assert.ok(await until(() => idx.generation > gen), 'rescanned'); + assert.equal(idx.hashOf('assets/a.png'), sha16(Buffer.from('two!'))); + assert.equal(changes, 2, 'one rescan for both misses'); + idx.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); From 893d17e725e4919b4cd5ae34d4e41e9ec1da3c5f Mon Sep 17 00:00:00 2001 From: sganggs Date: Mon, 12 Oct 2026 16:24:03 +0800 Subject: [PATCH 07/11] =?UTF-8?q?0.2.6-dev:=20/data/assets.json=20and=20/d?= =?UTF-8?q?ata/local-assets.json=20are=20served=20with=20content-hashed=20?= =?UTF-8?q?URLs=20(/assets/~/=E2=80=A6,=20one=20hash=20per=20Spine?= =?UTF-8?q?=20model,=20the=20committed=20files=20unchanged);=20the=20~hash?= =?UTF-8?q?=20routes=20of=20/assets/=20/fonts/=20/media/=20answer=20immuta?= =?UTF-8?q?ble=20when=20the=20hash=20names=20the=20file=20and=20the=20curr?= =?UTF-8?q?ent=20file=20with=20no-cache=20otherwise;=20content=20ETags=20?= =?UTF-8?q?=E2=80=94=20host=20bandwidth=20layer=201.2;=20golden:=20unchang?= =?UTF-8?q?ed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + docs/ARCHITECTURE.md | 5 +- docs/ASSETS.md | 35 +++++- docs/DEPLOY.md | 3 +- docs/design/overview.md | 7 +- server/http/assetViews.js | 185 ++++++++++++++++++++++++++++++ server/http/files.js | 54 +++++++-- server/http/media.js | 13 ++- server/http/static.js | 45 ++++++-- server/index.js | 14 ++- test/asset-index.test.js | 236 +++++++++++++++++++++++++++++++++++++- test/lobby.test.js | 7 +- 12 files changed, 571 insertions(+), 34 deletions(-) create mode 100644 server/http/assetViews.js diff --git a/README.md b/README.md index 53c9718e..6469872b 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ npm start # 启动服务器：http://localhost:3000 | `DEBUG` | 空 | 设为任意值输出详细日志 | | `SP_NO_BROWSER` | 空 | 设为 `1` 时启动脚本不自动打开浏览器 | | `SP_BOT_EMOTES` | 空 | AI 队友的对局表情回应（战斗、整备、合成、联防、收礼时即刻的 `m.emote` 广播）；设为 `0` 时整体关闭 | +| `SP_ASSET_INDEX` | 空 | 素材地址带内容哈希、浏览器长期缓存（0.2.6 起默认开启，哈希存在 `.cache/asset-hashes.json`）；设为 `0` 时关闭，素材清单原样下发 | 设置方式：macOS / Linux `PORT=8080 npm start`；PowerShell `$env:PORT=8080; npm start`；cmd `set "PORT=8080" && npm start`。健康检查：`GET /healthz`。 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f3a24258..58a4f80b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -77,7 +77,7 @@ so old imports keep working: `public/js/ui/gameLogic.js` (`public/js/ui/gameLogi | path | what | |---|---| | `server/index.js` | the process entry (`npm start`); `startServer()` wires `server/http/` | -| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `assetIndex.js` (the content hashes of `public/assets/` and `public/fonts/`, cached in `.cache/asset-hashes.json`), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | +| `server/http/` | `config.js` (environment), `websocket.js` (sessions, `/ws`, permessage-deflate), `static.js` (the mounts, the `~` asset routes), `media.js`, `files.js` (MIME, gzip, ETag, ranges), `buildTag.js`, `routes.js` (`/healthz`), `traffic.js` (bytes sent by category and on the WebSocket, `/healthz?traffic=1`), `assetIndex.js` (the content hashes of `public/assets/` and `public/fonts/`, cached in `.cache/asset-hashes.json`), `assetViews.js` (the served `/data/assets.json` and `/data/local-assets.json`: every art URL with its hash segment; ASSETS.md), `common.js`, `boot.js` (a pending update package first, banner, shutdown) | | `server/net.js` | sessions and reconnect tokens, rate limits, message validation | | `server/lobby.js` | rooms, seats, AI seats, spectators; starts a `Match` | | `server/data.js` | loads `data/*.json` once (frozen) | @@ -131,7 +131,8 @@ data/*.json ──────────┬─▶ server/data.js ──── ├─▶ public/js/data.js ──────────▶ the UI └─▶ public/js/battle/runner.js ─▶ the browser's battles (setSimData) data/i18n/en.json ──────▶ public/js/data.js ──────────▶ the English game texts -public/assets/ ─────────▶ public/js/render/, public/js/audio.js (URLs listed in data/assets.json) +public/assets/ ─────────▶ public/js/render/, public/js/audio.js (URLs listed in data/assets.json, + served with content-hash segments: server/http/assetViews.js) ``` - **Game data.** `tools/build-data.mjs` downloads the official zh_CN tables into `.cache/gamedata/` (or reads them diff --git a/docs/ASSETS.md b/docs/ASSETS.md index 32d6729c..16fcc6ec 100644 --- a/docs/ASSETS.md +++ b/docs/ASSETS.md @@ -215,7 +215,7 @@ The `stem` of a Spine model is the upstream file name. Two examples: `char_107_l ## Manifest schema (`data/assets.json`) -All paths are URL paths relative to the site root, for example `/assets/char/avatar/char_002_amiya.png`. **Only entries whose files exist on disk are emitted.** When a key is missing, the client should use its fallback (research 07 §5.6). +All paths are URL paths relative to the site root, for example `/assets/char/avatar/char_002_amiya.png` — in the committed file; a server hands them out with a content-hash segment ("The committed manifest and the served manifests" below). **Only entries whose files exist on disk are emitted.** When a key is missing, the client should use its fallback (research 07 §5.6). ```js { @@ -470,6 +470,39 @@ Other renderer rules from research 07 §5.4–5.5: - **Missing unit SFX:** use `audio.sfx.battle.enemyHit` or a WebAudio blip (research 07 §6.4). - **Bond icons:** if the real glyph ever fails, the stored file is the nation camp logo. +## The committed manifest and the served manifests + +`data/assets.json` in git (and as `tools/fetch-assets.mjs` writes it) and `data/local-assets.json` (as +`tools/local-extract` writes it) hold plain URLs. Since 0.2.6 (host bandwidth, layer 1) the server hands out a view of +each (`server/http/assetIndex.js`, `server/http/assetViews.js`): + +- Shortly after the start (1 s) the server hashes every file under `public/assets/` and `public/fonts/` in the + background: SHA-256, the first 16 hex digits. The hashes are kept in `.cache/asset-hashes.json` by path + size + + mtime, so a restart only stats the files and reads none of the unchanged ones (12,691 files / 724 MB on the + maintainer's machine: 1.3 s for the first build, 0.25 s after that). A file whose size or mtime changed is hashed + again; a re-extraction, `setup --force` or a Docker rebuild then costs the players only the files whose bytes changed. +- `GET /data/assets.json` and `GET /data/local-assets.json` answer with every art URL carrying its file's hash as the + first segment: `/assets/char/avatar/char_003_kalts.png` → `/assets/~398f32e2ff78d9e6/char/avatar/char_003_kalts.png`, + `/fonts/~/…`. Audio keeps its `/assets/~/audio/….mp3` form; `public/js/media.js` asks for it as + `/media/~/…` (the download-manager route, `shared/media.js`). +- A Spine model's skeleton, atlas and pages carry one hash, the model's (SHA-256 of the files' hashes): pixi-spine finds + the atlas by the skeleton's name and the pages next to the atlas, so the URLs share a folder. In `data/assets.json` + every `{ skel, atlas, textures }` object is a model; in `data/local-assets.json` every `spine/…` group (the files a + `spineLocal` record names). +- The server strips the segment and serves the file at the plain path. When the segment names the file's current bytes + (its own hash, or its model's) the answer is `Cache-Control: public, max-age=31536000, immutable`; any other segment — + a page loaded before an update, a URL derived from another file's — gets the current file with `no-cache`, so an old + page keeps working. The ETag of every indexed file is its content hash (`"398f32e2ff78d9e6"`, plus `-gz` when + compressed), on the plain URL too. +- A path segment, never `?v=`: `assets.js validSpine` matches `/\.skel$/`, `audio.js SKILL_MODE_FILE` the `.mp3` ending, + pixi-spine takes the atlas name from the skeleton's basename and resolves the pages against the atlas's folder — a + query string survives none of that. +- A URL the index has no hash for (the file is missing, or not hashed yet) stays plain; plain URLs work as before + (`public, max-age=86400`, revalidated after a day). +- Until the index is ready, with `SP_ASSET_INDEX=0`, or on a static host without this server, the manifests go out as + committed. The views are rebuilt in memory when a manifest or the hashes change; neither file is ever written by the + server, and the `hash` field is the committed one. The index lives in `.cache/` only and is never packaged. + ## Verification `node --test test/assets.test.js` covers the pure helpers: the resolver, the atlas normalizer, the format sniffers, WOFF2, audio banks, the plan id sets, the downloader against a fake network, and the self-heal of corrupt skeletons. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index aa68f551..887b6e4c 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -9,7 +9,7 @@ |---|---| | 服务器 CPU | 战斗在各玩家浏览器里模拟（DESIGN §14），服务器只负责回合、经济和校验：**每个房间每个作战回合约 1 ms CPU**。AI 队友 / 掉线玩家的战场由服务器模拟：作战开始时 3 个 AI 战场在开发机上约 0.2–0.5 s CPU，小主机上可能要几秒（分成 8 ms 小片执行，不会卡住其他房间）。`SP_VERIFY=all` 会复算每个真人战场，CPU 明显增加，小主机建议保持 `off` 或 `sample`。 | | 服务器内存 | 空闲约 100 MB，每个进行中的对局再增加几 MB。 | -| 网络 | 4 人对局中服务器每回合下行约 0.25 MB（DESIGN §14 实测；0.2.6 起 1 KB 以上的 WebSocket 消息用 permessage-deflate 压缩，实测一局单人对局的服务器消息从 169 KB 降到 63 KB，约为原来的四成）。首次进入游戏时浏览器要从主机下载所需的图片 / Spine 模型 / 音频（按需加载，之后走浏览器缓存），公网隧道带宽小时第一次会慢一些。 | +| 网络 | 4 人对局中服务器每回合下行约 0.25 MB（DESIGN §14 实测；0.2.6 起 1 KB 以上的 WebSocket 消息用 permessage-deflate 压缩，实测一局单人对局的服务器消息从 169 KB 降到 63 KB，约为原来的四成）。首次进入游戏时浏览器要从主机下载所需的图片 / Spine 模型 / 音频（按需加载，之后走浏览器缓存），公网隧道带宽小时第一次会慢一些。0.2.6 起服务器启动后在后台给素材算内容哈希（第一次约 1–20 秒，看机器和硬盘；之后只检查大小和修改时间），下发的素材清单里每个地址带上哈希：浏览器把图片、模型和音频缓存一年，服务器更新、重新解压或 Docker 重建之后，玩家只重新下载内容真的变了的文件。哈希存在 `.cache/asset-hashes.json`，删掉会重新计算；目录不可写（例如只读的容器）时只在内存里算。 | | 磁盘 | 素材约 550 MB（`public/assets`，含中文约 65 MB、日文约 85 MB 两套干员语音）+ 依赖约 125 MB（`node_modules`；整合包只带运行依赖，约 65 MB）；可选的本地提取约 40 MB（`.venv-extract`）+ 70 MB 贴图（见第 6 节）。完整包解压后约 710 MB。 | | 玩家设备 | 支持 WebGL 的现代浏览器（Chrome / Edge / Firefox / Safari 最新版），电脑或手机平板（横屏）。老旧设备可在设置里调低画质或访问 `/?board=2d`。 | @@ -295,6 +295,7 @@ services: | 表情显示成默认图标、「玩法说明」只有文字要点 | 素材没下载完整：重新运行 `node tools/setup.mjs`（表情和教程图随其他素材一起从公开镜像下载，不需要客户端）；缺失明细在 `.cache/assets-report.json` | | 本地提取失败 | 游戏照常运行，只是第 6 节表格里的几样换成替代样式。确认客户端已下载全部资源；Python 版本太新导致依赖安装失败时，安装 Python 3.12 后删除 `.venv-extract` 再运行 `node tools/setup.mjs --local` | | 3D 棋盘没出现 | 需要本地提取的棋盘贴图（`node tools/doctor.mjs` 会显示「3D 棋盘可用」），以及支持 WebGL2 的浏览器。没有客户端的服务器可以从同一版本的整合包复制本地素材（第 6 节） | +| 素材地址里的 `~` 加一串字母数字 | `/assets/~<哈希>/…` 是 0.2.6 起的带版本地址（见第 0 节「网络」），浏览器可以长期缓存，属于正常现象。怀疑它导致问题时，可以用 `SP_ASSET_INDEX=0` 启动服务器临时关闭（清单改回普通地址），并请在 issue 里说明 | | 想知道开服电脑上传了多少流量 | 浏览器打开 `http://localhost:3000/healthz?traffic=1`（换成你的端口）：`traffic.http` 是服务器启动以来各类内容（`assets` 图片和模型、`audio` 音频、`code` 页面和脚本、`data` 游戏数据和素材清单）发出的字节数和请求数（压缩后的实际字节，`notModified` 是只回了「没变」的 304），`traffic.ws` 是 WebSocket 的线上字节（`wireOut` / `wireIn`，压缩后）和压缩前的消息字节（`payloadOut` / `payloadIn`）。数字只在内存里，重启后从 0 开始。玩家那边：设置 →「问题反馈」复制的诊断里有一行 `traffic:`，是这个页面从打开起下载的字节数（`cached` 是浏览器缓存直接给的，不占流量；`external` 是其他网站，例如 Google 字体） | | 断线 | 同盟模拟 10 分钟内、独立模拟 24 小时内（`config.constants.singleReconnectTime`）用同一浏览器重新打开页面，自动回到原座位。同盟掉线期间按原阵容自动作战、到时自动准备（不会代为购买；想让 AI 代打请用「离开模拟 → 暂离（AI 托管）」）；独立模拟不计时，等你回来 | diff --git a/docs/design/overview.md b/docs/design/overview.md index e871de44..3b86d718 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -44,13 +44,16 @@ server/ config.js ROOT, served dirs, env (PORT, HOST, TRUST_PROXY, DEBUG), options handed to net.js / lobby.js websocket.js session wiring (SessionRegistry → Lobby → Network), WebSocket upgrade at /ws (64 KB frames, permessage-deflate from 1 KB) - static.js mounts / → public, /data/, /shared/, /sim/ (.js only), the /data.js browser stand-in, path guards + static.js mounts / → public, /data/, /shared/, /sim/ (.js only), the /data.js browser stand-in, path guards, + the ~ segment of /assets/ /fonts/ /media/ URLs stripped (immutable when it names the file) media.js /media/… extension-less audio → public/assets/audio - files.js one file → response: MIME, gzip (.skel/.atlas/.json/.js/.css …), ETag / 304, cache policy, ranges + files.js one file → response: MIME, gzip (.skel/.atlas/.json/.js/.css …), ETag (content hash when known) / 304, + cache policy, ranges; serveBuffer for the served manifests buildTag.js build tag of the served browser runtime (/healthz build, public/js/ui/buildGuard.js) routes.js request listener: 414 / 400 / 405 guards, GET /healthz (?traffic=1: the counters), then static files traffic.js traffic counters: HTTP body bytes by category (assets / audio / code / data), WebSocket wire / payload assetIndex.js content hashes (SHA-256, 16 hex) of public/assets + public/fonts, background build, .cache/asset-hashes.json + assetViews.js the served /data/assets.json + /data/local-assets.json: art URLs as /assets/~/… (immutable when it matches) common.js security headers, URL split, error page, JSON replies, bare 400 for unparseable requests boot.js boot banner (Local / LAN / tunnel URLs), port-in-use hint, graceful SIGINT / SIGTERM net.js session registry, send helpers, per-connection rate limit, message validation (uses shared/protocol.js) diff --git a/server/http/assetViews.js b/server/http/assetViews.js new file mode 100644 index 00000000..e908e6d2 --- /dev/null +++ b/server/http/assetViews.js @@ -0,0 +1,185 @@ +// server/http/assetViews.js — the served manifests (host bandwidth, layer 1; docs/ASSETS.md "The served manifests"): +// data/assets.json and data/local-assets.json as this server hands them out, every art URL carrying the content hash +// of its file as its first segment (shared/assetPaths.js) — `/assets/char/avatar/x.png` → +// `/assets/~/char/avatar/x.png`, `/fonts/~/…` — so the browser keeps the file for a year and asks again +// only when the bytes change. Audio keeps the `/assets/~/audio/….mp3` form: public/js/media.js turns it into +// `/media/~/…` when it fetches it. +// +// * A Spine model's skeleton, atlas and pages carry ONE hash, that of the model (assetIndex.js models): pixi-spine +// finds the atlas by the skeleton's name and the pages next to the atlas, so their URLs share a folder. A model is +// every `{ skel, atlas, textures }` object of data/assets.json (chars' front / back, tokens, enemies) and every +// `spine/…` group of data/local-assets.json (the files a `spineLocal` record names). +// * A URL the index has no hash for (not on disk, not hashed yet) stays as it is. +// * The committed data/assets.json is never written: a view is built in memory from the file and the index, again +// when either changes (the file's size / mtime, the index's generation), and once after every index build, so the +// index knows the model lists before any page asks. +// * No index, or not ready yet: no view (`get` → null) and the files go out as they are. + +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { hashedPath } from '../../shared/assetPaths.js'; +import { hash16 } from './assetIndex.js'; +import { noopLog } from './config.js'; + +/** The manifests that have a view, by their name under data/. */ +export const VIEW_MANIFESTS = Object.freeze(['assets.json', 'local-assets.json']); + +const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); +const ART_URL = /^\/(?:assets|fonts)\/./; + +/** URL path of a manifest → the index's rel path ('/assets/a%20b.png' → 'assets/a b.png'), or null. */ +export function relOf(url) { + if (typeof url !== 'string' || !ART_URL.test(url) || /[?#]/.test(url)) return null; + let p; + try { p = decodeURIComponent(url); } catch { return null; } + if (p.includes('\0') || p.includes('\\')) return null; + const segs = p.slice(1).split('/'); + if (segs.some((s) => s === '' || s === '.' || s === '..')) return null; + return segs.join('/'); +} + +/** Is `o` a Spine object of data/assets.json (`{ skel: '/….skel', atlas, textures? }`)? */ +const isSpine = (o) => isObj(o) && typeof o.skel === 'string' && o.skel.startsWith('/') && /\.skel$/.test(o.skel); + +/** The URLs of a Spine object: skeleton, atlas, pages (the `.png` when it lists none). */ +function spineUrls(o) { + const out = [o.skel]; + if (typeof o.atlas === 'string' && o.atlas) out.push(o.atlas); + const pages = Array.isArray(o.textures) ? o.textures.filter((u) => typeof u === 'string' && u) : []; + out.push(...(pages.length ? pages : [o.skel.replace(/\.skel$/, '.png')])); + return out; +} + +/** + * The served copy of a manifest and its model lists. `index`: hashOf(rel) / modelHash(rels) of assetIndex.js. + * @param {any} doc the parsed manifest (not changed) + * @param {'assets.json'|'local-assets.json'} name + * @param {{ hashOf: (rel: string) => string | null, modelHash: (rels: string[]) => string | null }} index + * @returns {{ doc: any, models: string[][], hashed: number, plain: number }} + */ +export function rewriteManifest(doc, name, index) { + const models = []; + let hashed = 0; + let plain = 0; + const own = (url) => { + const rel = relOf(url); + const h = rel ? index.hashOf(rel) : null; + if (h) { hashed++; return hashedPath(url, h); } + if (rel) plain++; + return url; + }; + /** All URLs of one model with the model's hash (or all left plain when a member is unknown). */ + const model = (urls) => { + const rels = urls.map(relOf); + if (rels.some((r) => !r)) return null; + models.push(rels); + const h = index.modelHash(rels); + if (!h) { plain += urls.length; return null; } + hashed += urls.length; + return h; + }; + const walk = (v) => { + if (typeof v === 'string') return ART_URL.test(v) ? own(v) : v; + if (Array.isArray(v)) return v.map(walk); + if (!isObj(v)) return v; + if (name === 'assets.json' && isSpine(v)) { + const h = model(spineUrls(v)); + const out = {}; + for (const [k, x] of Object.entries(v)) { + if (h && k === 'skel') out[k] = hashedPath(x, h); + else if (h && k === 'atlas' && typeof x === 'string') out[k] = hashedPath(x, h); + else if (h && k === 'textures' && Array.isArray(x)) out[k] = x.map((u) => (typeof u === 'string' ? hashedPath(u, h) : u)); + else out[k] = walk(x); + } + return out; + } + const out = {}; + for (const [k, x] of Object.entries(v)) out[k] = walk(x); + return out; + }; + if (name === 'local-assets.json' && isObj(doc) && isObj(doc.groups)) { + const groups = {}; + for (const [g, entries] of Object.entries(doc.groups)) { + if (g.startsWith('spine/') && isObj(entries)) { + const names = Object.keys(entries).filter((n) => isObj(entries[n]) && typeof entries[n].path === 'string').sort(); + const h = names.length ? model(names.map((n) => entries[n].path)) : null; + groups[g] = {}; + for (const [n, e] of Object.entries(entries)) { + groups[g][n] = h && isObj(e) && typeof e.path === 'string' ? { ...e, path: hashedPath(e.path, h) } : walk(e); + } + } else groups[g] = walk(entries); + } + return { doc: { ...walk({ ...doc, groups: null }), groups }, models, hashed, plain }; + } + return { doc: walk(doc), models, hashed, plain }; +} + +/** + * The views of one server. + * @param {{ dataDir: string, index: ReturnType, log?: object }} opts + */ +export function createManifestViews({ dataDir, index, log = noopLog }) { + const dir = path.resolve(dataDir); + /** @type {Map} */ + const cache = new Map(); + /** @type {Map>} */ + const pending = new Map(); + const warned = new Set(); + + const pushModels = () => index.setModels([...cache.values()].flatMap((c) => c.models)); + + /** Build (or reuse) the view of `name`; null without the file, while the index is not ready or empty, or for a broken file. */ + async function build(name) { + if (!index.ready || !index.size) return null; + const file = path.join(dir, name); + const st = await fsp.stat(file).catch(() => null); + if (!st || !st.isFile()) { + if (cache.delete(name)) pushModels(); + return null; + } + const key = `${index.generation}|${st.size}|${st.mtimeMs}`; + const hit = cache.get(name); + if (hit && hit.key === key) return hit.view; + let doc; + try { + doc = JSON.parse(await fsp.readFile(file, 'utf8')); + } catch (e) { + if (!warned.has(name)) { warned.add(name); log.warn?.(`[assets] data/${name} unreadable (${e?.message || e}); served as it is`); } + cache.set(name, { key, view: null, models: [] }); + pushModels(); + return null; + } + // one synchronous step: the view and the model lists the index checks a model's URLs against + const { doc: out, models, hashed, plain } = rewriteManifest(doc, name, index); + const body = Buffer.from(JSON.stringify(out)); + const view = { body, hash: hash16(body), hashed, plain }; + cache.set(name, { key, view, models }); + pushModels(); + return view; + } + + /** The view of `name` (one build at a time per manifest). */ + function get(name) { + if (!VIEW_MANIFESTS.includes(name)) return Promise.resolve(null); + const running = pending.get(name); + if (running) return running; + const p = build(name).catch((e) => { log.error?.('[assets] manifest view failed', e); return null; }).finally(() => pending.delete(name)); + pending.set(name, p); + return p; + } + + const rebuildAll = () => Promise.all(VIEW_MANIFESTS.map(get)); + // after every build that changed the hashes (and the first one): the views, and with them the index's model lists + index.onChange(() => { rebuildAll(); }); + index.whenReady().then(() => rebuildAll(), () => {}); + + return { + get, + rebuildAll, + stats() { + const out = {}; + for (const [name, c] of cache) out[name] = c.view ? { bytes: c.view.body.length, hashed: c.view.hashed, plain: c.view.plain } : null; + return out; + }, + }; +} diff --git a/server/http/files.js b/server/http/files.js index 98910651..f63a7a71 100644 --- a/server/http/files.js +++ b/server/http/files.js @@ -1,9 +1,11 @@ -// server/http/files.js — one file on disk → an HTTP response (static.js and media.js decide which file): +// server/http/files.js — one file on disk (serveFile) or one body in memory (serveBuffer, the served manifests of +// assetViews.js) → an HTTP response (static.js and media.js decide which file): // MIME types incl. .mjs/.js text/javascript, .skel application/octet-stream, .atlas text/plain; // gzip for text-like types, .skel and uncompressed fonts when the client accepts it (small files are -// compressed once and cached in memory); strong ETag + Last-Modified with 304s; Cache-Control -// (html & code/data: no-cache + revalidate; public/assets|fonts|vendor: 1 day; any `?v=` URL: immutable); -// single byte-range requests (206/416, used by