From f6794a2f2c8f30d1df889c91d941e88b26c3af26 Mon Sep 17 00:00:00 2001 From: Tome Chen <96970949+NekoTC@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:18:30 +0800 Subject: [PATCH] Asset downloads can use a gh-proxy mirror when chosen by hand (--asset-source / SP_ASSET_SOURCE): one try per URL, the original source as fallback, the mirror off for the run after 3 failures in a row (#24) Off by default; the original sources, retries, skipped files and the manifest-shrink guard are unchanged. Contributed by @NekoTC. --- docs/DEPLOY.md | 26 ++- test/asset-network.test.js | 370 ++++++++++++++++++++++++++++++++++++ tools/assets/cache.mjs | 59 ++++-- tools/assets/downloader.mjs | 36 ++-- tools/assets/manifest.mjs | 6 +- tools/assets/network.mjs | 115 +++++++++++ tools/assets/sources.mjs | 57 +++++- tools/fetch-assets.mjs | 36 +++- tools/setup.mjs | 11 +- 9 files changed, 671 insertions(+), 45 deletions(-) create mode 100644 test/asset-network.test.js create mode 100644 tools/assets/network.mjs diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index f201500e..945322d4 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -36,6 +36,30 @@ 等价的手动命令：`npm ci`、`node tools/setup.mjs`、`npm start`。 +#### 国内镜像下载 + +Setup 默认使用「GitHub 原始源 → jsDelivr」，不查询公网 IP，也不请求 gh-proxy.com。GitHub 下载失败时会提示如何手动开启镜像；仅添加提示，不自动切换到第三方代理。 + +镜像方法是在完整 GitHub 链接前加 `https://gh-proxy.com/`，例如： + +```text +https://gh-proxy.com/https://raw.githubusercontent.com/OWNER/REPO/BRANCH/file.png +``` + +手动开启后顺序为「前缀镜像 → 原始源 → jsDelivr」。索引、图片、Spine、音频和字体都使用此规则（音频 voice 分支跳过 jsDelivr）。镜像是第三方代理；当前只校验格式和大小，没有内容哈希校验，请自行决定是否信任并启用。npm / pip 依赖不使用 GitHub 前缀。 + +```powershell +node tools/setup.mjs --asset-source=mirror # 手动优先国内镜像 +node tools/setup.mjs --asset-source=direct # 默认：仅原始源和 jsDelivr，不使用前缀代理 +$env:SP_ASSET_SOURCE = 'mirror' # 也可用环境变量显式启用 +``` + +`node tools/fetch-assets.mjs` 同样支持 `--asset-source=direct|mirror`。命令行优先于 `SP_ASSET_SOURCE`。默认镜像前缀为 `https://gh-proxy.com/`，可通过 `SP_GITHUB_PROXY` 指定其他 HTTPS 前缀；仅配置前缀不会启用镜像。将 `SP_GITHUB_PROXY` 设为空字符串（或全空格）可彻底禁用前缀代理，即使选择了 `mirror` 模式；未设置此变量与显式设空不同，前者使用默认前缀。Windows PowerShell 的某些版本会将空值视为删除变量，可设置 `$env:SP_GITHUB_PROXY = ' '` 或使用 `--asset-source=direct` 来明确禁用。前缀只处理 GitHub 下载链接，不重复添加。 + +镜像请求每个 URL 只尝试一次，响应头超时 8 秒，响应体有独立的空闲超时，失败即尝试原始源。连续 3 次网络错误、HTTP 错误或无效内容会在本次运行中关闭镜像，后续索引、素材和字体共享该状态；正在进行的镜像请求也会中止并回退。成功会清零连续失败次数；404 / 410 是资源不存在，不触发熔断。再次运行脚本会重新尝试手动启用的镜像。原始源的重试、已有文件跳过和 0.1.1 的清单缩减保护保持不变。 + +从历史下载记录派生的 Spine 补充贴图也按本次设置重新选择来源，禁用后不会沿用旧代理地址。 + ### 1.2 防火墙 - 第一次启动时 Windows 会弹出「Windows 安全中心警报」：勾选**专用网络**并点「允许访问」。 @@ -235,7 +259,7 @@ services: | 任何问题 | `node tools/doctor.mjs`：Node 版本、依赖、素材完整性、端口、局域网地址、防火墙、网络类型 | | `端口已被占用 / EADDRINUSE` | 已经有一个服务器在运行（自启任务？）或其他程序占用 3000：换端口 `scripts\start-windows.bat --port 3001` | | 朋友打不开页面 | 防火墙规则 / 网络类型（1.2）；确认用的是 `LAN` 地址而不是 `localhost`；访客 Wi-Fi 常开启「AP 隔离」；不在同一网络请看第 2 节 | -| 画面是占位图、没有声音 | 素材没下完：重新运行 `node tools/setup.mjs`（会续传）；缺失明细在 `.cache/assets-report.json`。GitHub 原始地址访问失败时会自动改用 jsDelivr 镜像 | +| 画面是占位图、没有声音 | 素材没下完：重新运行 `node tools/setup.mjs`（会续传）；缺失明细在 `.cache/assets-report.json`。默认仅原始源和 jsDelivr；可用 `--asset-source=mirror` 手动开启前缀镜像（见上文） | | 素材下载很慢 / 失败 | 网络问题可随时中断，重新运行会跳过已完成的文件；`node tools/fetch-assets.mjs --concurrency=4` 降低并发。有文件没下载成功时，素材清单 `data/assets.json` 保持不变（脚本列出缺少的条目并以非零状态结束；游戏里缺的图片用占位图，缺的声音不播放），重新运行即可补齐 | | 表情显示成默认图标、「玩法说明」只有文字要点 | 素材没下载完整：重新运行 `node tools/setup.mjs`（表情和教程图随其他素材一起从公开镜像下载，不需要客户端）；缺失明细在 `.cache/assets-report.json` | | 本地提取失败 | 游戏照常运行，只是第 6 节表格里的几样换成替代样式。确认客户端已下载全部资源；Python 版本太新导致依赖安装失败时，安装 Python 3.12 后删除 `.venv-extract` 再运行 `node tools/setup.mjs --local` | diff --git a/test/asset-network.test.js b/test/asset-network.test.js new file mode 100644 index 00000000..afad7ccb --- /dev/null +++ b/test/asset-network.test.js @@ -0,0 +1,370 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { MirrorPolicy, selectDownloadSource } from '../tools/assets/network.mjs'; +import { downloadUrls, githubProxyUrl, githubSourceUrl, normalizeProxyPrefix } from '../tools/assets/sources.mjs'; +import { Downloader } from '../tools/assets/downloader.mjs'; +import { cachedJson, loadIndexes } from '../tools/assets/cache.mjs'; +import { resolveTemplate } from '../tools/assets/manifest.mjs'; +import { processModels } from '../tools/assets/spine.mjs'; + +const RAW = 'https://raw.githubusercontent.com/o/r/main/a.json'; +const PROXY = 'https://gh-proxy.com/' + RAW; +const CDN = 'https://cdn.jsdelivr.net/gh/o/r@main/a.json'; +const quiet = () => {}; + +async function fixture(t) { + const dir = await mkdtemp(join(tmpdir(), 'sp-network-')); + t.after(() => rm(dir, { recursive: true, force: true })); + return dir; +} + +test('default, manual source and offline mode perform no public IP lookup', async (t) => { + const fetch = t.mock.method(globalThis, 'fetch', async () => { assert.fail('unexpected network request'); }); + assert.equal(await selectDownloadSource({ log: quiet }), 'direct'); + for (const mode of ['direct', 'mirror']) assert.equal(await selectDownloadSource({ mode, log: quiet }), mode); + assert.equal(await selectDownloadSource({ offline: true }), 'direct'); + await assert.rejects(selectDownloadSource({ mode: 'invalid' }), /unknown asset source/); + await assert.rejects(selectDownloadSource({ mode: 'auto' }), /unknown asset source/); + assert.equal(fetch.mock.callCount(), 0); +}); + +test('mirror startup log names its proxy and states the validation scope', async () => { + const logs = []; + assert.equal(await selectDownloadSource({ mode: 'mirror', proxyPrefix: 'https://mirror.example/p/', log: (msg) => logs.push(msg) }), 'mirror'); + assert.match(logs[0], /https:\/\/mirror\.example\/p\//); + assert.match(logs[0], /仅校验格式和大小/); +}); + +test('URL order, voice support, custom prefix, and no double prefix', () => { + assert.deepEqual(downloadUrls(RAW, { source: 'mirror' }), [PROXY, RAW, CDN]); + assert.deepEqual(downloadUrls(RAW), [RAW, CDN]); + assert.deepEqual(downloadUrls(RAW, { source: 'direct', proxyPrefix: 'https://custom.example/' }), [RAW, CDN]); + const voice = 'https://raw.githubusercontent.com/ArknightsAssets/ArknightsAssets2/voice/assets/x.mp3'; + assert.deepEqual(downloadUrls(voice, { source: 'mirror' }), ['https://gh-proxy.com/' + voice, voice]); + assert.deepEqual(downloadUrls(PROXY, { source: 'mirror' }), [PROXY, RAW, CDN]); + assert.deepEqual(downloadUrls(PROXY), [RAW, CDN]); + assert.deepEqual(downloadUrls('https://old-proxy.example/prefix/' + RAW, { source: 'mirror', proxyPrefix: '' }), [RAW, CDN]); + assert.deepEqual(downloadUrls(PROXY, { source: 'mirror', proxyPrefix: 'https://new-proxy.example/' }), ['https://new-proxy.example/' + RAW, RAW, CDN]); + assert.equal(githubSourceUrl('https://old-proxy.example/' + PROXY), RAW); + assert.equal(githubSourceUrl(RAW + '/https://example.com/path'), RAW + '/https://example.com/path'); + for (const url of ['https://example.com/a', 'https://github.com.evil.example/a', 'https://user:secret@github.com/a']) { + assert.equal(githubProxyUrl(url), null); + } + assert.equal(githubProxyUrl(RAW, 'https://mirror.example/prefix'), 'https://mirror.example/prefix/' + RAW); + assert.equal(normalizeProxyPrefix(), 'https://gh-proxy.com/'); + for (const proxyPrefix of ['', ' ']) { + assert.equal(normalizeProxyPrefix(proxyPrefix), ''); + assert.equal(githubProxyUrl(RAW, proxyPrefix), null); + assert.deepEqual(downloadUrls(RAW, { source: 'mirror', proxyPrefix }), [RAW, CDN]); + } + for (const prefix of ['http://mirror.example/', 'https://mirror.example/?q=x', 'https://user:secret@mirror.example/']) { + assert.throws(() => normalizeProxyPrefix(prefix), /HTTPS URL/); + } +}); + +test('downloader uses proxy, validates payload, falls back, and skips existing files', async (t) => { + const dir = await fixture(t); + const calls = []; + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', retries: 1, backoffMs: 0, log: quiet, + fetchImpl: async (url) => { + calls.push(url); + return url === PROXY ? new Response('bad gateway') : Response.json({ ok: true }); + } }); + const job = { rel: 'a.json', urls: [RAW], kind: 'json' }; + assert.equal((await dl.run([job])).get(job.rel).url, RAW); + assert.deepEqual(calls, [PROXY, RAW]); + assert.equal((await dl.run([job])).get(job.rel).status, 'skip'); + assert.equal(calls.length, 2); +}); + +test('proxy provenance is recognized as the primary asset, not a fallback asset', async (t) => { + const dir = await fixture(t); + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', log: quiet, + fetchImpl: async () => Response.json({ ok: true }) }); + const job = { rel: 'a.json', urls: [RAW], kind: 'json' }; + assert.equal((await dl.run([job])).get(job.rel).url, PROXY); + for (const source of [PROXY, 'https://old-proxy.example/prefix/' + RAW]) { + const resolved = resolveTemplate({ icon: { alts: [job] } }, { root: dir, spine: new Map(), sourceOf: () => source }); + assert.deepEqual(resolved.fallbacks, [], 'disabling/changing the proxy does not change default proxy provenance'); + assert.equal(resolved.value.icon, '/assets/a.json'); + } +}); + +test('Spine pages from old proxy ledger entries respect direct and explicitly disabled mirror mode', async (t) => { + const base = 'https://raw.githubusercontent.com/o/r/main/'; + const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII=', 'base64'); + for (const prefix of ['https://gh-proxy.com/', 'https://old-proxy.example/prefix/']) { + for (const opts of [{ source: 'direct' }, { source: 'mirror', proxyPrefix: '' }]) { + const dir = await fixture(t); + const atlas = 'extra.png\nsize: 1,1\nformat: RGBA8888\nfilter: Linear,Linear\nrepeat: none\n'; + await writeFile(join(dir, 'model.atlas'), atlas); + const calls = []; + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), log: quiet, ...opts, + fetchImpl: async (url) => { + calls.push(url); + return url === base + 'extra.png' ? new Response(png) : new Response('', { status: 404 }); + } }); + dl.ledger.files['model.atlas'] = { url: prefix + base + 'model.atlas', bytes: Buffer.byteLength(atlas) }; + const model = { key: 'model', dir: '', baseUrl: base, pngs: [], + skel: { rel: 'model.skel', urls: [base + 'model.skel'], kind: 'skel' }, + atlas: { rel: 'model.atlas', urls: [base + 'model.atlas'], kind: 'atlas' } }; + await processModels(new Map([['model', model]]), { root: dir, dl, cachePath: join(dir, 'spine.json'), log: quiet }); + assert.ok(calls.includes(base + 'extra.png')); + assert.ok(calls.every((url) => url.startsWith(base) || url.startsWith('https://cdn.jsdelivr.net/'))); + assert.equal(dl.ledger.files['extra.png'].url, base + 'extra.png'); + } + } +}); + +test('all three indexes use the selected proxy and cached/offline reads make no request', async (t) => { + const dir = await fixture(t); + const calls = []; + const opts = { source: 'mirror', log: quiet, fetchImpl: async (url) => { calls.push(url); return Response.json({ index: true }); } }; + const result = await loadIndexes(dir, opts); + assert.deepEqual(result, { audioData: { index: true }, modelsData: { index: true }, charword: { index: true } }); + assert.equal(calls.length, 3); + assert.ok(calls.some((url) => url.endsWith('/charword_table.json'))); + assert.ok(calls.every((url) => url.startsWith('https://gh-proxy.com/https://raw.githubusercontent.com/'))); + await loadIndexes(dir, { ...opts, offline: true, refresh: true }); + assert.equal(calls.length, 3); + assert.deepEqual(JSON.parse(await readFile(join(dir, '.cache', 'ark-models', 'models_data.json'), 'utf8')), { index: true }); +}); + +test('index proxy miss falls back to raw', async (t) => { + const dir = await fixture(t); + const calls = []; + assert.deepEqual(await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, source: 'mirror', log: quiet, + fetchImpl: async (url) => { calls.push(url); return url === PROXY ? new Response('', { status: 404 }) : Response.json({ ok: true }); } }), { ok: true }); + assert.deepEqual(calls, [PROXY, RAW]); +}); + +test('direct retries that recover do not print a failure hint or contact the prefix proxy', async (t) => { + const dir = await fixture(t); + const calls = []; + const logs = []; + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), retries: 1, backoffMs: 0, + log: (msg) => logs.push(msg), fetchImpl: async (url) => { + calls.push(url); + return url.startsWith('https://raw.githubusercontent.com/') ? new Response('', { status: 503 }) : Response.json({ ok: true }); + } }); + await dl.run([1, 2].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); + assert.equal(calls.length, 4); + assert.ok(calls.every((url) => url.startsWith('https://raw.githubusercontent.com/') || url.startsWith('https://cdn.jsdelivr.net/'))); + assert.equal(logs.filter((msg) => msg.includes('--asset-source=mirror')).length, 0); +}); + +test('default index retries that fall back successfully do not print a failure hint', async (t) => { + const dir = await fixture(t); + const calls = []; + const logs = []; + const result = await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, backoffMs: 0, + log: (msg) => logs.push(msg), fetchImpl: async (url) => { + calls.push(url); + return url === RAW ? new Response('', { status: 503 }) : Response.json({ ok: true }); + } }); + assert.deepEqual(result, { ok: true }); + assert.deepEqual(calls, [RAW, RAW, RAW, CDN]); + assert.equal(logs.filter((msg) => msg.includes('--asset-source=mirror')).length, 0); +}); + +test('empty proxy setting disables every downloader and index proxy request in mirror mode', async (t) => { + const dir = await fixture(t); + const calls = []; + const opts = { source: 'mirror', proxyPrefix: '', log: quiet, fetchImpl: async (url) => { calls.push(url); return Response.json({ ok: true }); } }; + await loadIndexes(dir, opts); + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), ...opts }); + await dl.run([{ rel: 'a.json', urls: [RAW], kind: 'json' }]); + assert.equal(calls.length, 4); + assert.ok(calls.every((url) => url.startsWith('https://raw.githubusercontent.com/'))); +}); + +test('one circuit spans indexes, assets and fonts and stops after three failed proxy attempts', async (t) => { + const dir = await fixture(t); + const calls = []; + const log = []; + const mirrorPolicy = new MirrorPolicy({ source: 'mirror', log: (msg) => log.push(msg) }); + const opts = { source: 'mirror', mirrorPolicy, log: quiet, fetchImpl: async (url) => { + calls.push(url); + return url.startsWith('https://gh-proxy.com/') ? new Response('', { status: 503 }) : Response.json({ ok: true }); + } }; + await loadIndexes(dir, opts); // three consecutive proxy failures across the indexes + assert.equal(mirrorPolicy.disabled, true); + const dl = new Downloader({ root: join(dir, 'assets'), ledgerPath: join(dir, 'ledger.json'), concurrency: 1, ...opts }); + await dl.run([1, 2, 3].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); + const fonts = new Downloader({ root: join(dir, 'fonts'), ledgerPath: join(dir, 'fonts-ledger.json'), ...opts }); + await fonts.run([{ rel: 'font.json', urls: [RAW + '/font'], kind: 'json' }]); + assert.equal(calls.filter((url) => url.startsWith('https://gh-proxy.com/')).length, 3, 'no retries on the proxy, no later proxy requests'); + assert.equal(dl.totals.ok, 3); + assert.equal(fonts.totals.ok, 1); + assert.equal(log.length, 1, 'one circuit-open warning for the shared invocation'); +}); + +test('successful and not-found proxy responses reset the failure streak, opened circuit stays off', async (t) => { + const dir = await fixture(t); + const statuses = [503, 503, 200, 503, 404, 503, 503, 503]; + let proxyCalls = 0; + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', concurrency: 1, log: quiet, + fetchImpl: async (url) => { + if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true }); + const status = statuses[proxyCalls++] ?? 200; + return status === 200 ? Response.json({ ok: true }) : new Response('', { status }); + } }); + await dl.run(Array.from({ length: 10 }, (_, i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); + assert.equal(proxyCalls, 8); + assert.equal(dl.totals.ok, 10); +}); + +test('invalid payloads trip the same circuit as HTTP failures', async (t) => { + const dir = await fixture(t); + let proxyCalls = 0; + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', concurrency: 1, log: quiet, + fetchImpl: async (url) => { + if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true }); + proxyCalls++; + return new Response('upstream error'); + } }); + await dl.run([1, 2, 3, 4].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); + assert.equal(proxyCalls, 3); + assert.equal(dl.totals.ok, 4); +}); + +test('slow proxy requests use a short deadline, abort, and fall back without repeated waits', async (t) => { + const dir = await fixture(t); + let proxyCalls = 0; + const mirrorPolicy = new MirrorPolicy({ source: 'mirror', timeoutMs: 10, failureLimit: 2, log: quiet }); + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', mirrorPolicy, concurrency: 1, log: quiet, + fetchImpl: async (url, { signal }) => { + if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true }); + proxyCalls++; + return new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), { once: true })); + } }); + const keepAlive = setInterval(quiet, 1000); + const started = Date.now(); + try { await dl.run([1, 2, 3].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); } + finally { clearInterval(keepAlive); } + assert.equal(proxyCalls, 2); + assert.equal(dl.totals.ok, 3); + assert.ok(Date.now() - started < 3000, 'uses the mirror deadline, not 120 seconds times three'); +}); + +test('opening the circuit aborts concurrent proxy work and later jobs go direct', async (t) => { + const dir = await fixture(t); + let proxyCalls = 0; + let aborted = 0; + const mirrorPolicy = new MirrorPolicy({ source: 'mirror', failureLimit: 1, log: quiet }); + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', mirrorPolicy, concurrency: 3, log: quiet, + fetchImpl: async (url, { signal }) => { + if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true }); + proxyCalls++; + if (proxyCalls === 3) return new Response('', { status: 503 }); + return new Promise((_, reject) => signal.addEventListener('abort', () => { aborted++; reject(signal.reason); }, { once: true })); + } }); + await dl.run([1, 2, 3, 4, 5].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); + assert.equal(proxyCalls, 3); + assert.equal(aborted, 2); + assert.equal(dl.totals.ok, 5); +}); + +test('the mirror body idle deadline covers stalled response bodies for indexes and files', async (t) => { + const dir = await fixture(t); + const server = createServer((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{"ok":'); // Headers arrive, but the body never finishes. + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + t.after(() => new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); })); + let proxyCalls = 0; + const mirrorPolicy = new MirrorPolicy({ source: 'mirror', timeoutMs: 100, failureLimit: 2, log: quiet }); + const opts = { mirrorPolicy, log: quiet, fetchImpl: async (url, options) => { + if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true }); + proxyCalls++; + return fetch(`http://127.0.0.1:${server.address().port}`, options); + } }; + const started = Date.now(); + assert.deepEqual(await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, ...opts }), { ok: true }); + const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), concurrency: 1, ...opts }); + await dl.run([1, 2].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); + assert.equal(proxyCalls, 2); + assert.equal(dl.totals.ok, 2); + assert.ok(Date.now() - started < 4000, 'stalled body reads time out instead of waiting indefinitely'); +}); + +test('mirror header timeout does not cap a steadily streaming index or asset body', async (t) => { + const dir = await fixture(t); + const payload = Buffer.from('{"streamed":true}'); + const delayedResponse = () => { + let offset = 0; + let timer; + return new Response(new ReadableStream({ + start(controller) { + timer = setInterval(() => { + if (offset >= payload.length) { + clearInterval(timer); + controller.close(); + return; + } + controller.enqueue(payload.subarray(offset, ++offset)); + }, 20); + }, + cancel() { clearInterval(timer); }, + }), { headers: { 'content-type': 'application/json', 'content-length': String(payload.length) } }); + }; + const mirrorPolicy = new MirrorPolicy({ source: 'mirror', timeoutMs: 100, log: quiet }); + const opts = { mirrorPolicy, log: quiet, fetchImpl: async (url) => url.startsWith('https://gh-proxy.com/') ? delayedResponse() : Response.json({ direct: true }) }; + const started = Date.now(); + assert.deepEqual(await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, ...opts }), { streamed: true }); + const dl = new Downloader({ root: join(dir, 'assets'), ledgerPath: join(dir, 'ledger.json'), ...opts }); + const result = await dl.run([{ rel: 'slow.json', urls: [RAW], kind: 'json' }]); + assert.equal(result.get('slow.json').status, 'ok'); + assert.ok(Date.now() - started > 100, 'the body takes longer than the header deadline while making progress'); +}); + +test('CLI source defaults to direct and retains the 0.1.1 shrink/local-spine flags', async () => { + const { parseArgs } = await import('../tools/fetch-assets.mjs'); + const old = process.env.SP_ASSET_SOURCE; + try { + delete process.env.SP_ASSET_SOURCE; + assert.equal(parseArgs([]).source, 'direct'); + process.env.SP_ASSET_SOURCE = 'mirror'; + assert.equal(parseArgs([]).source, 'mirror'); + assert.equal(parseArgs([]).voiceLang, 'cn'); + assert.equal(parseArgs([]).voiceAll, false); + const opts = parseArgs(['--asset-source=direct', '--allow-shrink', '--local-spines', '--voice-lang=jp', '--voice-all']); + assert.equal(opts.source, 'direct'); + assert.equal(opts.allowShrink, true); + assert.equal(opts.localSpines, true); + assert.equal(opts.voiceLang, 'jp'); + assert.equal(opts.voiceAll, true); + } finally { + if (old === undefined) delete process.env.SP_ASSET_SOURCE; + else process.env.SP_ASSET_SOURCE = old; + } +}); + +test('SP_GITHUB_PROXY is parsed only for online mirror mode', async () => { + const { resolveProxyPrefix } = await import('../tools/fetch-assets.mjs'); + assert.equal(resolveProxyPrefix('direct', false, 'not-an-https-url'), ''); + assert.equal(resolveProxyPrefix('mirror', true, 'not-an-https-url'), ''); + assert.equal(resolveProxyPrefix('mirror', false, ''), ''); + assert.throws(() => resolveProxyPrefix('mirror', false, 'not-an-https-url'), /Invalid URL/); +}); + +test('setup and asset CLI document and validate source options before doing work', () => { + for (const script of ['tools/setup.mjs', 'tools/fetch-assets.mjs']) { + const help = spawnSync(process.execPath, [script, '--help'], { encoding: 'utf8' }); + assert.equal(help.status, 0, help.stderr); + assert.match(help.stdout, /--asset-source/); + assert.match(help.stdout, /SP_GITHUB_PROXY/); + const invalid = spawnSync(process.execPath, [script, '--asset-source=invalid'], { encoding: 'utf8' }); + assert.notEqual(invalid.status, 0); + assert.match(invalid.stderr, /unknown asset source/); + } +}); diff --git a/tools/assets/cache.mjs b/tools/assets/cache.mjs index 65fe437a..db8fc325 100644 --- a/tools/assets/cache.mjs +++ b/tools/assets/cache.mjs @@ -6,7 +6,8 @@ import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; -import { RAW, mirrorUrl } from './sources.mjs'; +import { RAW } from './sources.mjs'; +import { MirrorPolicy } from './network.mjs'; /** * Read a cached JSON file, downloading it first when missing or unparsable. @@ -16,44 +17,70 @@ import { RAW, mirrorUrl } from './sources.mjs'; * @param {boolean} [o.refresh] force re-download * @param {boolean} [o.offline] never download (throw when the cache is missing) * @param {(m:string)=>void} [o.log] + * @param {'direct'|'mirror'} [o.source] + * @param {string} [o.proxyPrefix] + * @param {typeof fetch} [o.fetchImpl] + * @param {MirrorPolicy} [o.mirrorPolicy] shared invocation-wide mirror circuit breaker + * @param {number} [o.timeoutMs] + * @param {number} [o.backoffMs] * @returns {Promise} parsed JSON */ -export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log }) { +export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log, source = 'direct', proxyPrefix, fetchImpl = globalThis.fetch, mirrorPolicy, timeoutMs = 180000, backoffMs = 500 }) { if (!refresh || offline) { try { return JSON.parse(await readFile(cacheFile, 'utf8')); } catch (e) { if (offline) throw new Error(`--offline: cached index ${cacheFile} is missing or corrupt (${e.message}); run once online`); } } + const network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log }); let lastErr = null; - for (const src of [url, mirrorUrl(url)].filter(Boolean)) { - for (let attempt = 1; attempt <= 3; attempt++) { + for (const src of network.urls(url)) { + if (network.skip(src)) continue; + const attempts = network.isProxy(src) ? 1 : 3; + for (let attempt = 1; attempt <= attempts; attempt++) { + let text, json; try { log(`[cache] downloading ${src}`); - const res = await fetch(src, { signal: AbortSignal.timeout(180000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const text = await res.text(); - const json = JSON.parse(text); - await mkdir(dirname(cacheFile), { recursive: true }); - await writeFile(cacheFile + '.tmp', text); - await rename(cacheFile + '.tmp', cacheFile); - return json; + const res = await network.request(src, fetchImpl, {}, timeoutMs); + if (!res.ok) { + await res.body?.cancel(); + if (res.status === 404 || res.status === 410) { + network.succeeded(src); // missing index, not a proxy outage + lastErr = new Error(`HTTP ${res.status}`); + break; + } + throw new Error(`HTTP ${res.status}`); + } + text = (await network.readBody(src, res)).toString('utf8'); + json = JSON.parse(text); } catch (e) { lastErr = e; - await new Promise((r) => setTimeout(r, 500 * attempt)); + network.failed(src); + if (attempt < attempts && backoffMs) await new Promise((r) => setTimeout(r, backoffMs * attempt)); + continue; } + network.succeeded(src); + // A local disk failure is not evidence that the mirror is unhealthy. + await mkdir(dirname(cacheFile), { recursive: true }); + await writeFile(cacheFile + '.tmp', text); + await rename(cacheFile + '.tmp', cacheFile); + return json; } } + if (lastErr) network.directFailureHint(); throw new Error(`cannot fetch ${url}: ${lastErr?.message}`); } /** * Load audio_data.json (official), charword_table.json (official voice slots) and Ark-Models models_data.json. * @param {string} root project root - * @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void}} [opts] + * @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void, source?:'direct'|'mirror', proxyPrefix?:string, fetchImpl?:typeof fetch, mirrorPolicy?:MirrorPolicy}} [opts] * @returns {Promise<{ audioData: any, modelsData: any, charword: any }>} */ export async function loadIndexes(root, opts = {}) { + const mirrorPolicy = opts.mirrorPolicy ?? new MirrorPolicy(opts); const audioData = await cachedJson({ + ...opts, + mirrorPolicy, cacheFile: join(root, '.cache', 'gamedata', 'excel', 'audio_data.json'), url: RAW.gamedata + 'excel/audio_data.json', refresh: opts.refresh, @@ -63,6 +90,8 @@ export async function loadIndexes(root, opts = {}) { // 11 MB, and raw.githubusercontent stalls on it often: the jsDelivr mirror cachedJson() falls back to is the // reliable path (the operator battle voice needs it, plan.mjs indexVoice). const charword = await cachedJson({ + ...opts, + mirrorPolicy, cacheFile: join(root, '.cache', 'gamedata', 'excel', 'charword_table.json'), url: RAW.gamedata + 'excel/charword_table.json', refresh: opts.refresh, @@ -70,6 +99,8 @@ export async function loadIndexes(root, opts = {}) { log: opts.log, }); const modelsData = await cachedJson({ + ...opts, + mirrorPolicy, cacheFile: join(root, '.cache', 'ark-models', 'models_data.json'), url: RAW.arkModels + 'models_data.json', refresh: opts.refresh, diff --git a/tools/assets/downloader.mjs b/tools/assets/downloader.mjs index 96efa4a9..51257e16 100644 --- a/tools/assets/downloader.mjs +++ b/tools/assets/downloader.mjs @@ -5,8 +5,9 @@ // download never appears at the final path. // - Per source: up to `retries` attempts with exponential backoff on network // errors, timeouts, 403/429/5xx and payloads that fail format validation; -// a 404/410 moves on immediately. Sources: each candidate URL, then its -// jsDelivr mirror (sources.mirrorUrl). +// a 404/410 moves on immediately. Each candidate uses its opt-in prefix proxy +// first (when enabled), then the original URL and jsDelivr fallback. The proxy gets one short attempt +// per file and shares a run-wide circuit breaker with the index/font fetchers. // - Idempotent: an existing file is kept when its size matches the ledger entry // of a previous download or the expected byte count from research, or (when // neither is known) when it passes format validation. The ledger lives in @@ -14,7 +15,7 @@ import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; -import { mirrorUrl } from './sources.mjs'; +import { MirrorPolicy } from './network.mjs'; import { validate } from './formats.mjs'; /** @@ -42,8 +43,11 @@ export class Downloader { * @param {(msg:string)=>void} [o.log] * @param {typeof fetch} [o.fetchImpl] * @param {number} [o.backoffMs] base retry delay (doubles per attempt) + * @param {'direct'|'mirror'} [o.source] preferred download source + * @param {string} [o.proxyPrefix] HTTPS prefix for GitHub downloads + * @param {MirrorPolicy} [o.mirrorPolicy] invocation-wide policy and mirror circuit breaker */ - constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400 }) { + constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400, source = 'direct', proxyPrefix, mirrorPolicy }) { this.root = root; this.ledgerPath = ledgerPath; this.concurrency = Math.max(1, Math.min(64, Number(concurrency) || 16)); @@ -52,6 +56,7 @@ export class Downloader { this.force = force; this.log = log; this.fetch = fetchImpl; + this.network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log }); this.backoffMs = Math.max(0, Number(backoffMs) || 0); this.ledger = { files: {} }; this.totals = { ok: 0, skip: 0, miss: 0, error: 0, bytesDownloaded: 0, sizeChanged: 0 }; @@ -101,36 +106,40 @@ export class Downloader { /** * Fetch one URL with retries. - * @returns {Promise<{buf:Buffer}|{notFound:true}|{error:string}>} + * @returns {Promise<{buf:Buffer}|{notFound:true}|{skipped:true}|{error:string}>} */ async fetchWithRetries(url, kind) { + if (this.network.skip(url)) return { skipped: true }; let lastErr = 'unknown error'; - for (let attempt = 1; attempt <= this.retries; attempt++) { + const attempts = this.network.isProxy(url) ? 1 : this.retries; + for (let attempt = 1; attempt <= attempts; attempt++) { try { - const res = await this.fetch(url, { + const res = await this.network.request(url, this.fetch, { headers: { 'user-agent': 'stronghold-protocol-fetch-assets/1.0' }, - signal: AbortSignal.timeout(this.timeoutMs), redirect: 'follow', - }); + }, this.timeoutMs); if (res.status === 404 || res.status === 410) { try { await res.body?.cancel(); } catch { /* ignore */ } + this.network.succeeded(url); // a missing asset is not a proxy outage return { notFound: true }; } if (!res.ok) { try { await res.body?.cancel(); } catch { /* ignore */ } throw new HttpError(res.status, url); } - const buf = Buffer.from(await res.arrayBuffer()); + const buf = await this.network.readBody(url, res); const len = Number(res.headers.get('content-length')); const enc = res.headers.get('content-encoding'); if (!enc && Number.isFinite(len) && len > 0 && len !== buf.length) { throw new Error(`truncated body ${buf.length}/${len} ${url}`); } if (!validate(kind, buf)) throw new Error(`invalid ${kind} payload (${buf.length} B) ${url}`); + this.network.succeeded(url); return { buf }; } catch (e) { lastErr = e?.message || String(e); - if (attempt < this.retries && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs)); + this.network.failed(url); + if (attempt < attempts && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs)); } } return { error: lastErr }; @@ -146,10 +155,10 @@ export class Downloader { if (kept >= 0) return { status: 'skip', bytes: kept }; let lastError = null; for (const url of job.urls) { - const sources = [url, mirrorUrl(url)].filter(Boolean); + const sources = this.network.urls(url); for (const src of sources) { const r = await this.fetchWithRetries(src, job.kind); - if (r.notFound) continue; + if (r.notFound || r.skipped) continue; if (r.error) { lastError = r.error; continue; } const abs = join(this.root, job.rel); await mkdir(dirname(abs), { recursive: true }); @@ -167,6 +176,7 @@ export class Downloader { return { status: 'ok', bytes: r.buf.length, url: src, sizeChanged }; } } + if (lastError) this.network.directFailureHint(); return lastError ? { status: 'error', bytes: 0, error: lastError } : { status: 'miss', bytes: 0, error: 'not found (404) on all sources' }; } diff --git a/tools/assets/manifest.mjs b/tools/assets/manifest.mjs index 7af766de..38b17b40 100644 --- a/tools/assets/manifest.mjs +++ b/tools/assets/manifest.mjs @@ -7,7 +7,7 @@ import { existsSync, statSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { join } from 'node:path'; -import { assetUrl, mirrorUrl } from './sources.mjs'; +import { assetUrl, githubSourceUrl, mirrorUrl } from './sources.mjs'; /** Manifest schema version (bump on breaking shape changes). */ export const MANIFEST_VERSION = 1; @@ -94,9 +94,11 @@ export function resolveTemplate(template, { root, spine, sourceOf = () => undefi const a = node.alts[i]; if (existsSync(join(root, a.rel))) { const src = sourceOf(a.rel); + // Provenance only: disabling/changing a proxy does not change the + // identity of a previously downloaded asset. const primary = node.alts[0].urls.flatMap((u) => [u, mirrorUrl(u)]); if (i > 0) fallbacks.push(`${path} ← ${src || a.urls[0]}`); - else if (src && !primary.includes(src)) fallbacks.push(`${path} ← ${src}`); + else if (src && !primary.includes(githubSourceUrl(src))) fallbacks.push(`${path} ← ${src}`); files.add(a.rel); return assetUrl(a.rel); } diff --git a/tools/assets/network.mjs b/tools/assets/network.mjs new file mode 100644 index 00000000..d257a715 --- /dev/null +++ b/tools/assets/network.mjs @@ -0,0 +1,115 @@ +// Explicit download-source selection and one shared policy per asset invocation. +// Selecting a source never looks up the user's public IP or makes a request. +import { downloadUrls, normalizeProxyPrefix } from './sources.mjs'; + +export function validateSource(mode) { + if (!['direct', 'mirror'].includes(mode)) throw new Error(`unknown asset source: ${mode} (direct|mirror)`); + return mode; +} + +export async function selectDownloadSource({ mode = 'direct', offline = false, proxyPrefix, log = console.log } = {}) { + validateSource(mode); + if (offline) return 'direct'; + const normalizedProxy = mode === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : ''; + log(mode === 'mirror' + ? `[network] 手动开启 GitHub 镜像：${normalizedProxy || '代理已禁用'}；文件仅校验格式和大小。` + : '[network] 原始源（未开启 GitHub 镜像）'); + return mode; +} + +/** Shared by indexes, files, Spine follow-ups and fonts; a tripped mirror stays off. */ +export class MirrorPolicy { + constructor({ source = 'direct', proxyPrefix, failureLimit = 3, timeoutMs = 8000, log = console.log } = {}) { + this.source = validateSource(source); + this.proxyPrefix = this.source === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : ''; + this.failureLimit = Math.max(1, Number(failureLimit) || 3); + this.timeoutMs = Math.max(1, Number(timeoutMs) || 8000); + this.log = log; + this.failures = 0; + this.disabled = false; + this.hintShown = false; + this.abortMirror = new AbortController(); + } + + urls(url) { + return downloadUrls(url, { source: this.disabled ? 'direct' : this.source, proxyPrefix: this.proxyPrefix }); + } + + isProxy(url) { + return this.source === 'mirror' && !!this.proxyPrefix && url.startsWith(this.proxyPrefix); + } + + skip(url) { return this.isProxy(url) && this.disabled; } + + async request(url, fetchImpl, options = {}, timeoutMs = 120000) { + if (!this.isProxy(url)) { + return fetchImpl(url, { ...options, signal: options.signal ?? AbortSignal.timeout(timeoutMs) }); + } + const headerDeadline = new AbortController(); + const signal = AbortSignal.any([headerDeadline.signal, this.abortMirror.signal]); + const timer = setTimeout(() => headerDeadline.abort(new Error('GitHub mirror response headers timed out')), Math.min(timeoutMs, this.timeoutMs)); + try { + return await fetchImpl(url, { ...options, signal }); + } finally { + clearTimeout(timer); + } + } + + async readBody(url, response) { + if (!this.isProxy(url)) return Buffer.from(await response.arrayBuffer()); + if (!response.body) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks = []; + let size = 0; + try { + while (true) { + const { value, done } = await readChunk(reader, this.timeoutMs, this.abortMirror.signal); + if (done) break; + const chunk = Buffer.from(value); + chunks.push(chunk); + size += chunk.length; + } + } catch (error) { + try { await reader.cancel(error); } catch { /* ignore */ } + throw error; + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks, size); + } + + succeeded(url) { + // A success from a request already in flight must not reopen a tripped circuit. + if (this.isProxy(url) && !this.disabled) this.failures = 0; + } + + failed(url) { + if (this.isProxy(url)) { + if (this.disabled || ++this.failures < this.failureLimit) return; + this.disabled = true; + this.log(`[network] GitHub 镜像连续失败 ${this.failures} 次：本次运行停用镜像，继续使用原始源 / jsDelivr。`); + this.abortMirror.abort(new Error('GitHub mirror disabled for this run')); + } + } + + directFailureHint() { + if (this.source !== 'direct' || this.hintShown) return; + this.hintShown = true; + this.log('[network] GitHub 下载失败；如需使用第三方镜像，可重新运行并加 --asset-source=mirror（或 SP_ASSET_SOURCE=mirror）。'); + } +} + +function readChunk(reader, timeoutMs, signal) { + return new Promise((resolve, reject) => { + let timer; + const cleanup = () => { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { cleanup(); reject(signal.reason ?? new Error('GitHub mirror request aborted')); }; + if (signal?.aborted) return onAbort(); + timer = setTimeout(() => { cleanup(); reject(new Error(`GitHub mirror response body idle for ${timeoutMs} ms`)); }, timeoutMs); + signal?.addEventListener('abort', onAbort, { once: true }); + reader.read().then((value) => { cleanup(); resolve(value); }, (error) => { cleanup(); reject(error); }); + }); +} diff --git a/tools/assets/sources.mjs b/tools/assets/sources.mjs index 130b7228..56c667b3 100644 --- a/tools/assets/sources.mjs +++ b/tools/assets/sources.mjs @@ -1,10 +1,59 @@ // Upstream asset sources (GitHub dumps of the official client), URL helpers and // local-path helpers shared by the asset pipeline (tools/fetch-assets.mjs). // -// Every URL we download from is a raw.githubusercontent.com URL; mirrorUrl() -// maps it to the equivalent jsDelivr URL used as a fallback when GitHub raw -// fails. jsDelivr 404s on the ArknightsAssets2 `voice` branch, so no mirror is -// offered for audio. See docs/ASSETS.md for the full source list and credits. +// GitHub URLs support a prefix proxy and a jsDelivr fallback. jsDelivr does +// not serve the voice branch, but the prefix proxy can still be tried there. +// See docs/ASSETS.md for the full source list and credits. + +export const DEFAULT_GITHUB_PROXY = 'https://gh-proxy.com/'; + +export function normalizeProxyPrefix(prefix = DEFAULT_GITHUB_PROXY) { + if (prefix.trim() === '') return ''; + const url = new URL(prefix); + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { + throw new Error('GitHub proxy prefix must be an HTTPS URL without credentials, query or fragment'); + } + return url.href.endsWith('/') ? url.href : url.href + '/'; +} + +function isGithubDownloadUrl(url) { + try { + const parsed = new URL(url); + return parsed.protocol === 'https:' && !parsed.username && !parsed.password && + ['github.com', 'raw.githubusercontent.com', 'objects.githubusercontent.com'].includes(parsed.host); + } catch { return false; } +} + +/** + * Legacy ledgers store the successful proxy URL. Spine page jobs derived from + * those URLs must use today's policy, not implicitly opt in to the old proxy. + * Recognize embedded public GitHub URLs even when the old custom prefix is no + * longer configured. An ordinary GitHub URL (including its path) stays intact. + */ +export function githubSourceUrl(url) { + let candidate = String(url); + while (candidate.startsWith('https://')) { + if (isGithubDownloadUrl(candidate)) return candidate; + const marker = candidate.indexOf('/https://', 8); + if (marker < 0) break; + candidate = candidate.slice(marker + 1); + } + return url; +} + +/** Only public GitHub download URLs are sent to the proxy; never prefix twice. */ +export function githubProxyUrl(url, prefix = DEFAULT_GITHUB_PROXY) { + const normalized = normalizeProxyPrefix(prefix); + if (!normalized || !isGithubDownloadUrl(url)) return null; + return normalized + url; +} + +/** Ordered transport alternatives; direct mode retains the existing jsDelivr order. */ +export function downloadUrls(url, { source = 'direct', proxyPrefix = DEFAULT_GITHUB_PROXY } = {}) { + url = githubSourceUrl(url); + const proxy = source === 'mirror' ? githubProxyUrl(url, proxyPrefix) : null; + return [...new Set([proxy, url, mirrorUrl(url)].filter(Boolean))]; +} /** Raw base URLs (always end with '/'). */ export const RAW = Object.freeze({ diff --git a/tools/fetch-assets.mjs b/tools/fetch-assets.mjs index 068edb75..675a9852 100644 --- a/tools/fetch-assets.mjs +++ b/tools/fetch-assets.mjs @@ -13,9 +13,10 @@ // that file from the extracted models (after a game update). // // Idempotent: existing files with the right size are skipped, so re-running is -// cheap. Downloads use ~16 parallel connections, 3 retries per source and a -// jsDelivr mirror fallback. Spine atlases get `size:` (and `pma: true` for -// enemies); every skeleton is parsed to resolve animation roles. +// cheap. Downloads use ~16 parallel connections, 3 retries per direct source, +// a jsDelivr fallback and an opt-in GitHub proxy (one short attempt per URL). +// Spine atlases get `size:` (and `pma: true` for enemies); every skeleton is +// parsed to resolve animation roles. // // The committed data/assets.json never shrinks by accident: an entry whose files // are missing here is left out of a rebuilt manifest, so a run on a machine where @@ -33,6 +34,8 @@ import { existsSync, realpathSync } from 'node:fs'; import { dirname, join, relative, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { Downloader } from './assets/downloader.mjs'; +import { MirrorPolicy, selectDownloadSource, validateSource } from './assets/network.mjs'; +import { normalizeProxyPrefix } from './assets/sources.mjs'; import { loadIndexes } from './assets/cache.mjs'; import { indexAudio, VOICE_DIRS } from './assets/audio.mjs'; import { buildPlan } from './assets/plan.mjs'; @@ -51,6 +54,7 @@ const LOCAL_SPINES = join(ROOT, LOCAL_ENEMY_SPINES_FILE); const HELP = `Usage: node tools/fetch-assets.mjs [options] --concurrency=N parallel downloads (default 16) + --asset-source=M direct (default) or mirror (opt-in; no public-IP lookup) --force re-download files even when present --offline no network: post-process what is on disk and rebuild data/assets.json --dry-run print the plan and exit @@ -64,18 +68,23 @@ const HELP = `Usage: node tools/fetch-assets.mjs [options] (without it such a run keeps the current manifest, lists the entries and exits 1) --local-spines rewrite ${LOCAL_ENEMY_SPINES_FILE} from the enemy models extracted by tools/local-extract/extract.py (public/assets/local/spine/enemy/) - --help this text`; + --help this text +Environment: SP_ASSET_SOURCE sets the default source; SP_GITHUB_PROXY sets the +HTTPS mirror prefix (default https://gh-proxy.com/; empty disables the proxy). +Mirror attempts have an 8 s response header timeout; response body has a separate idle timeout. Stops for this run after 3 consecutive +failures. Only explicitly enabled GitHub downloads use the third-party proxy.`; /** * Parse CLI flags. * @param {string[]} argv - * @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, allowShrink:boolean, localSpines:boolean, voiceLang:string, voiceAll:boolean, help:boolean}} + * @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, allowShrink:boolean, localSpines:boolean, voiceLang:string, voiceAll:boolean, help:boolean, source:string}} */ export function parseArgs(argv) { - const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, allowShrink: false, localSpines: false, voiceLang: 'cn', voiceAll: false, help: false }; + const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, allowShrink: false, localSpines: false, voiceLang: 'cn', voiceAll: false, help: false, source: process.env.SP_ASSET_SOURCE || 'direct' }; for (const a of argv) { const [k, v] = a.split('='); if (k === '--concurrency') o.concurrency = Math.max(1, Math.min(64, parseInt(v, 10) || 16)); + else if (k === '--asset-source') o.source = v; else if (k === '--force') o.force = true; else if (k === '--offline') o.offline = true; else if (k === '--dry-run') o.dryRun = true; @@ -88,9 +97,14 @@ export function parseArgs(argv) { else if (k === '--help' || k === '-h') o.help = true; else throw new Error(`unknown option ${a}\n${HELP}`); } + if (!o.help) validateSource(o.source); return o; } +export function resolveProxyPrefix(source, offline = false, value = process.env.SP_GITHUB_PROXY) { + return offline || source !== 'mirror' ? '' : normalizeProxyPrefix(value); +} + /** * The shrink guard of data/assets.json (header): the entries of the current manifest `prev` that `next` would drop, * and whether `next` may be written — always when nothing is dropped (or there is no current manifest), otherwise @@ -224,7 +238,11 @@ async function main() { readJson('docs/research/05-enemies.json'), readJson('docs/research/05-maps.json'), ]); - const { audioData, modelsData, charword } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log }); + const proxyPrefix = resolveProxyPrefix(opts.source, opts.offline); + const source = await selectDownloadSource({ mode: opts.source, offline: opts.offline, proxyPrefix, log }); + const mirrorPolicy = new MirrorPolicy({ source, proxyPrefix, log }); + const network = { source, proxyPrefix, mirrorPolicy }; + const { audioData, modelsData, charword } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log, ...network }); const audio = indexAudio(audioData); // The game data built by tools/build-data.mjs (when present) may reference more // spawnable enemies/tokens than research lists (e.g. 机变 enemy swaps): cover them too. @@ -255,7 +273,7 @@ async function main() { const dl = new Downloader({ root: ASSETS, ledgerPath: join(CACHE, 'assets-ledger.json'), - concurrency: opts.concurrency, force: opts.force, log, + concurrency: opts.concurrency, force: opts.force, log, ...network, }); await dl.loadLedger(); const downloadErrors = opts.offline ? [] : await downloadLeaves(leaves, dl, ASSETS, 'files'); @@ -263,7 +281,7 @@ async function main() { // Fonts let fontErrors = []; if (!opts.offline) { - const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log }); + const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log, ...network }); await fdl.loadLedger(); await fdl.run(fontJobs(), 'fonts'); dl.totals.bytesDownloaded += fdl.totals.bytesDownloaded; diff --git a/tools/setup.mjs b/tools/setup.mjs index 45dce49e..fd4de332 100755 --- a/tools/setup.mjs +++ b/tools/setup.mjs @@ -19,6 +19,9 @@ // Options: // --check report only, change nothing (exit 1 when something essential is missing) // --no-assets skip the art/audio download +// --asset-source=M direct (default) or mirror (opt-in; no public-IP lookup) +// SP_ASSET_SOURCE sets the default; SP_GITHUB_PROXY sets the HTTPS prefix (https://gh-proxy.com/) +// An empty SP_GITHUB_PROXY disables the proxy, including in mirror mode. // --no-local skip the local-client detection and extraction // --local extract from the local client without asking (re-extracts when already done) // --game AssetBundle root of the local client (…/StreamingAssets/AB/Windows or PlayCover …/Documents/Bundles) @@ -35,6 +38,7 @@ import path from 'node:path'; import readline from 'node:readline'; import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; +import { validateSource } from './assets/network.mjs'; export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); export const MIN_NODE = 22; @@ -328,11 +332,13 @@ function ensureVenv(py, log) { // --------------------------------------------------------------------------------------------------- function parseArgs(argv) { - const o = { check: false, assets: true, local: 'ask', game: null, yes: false, quiet: false, help: false }; + const o = { check: false, assets: true, local: 'ask', game: null, yes: false, quiet: false, help: false, source: process.env.SP_ASSET_SOURCE || 'direct' }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--check') o.check = true; else if (a === '--no-assets') o.assets = false; + else if (a === '--asset-source') o.source = argv[++i]; + else if (a.startsWith('--asset-source=')) o.source = a.slice('--asset-source='.length); else if (a === '--no-local') o.local = 'no'; else if (a === '--local') o.local = 'force'; else if (a === '--game') { o.game = argv[++i] || null; if (o.local !== 'no') o.local = 'force'; } @@ -342,6 +348,7 @@ function parseArgs(argv) { else if (a === '-h' || a === '--help') o.help = true; else throw new Error(`未知参数 / unknown option: ${a}（--help 查看用法）`); } + if (!o.help) validateSource(o.source); return o; } @@ -413,7 +420,7 @@ async function main() { const what = !assets.present ? `首次下载约 ${assets.bytes ? mb(assets.bytes) : '270 MB'}，可随时中断，重新运行会续传` : `补全缺失的 ${assets.missing} 个文件`; log(`\n${c.cyan('▶')} 下载美术与音频素材（${what}）…`); - const r = run(process.execPath, [path.join(ROOT, 'tools', 'fetch-assets.mjs')]); + const r = run(process.execPath, [path.join(ROOT, 'tools', 'fetch-assets.mjs'), `--asset-source=${opts.source}`]); assets = checkAssets(); if (!r.ok && !assets.ok) log(c.warn(' 素材下载未完成（网络问题？）。游戏仍可运行（使用占位图），稍后重新运行 setup 即可续传。')); }