Asset downloads can use a gh-proxy mirror when chosen by hand (--asset-source / SP_ASSET_SOURCE): one try per URL, the original source as fallback, the mirror off for the run after 3 failures in a row (#24)
Off by default; the original sources, retries, skipped files and the manifest-shrink guard are unchanged. Contributed by @NekoTC.
This commit is contained in:
1 parent
f6f5ed50cd
commit
f6794a2f2c
9 files changed
+671
-45
No files matched your search
+25
-1
@@ -36,6 +36,30 @@
|
||||
|
||||
等价的手动命令:`npm ci`、`node tools/setup.mjs`、`npm start`。
|
||||
|
||||
#### 国内镜像下载
|
||||
|
||||
Setup 默认使用「GitHub 原始源 → jsDelivr」,不查询公网 IP,也不请求 gh-proxy.com。GitHub 下载失败时会提示如何手动开启镜像;仅添加提示,不自动切换到第三方代理。
|
||||
|
||||
镜像方法是在完整 GitHub 链接前加 `https://gh-proxy.com/`,例如:
|
||||
|
||||
```text
|
||||
https://gh-proxy.com/https://raw.githubusercontent.com/OWNER/REPO/BRANCH/file.png
|
||||
```
|
||||
|
||||
手动开启后顺序为「前缀镜像 → 原始源 → jsDelivr」。索引、图片、Spine、音频和字体都使用此规则(音频 voice 分支跳过 jsDelivr)。镜像是第三方代理;当前只校验格式和大小,没有内容哈希校验,请自行决定是否信任并启用。npm / pip 依赖不使用 GitHub 前缀。
|
||||
|
||||
```powershell
|
||||
node tools/setup.mjs --asset-source=mirror # 手动优先国内镜像
|
||||
node tools/setup.mjs --asset-source=direct # 默认:仅原始源和 jsDelivr,不使用前缀代理
|
||||
$env:SP_ASSET_SOURCE = 'mirror' # 也可用环境变量显式启用
|
||||
```
|
||||
|
||||
`node tools/fetch-assets.mjs` 同样支持 `--asset-source=direct|mirror`。命令行优先于 `SP_ASSET_SOURCE`。默认镜像前缀为 `https://gh-proxy.com/`,可通过 `SP_GITHUB_PROXY` 指定其他 HTTPS 前缀;仅配置前缀不会启用镜像。将 `SP_GITHUB_PROXY` 设为空字符串(或全空格)可彻底禁用前缀代理,即使选择了 `mirror` 模式;未设置此变量与显式设空不同,前者使用默认前缀。Windows PowerShell 的某些版本会将空值视为删除变量,可设置 `$env:SP_GITHUB_PROXY = ' '` 或使用 `--asset-source=direct` 来明确禁用。前缀只处理 GitHub 下载链接,不重复添加。
|
||||
|
||||
镜像请求每个 URL 只尝试一次,响应头超时 8 秒,响应体有独立的空闲超时,失败即尝试原始源。连续 3 次网络错误、HTTP 错误或无效内容会在本次运行中关闭镜像,后续索引、素材和字体共享该状态;正在进行的镜像请求也会中止并回退。成功会清零连续失败次数;404 / 410 是资源不存在,不触发熔断。再次运行脚本会重新尝试手动启用的镜像。原始源的重试、已有文件跳过和 0.1.1 的清单缩减保护保持不变。
|
||||
|
||||
从历史下载记录派生的 Spine 补充贴图也按本次设置重新选择来源,禁用后不会沿用旧代理地址。
|
||||
|
||||
### 1.2 防火墙
|
||||
|
||||
- 第一次启动时 Windows 会弹出「Windows 安全中心警报」:勾选**专用网络**并点「允许访问」。
|
||||
@@ -235,7 +259,7 @@ services:
|
||||
| 任何问题 | `node tools/doctor.mjs`:Node 版本、依赖、素材完整性、端口、局域网地址、防火墙、网络类型 |
|
||||
| `端口已被占用 / EADDRINUSE` | 已经有一个服务器在运行(自启任务?)或其他程序占用 3000:换端口 `scripts\start-windows.bat --port 3001` |
|
||||
| 朋友打不开页面 | 防火墙规则 / 网络类型(1.2);确认用的是 `LAN` 地址而不是 `localhost`;访客 Wi-Fi 常开启「AP 隔离」;不在同一网络请看第 2 节 |
|
||||
| 画面是占位图、没有声音 | 素材没下完:重新运行 `node tools/setup.mjs`(会续传);缺失明细在 `.cache/assets-report.json`。GitHub 原始地址访问失败时会自动改用 jsDelivr 镜像 |
|
||||
| 画面是占位图、没有声音 | 素材没下完:重新运行 `node tools/setup.mjs`(会续传);缺失明细在 `.cache/assets-report.json`。默认仅原始源和 jsDelivr;可用 `--asset-source=mirror` 手动开启前缀镜像(见上文) |
|
||||
| 素材下载很慢 / 失败 | 网络问题可随时中断,重新运行会跳过已完成的文件;`node tools/fetch-assets.mjs --concurrency=4` 降低并发。有文件没下载成功时,素材清单 `data/assets.json` 保持不变(脚本列出缺少的条目并以非零状态结束;游戏里缺的图片用占位图,缺的声音不播放),重新运行即可补齐 |
|
||||
| 表情显示成默认图标、「玩法说明」只有文字要点 | 素材没下载完整:重新运行 `node tools/setup.mjs`(表情和教程图随其他素材一起从公开镜像下载,不需要客户端);缺失明细在 `.cache/assets-report.json` |
|
||||
| 本地提取失败 | 游戏照常运行,只是第 6 节表格里的几样换成替代样式。确认客户端已下载全部资源;Python 版本太新导致依赖安装失败时,安装 Python 3.12 后删除 `.venv-extract` 再运行 `node tools/setup.mjs --local` |
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createServer } from 'node:http';
|
||||
import { MirrorPolicy, selectDownloadSource } from '../tools/assets/network.mjs';
|
||||
import { downloadUrls, githubProxyUrl, githubSourceUrl, normalizeProxyPrefix } from '../tools/assets/sources.mjs';
|
||||
import { Downloader } from '../tools/assets/downloader.mjs';
|
||||
import { cachedJson, loadIndexes } from '../tools/assets/cache.mjs';
|
||||
import { resolveTemplate } from '../tools/assets/manifest.mjs';
|
||||
import { processModels } from '../tools/assets/spine.mjs';
|
||||
|
||||
const RAW = 'https://raw.githubusercontent.com/o/r/main/a.json';
|
||||
const PROXY = 'https://gh-proxy.com/' + RAW;
|
||||
const CDN = 'https://cdn.jsdelivr.net/gh/o/r@main/a.json';
|
||||
const quiet = () => {};
|
||||
|
||||
async function fixture(t) {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'sp-network-'));
|
||||
t.after(() => rm(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test('default, manual source and offline mode perform no public IP lookup', async (t) => {
|
||||
const fetch = t.mock.method(globalThis, 'fetch', async () => { assert.fail('unexpected network request'); });
|
||||
assert.equal(await selectDownloadSource({ log: quiet }), 'direct');
|
||||
for (const mode of ['direct', 'mirror']) assert.equal(await selectDownloadSource({ mode, log: quiet }), mode);
|
||||
assert.equal(await selectDownloadSource({ offline: true }), 'direct');
|
||||
await assert.rejects(selectDownloadSource({ mode: 'invalid' }), /unknown asset source/);
|
||||
await assert.rejects(selectDownloadSource({ mode: 'auto' }), /unknown asset source/);
|
||||
assert.equal(fetch.mock.callCount(), 0);
|
||||
});
|
||||
|
||||
test('mirror startup log names its proxy and states the validation scope', async () => {
|
||||
const logs = [];
|
||||
assert.equal(await selectDownloadSource({ mode: 'mirror', proxyPrefix: 'https://mirror.example/p/', log: (msg) => logs.push(msg) }), 'mirror');
|
||||
assert.match(logs[0], /https:\/\/mirror\.example\/p\//);
|
||||
assert.match(logs[0], /仅校验格式和大小/);
|
||||
});
|
||||
|
||||
test('URL order, voice support, custom prefix, and no double prefix', () => {
|
||||
assert.deepEqual(downloadUrls(RAW, { source: 'mirror' }), [PROXY, RAW, CDN]);
|
||||
assert.deepEqual(downloadUrls(RAW), [RAW, CDN]);
|
||||
assert.deepEqual(downloadUrls(RAW, { source: 'direct', proxyPrefix: 'https://custom.example/' }), [RAW, CDN]);
|
||||
const voice = 'https://raw.githubusercontent.com/ArknightsAssets/ArknightsAssets2/voice/assets/x.mp3';
|
||||
assert.deepEqual(downloadUrls(voice, { source: 'mirror' }), ['https://gh-proxy.com/' + voice, voice]);
|
||||
assert.deepEqual(downloadUrls(PROXY, { source: 'mirror' }), [PROXY, RAW, CDN]);
|
||||
assert.deepEqual(downloadUrls(PROXY), [RAW, CDN]);
|
||||
assert.deepEqual(downloadUrls('https://old-proxy.example/prefix/' + RAW, { source: 'mirror', proxyPrefix: '' }), [RAW, CDN]);
|
||||
assert.deepEqual(downloadUrls(PROXY, { source: 'mirror', proxyPrefix: 'https://new-proxy.example/' }), ['https://new-proxy.example/' + RAW, RAW, CDN]);
|
||||
assert.equal(githubSourceUrl('https://old-proxy.example/' + PROXY), RAW);
|
||||
assert.equal(githubSourceUrl(RAW + '/https://example.com/path'), RAW + '/https://example.com/path');
|
||||
for (const url of ['https://example.com/a', 'https://github.com.evil.example/a', 'https://user:secret@github.com/a']) {
|
||||
assert.equal(githubProxyUrl(url), null);
|
||||
}
|
||||
assert.equal(githubProxyUrl(RAW, 'https://mirror.example/prefix'), 'https://mirror.example/prefix/' + RAW);
|
||||
assert.equal(normalizeProxyPrefix(), 'https://gh-proxy.com/');
|
||||
for (const proxyPrefix of ['', ' ']) {
|
||||
assert.equal(normalizeProxyPrefix(proxyPrefix), '');
|
||||
assert.equal(githubProxyUrl(RAW, proxyPrefix), null);
|
||||
assert.deepEqual(downloadUrls(RAW, { source: 'mirror', proxyPrefix }), [RAW, CDN]);
|
||||
}
|
||||
for (const prefix of ['http://mirror.example/', 'https://mirror.example/?q=x', 'https://user:secret@mirror.example/']) {
|
||||
assert.throws(() => normalizeProxyPrefix(prefix), /HTTPS URL/);
|
||||
}
|
||||
});
|
||||
|
||||
test('downloader uses proxy, validates payload, falls back, and skips existing files', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', retries: 1, backoffMs: 0, log: quiet,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
return url === PROXY ? new Response('<html>bad gateway</html>') : Response.json({ ok: true });
|
||||
} });
|
||||
const job = { rel: 'a.json', urls: [RAW], kind: 'json' };
|
||||
assert.equal((await dl.run([job])).get(job.rel).url, RAW);
|
||||
assert.deepEqual(calls, [PROXY, RAW]);
|
||||
assert.equal((await dl.run([job])).get(job.rel).status, 'skip');
|
||||
assert.equal(calls.length, 2);
|
||||
});
|
||||
|
||||
test('proxy provenance is recognized as the primary asset, not a fallback asset', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', log: quiet,
|
||||
fetchImpl: async () => Response.json({ ok: true }) });
|
||||
const job = { rel: 'a.json', urls: [RAW], kind: 'json' };
|
||||
assert.equal((await dl.run([job])).get(job.rel).url, PROXY);
|
||||
for (const source of [PROXY, 'https://old-proxy.example/prefix/' + RAW]) {
|
||||
const resolved = resolveTemplate({ icon: { alts: [job] } }, { root: dir, spine: new Map(), sourceOf: () => source });
|
||||
assert.deepEqual(resolved.fallbacks, [], 'disabling/changing the proxy does not change default proxy provenance');
|
||||
assert.equal(resolved.value.icon, '/assets/a.json');
|
||||
}
|
||||
});
|
||||
|
||||
test('Spine pages from old proxy ledger entries respect direct and explicitly disabled mirror mode', async (t) => {
|
||||
const base = 'https://raw.githubusercontent.com/o/r/main/';
|
||||
const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII=', 'base64');
|
||||
for (const prefix of ['https://gh-proxy.com/', 'https://old-proxy.example/prefix/']) {
|
||||
for (const opts of [{ source: 'direct' }, { source: 'mirror', proxyPrefix: '' }]) {
|
||||
const dir = await fixture(t);
|
||||
const atlas = 'extra.png\nsize: 1,1\nformat: RGBA8888\nfilter: Linear,Linear\nrepeat: none\n';
|
||||
await writeFile(join(dir, 'model.atlas'), atlas);
|
||||
const calls = [];
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), log: quiet, ...opts,
|
||||
fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
return url === base + 'extra.png' ? new Response(png) : new Response('', { status: 404 });
|
||||
} });
|
||||
dl.ledger.files['model.atlas'] = { url: prefix + base + 'model.atlas', bytes: Buffer.byteLength(atlas) };
|
||||
const model = { key: 'model', dir: '', baseUrl: base, pngs: [],
|
||||
skel: { rel: 'model.skel', urls: [base + 'model.skel'], kind: 'skel' },
|
||||
atlas: { rel: 'model.atlas', urls: [base + 'model.atlas'], kind: 'atlas' } };
|
||||
await processModels(new Map([['model', model]]), { root: dir, dl, cachePath: join(dir, 'spine.json'), log: quiet });
|
||||
assert.ok(calls.includes(base + 'extra.png'));
|
||||
assert.ok(calls.every((url) => url.startsWith(base) || url.startsWith('https://cdn.jsdelivr.net/')));
|
||||
assert.equal(dl.ledger.files['extra.png'].url, base + 'extra.png');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('all three indexes use the selected proxy and cached/offline reads make no request', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
const opts = { source: 'mirror', log: quiet, fetchImpl: async (url) => { calls.push(url); return Response.json({ index: true }); } };
|
||||
const result = await loadIndexes(dir, opts);
|
||||
assert.deepEqual(result, { audioData: { index: true }, modelsData: { index: true }, charword: { index: true } });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.ok(calls.some((url) => url.endsWith('/charword_table.json')));
|
||||
assert.ok(calls.every((url) => url.startsWith('https://gh-proxy.com/https://raw.githubusercontent.com/')));
|
||||
await loadIndexes(dir, { ...opts, offline: true, refresh: true });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.deepEqual(JSON.parse(await readFile(join(dir, '.cache', 'ark-models', 'models_data.json'), 'utf8')), { index: true });
|
||||
});
|
||||
|
||||
test('index proxy miss falls back to raw', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
assert.deepEqual(await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, source: 'mirror', log: quiet,
|
||||
fetchImpl: async (url) => { calls.push(url); return url === PROXY ? new Response('', { status: 404 }) : Response.json({ ok: true }); } }), { ok: true });
|
||||
assert.deepEqual(calls, [PROXY, RAW]);
|
||||
});
|
||||
|
||||
test('direct retries that recover do not print a failure hint or contact the prefix proxy', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
const logs = [];
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), retries: 1, backoffMs: 0,
|
||||
log: (msg) => logs.push(msg), fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
return url.startsWith('https://raw.githubusercontent.com/') ? new Response('', { status: 503 }) : Response.json({ ok: true });
|
||||
} });
|
||||
await dl.run([1, 2].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' })));
|
||||
assert.equal(calls.length, 4);
|
||||
assert.ok(calls.every((url) => url.startsWith('https://raw.githubusercontent.com/') || url.startsWith('https://cdn.jsdelivr.net/')));
|
||||
assert.equal(logs.filter((msg) => msg.includes('--asset-source=mirror')).length, 0);
|
||||
});
|
||||
|
||||
test('default index retries that fall back successfully do not print a failure hint', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
const logs = [];
|
||||
const result = await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, backoffMs: 0,
|
||||
log: (msg) => logs.push(msg), fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
return url === RAW ? new Response('', { status: 503 }) : Response.json({ ok: true });
|
||||
} });
|
||||
assert.deepEqual(result, { ok: true });
|
||||
assert.deepEqual(calls, [RAW, RAW, RAW, CDN]);
|
||||
assert.equal(logs.filter((msg) => msg.includes('--asset-source=mirror')).length, 0);
|
||||
});
|
||||
|
||||
test('empty proxy setting disables every downloader and index proxy request in mirror mode', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
const opts = { source: 'mirror', proxyPrefix: '', log: quiet, fetchImpl: async (url) => { calls.push(url); return Response.json({ ok: true }); } };
|
||||
await loadIndexes(dir, opts);
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), ...opts });
|
||||
await dl.run([{ rel: 'a.json', urls: [RAW], kind: 'json' }]);
|
||||
assert.equal(calls.length, 4);
|
||||
assert.ok(calls.every((url) => url.startsWith('https://raw.githubusercontent.com/')));
|
||||
});
|
||||
|
||||
test('one circuit spans indexes, assets and fonts and stops after three failed proxy attempts', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const calls = [];
|
||||
const log = [];
|
||||
const mirrorPolicy = new MirrorPolicy({ source: 'mirror', log: (msg) => log.push(msg) });
|
||||
const opts = { source: 'mirror', mirrorPolicy, log: quiet, fetchImpl: async (url) => {
|
||||
calls.push(url);
|
||||
return url.startsWith('https://gh-proxy.com/') ? new Response('', { status: 503 }) : Response.json({ ok: true });
|
||||
} };
|
||||
await loadIndexes(dir, opts); // three consecutive proxy failures across the indexes
|
||||
assert.equal(mirrorPolicy.disabled, true);
|
||||
const dl = new Downloader({ root: join(dir, 'assets'), ledgerPath: join(dir, 'ledger.json'), concurrency: 1, ...opts });
|
||||
await dl.run([1, 2, 3].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' })));
|
||||
const fonts = new Downloader({ root: join(dir, 'fonts'), ledgerPath: join(dir, 'fonts-ledger.json'), ...opts });
|
||||
await fonts.run([{ rel: 'font.json', urls: [RAW + '/font'], kind: 'json' }]);
|
||||
assert.equal(calls.filter((url) => url.startsWith('https://gh-proxy.com/')).length, 3, 'no retries on the proxy, no later proxy requests');
|
||||
assert.equal(dl.totals.ok, 3);
|
||||
assert.equal(fonts.totals.ok, 1);
|
||||
assert.equal(log.length, 1, 'one circuit-open warning for the shared invocation');
|
||||
});
|
||||
|
||||
test('successful and not-found proxy responses reset the failure streak, opened circuit stays off', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const statuses = [503, 503, 200, 503, 404, 503, 503, 503];
|
||||
let proxyCalls = 0;
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', concurrency: 1, log: quiet,
|
||||
fetchImpl: async (url) => {
|
||||
if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true });
|
||||
const status = statuses[proxyCalls++] ?? 200;
|
||||
return status === 200 ? Response.json({ ok: true }) : new Response('', { status });
|
||||
} });
|
||||
await dl.run(Array.from({ length: 10 }, (_, i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' })));
|
||||
assert.equal(proxyCalls, 8);
|
||||
assert.equal(dl.totals.ok, 10);
|
||||
});
|
||||
|
||||
test('invalid payloads trip the same circuit as HTTP failures', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
let proxyCalls = 0;
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', concurrency: 1, log: quiet,
|
||||
fetchImpl: async (url) => {
|
||||
if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true });
|
||||
proxyCalls++;
|
||||
return new Response('<html>upstream error</html>');
|
||||
} });
|
||||
await dl.run([1, 2, 3, 4].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' })));
|
||||
assert.equal(proxyCalls, 3);
|
||||
assert.equal(dl.totals.ok, 4);
|
||||
});
|
||||
|
||||
test('slow proxy requests use a short deadline, abort, and fall back without repeated waits', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
let proxyCalls = 0;
|
||||
const mirrorPolicy = new MirrorPolicy({ source: 'mirror', timeoutMs: 10, failureLimit: 2, log: quiet });
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', mirrorPolicy, concurrency: 1, log: quiet,
|
||||
fetchImpl: async (url, { signal }) => {
|
||||
if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true });
|
||||
proxyCalls++;
|
||||
return new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), { once: true }));
|
||||
} });
|
||||
const keepAlive = setInterval(quiet, 1000);
|
||||
const started = Date.now();
|
||||
try { await dl.run([1, 2, 3].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' }))); }
|
||||
finally { clearInterval(keepAlive); }
|
||||
assert.equal(proxyCalls, 2);
|
||||
assert.equal(dl.totals.ok, 3);
|
||||
assert.ok(Date.now() - started < 3000, 'uses the mirror deadline, not 120 seconds times three');
|
||||
});
|
||||
|
||||
test('opening the circuit aborts concurrent proxy work and later jobs go direct', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
let proxyCalls = 0;
|
||||
let aborted = 0;
|
||||
const mirrorPolicy = new MirrorPolicy({ source: 'mirror', failureLimit: 1, log: quiet });
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), source: 'mirror', mirrorPolicy, concurrency: 3, log: quiet,
|
||||
fetchImpl: async (url, { signal }) => {
|
||||
if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true });
|
||||
proxyCalls++;
|
||||
if (proxyCalls === 3) return new Response('', { status: 503 });
|
||||
return new Promise((_, reject) => signal.addEventListener('abort', () => { aborted++; reject(signal.reason); }, { once: true }));
|
||||
} });
|
||||
await dl.run([1, 2, 3, 4, 5].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' })));
|
||||
assert.equal(proxyCalls, 3);
|
||||
assert.equal(aborted, 2);
|
||||
assert.equal(dl.totals.ok, 5);
|
||||
});
|
||||
|
||||
test('the mirror body idle deadline covers stalled response bodies for indexes and files', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const server = createServer((req, res) => {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.write('{"ok":'); // Headers arrive, but the body never finishes.
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', resolve);
|
||||
});
|
||||
t.after(() => new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); }));
|
||||
let proxyCalls = 0;
|
||||
const mirrorPolicy = new MirrorPolicy({ source: 'mirror', timeoutMs: 100, failureLimit: 2, log: quiet });
|
||||
const opts = { mirrorPolicy, log: quiet, fetchImpl: async (url, options) => {
|
||||
if (!url.startsWith('https://gh-proxy.com/')) return Response.json({ ok: true });
|
||||
proxyCalls++;
|
||||
return fetch(`http://127.0.0.1:${server.address().port}`, options);
|
||||
} };
|
||||
const started = Date.now();
|
||||
assert.deepEqual(await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, ...opts }), { ok: true });
|
||||
const dl = new Downloader({ root: dir, ledgerPath: join(dir, 'ledger.json'), concurrency: 1, ...opts });
|
||||
await dl.run([1, 2].map((i) => ({ rel: `${i}.json`, urls: [RAW + i], kind: 'json' })));
|
||||
assert.equal(proxyCalls, 2);
|
||||
assert.equal(dl.totals.ok, 2);
|
||||
assert.ok(Date.now() - started < 4000, 'stalled body reads time out instead of waiting indefinitely');
|
||||
});
|
||||
|
||||
test('mirror header timeout does not cap a steadily streaming index or asset body', async (t) => {
|
||||
const dir = await fixture(t);
|
||||
const payload = Buffer.from('{"streamed":true}');
|
||||
const delayedResponse = () => {
|
||||
let offset = 0;
|
||||
let timer;
|
||||
return new Response(new ReadableStream({
|
||||
start(controller) {
|
||||
timer = setInterval(() => {
|
||||
if (offset >= payload.length) {
|
||||
clearInterval(timer);
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(payload.subarray(offset, ++offset));
|
||||
}, 20);
|
||||
},
|
||||
cancel() { clearInterval(timer); },
|
||||
}), { headers: { 'content-type': 'application/json', 'content-length': String(payload.length) } });
|
||||
};
|
||||
const mirrorPolicy = new MirrorPolicy({ source: 'mirror', timeoutMs: 100, log: quiet });
|
||||
const opts = { mirrorPolicy, log: quiet, fetchImpl: async (url) => url.startsWith('https://gh-proxy.com/') ? delayedResponse() : Response.json({ direct: true }) };
|
||||
const started = Date.now();
|
||||
assert.deepEqual(await cachedJson({ cacheFile: join(dir, 'index.json'), url: RAW, ...opts }), { streamed: true });
|
||||
const dl = new Downloader({ root: join(dir, 'assets'), ledgerPath: join(dir, 'ledger.json'), ...opts });
|
||||
const result = await dl.run([{ rel: 'slow.json', urls: [RAW], kind: 'json' }]);
|
||||
assert.equal(result.get('slow.json').status, 'ok');
|
||||
assert.ok(Date.now() - started > 100, 'the body takes longer than the header deadline while making progress');
|
||||
});
|
||||
|
||||
test('CLI source defaults to direct and retains the 0.1.1 shrink/local-spine flags', async () => {
|
||||
const { parseArgs } = await import('../tools/fetch-assets.mjs');
|
||||
const old = process.env.SP_ASSET_SOURCE;
|
||||
try {
|
||||
delete process.env.SP_ASSET_SOURCE;
|
||||
assert.equal(parseArgs([]).source, 'direct');
|
||||
process.env.SP_ASSET_SOURCE = 'mirror';
|
||||
assert.equal(parseArgs([]).source, 'mirror');
|
||||
assert.equal(parseArgs([]).voiceLang, 'cn');
|
||||
assert.equal(parseArgs([]).voiceAll, false);
|
||||
const opts = parseArgs(['--asset-source=direct', '--allow-shrink', '--local-spines', '--voice-lang=jp', '--voice-all']);
|
||||
assert.equal(opts.source, 'direct');
|
||||
assert.equal(opts.allowShrink, true);
|
||||
assert.equal(opts.localSpines, true);
|
||||
assert.equal(opts.voiceLang, 'jp');
|
||||
assert.equal(opts.voiceAll, true);
|
||||
} finally {
|
||||
if (old === undefined) delete process.env.SP_ASSET_SOURCE;
|
||||
else process.env.SP_ASSET_SOURCE = old;
|
||||
}
|
||||
});
|
||||
|
||||
test('SP_GITHUB_PROXY is parsed only for online mirror mode', async () => {
|
||||
const { resolveProxyPrefix } = await import('../tools/fetch-assets.mjs');
|
||||
assert.equal(resolveProxyPrefix('direct', false, 'not-an-https-url'), '');
|
||||
assert.equal(resolveProxyPrefix('mirror', true, 'not-an-https-url'), '');
|
||||
assert.equal(resolveProxyPrefix('mirror', false, ''), '');
|
||||
assert.throws(() => resolveProxyPrefix('mirror', false, 'not-an-https-url'), /Invalid URL/);
|
||||
});
|
||||
|
||||
test('setup and asset CLI document and validate source options before doing work', () => {
|
||||
for (const script of ['tools/setup.mjs', 'tools/fetch-assets.mjs']) {
|
||||
const help = spawnSync(process.execPath, [script, '--help'], { encoding: 'utf8' });
|
||||
assert.equal(help.status, 0, help.stderr);
|
||||
assert.match(help.stdout, /--asset-source/);
|
||||
assert.match(help.stdout, /SP_GITHUB_PROXY/);
|
||||
const invalid = spawnSync(process.execPath, [script, '--asset-source=invalid'], { encoding: 'utf8' });
|
||||
assert.notEqual(invalid.status, 0);
|
||||
assert.match(invalid.stderr, /unknown asset source/);
|
||||
}
|
||||
});
|
||||
+45
-14
@@ -6,7 +6,8 @@
|
||||
|
||||
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { RAW, mirrorUrl } from './sources.mjs';
|
||||
import { RAW } from './sources.mjs';
|
||||
import { MirrorPolicy } from './network.mjs';
|
||||
|
||||
/**
|
||||
* Read a cached JSON file, downloading it first when missing or unparsable.
|
||||
@@ -16,44 +17,70 @@ import { RAW, mirrorUrl } from './sources.mjs';
|
||||
* @param {boolean} [o.refresh] force re-download
|
||||
* @param {boolean} [o.offline] never download (throw when the cache is missing)
|
||||
* @param {(m:string)=>void} [o.log]
|
||||
* @param {'direct'|'mirror'} [o.source]
|
||||
* @param {string} [o.proxyPrefix]
|
||||
* @param {typeof fetch} [o.fetchImpl]
|
||||
* @param {MirrorPolicy} [o.mirrorPolicy] shared invocation-wide mirror circuit breaker
|
||||
* @param {number} [o.timeoutMs]
|
||||
* @param {number} [o.backoffMs]
|
||||
* @returns {Promise<any>} parsed JSON
|
||||
*/
|
||||
export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log }) {
|
||||
export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log, source = 'direct', proxyPrefix, fetchImpl = globalThis.fetch, mirrorPolicy, timeoutMs = 180000, backoffMs = 500 }) {
|
||||
if (!refresh || offline) {
|
||||
try { return JSON.parse(await readFile(cacheFile, 'utf8')); } catch (e) {
|
||||
if (offline) throw new Error(`--offline: cached index ${cacheFile} is missing or corrupt (${e.message}); run once online`);
|
||||
}
|
||||
}
|
||||
const network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log });
|
||||
let lastErr = null;
|
||||
for (const src of [url, mirrorUrl(url)].filter(Boolean)) {
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
for (const src of network.urls(url)) {
|
||||
if (network.skip(src)) continue;
|
||||
const attempts = network.isProxy(src) ? 1 : 3;
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
let text, json;
|
||||
try {
|
||||
log(`[cache] downloading ${src}`);
|
||||
const res = await fetch(src, { signal: AbortSignal.timeout(180000) });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const text = await res.text();
|
||||
const json = JSON.parse(text);
|
||||
await mkdir(dirname(cacheFile), { recursive: true });
|
||||
await writeFile(cacheFile + '.tmp', text);
|
||||
await rename(cacheFile + '.tmp', cacheFile);
|
||||
return json;
|
||||
const res = await network.request(src, fetchImpl, {}, timeoutMs);
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel();
|
||||
if (res.status === 404 || res.status === 410) {
|
||||
network.succeeded(src); // missing index, not a proxy outage
|
||||
lastErr = new Error(`HTTP ${res.status}`);
|
||||
break;
|
||||
}
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
text = (await network.readBody(src, res)).toString('utf8');
|
||||
json = JSON.parse(text);
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
await new Promise((r) => setTimeout(r, 500 * attempt));
|
||||
network.failed(src);
|
||||
if (attempt < attempts && backoffMs) await new Promise((r) => setTimeout(r, backoffMs * attempt));
|
||||
continue;
|
||||
}
|
||||
network.succeeded(src);
|
||||
// A local disk failure is not evidence that the mirror is unhealthy.
|
||||
await mkdir(dirname(cacheFile), { recursive: true });
|
||||
await writeFile(cacheFile + '.tmp', text);
|
||||
await rename(cacheFile + '.tmp', cacheFile);
|
||||
return json;
|
||||
}
|
||||
}
|
||||
if (lastErr) network.directFailureHint();
|
||||
throw new Error(`cannot fetch ${url}: ${lastErr?.message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load audio_data.json (official), charword_table.json (official voice slots) and Ark-Models models_data.json.
|
||||
* @param {string} root project root
|
||||
* @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void}} [opts]
|
||||
* @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void, source?:'direct'|'mirror', proxyPrefix?:string, fetchImpl?:typeof fetch, mirrorPolicy?:MirrorPolicy}} [opts]
|
||||
* @returns {Promise<{ audioData: any, modelsData: any, charword: any }>}
|
||||
*/
|
||||
export async function loadIndexes(root, opts = {}) {
|
||||
const mirrorPolicy = opts.mirrorPolicy ?? new MirrorPolicy(opts);
|
||||
const audioData = await cachedJson({
|
||||
...opts,
|
||||
mirrorPolicy,
|
||||
cacheFile: join(root, '.cache', 'gamedata', 'excel', 'audio_data.json'),
|
||||
url: RAW.gamedata + 'excel/audio_data.json',
|
||||
refresh: opts.refresh,
|
||||
@@ -63,6 +90,8 @@ export async function loadIndexes(root, opts = {}) {
|
||||
// 11 MB, and raw.githubusercontent stalls on it often: the jsDelivr mirror cachedJson() falls back to is the
|
||||
// reliable path (the operator battle voice needs it, plan.mjs indexVoice).
|
||||
const charword = await cachedJson({
|
||||
...opts,
|
||||
mirrorPolicy,
|
||||
cacheFile: join(root, '.cache', 'gamedata', 'excel', 'charword_table.json'),
|
||||
url: RAW.gamedata + 'excel/charword_table.json',
|
||||
refresh: opts.refresh,
|
||||
@@ -70,6 +99,8 @@ export async function loadIndexes(root, opts = {}) {
|
||||
log: opts.log,
|
||||
});
|
||||
const modelsData = await cachedJson({
|
||||
...opts,
|
||||
mirrorPolicy,
|
||||
cacheFile: join(root, '.cache', 'ark-models', 'models_data.json'),
|
||||
url: RAW.arkModels + 'models_data.json',
|
||||
refresh: opts.refresh,
|
||||
|
||||
+23
-13
@@ -5,8 +5,9 @@
|
||||
// download never appears at the final path.
|
||||
// - Per source: up to `retries` attempts with exponential backoff on network
|
||||
// errors, timeouts, 403/429/5xx and payloads that fail format validation;
|
||||
// a 404/410 moves on immediately. Sources: each candidate URL, then its
|
||||
// jsDelivr mirror (sources.mirrorUrl).
|
||||
// a 404/410 moves on immediately. Each candidate uses its opt-in prefix proxy
|
||||
// first (when enabled), then the original URL and jsDelivr fallback. The proxy gets one short attempt
|
||||
// per file and shares a run-wide circuit breaker with the index/font fetchers.
|
||||
// - Idempotent: an existing file is kept when its size matches the ledger entry
|
||||
// of a previous download or the expected byte count from research, or (when
|
||||
// neither is known) when it passes format validation. The ledger lives in
|
||||
@@ -14,7 +15,7 @@
|
||||
|
||||
import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { mirrorUrl } from './sources.mjs';
|
||||
import { MirrorPolicy } from './network.mjs';
|
||||
import { validate } from './formats.mjs';
|
||||
|
||||
/**
|
||||
@@ -42,8 +43,11 @@ export class Downloader {
|
||||
* @param {(msg:string)=>void} [o.log]
|
||||
* @param {typeof fetch} [o.fetchImpl]
|
||||
* @param {number} [o.backoffMs] base retry delay (doubles per attempt)
|
||||
* @param {'direct'|'mirror'} [o.source] preferred download source
|
||||
* @param {string} [o.proxyPrefix] HTTPS prefix for GitHub downloads
|
||||
* @param {MirrorPolicy} [o.mirrorPolicy] invocation-wide policy and mirror circuit breaker
|
||||
*/
|
||||
constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400 }) {
|
||||
constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400, source = 'direct', proxyPrefix, mirrorPolicy }) {
|
||||
this.root = root;
|
||||
this.ledgerPath = ledgerPath;
|
||||
this.concurrency = Math.max(1, Math.min(64, Number(concurrency) || 16));
|
||||
@@ -52,6 +56,7 @@ export class Downloader {
|
||||
this.force = force;
|
||||
this.log = log;
|
||||
this.fetch = fetchImpl;
|
||||
this.network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log });
|
||||
this.backoffMs = Math.max(0, Number(backoffMs) || 0);
|
||||
this.ledger = { files: {} };
|
||||
this.totals = { ok: 0, skip: 0, miss: 0, error: 0, bytesDownloaded: 0, sizeChanged: 0 };
|
||||
@@ -101,36 +106,40 @@ export class Downloader {
|
||||
|
||||
/**
|
||||
* Fetch one URL with retries.
|
||||
* @returns {Promise<{buf:Buffer}|{notFound:true}|{error:string}>}
|
||||
* @returns {Promise<{buf:Buffer}|{notFound:true}|{skipped:true}|{error:string}>}
|
||||
*/
|
||||
async fetchWithRetries(url, kind) {
|
||||
if (this.network.skip(url)) return { skipped: true };
|
||||
let lastErr = 'unknown error';
|
||||
for (let attempt = 1; attempt <= this.retries; attempt++) {
|
||||
const attempts = this.network.isProxy(url) ? 1 : this.retries;
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
const res = await this.fetch(url, {
|
||||
const res = await this.network.request(url, this.fetch, {
|
||||
headers: { 'user-agent': 'stronghold-protocol-fetch-assets/1.0' },
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
redirect: 'follow',
|
||||
});
|
||||
}, this.timeoutMs);
|
||||
if (res.status === 404 || res.status === 410) {
|
||||
try { await res.body?.cancel(); } catch { /* ignore */ }
|
||||
this.network.succeeded(url); // a missing asset is not a proxy outage
|
||||
return { notFound: true };
|
||||
}
|
||||
if (!res.ok) {
|
||||
try { await res.body?.cancel(); } catch { /* ignore */ }
|
||||
throw new HttpError(res.status, url);
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
const buf = await this.network.readBody(url, res);
|
||||
const len = Number(res.headers.get('content-length'));
|
||||
const enc = res.headers.get('content-encoding');
|
||||
if (!enc && Number.isFinite(len) && len > 0 && len !== buf.length) {
|
||||
throw new Error(`truncated body ${buf.length}/${len} ${url}`);
|
||||
}
|
||||
if (!validate(kind, buf)) throw new Error(`invalid ${kind} payload (${buf.length} B) ${url}`);
|
||||
this.network.succeeded(url);
|
||||
return { buf };
|
||||
} catch (e) {
|
||||
lastErr = e?.message || String(e);
|
||||
if (attempt < this.retries && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs));
|
||||
this.network.failed(url);
|
||||
if (attempt < attempts && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs));
|
||||
}
|
||||
}
|
||||
return { error: lastErr };
|
||||
@@ -146,10 +155,10 @@ export class Downloader {
|
||||
if (kept >= 0) return { status: 'skip', bytes: kept };
|
||||
let lastError = null;
|
||||
for (const url of job.urls) {
|
||||
const sources = [url, mirrorUrl(url)].filter(Boolean);
|
||||
const sources = this.network.urls(url);
|
||||
for (const src of sources) {
|
||||
const r = await this.fetchWithRetries(src, job.kind);
|
||||
if (r.notFound) continue;
|
||||
if (r.notFound || r.skipped) continue;
|
||||
if (r.error) { lastError = r.error; continue; }
|
||||
const abs = join(this.root, job.rel);
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
@@ -167,6 +176,7 @@ export class Downloader {
|
||||
return { status: 'ok', bytes: r.buf.length, url: src, sizeChanged };
|
||||
}
|
||||
}
|
||||
if (lastError) this.network.directFailureHint();
|
||||
return lastError ? { status: 'error', bytes: 0, error: lastError } : { status: 'miss', bytes: 0, error: 'not found (404) on all sources' };
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
import { existsSync, statSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { assetUrl, mirrorUrl } from './sources.mjs';
|
||||
import { assetUrl, githubSourceUrl, mirrorUrl } from './sources.mjs';
|
||||
|
||||
/** Manifest schema version (bump on breaking shape changes). */
|
||||
export const MANIFEST_VERSION = 1;
|
||||
@@ -94,9 +94,11 @@ export function resolveTemplate(template, { root, spine, sourceOf = () => undefi
|
||||
const a = node.alts[i];
|
||||
if (existsSync(join(root, a.rel))) {
|
||||
const src = sourceOf(a.rel);
|
||||
// Provenance only: disabling/changing a proxy does not change the
|
||||
// identity of a previously downloaded asset.
|
||||
const primary = node.alts[0].urls.flatMap((u) => [u, mirrorUrl(u)]);
|
||||
if (i > 0) fallbacks.push(`${path} ← ${src || a.urls[0]}`);
|
||||
else if (src && !primary.includes(src)) fallbacks.push(`${path} ← ${src}`);
|
||||
else if (src && !primary.includes(githubSourceUrl(src))) fallbacks.push(`${path} ← ${src}`);
|
||||
files.add(a.rel);
|
||||
return assetUrl(a.rel);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// Explicit download-source selection and one shared policy per asset invocation.
|
||||
// Selecting a source never looks up the user's public IP or makes a request.
|
||||
import { downloadUrls, normalizeProxyPrefix } from './sources.mjs';
|
||||
|
||||
export function validateSource(mode) {
|
||||
if (!['direct', 'mirror'].includes(mode)) throw new Error(`unknown asset source: ${mode} (direct|mirror)`);
|
||||
return mode;
|
||||
}
|
||||
|
||||
export async function selectDownloadSource({ mode = 'direct', offline = false, proxyPrefix, log = console.log } = {}) {
|
||||
validateSource(mode);
|
||||
if (offline) return 'direct';
|
||||
const normalizedProxy = mode === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : '';
|
||||
log(mode === 'mirror'
|
||||
? `[network] 手动开启 GitHub 镜像:${normalizedProxy || '代理已禁用'};文件仅校验格式和大小。`
|
||||
: '[network] 原始源(未开启 GitHub 镜像)');
|
||||
return mode;
|
||||
}
|
||||
|
||||
/** Shared by indexes, files, Spine follow-ups and fonts; a tripped mirror stays off. */
|
||||
export class MirrorPolicy {
|
||||
constructor({ source = 'direct', proxyPrefix, failureLimit = 3, timeoutMs = 8000, log = console.log } = {}) {
|
||||
this.source = validateSource(source);
|
||||
this.proxyPrefix = this.source === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : '';
|
||||
this.failureLimit = Math.max(1, Number(failureLimit) || 3);
|
||||
this.timeoutMs = Math.max(1, Number(timeoutMs) || 8000);
|
||||
this.log = log;
|
||||
this.failures = 0;
|
||||
this.disabled = false;
|
||||
this.hintShown = false;
|
||||
this.abortMirror = new AbortController();
|
||||
}
|
||||
|
||||
urls(url) {
|
||||
return downloadUrls(url, { source: this.disabled ? 'direct' : this.source, proxyPrefix: this.proxyPrefix });
|
||||
}
|
||||
|
||||
isProxy(url) {
|
||||
return this.source === 'mirror' && !!this.proxyPrefix && url.startsWith(this.proxyPrefix);
|
||||
}
|
||||
|
||||
skip(url) { return this.isProxy(url) && this.disabled; }
|
||||
|
||||
async request(url, fetchImpl, options = {}, timeoutMs = 120000) {
|
||||
if (!this.isProxy(url)) {
|
||||
return fetchImpl(url, { ...options, signal: options.signal ?? AbortSignal.timeout(timeoutMs) });
|
||||
}
|
||||
const headerDeadline = new AbortController();
|
||||
const signal = AbortSignal.any([headerDeadline.signal, this.abortMirror.signal]);
|
||||
const timer = setTimeout(() => headerDeadline.abort(new Error('GitHub mirror response headers timed out')), Math.min(timeoutMs, this.timeoutMs));
|
||||
try {
|
||||
return await fetchImpl(url, { ...options, signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async readBody(url, response) {
|
||||
if (!this.isProxy(url)) return Buffer.from(await response.arrayBuffer());
|
||||
if (!response.body) return Buffer.alloc(0);
|
||||
const reader = response.body.getReader();
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await readChunk(reader, this.timeoutMs, this.abortMirror.signal);
|
||||
if (done) break;
|
||||
const chunk = Buffer.from(value);
|
||||
chunks.push(chunk);
|
||||
size += chunk.length;
|
||||
}
|
||||
} catch (error) {
|
||||
try { await reader.cancel(error); } catch { /* ignore */ }
|
||||
throw error;
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
return Buffer.concat(chunks, size);
|
||||
}
|
||||
|
||||
succeeded(url) {
|
||||
// A success from a request already in flight must not reopen a tripped circuit.
|
||||
if (this.isProxy(url) && !this.disabled) this.failures = 0;
|
||||
}
|
||||
|
||||
failed(url) {
|
||||
if (this.isProxy(url)) {
|
||||
if (this.disabled || ++this.failures < this.failureLimit) return;
|
||||
this.disabled = true;
|
||||
this.log(`[network] GitHub 镜像连续失败 ${this.failures} 次:本次运行停用镜像,继续使用原始源 / jsDelivr。`);
|
||||
this.abortMirror.abort(new Error('GitHub mirror disabled for this run'));
|
||||
}
|
||||
}
|
||||
|
||||
directFailureHint() {
|
||||
if (this.source !== 'direct' || this.hintShown) return;
|
||||
this.hintShown = true;
|
||||
this.log('[network] GitHub 下载失败;如需使用第三方镜像,可重新运行并加 --asset-source=mirror(或 SP_ASSET_SOURCE=mirror)。');
|
||||
}
|
||||
}
|
||||
|
||||
function readChunk(reader, timeoutMs, signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let timer;
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
const onAbort = () => { cleanup(); reject(signal.reason ?? new Error('GitHub mirror request aborted')); };
|
||||
if (signal?.aborted) return onAbort();
|
||||
timer = setTimeout(() => { cleanup(); reject(new Error(`GitHub mirror response body idle for ${timeoutMs} ms`)); }, timeoutMs);
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
reader.read().then((value) => { cleanup(); resolve(value); }, (error) => { cleanup(); reject(error); });
|
||||
});
|
||||
}
|
||||
@@ -1,10 +1,59 @@
|
||||
// Upstream asset sources (GitHub dumps of the official client), URL helpers and
|
||||
// local-path helpers shared by the asset pipeline (tools/fetch-assets.mjs).
|
||||
//
|
||||
// Every URL we download from is a raw.githubusercontent.com URL; mirrorUrl()
|
||||
// maps it to the equivalent jsDelivr URL used as a fallback when GitHub raw
|
||||
// fails. jsDelivr 404s on the ArknightsAssets2 `voice` branch, so no mirror is
|
||||
// offered for audio. See docs/ASSETS.md for the full source list and credits.
|
||||
// GitHub URLs support a prefix proxy and a jsDelivr fallback. jsDelivr does
|
||||
// not serve the voice branch, but the prefix proxy can still be tried there.
|
||||
// See docs/ASSETS.md for the full source list and credits.
|
||||
|
||||
export const DEFAULT_GITHUB_PROXY = 'https://gh-proxy.com/';
|
||||
|
||||
export function normalizeProxyPrefix(prefix = DEFAULT_GITHUB_PROXY) {
|
||||
if (prefix.trim() === '') return '';
|
||||
const url = new URL(prefix);
|
||||
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) {
|
||||
throw new Error('GitHub proxy prefix must be an HTTPS URL without credentials, query or fragment');
|
||||
}
|
||||
return url.href.endsWith('/') ? url.href : url.href + '/';
|
||||
}
|
||||
|
||||
function isGithubDownloadUrl(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === 'https:' && !parsed.username && !parsed.password &&
|
||||
['github.com', 'raw.githubusercontent.com', 'objects.githubusercontent.com'].includes(parsed.host);
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy ledgers store the successful proxy URL. Spine page jobs derived from
|
||||
* those URLs must use today's policy, not implicitly opt in to the old proxy.
|
||||
* Recognize embedded public GitHub URLs even when the old custom prefix is no
|
||||
* longer configured. An ordinary GitHub URL (including its path) stays intact.
|
||||
*/
|
||||
export function githubSourceUrl(url) {
|
||||
let candidate = String(url);
|
||||
while (candidate.startsWith('https://')) {
|
||||
if (isGithubDownloadUrl(candidate)) return candidate;
|
||||
const marker = candidate.indexOf('/https://', 8);
|
||||
if (marker < 0) break;
|
||||
candidate = candidate.slice(marker + 1);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Only public GitHub download URLs are sent to the proxy; never prefix twice. */
|
||||
export function githubProxyUrl(url, prefix = DEFAULT_GITHUB_PROXY) {
|
||||
const normalized = normalizeProxyPrefix(prefix);
|
||||
if (!normalized || !isGithubDownloadUrl(url)) return null;
|
||||
return normalized + url;
|
||||
}
|
||||
|
||||
/** Ordered transport alternatives; direct mode retains the existing jsDelivr order. */
|
||||
export function downloadUrls(url, { source = 'direct', proxyPrefix = DEFAULT_GITHUB_PROXY } = {}) {
|
||||
url = githubSourceUrl(url);
|
||||
const proxy = source === 'mirror' ? githubProxyUrl(url, proxyPrefix) : null;
|
||||
return [...new Set([proxy, url, mirrorUrl(url)].filter(Boolean))];
|
||||
}
|
||||
|
||||
/** Raw base URLs (always end with '/'). */
|
||||
export const RAW = Object.freeze({
|
||||
|
||||
+27
-9
@@ -13,9 +13,10 @@
|
||||
// that file from the extracted models (after a game update).
|
||||
//
|
||||
// Idempotent: existing files with the right size are skipped, so re-running is
|
||||
// cheap. Downloads use ~16 parallel connections, 3 retries per source and a
|
||||
// jsDelivr mirror fallback. Spine atlases get `size:` (and `pma: true` for
|
||||
// enemies); every skeleton is parsed to resolve animation roles.
|
||||
// cheap. Downloads use ~16 parallel connections, 3 retries per direct source,
|
||||
// a jsDelivr fallback and an opt-in GitHub proxy (one short attempt per URL).
|
||||
// Spine atlases get `size:` (and `pma: true` for enemies); every skeleton is
|
||||
// parsed to resolve animation roles.
|
||||
//
|
||||
// The committed data/assets.json never shrinks by accident: an entry whose files
|
||||
// are missing here is left out of a rebuilt manifest, so a run on a machine where
|
||||
@@ -33,6 +34,8 @@ import { existsSync, realpathSync } from 'node:fs';
|
||||
import { dirname, join, relative, sep } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { Downloader } from './assets/downloader.mjs';
|
||||
import { MirrorPolicy, selectDownloadSource, validateSource } from './assets/network.mjs';
|
||||
import { normalizeProxyPrefix } from './assets/sources.mjs';
|
||||
import { loadIndexes } from './assets/cache.mjs';
|
||||
import { indexAudio, VOICE_DIRS } from './assets/audio.mjs';
|
||||
import { buildPlan } from './assets/plan.mjs';
|
||||
@@ -51,6 +54,7 @@ const LOCAL_SPINES = join(ROOT, LOCAL_ENEMY_SPINES_FILE);
|
||||
|
||||
const HELP = `Usage: node tools/fetch-assets.mjs [options]
|
||||
--concurrency=N parallel downloads (default 16)
|
||||
--asset-source=M direct (default) or mirror (opt-in; no public-IP lookup)
|
||||
--force re-download files even when present
|
||||
--offline no network: post-process what is on disk and rebuild data/assets.json
|
||||
--dry-run print the plan and exit
|
||||
@@ -64,18 +68,23 @@ const HELP = `Usage: node tools/fetch-assets.mjs [options]
|
||||
(without it such a run keeps the current manifest, lists the entries and exits 1)
|
||||
--local-spines rewrite ${LOCAL_ENEMY_SPINES_FILE} from the enemy models extracted
|
||||
by tools/local-extract/extract.py (public/assets/local/spine/enemy/)
|
||||
--help this text`;
|
||||
--help this text
|
||||
Environment: SP_ASSET_SOURCE sets the default source; SP_GITHUB_PROXY sets the
|
||||
HTTPS mirror prefix (default https://gh-proxy.com/; empty disables the proxy).
|
||||
Mirror attempts have an 8 s response header timeout; response body has a separate idle timeout. Stops for this run after 3 consecutive
|
||||
failures. Only explicitly enabled GitHub downloads use the third-party proxy.`;
|
||||
|
||||
/**
|
||||
* Parse CLI flags.
|
||||
* @param {string[]} argv
|
||||
* @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, allowShrink:boolean, localSpines:boolean, voiceLang:string, voiceAll:boolean, help:boolean}}
|
||||
* @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, allowShrink:boolean, localSpines:boolean, voiceLang:string, voiceAll:boolean, help:boolean, source:string}}
|
||||
*/
|
||||
export function parseArgs(argv) {
|
||||
const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, allowShrink: false, localSpines: false, voiceLang: 'cn', voiceAll: false, help: false };
|
||||
const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, allowShrink: false, localSpines: false, voiceLang: 'cn', voiceAll: false, help: false, source: process.env.SP_ASSET_SOURCE || 'direct' };
|
||||
for (const a of argv) {
|
||||
const [k, v] = a.split('=');
|
||||
if (k === '--concurrency') o.concurrency = Math.max(1, Math.min(64, parseInt(v, 10) || 16));
|
||||
else if (k === '--asset-source') o.source = v;
|
||||
else if (k === '--force') o.force = true;
|
||||
else if (k === '--offline') o.offline = true;
|
||||
else if (k === '--dry-run') o.dryRun = true;
|
||||
@@ -88,9 +97,14 @@ export function parseArgs(argv) {
|
||||
else if (k === '--help' || k === '-h') o.help = true;
|
||||
else throw new Error(`unknown option ${a}\n${HELP}`);
|
||||
}
|
||||
if (!o.help) validateSource(o.source);
|
||||
return o;
|
||||
}
|
||||
|
||||
export function resolveProxyPrefix(source, offline = false, value = process.env.SP_GITHUB_PROXY) {
|
||||
return offline || source !== 'mirror' ? '' : normalizeProxyPrefix(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* The shrink guard of data/assets.json (header): the entries of the current manifest `prev` that `next` would drop,
|
||||
* and whether `next` may be written — always when nothing is dropped (or there is no current manifest), otherwise
|
||||
@@ -224,7 +238,11 @@ async function main() {
|
||||
readJson('docs/research/05-enemies.json'),
|
||||
readJson('docs/research/05-maps.json'),
|
||||
]);
|
||||
const { audioData, modelsData, charword } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log });
|
||||
const proxyPrefix = resolveProxyPrefix(opts.source, opts.offline);
|
||||
const source = await selectDownloadSource({ mode: opts.source, offline: opts.offline, proxyPrefix, log });
|
||||
const mirrorPolicy = new MirrorPolicy({ source, proxyPrefix, log });
|
||||
const network = { source, proxyPrefix, mirrorPolicy };
|
||||
const { audioData, modelsData, charword } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log, ...network });
|
||||
const audio = indexAudio(audioData);
|
||||
// The game data built by tools/build-data.mjs (when present) may reference more
|
||||
// spawnable enemies/tokens than research lists (e.g. 机变 enemy swaps): cover them too.
|
||||
@@ -255,7 +273,7 @@ async function main() {
|
||||
|
||||
const dl = new Downloader({
|
||||
root: ASSETS, ledgerPath: join(CACHE, 'assets-ledger.json'),
|
||||
concurrency: opts.concurrency, force: opts.force, log,
|
||||
concurrency: opts.concurrency, force: opts.force, log, ...network,
|
||||
});
|
||||
await dl.loadLedger();
|
||||
const downloadErrors = opts.offline ? [] : await downloadLeaves(leaves, dl, ASSETS, 'files');
|
||||
@@ -263,7 +281,7 @@ async function main() {
|
||||
// Fonts
|
||||
let fontErrors = [];
|
||||
if (!opts.offline) {
|
||||
const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log });
|
||||
const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log, ...network });
|
||||
await fdl.loadLedger();
|
||||
await fdl.run(fontJobs(), 'fonts');
|
||||
dl.totals.bytesDownloaded += fdl.totals.bytesDownloaded;
|
||||
|
||||
+9
-2
@@ -19,6 +19,9 @@
|
||||
// Options:
|
||||
// --check report only, change nothing (exit 1 when something essential is missing)
|
||||
// --no-assets skip the art/audio download
|
||||
// --asset-source=M direct (default) or mirror (opt-in; no public-IP lookup)
|
||||
// SP_ASSET_SOURCE sets the default; SP_GITHUB_PROXY sets the HTTPS prefix (https://gh-proxy.com/)
|
||||
// An empty SP_GITHUB_PROXY disables the proxy, including in mirror mode.
|
||||
// --no-local skip the local-client detection and extraction
|
||||
// --local extract from the local client without asking (re-extracts when already done)
|
||||
// --game <dir> AssetBundle root of the local client (…/StreamingAssets/AB/Windows or PlayCover …/Documents/Bundles)
|
||||
@@ -35,6 +38,7 @@ import path from 'node:path';
|
||||
import readline from 'node:readline';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { validateSource } from './assets/network.mjs';
|
||||
|
||||
export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
export const MIN_NODE = 22;
|
||||
@@ -328,11 +332,13 @@ function ensureVenv(py, log) {
|
||||
// ---------------------------------------------------------------------------------------------------
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { check: false, assets: true, local: 'ask', game: null, yes: false, quiet: false, help: false };
|
||||
const o = { check: false, assets: true, local: 'ask', game: null, yes: false, quiet: false, help: false, source: process.env.SP_ASSET_SOURCE || 'direct' };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--check') o.check = true;
|
||||
else if (a === '--no-assets') o.assets = false;
|
||||
else if (a === '--asset-source') o.source = argv[++i];
|
||||
else if (a.startsWith('--asset-source=')) o.source = a.slice('--asset-source='.length);
|
||||
else if (a === '--no-local') o.local = 'no';
|
||||
else if (a === '--local') o.local = 'force';
|
||||
else if (a === '--game') { o.game = argv[++i] || null; if (o.local !== 'no') o.local = 'force'; }
|
||||
@@ -342,6 +348,7 @@ function parseArgs(argv) {
|
||||
else if (a === '-h' || a === '--help') o.help = true;
|
||||
else throw new Error(`未知参数 / unknown option: ${a}(--help 查看用法)`);
|
||||
}
|
||||
if (!o.help) validateSource(o.source);
|
||||
return o;
|
||||
}
|
||||
|
||||
@@ -413,7 +420,7 @@ async function main() {
|
||||
const what = !assets.present ? `首次下载约 ${assets.bytes ? mb(assets.bytes) : '270 MB'},可随时中断,重新运行会续传`
|
||||
: `补全缺失的 ${assets.missing} 个文件`;
|
||||
log(`\n${c.cyan('▶')} 下载美术与音频素材(${what})…`);
|
||||
const r = run(process.execPath, [path.join(ROOT, 'tools', 'fetch-assets.mjs')]);
|
||||
const r = run(process.execPath, [path.join(ROOT, 'tools', 'fetch-assets.mjs'), `--asset-source=${opts.source}`]);
|
||||
assets = checkAssets();
|
||||
if (!r.ok && !assets.ok) log(c.warn(' 素材下载未完成(网络问题?)。游戏仍可运行(使用占位图),稍后重新运行 setup 即可续传。'));
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user