feedback4: merge master — PR #24 (an opt-in gh-proxy mirror for asset downloads); golden unchanged (tools only)
This commit is contained in:
commit
0df1391859
9 files changed
+671
-45
No files matched your search
+45
-14
@@ -6,7 +6,8 @@
|
||||
|
||||
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { RAW, mirrorUrl } from './sources.mjs';
|
||||
import { RAW } from './sources.mjs';
|
||||
import { MirrorPolicy } from './network.mjs';
|
||||
|
||||
/**
|
||||
* Read a cached JSON file, downloading it first when missing or unparsable.
|
||||
@@ -16,44 +17,70 @@ import { RAW, mirrorUrl } from './sources.mjs';
|
||||
* @param {boolean} [o.refresh] force re-download
|
||||
* @param {boolean} [o.offline] never download (throw when the cache is missing)
|
||||
* @param {(m:string)=>void} [o.log]
|
||||
* @param {'direct'|'mirror'} [o.source]
|
||||
* @param {string} [o.proxyPrefix]
|
||||
* @param {typeof fetch} [o.fetchImpl]
|
||||
* @param {MirrorPolicy} [o.mirrorPolicy] shared invocation-wide mirror circuit breaker
|
||||
* @param {number} [o.timeoutMs]
|
||||
* @param {number} [o.backoffMs]
|
||||
* @returns {Promise<any>} parsed JSON
|
||||
*/
|
||||
export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log }) {
|
||||
export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log, source = 'direct', proxyPrefix, fetchImpl = globalThis.fetch, mirrorPolicy, timeoutMs = 180000, backoffMs = 500 }) {
|
||||
if (!refresh || offline) {
|
||||
try { return JSON.parse(await readFile(cacheFile, 'utf8')); } catch (e) {
|
||||
if (offline) throw new Error(`--offline: cached index ${cacheFile} is missing or corrupt (${e.message}); run once online`);
|
||||
}
|
||||
}
|
||||
const network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log });
|
||||
let lastErr = null;
|
||||
for (const src of [url, mirrorUrl(url)].filter(Boolean)) {
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
for (const src of network.urls(url)) {
|
||||
if (network.skip(src)) continue;
|
||||
const attempts = network.isProxy(src) ? 1 : 3;
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
let text, json;
|
||||
try {
|
||||
log(`[cache] downloading ${src}`);
|
||||
const res = await fetch(src, { signal: AbortSignal.timeout(180000) });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const text = await res.text();
|
||||
const json = JSON.parse(text);
|
||||
await mkdir(dirname(cacheFile), { recursive: true });
|
||||
await writeFile(cacheFile + '.tmp', text);
|
||||
await rename(cacheFile + '.tmp', cacheFile);
|
||||
return json;
|
||||
const res = await network.request(src, fetchImpl, {}, timeoutMs);
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel();
|
||||
if (res.status === 404 || res.status === 410) {
|
||||
network.succeeded(src); // missing index, not a proxy outage
|
||||
lastErr = new Error(`HTTP ${res.status}`);
|
||||
break;
|
||||
}
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
text = (await network.readBody(src, res)).toString('utf8');
|
||||
json = JSON.parse(text);
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
await new Promise((r) => setTimeout(r, 500 * attempt));
|
||||
network.failed(src);
|
||||
if (attempt < attempts && backoffMs) await new Promise((r) => setTimeout(r, backoffMs * attempt));
|
||||
continue;
|
||||
}
|
||||
network.succeeded(src);
|
||||
// A local disk failure is not evidence that the mirror is unhealthy.
|
||||
await mkdir(dirname(cacheFile), { recursive: true });
|
||||
await writeFile(cacheFile + '.tmp', text);
|
||||
await rename(cacheFile + '.tmp', cacheFile);
|
||||
return json;
|
||||
}
|
||||
}
|
||||
if (lastErr) network.directFailureHint();
|
||||
throw new Error(`cannot fetch ${url}: ${lastErr?.message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load audio_data.json (official), charword_table.json (official voice slots) and Ark-Models models_data.json.
|
||||
* @param {string} root project root
|
||||
* @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void}} [opts]
|
||||
* @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void, source?:'direct'|'mirror', proxyPrefix?:string, fetchImpl?:typeof fetch, mirrorPolicy?:MirrorPolicy}} [opts]
|
||||
* @returns {Promise<{ audioData: any, modelsData: any, charword: any }>}
|
||||
*/
|
||||
export async function loadIndexes(root, opts = {}) {
|
||||
const mirrorPolicy = opts.mirrorPolicy ?? new MirrorPolicy(opts);
|
||||
const audioData = await cachedJson({
|
||||
...opts,
|
||||
mirrorPolicy,
|
||||
cacheFile: join(root, '.cache', 'gamedata', 'excel', 'audio_data.json'),
|
||||
url: RAW.gamedata + 'excel/audio_data.json',
|
||||
refresh: opts.refresh,
|
||||
@@ -63,6 +90,8 @@ export async function loadIndexes(root, opts = {}) {
|
||||
// 11 MB, and raw.githubusercontent stalls on it often: the jsDelivr mirror cachedJson() falls back to is the
|
||||
// reliable path (the operator battle voice needs it, plan.mjs indexVoice).
|
||||
const charword = await cachedJson({
|
||||
...opts,
|
||||
mirrorPolicy,
|
||||
cacheFile: join(root, '.cache', 'gamedata', 'excel', 'charword_table.json'),
|
||||
url: RAW.gamedata + 'excel/charword_table.json',
|
||||
refresh: opts.refresh,
|
||||
@@ -70,6 +99,8 @@ export async function loadIndexes(root, opts = {}) {
|
||||
log: opts.log,
|
||||
});
|
||||
const modelsData = await cachedJson({
|
||||
...opts,
|
||||
mirrorPolicy,
|
||||
cacheFile: join(root, '.cache', 'ark-models', 'models_data.json'),
|
||||
url: RAW.arkModels + 'models_data.json',
|
||||
refresh: opts.refresh,
|
||||
|
||||
+23
-13
@@ -5,8 +5,9 @@
|
||||
// download never appears at the final path.
|
||||
// - Per source: up to `retries` attempts with exponential backoff on network
|
||||
// errors, timeouts, 403/429/5xx and payloads that fail format validation;
|
||||
// a 404/410 moves on immediately. Sources: each candidate URL, then its
|
||||
// jsDelivr mirror (sources.mirrorUrl).
|
||||
// a 404/410 moves on immediately. Each candidate uses its opt-in prefix proxy
|
||||
// first (when enabled), then the original URL and jsDelivr fallback. The proxy gets one short attempt
|
||||
// per file and shares a run-wide circuit breaker with the index/font fetchers.
|
||||
// - Idempotent: an existing file is kept when its size matches the ledger entry
|
||||
// of a previous download or the expected byte count from research, or (when
|
||||
// neither is known) when it passes format validation. The ledger lives in
|
||||
@@ -14,7 +15,7 @@
|
||||
|
||||
import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { mirrorUrl } from './sources.mjs';
|
||||
import { MirrorPolicy } from './network.mjs';
|
||||
import { validate } from './formats.mjs';
|
||||
|
||||
/**
|
||||
@@ -42,8 +43,11 @@ export class Downloader {
|
||||
* @param {(msg:string)=>void} [o.log]
|
||||
* @param {typeof fetch} [o.fetchImpl]
|
||||
* @param {number} [o.backoffMs] base retry delay (doubles per attempt)
|
||||
* @param {'direct'|'mirror'} [o.source] preferred download source
|
||||
* @param {string} [o.proxyPrefix] HTTPS prefix for GitHub downloads
|
||||
* @param {MirrorPolicy} [o.mirrorPolicy] invocation-wide policy and mirror circuit breaker
|
||||
*/
|
||||
constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400 }) {
|
||||
constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400, source = 'direct', proxyPrefix, mirrorPolicy }) {
|
||||
this.root = root;
|
||||
this.ledgerPath = ledgerPath;
|
||||
this.concurrency = Math.max(1, Math.min(64, Number(concurrency) || 16));
|
||||
@@ -52,6 +56,7 @@ export class Downloader {
|
||||
this.force = force;
|
||||
this.log = log;
|
||||
this.fetch = fetchImpl;
|
||||
this.network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log });
|
||||
this.backoffMs = Math.max(0, Number(backoffMs) || 0);
|
||||
this.ledger = { files: {} };
|
||||
this.totals = { ok: 0, skip: 0, miss: 0, error: 0, bytesDownloaded: 0, sizeChanged: 0 };
|
||||
@@ -101,36 +106,40 @@ export class Downloader {
|
||||
|
||||
/**
|
||||
* Fetch one URL with retries.
|
||||
* @returns {Promise<{buf:Buffer}|{notFound:true}|{error:string}>}
|
||||
* @returns {Promise<{buf:Buffer}|{notFound:true}|{skipped:true}|{error:string}>}
|
||||
*/
|
||||
async fetchWithRetries(url, kind) {
|
||||
if (this.network.skip(url)) return { skipped: true };
|
||||
let lastErr = 'unknown error';
|
||||
for (let attempt = 1; attempt <= this.retries; attempt++) {
|
||||
const attempts = this.network.isProxy(url) ? 1 : this.retries;
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
const res = await this.fetch(url, {
|
||||
const res = await this.network.request(url, this.fetch, {
|
||||
headers: { 'user-agent': 'stronghold-protocol-fetch-assets/1.0' },
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
redirect: 'follow',
|
||||
});
|
||||
}, this.timeoutMs);
|
||||
if (res.status === 404 || res.status === 410) {
|
||||
try { await res.body?.cancel(); } catch { /* ignore */ }
|
||||
this.network.succeeded(url); // a missing asset is not a proxy outage
|
||||
return { notFound: true };
|
||||
}
|
||||
if (!res.ok) {
|
||||
try { await res.body?.cancel(); } catch { /* ignore */ }
|
||||
throw new HttpError(res.status, url);
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
const buf = await this.network.readBody(url, res);
|
||||
const len = Number(res.headers.get('content-length'));
|
||||
const enc = res.headers.get('content-encoding');
|
||||
if (!enc && Number.isFinite(len) && len > 0 && len !== buf.length) {
|
||||
throw new Error(`truncated body ${buf.length}/${len} ${url}`);
|
||||
}
|
||||
if (!validate(kind, buf)) throw new Error(`invalid ${kind} payload (${buf.length} B) ${url}`);
|
||||
this.network.succeeded(url);
|
||||
return { buf };
|
||||
} catch (e) {
|
||||
lastErr = e?.message || String(e);
|
||||
if (attempt < this.retries && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs));
|
||||
this.network.failed(url);
|
||||
if (attempt < attempts && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs));
|
||||
}
|
||||
}
|
||||
return { error: lastErr };
|
||||
@@ -146,10 +155,10 @@ export class Downloader {
|
||||
if (kept >= 0) return { status: 'skip', bytes: kept };
|
||||
let lastError = null;
|
||||
for (const url of job.urls) {
|
||||
const sources = [url, mirrorUrl(url)].filter(Boolean);
|
||||
const sources = this.network.urls(url);
|
||||
for (const src of sources) {
|
||||
const r = await this.fetchWithRetries(src, job.kind);
|
||||
if (r.notFound) continue;
|
||||
if (r.notFound || r.skipped) continue;
|
||||
if (r.error) { lastError = r.error; continue; }
|
||||
const abs = join(this.root, job.rel);
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
@@ -167,6 +176,7 @@ export class Downloader {
|
||||
return { status: 'ok', bytes: r.buf.length, url: src, sizeChanged };
|
||||
}
|
||||
}
|
||||
if (lastError) this.network.directFailureHint();
|
||||
return lastError ? { status: 'error', bytes: 0, error: lastError } : { status: 'miss', bytes: 0, error: 'not found (404) on all sources' };
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
import { existsSync, statSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { assetUrl, mirrorUrl } from './sources.mjs';
|
||||
import { assetUrl, githubSourceUrl, mirrorUrl } from './sources.mjs';
|
||||
|
||||
/** Manifest schema version (bump on breaking shape changes). */
|
||||
export const MANIFEST_VERSION = 1;
|
||||
@@ -94,9 +94,11 @@ export function resolveTemplate(template, { root, spine, sourceOf = () => undefi
|
||||
const a = node.alts[i];
|
||||
if (existsSync(join(root, a.rel))) {
|
||||
const src = sourceOf(a.rel);
|
||||
// Provenance only: disabling/changing a proxy does not change the
|
||||
// identity of a previously downloaded asset.
|
||||
const primary = node.alts[0].urls.flatMap((u) => [u, mirrorUrl(u)]);
|
||||
if (i > 0) fallbacks.push(`${path} ← ${src || a.urls[0]}`);
|
||||
else if (src && !primary.includes(src)) fallbacks.push(`${path} ← ${src}`);
|
||||
else if (src && !primary.includes(githubSourceUrl(src))) fallbacks.push(`${path} ← ${src}`);
|
||||
files.add(a.rel);
|
||||
return assetUrl(a.rel);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// Explicit download-source selection and one shared policy per asset invocation.
|
||||
// Selecting a source never looks up the user's public IP or makes a request.
|
||||
import { downloadUrls, normalizeProxyPrefix } from './sources.mjs';
|
||||
|
||||
export function validateSource(mode) {
|
||||
if (!['direct', 'mirror'].includes(mode)) throw new Error(`unknown asset source: ${mode} (direct|mirror)`);
|
||||
return mode;
|
||||
}
|
||||
|
||||
export async function selectDownloadSource({ mode = 'direct', offline = false, proxyPrefix, log = console.log } = {}) {
|
||||
validateSource(mode);
|
||||
if (offline) return 'direct';
|
||||
const normalizedProxy = mode === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : '';
|
||||
log(mode === 'mirror'
|
||||
? `[network] 手动开启 GitHub 镜像:${normalizedProxy || '代理已禁用'};文件仅校验格式和大小。`
|
||||
: '[network] 原始源(未开启 GitHub 镜像)');
|
||||
return mode;
|
||||
}
|
||||
|
||||
/** Shared by indexes, files, Spine follow-ups and fonts; a tripped mirror stays off. */
|
||||
export class MirrorPolicy {
|
||||
constructor({ source = 'direct', proxyPrefix, failureLimit = 3, timeoutMs = 8000, log = console.log } = {}) {
|
||||
this.source = validateSource(source);
|
||||
this.proxyPrefix = this.source === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : '';
|
||||
this.failureLimit = Math.max(1, Number(failureLimit) || 3);
|
||||
this.timeoutMs = Math.max(1, Number(timeoutMs) || 8000);
|
||||
this.log = log;
|
||||
this.failures = 0;
|
||||
this.disabled = false;
|
||||
this.hintShown = false;
|
||||
this.abortMirror = new AbortController();
|
||||
}
|
||||
|
||||
urls(url) {
|
||||
return downloadUrls(url, { source: this.disabled ? 'direct' : this.source, proxyPrefix: this.proxyPrefix });
|
||||
}
|
||||
|
||||
isProxy(url) {
|
||||
return this.source === 'mirror' && !!this.proxyPrefix && url.startsWith(this.proxyPrefix);
|
||||
}
|
||||
|
||||
skip(url) { return this.isProxy(url) && this.disabled; }
|
||||
|
||||
async request(url, fetchImpl, options = {}, timeoutMs = 120000) {
|
||||
if (!this.isProxy(url)) {
|
||||
return fetchImpl(url, { ...options, signal: options.signal ?? AbortSignal.timeout(timeoutMs) });
|
||||
}
|
||||
const headerDeadline = new AbortController();
|
||||
const signal = AbortSignal.any([headerDeadline.signal, this.abortMirror.signal]);
|
||||
const timer = setTimeout(() => headerDeadline.abort(new Error('GitHub mirror response headers timed out')), Math.min(timeoutMs, this.timeoutMs));
|
||||
try {
|
||||
return await fetchImpl(url, { ...options, signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async readBody(url, response) {
|
||||
if (!this.isProxy(url)) return Buffer.from(await response.arrayBuffer());
|
||||
if (!response.body) return Buffer.alloc(0);
|
||||
const reader = response.body.getReader();
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await readChunk(reader, this.timeoutMs, this.abortMirror.signal);
|
||||
if (done) break;
|
||||
const chunk = Buffer.from(value);
|
||||
chunks.push(chunk);
|
||||
size += chunk.length;
|
||||
}
|
||||
} catch (error) {
|
||||
try { await reader.cancel(error); } catch { /* ignore */ }
|
||||
throw error;
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
return Buffer.concat(chunks, size);
|
||||
}
|
||||
|
||||
succeeded(url) {
|
||||
// A success from a request already in flight must not reopen a tripped circuit.
|
||||
if (this.isProxy(url) && !this.disabled) this.failures = 0;
|
||||
}
|
||||
|
||||
failed(url) {
|
||||
if (this.isProxy(url)) {
|
||||
if (this.disabled || ++this.failures < this.failureLimit) return;
|
||||
this.disabled = true;
|
||||
this.log(`[network] GitHub 镜像连续失败 ${this.failures} 次:本次运行停用镜像,继续使用原始源 / jsDelivr。`);
|
||||
this.abortMirror.abort(new Error('GitHub mirror disabled for this run'));
|
||||
}
|
||||
}
|
||||
|
||||
directFailureHint() {
|
||||
if (this.source !== 'direct' || this.hintShown) return;
|
||||
this.hintShown = true;
|
||||
this.log('[network] GitHub 下载失败;如需使用第三方镜像,可重新运行并加 --asset-source=mirror(或 SP_ASSET_SOURCE=mirror)。');
|
||||
}
|
||||
}
|
||||
|
||||
function readChunk(reader, timeoutMs, signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let timer;
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
const onAbort = () => { cleanup(); reject(signal.reason ?? new Error('GitHub mirror request aborted')); };
|
||||
if (signal?.aborted) return onAbort();
|
||||
timer = setTimeout(() => { cleanup(); reject(new Error(`GitHub mirror response body idle for ${timeoutMs} ms`)); }, timeoutMs);
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
reader.read().then((value) => { cleanup(); resolve(value); }, (error) => { cleanup(); reject(error); });
|
||||
});
|
||||
}
|
||||
@@ -1,10 +1,59 @@
|
||||
// Upstream asset sources (GitHub dumps of the official client), URL helpers and
|
||||
// local-path helpers shared by the asset pipeline (tools/fetch-assets.mjs).
|
||||
//
|
||||
// Every URL we download from is a raw.githubusercontent.com URL; mirrorUrl()
|
||||
// maps it to the equivalent jsDelivr URL used as a fallback when GitHub raw
|
||||
// fails. jsDelivr 404s on the ArknightsAssets2 `voice` branch, so no mirror is
|
||||
// offered for audio. See docs/ASSETS.md for the full source list and credits.
|
||||
// GitHub URLs support a prefix proxy and a jsDelivr fallback. jsDelivr does
|
||||
// not serve the voice branch, but the prefix proxy can still be tried there.
|
||||
// See docs/ASSETS.md for the full source list and credits.
|
||||
|
||||
export const DEFAULT_GITHUB_PROXY = 'https://gh-proxy.com/';
|
||||
|
||||
export function normalizeProxyPrefix(prefix = DEFAULT_GITHUB_PROXY) {
|
||||
if (prefix.trim() === '') return '';
|
||||
const url = new URL(prefix);
|
||||
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) {
|
||||
throw new Error('GitHub proxy prefix must be an HTTPS URL without credentials, query or fragment');
|
||||
}
|
||||
return url.href.endsWith('/') ? url.href : url.href + '/';
|
||||
}
|
||||
|
||||
function isGithubDownloadUrl(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === 'https:' && !parsed.username && !parsed.password &&
|
||||
['github.com', 'raw.githubusercontent.com', 'objects.githubusercontent.com'].includes(parsed.host);
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy ledgers store the successful proxy URL. Spine page jobs derived from
|
||||
* those URLs must use today's policy, not implicitly opt in to the old proxy.
|
||||
* Recognize embedded public GitHub URLs even when the old custom prefix is no
|
||||
* longer configured. An ordinary GitHub URL (including its path) stays intact.
|
||||
*/
|
||||
export function githubSourceUrl(url) {
|
||||
let candidate = String(url);
|
||||
while (candidate.startsWith('https://')) {
|
||||
if (isGithubDownloadUrl(candidate)) return candidate;
|
||||
const marker = candidate.indexOf('/https://', 8);
|
||||
if (marker < 0) break;
|
||||
candidate = candidate.slice(marker + 1);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Only public GitHub download URLs are sent to the proxy; never prefix twice. */
|
||||
export function githubProxyUrl(url, prefix = DEFAULT_GITHUB_PROXY) {
|
||||
const normalized = normalizeProxyPrefix(prefix);
|
||||
if (!normalized || !isGithubDownloadUrl(url)) return null;
|
||||
return normalized + url;
|
||||
}
|
||||
|
||||
/** Ordered transport alternatives; direct mode retains the existing jsDelivr order. */
|
||||
export function downloadUrls(url, { source = 'direct', proxyPrefix = DEFAULT_GITHUB_PROXY } = {}) {
|
||||
url = githubSourceUrl(url);
|
||||
const proxy = source === 'mirror' ? githubProxyUrl(url, proxyPrefix) : null;
|
||||
return [...new Set([proxy, url, mirrorUrl(url)].filter(Boolean))];
|
||||
}
|
||||
|
||||
/** Raw base URLs (always end with '/'). */
|
||||
export const RAW = Object.freeze({
|
||||
|
||||
+27
-9
@@ -13,9 +13,10 @@
|
||||
// that file from the extracted models (after a game update).
|
||||
//
|
||||
// Idempotent: existing files with the right size are skipped, so re-running is
|
||||
// cheap. Downloads use ~16 parallel connections, 3 retries per source and a
|
||||
// jsDelivr mirror fallback. Spine atlases get `size:` (and `pma: true` for
|
||||
// enemies); every skeleton is parsed to resolve animation roles.
|
||||
// cheap. Downloads use ~16 parallel connections, 3 retries per direct source,
|
||||
// a jsDelivr fallback and an opt-in GitHub proxy (one short attempt per URL).
|
||||
// Spine atlases get `size:` (and `pma: true` for enemies); every skeleton is
|
||||
// parsed to resolve animation roles.
|
||||
//
|
||||
// The committed data/assets.json never shrinks by accident: an entry whose files
|
||||
// are missing here is left out of a rebuilt manifest, so a run on a machine where
|
||||
@@ -33,6 +34,8 @@ import { existsSync, realpathSync } from 'node:fs';
|
||||
import { dirname, join, relative, sep } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { Downloader } from './assets/downloader.mjs';
|
||||
import { MirrorPolicy, selectDownloadSource, validateSource } from './assets/network.mjs';
|
||||
import { normalizeProxyPrefix } from './assets/sources.mjs';
|
||||
import { loadIndexes } from './assets/cache.mjs';
|
||||
import { indexAudio, VOICE_DIRS } from './assets/audio.mjs';
|
||||
import { buildPlan } from './assets/plan.mjs';
|
||||
@@ -51,6 +54,7 @@ const LOCAL_SPINES = join(ROOT, LOCAL_ENEMY_SPINES_FILE);
|
||||
|
||||
const HELP = `Usage: node tools/fetch-assets.mjs [options]
|
||||
--concurrency=N parallel downloads (default 16)
|
||||
--asset-source=M direct (default) or mirror (opt-in; no public-IP lookup)
|
||||
--force re-download files even when present
|
||||
--offline no network: post-process what is on disk and rebuild data/assets.json
|
||||
--dry-run print the plan and exit
|
||||
@@ -64,18 +68,23 @@ const HELP = `Usage: node tools/fetch-assets.mjs [options]
|
||||
(without it such a run keeps the current manifest, lists the entries and exits 1)
|
||||
--local-spines rewrite ${LOCAL_ENEMY_SPINES_FILE} from the enemy models extracted
|
||||
by tools/local-extract/extract.py (public/assets/local/spine/enemy/)
|
||||
--help this text`;
|
||||
--help this text
|
||||
Environment: SP_ASSET_SOURCE sets the default source; SP_GITHUB_PROXY sets the
|
||||
HTTPS mirror prefix (default https://gh-proxy.com/; empty disables the proxy).
|
||||
Mirror attempts have an 8 s response header timeout; response body has a separate idle timeout. Stops for this run after 3 consecutive
|
||||
failures. Only explicitly enabled GitHub downloads use the third-party proxy.`;
|
||||
|
||||
/**
|
||||
* Parse CLI flags.
|
||||
* @param {string[]} argv
|
||||
* @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, allowShrink:boolean, localSpines:boolean, voiceLang:string, voiceAll:boolean, help:boolean}}
|
||||
* @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, allowShrink:boolean, localSpines:boolean, voiceLang:string, voiceAll:boolean, help:boolean, source:string}}
|
||||
*/
|
||||
export function parseArgs(argv) {
|
||||
const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, allowShrink: false, localSpines: false, voiceLang: 'cn', voiceAll: false, help: false };
|
||||
const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, allowShrink: false, localSpines: false, voiceLang: 'cn', voiceAll: false, help: false, source: process.env.SP_ASSET_SOURCE || 'direct' };
|
||||
for (const a of argv) {
|
||||
const [k, v] = a.split('=');
|
||||
if (k === '--concurrency') o.concurrency = Math.max(1, Math.min(64, parseInt(v, 10) || 16));
|
||||
else if (k === '--asset-source') o.source = v;
|
||||
else if (k === '--force') o.force = true;
|
||||
else if (k === '--offline') o.offline = true;
|
||||
else if (k === '--dry-run') o.dryRun = true;
|
||||
@@ -88,9 +97,14 @@ export function parseArgs(argv) {
|
||||
else if (k === '--help' || k === '-h') o.help = true;
|
||||
else throw new Error(`unknown option ${a}\n${HELP}`);
|
||||
}
|
||||
if (!o.help) validateSource(o.source);
|
||||
return o;
|
||||
}
|
||||
|
||||
export function resolveProxyPrefix(source, offline = false, value = process.env.SP_GITHUB_PROXY) {
|
||||
return offline || source !== 'mirror' ? '' : normalizeProxyPrefix(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* The shrink guard of data/assets.json (header): the entries of the current manifest `prev` that `next` would drop,
|
||||
* and whether `next` may be written — always when nothing is dropped (or there is no current manifest), otherwise
|
||||
@@ -224,7 +238,11 @@ async function main() {
|
||||
readJson('docs/research/05-enemies.json'),
|
||||
readJson('docs/research/05-maps.json'),
|
||||
]);
|
||||
const { audioData, modelsData, charword } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log });
|
||||
const proxyPrefix = resolveProxyPrefix(opts.source, opts.offline);
|
||||
const source = await selectDownloadSource({ mode: opts.source, offline: opts.offline, proxyPrefix, log });
|
||||
const mirrorPolicy = new MirrorPolicy({ source, proxyPrefix, log });
|
||||
const network = { source, proxyPrefix, mirrorPolicy };
|
||||
const { audioData, modelsData, charword } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log, ...network });
|
||||
const audio = indexAudio(audioData);
|
||||
// The game data built by tools/build-data.mjs (when present) may reference more
|
||||
// spawnable enemies/tokens than research lists (e.g. 机变 enemy swaps): cover them too.
|
||||
@@ -255,7 +273,7 @@ async function main() {
|
||||
|
||||
const dl = new Downloader({
|
||||
root: ASSETS, ledgerPath: join(CACHE, 'assets-ledger.json'),
|
||||
concurrency: opts.concurrency, force: opts.force, log,
|
||||
concurrency: opts.concurrency, force: opts.force, log, ...network,
|
||||
});
|
||||
await dl.loadLedger();
|
||||
const downloadErrors = opts.offline ? [] : await downloadLeaves(leaves, dl, ASSETS, 'files');
|
||||
@@ -263,7 +281,7 @@ async function main() {
|
||||
// Fonts
|
||||
let fontErrors = [];
|
||||
if (!opts.offline) {
|
||||
const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log });
|
||||
const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log, ...network });
|
||||
await fdl.loadLedger();
|
||||
await fdl.run(fontJobs(), 'fonts');
|
||||
dl.totals.bytesDownloaded += fdl.totals.bytesDownloaded;
|
||||
|
||||
+9
-2
@@ -19,6 +19,9 @@
|
||||
// Options:
|
||||
// --check report only, change nothing (exit 1 when something essential is missing)
|
||||
// --no-assets skip the art/audio download
|
||||
// --asset-source=M direct (default) or mirror (opt-in; no public-IP lookup)
|
||||
// SP_ASSET_SOURCE sets the default; SP_GITHUB_PROXY sets the HTTPS prefix (https://gh-proxy.com/)
|
||||
// An empty SP_GITHUB_PROXY disables the proxy, including in mirror mode.
|
||||
// --no-local skip the local-client detection and extraction
|
||||
// --local extract from the local client without asking (re-extracts when already done)
|
||||
// --game <dir> AssetBundle root of the local client (…/StreamingAssets/AB/Windows or PlayCover …/Documents/Bundles)
|
||||
@@ -35,6 +38,7 @@ import path from 'node:path';
|
||||
import readline from 'node:readline';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { validateSource } from './assets/network.mjs';
|
||||
|
||||
export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
export const MIN_NODE = 22;
|
||||
@@ -328,11 +332,13 @@ function ensureVenv(py, log) {
|
||||
// ---------------------------------------------------------------------------------------------------
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { check: false, assets: true, local: 'ask', game: null, yes: false, quiet: false, help: false };
|
||||
const o = { check: false, assets: true, local: 'ask', game: null, yes: false, quiet: false, help: false, source: process.env.SP_ASSET_SOURCE || 'direct' };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--check') o.check = true;
|
||||
else if (a === '--no-assets') o.assets = false;
|
||||
else if (a === '--asset-source') o.source = argv[++i];
|
||||
else if (a.startsWith('--asset-source=')) o.source = a.slice('--asset-source='.length);
|
||||
else if (a === '--no-local') o.local = 'no';
|
||||
else if (a === '--local') o.local = 'force';
|
||||
else if (a === '--game') { o.game = argv[++i] || null; if (o.local !== 'no') o.local = 'force'; }
|
||||
@@ -342,6 +348,7 @@ function parseArgs(argv) {
|
||||
else if (a === '-h' || a === '--help') o.help = true;
|
||||
else throw new Error(`未知参数 / unknown option: ${a}(--help 查看用法)`);
|
||||
}
|
||||
if (!o.help) validateSource(o.source);
|
||||
return o;
|
||||
}
|
||||
|
||||
@@ -413,7 +420,7 @@ async function main() {
|
||||
const what = !assets.present ? `首次下载约 ${assets.bytes ? mb(assets.bytes) : '270 MB'},可随时中断,重新运行会续传`
|
||||
: `补全缺失的 ${assets.missing} 个文件`;
|
||||
log(`\n${c.cyan('▶')} 下载美术与音频素材(${what})…`);
|
||||
const r = run(process.execPath, [path.join(ROOT, 'tools', 'fetch-assets.mjs')]);
|
||||
const r = run(process.execPath, [path.join(ROOT, 'tools', 'fetch-assets.mjs'), `--asset-source=${opts.source}`]);
|
||||
assets = checkAssets();
|
||||
if (!r.ok && !assets.ok) log(c.warn(' 素材下载未完成(网络问题?)。游戏仍可运行(使用占位图),稍后重新运行 setup 即可续传。'));
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user