Asset downloads can use a gh-proxy mirror when chosen by hand (--asset-source / SP_ASSET_SOURCE): one try per URL, the original source as fallback, the mirror off for the run after 3 failures in a row (#24)

Off by default; the original sources, retries, skipped files and the manifest-shrink guard are unchanged. Contributed by @NekoTC.

tools/assets/cache.mjs

@@ -6,7 +6,8 @@
 
 import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { RAW, mirrorUrl } from './sources.mjs';
+import { RAW } from './sources.mjs';
+import { MirrorPolicy } from './network.mjs';
 
 /**
  * Read a cached JSON file, downloading it first when missing or unparsable.
@@ -16,44 +17,70 @@ import { RAW, mirrorUrl } from './sources.mjs';
  * @param {boolean} [o.refresh] force re-download
  * @param {boolean} [o.offline] never download (throw when the cache is missing)
  * @param {(m:string)=>void} [o.log]
+ * @param {'direct'|'mirror'} [o.source]
+ * @param {string} [o.proxyPrefix]
+ * @param {typeof fetch} [o.fetchImpl]
+ * @param {MirrorPolicy} [o.mirrorPolicy] shared invocation-wide mirror circuit breaker
+ * @param {number} [o.timeoutMs]
+ * @param {number} [o.backoffMs]
  * @returns {Promise<any>} parsed JSON
  */
-export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log }) {
+export async function cachedJson({ cacheFile, url, refresh = false, offline = false, log = console.log, source = 'direct', proxyPrefix, fetchImpl = globalThis.fetch, mirrorPolicy, timeoutMs = 180000, backoffMs = 500 }) {
   if (!refresh || offline) {
     try { return JSON.parse(await readFile(cacheFile, 'utf8')); } catch (e) {
       if (offline) throw new Error(`--offline: cached index ${cacheFile} is missing or corrupt (${e.message}); run once online`);
     }
   }
+  const network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log });
   let lastErr = null;
-  for (const src of [url, mirrorUrl(url)].filter(Boolean)) {
-    for (let attempt = 1; attempt <= 3; attempt++) {
+  for (const src of network.urls(url)) {
+    if (network.skip(src)) continue;
+    const attempts = network.isProxy(src) ? 1 : 3;
+    for (let attempt = 1; attempt <= attempts; attempt++) {
+      let text, json;
       try {
         log(`[cache] downloading ${src}`);
-        const res = await fetch(src, { signal: AbortSignal.timeout(180000) });
-        if (!res.ok) throw new Error(`HTTP ${res.status}`);
-        const text = await res.text();
-        const json = JSON.parse(text);
-        await mkdir(dirname(cacheFile), { recursive: true });
-        await writeFile(cacheFile + '.tmp', text);
-        await rename(cacheFile + '.tmp', cacheFile);
-        return json;
+        const res = await network.request(src, fetchImpl, {}, timeoutMs);
+        if (!res.ok) {
+          await res.body?.cancel();
+          if (res.status === 404 || res.status === 410) {
+            network.succeeded(src); // missing index, not a proxy outage
+            lastErr = new Error(`HTTP ${res.status}`);
+            break;
+          }
+          throw new Error(`HTTP ${res.status}`);
+        }
+        text = (await network.readBody(src, res)).toString('utf8');
+        json = JSON.parse(text);
       } catch (e) {
         lastErr = e;
-        await new Promise((r) => setTimeout(r, 500 * attempt));
+        network.failed(src);
+        if (attempt < attempts && backoffMs) await new Promise((r) => setTimeout(r, backoffMs * attempt));
+        continue;
       }
+      network.succeeded(src);
+      // A local disk failure is not evidence that the mirror is unhealthy.
+      await mkdir(dirname(cacheFile), { recursive: true });
+      await writeFile(cacheFile + '.tmp', text);
+      await rename(cacheFile + '.tmp', cacheFile);
+      return json;
     }
   }
+  if (lastErr) network.directFailureHint();
   throw new Error(`cannot fetch ${url}: ${lastErr?.message}`);
 }
 
 /**
  * Load audio_data.json (official), charword_table.json (official voice slots) and Ark-Models models_data.json.
  * @param {string} root project root
- * @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void}} [opts]
+ * @param {{refresh?:boolean, offline?:boolean, log?:(m:string)=>void, source?:'direct'|'mirror', proxyPrefix?:string, fetchImpl?:typeof fetch, mirrorPolicy?:MirrorPolicy}} [opts]
  * @returns {Promise<{ audioData: any, modelsData: any, charword: any }>}
  */
 export async function loadIndexes(root, opts = {}) {
+  const mirrorPolicy = opts.mirrorPolicy ?? new MirrorPolicy(opts);
   const audioData = await cachedJson({
+    ...opts,
+    mirrorPolicy,
     cacheFile: join(root, '.cache', 'gamedata', 'excel', 'audio_data.json'),
     url: RAW.gamedata + 'excel/audio_data.json',
     refresh: opts.refresh,
@@ -63,6 +90,8 @@ export async function loadIndexes(root, opts = {}) {
   // 11 MB, and raw.githubusercontent stalls on it often: the jsDelivr mirror cachedJson() falls back to is the
   // reliable path (the operator battle voice needs it, plan.mjs indexVoice).
   const charword = await cachedJson({
+    ...opts,
+    mirrorPolicy,
     cacheFile: join(root, '.cache', 'gamedata', 'excel', 'charword_table.json'),
     url: RAW.gamedata + 'excel/charword_table.json',
     refresh: opts.refresh,
@@ -70,6 +99,8 @@ export async function loadIndexes(root, opts = {}) {
     log: opts.log,
   });
   const modelsData = await cachedJson({
+    ...opts,
+    mirrorPolicy,
     cacheFile: join(root, '.cache', 'ark-models', 'models_data.json'),
     url: RAW.arkModels + 'models_data.json',
     refresh: opts.refresh,

tools/assets/downloader.mjs

@@ -5,8 +5,9 @@
 //   download never appears at the final path.
 // - Per source: up to `retries` attempts with exponential backoff on network
 //   errors, timeouts, 403/429/5xx and payloads that fail format validation;
-//   a 404/410 moves on immediately. Sources: each candidate URL, then its
-//   jsDelivr mirror (sources.mirrorUrl).
+//   a 404/410 moves on immediately. Each candidate uses its opt-in prefix proxy
+//   first (when enabled), then the original URL and jsDelivr fallback. The proxy gets one short attempt
+//   per file and shares a run-wide circuit breaker with the index/font fetchers.
 // - Idempotent: an existing file is kept when its size matches the ledger entry
 //   of a previous download or the expected byte count from research, or (when
 //   neither is known) when it passes format validation. The ledger lives in
@@ -14,7 +15,7 @@
 
 import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { mirrorUrl } from './sources.mjs';
+import { MirrorPolicy } from './network.mjs';
 import { validate } from './formats.mjs';
 
 /**
@@ -42,8 +43,11 @@ export class Downloader {
    * @param {(msg:string)=>void} [o.log]
    * @param {typeof fetch} [o.fetchImpl]
    * @param {number} [o.backoffMs] base retry delay (doubles per attempt)
+   * @param {'direct'|'mirror'} [o.source] preferred download source
+   * @param {string} [o.proxyPrefix] HTTPS prefix for GitHub downloads
+   * @param {MirrorPolicy} [o.mirrorPolicy] invocation-wide policy and mirror circuit breaker
    */
-  constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400 }) {
+  constructor({ root, ledgerPath, concurrency = 16, retries = 3, timeoutMs = 120000, force = false, log = console.log, fetchImpl = globalThis.fetch, backoffMs = 400, source = 'direct', proxyPrefix, mirrorPolicy }) {
     this.root = root;
     this.ledgerPath = ledgerPath;
     this.concurrency = Math.max(1, Math.min(64, Number(concurrency) || 16));
@@ -52,6 +56,7 @@ export class Downloader {
     this.force = force;
     this.log = log;
     this.fetch = fetchImpl;
+    this.network = mirrorPolicy ?? new MirrorPolicy({ source, proxyPrefix, log });
     this.backoffMs = Math.max(0, Number(backoffMs) || 0);
     this.ledger = { files: {} };
     this.totals = { ok: 0, skip: 0, miss: 0, error: 0, bytesDownloaded: 0, sizeChanged: 0 };
@@ -101,36 +106,40 @@ export class Downloader {
 
   /**
    * Fetch one URL with retries.
-   * @returns {Promise<{buf:Buffer}|{notFound:true}|{error:string}>}
+   * @returns {Promise<{buf:Buffer}|{notFound:true}|{skipped:true}|{error:string}>}
    */
   async fetchWithRetries(url, kind) {
+    if (this.network.skip(url)) return { skipped: true };
     let lastErr = 'unknown error';
-    for (let attempt = 1; attempt <= this.retries; attempt++) {
+    const attempts = this.network.isProxy(url) ? 1 : this.retries;
+    for (let attempt = 1; attempt <= attempts; attempt++) {
       try {
-        const res = await this.fetch(url, {
+        const res = await this.network.request(url, this.fetch, {
           headers: { 'user-agent': 'stronghold-protocol-fetch-assets/1.0' },
-          signal: AbortSignal.timeout(this.timeoutMs),
           redirect: 'follow',
-        });
+        }, this.timeoutMs);
         if (res.status === 404 || res.status === 410) {
           try { await res.body?.cancel(); } catch { /* ignore */ }
+          this.network.succeeded(url); // a missing asset is not a proxy outage
           return { notFound: true };
         }
         if (!res.ok) {
           try { await res.body?.cancel(); } catch { /* ignore */ }
           throw new HttpError(res.status, url);
         }
-        const buf = Buffer.from(await res.arrayBuffer());
+        const buf = await this.network.readBody(url, res);
         const len = Number(res.headers.get('content-length'));
         const enc = res.headers.get('content-encoding');
         if (!enc && Number.isFinite(len) && len > 0 && len !== buf.length) {
           throw new Error(`truncated body ${buf.length}/${len} ${url}`);
         }
         if (!validate(kind, buf)) throw new Error(`invalid ${kind} payload (${buf.length} B) ${url}`);
+        this.network.succeeded(url);
         return { buf };
       } catch (e) {
         lastErr = e?.message || String(e);
-        if (attempt < this.retries && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs));
+        this.network.failed(url);
+        if (attempt < attempts && this.backoffMs) await sleep(this.backoffMs * 2 ** (attempt - 1) + Math.floor(Math.random() * this.backoffMs));
       }
     }
     return { error: lastErr };
@@ -146,10 +155,10 @@ export class Downloader {
     if (kept >= 0) return { status: 'skip', bytes: kept };
     let lastError = null;
     for (const url of job.urls) {
-      const sources = [url, mirrorUrl(url)].filter(Boolean);
+      const sources = this.network.urls(url);
       for (const src of sources) {
         const r = await this.fetchWithRetries(src, job.kind);
-        if (r.notFound) continue;
+        if (r.notFound || r.skipped) continue;
         if (r.error) { lastError = r.error; continue; }
         const abs = join(this.root, job.rel);
         await mkdir(dirname(abs), { recursive: true });
@@ -167,6 +176,7 @@ export class Downloader {
         return { status: 'ok', bytes: r.buf.length, url: src, sizeChanged };
       }
     }
+    if (lastError) this.network.directFailureHint();
     return lastError ? { status: 'error', bytes: 0, error: lastError } : { status: 'miss', bytes: 0, error: 'not found (404) on all sources' };
   }
 

tools/assets/manifest.mjs

@@ -7,7 +7,7 @@
 import { existsSync, statSync } from 'node:fs';
 import { createHash } from 'node:crypto';
 import { join } from 'node:path';
-import { assetUrl, mirrorUrl } from './sources.mjs';
+import { assetUrl, githubSourceUrl, mirrorUrl } from './sources.mjs';
 
 /** Manifest schema version (bump on breaking shape changes). */
 export const MANIFEST_VERSION = 1;
@@ -94,9 +94,11 @@ export function resolveTemplate(template, { root, spine, sourceOf = () => undefi
         const a = node.alts[i];
         if (existsSync(join(root, a.rel))) {
           const src = sourceOf(a.rel);
+          // Provenance only: disabling/changing a proxy does not change the
+          // identity of a previously downloaded asset.
           const primary = node.alts[0].urls.flatMap((u) => [u, mirrorUrl(u)]);
           if (i > 0) fallbacks.push(`${path} ← ${src || a.urls[0]}`);
-          else if (src && !primary.includes(src)) fallbacks.push(`${path} ← ${src}`);
+          else if (src && !primary.includes(githubSourceUrl(src))) fallbacks.push(`${path} ← ${src}`);
           files.add(a.rel);
           return assetUrl(a.rel);
         }

tools/assets/network.mjs

@@ -0,0 +1,115 @@
+// Explicit download-source selection and one shared policy per asset invocation.
+// Selecting a source never looks up the user's public IP or makes a request.
+import { downloadUrls, normalizeProxyPrefix } from './sources.mjs';
+
+export function validateSource(mode) {
+  if (!['direct', 'mirror'].includes(mode)) throw new Error(`unknown asset source: ${mode} (direct|mirror)`);
+  return mode;
+}
+
+export async function selectDownloadSource({ mode = 'direct', offline = false, proxyPrefix, log = console.log } = {}) {
+  validateSource(mode);
+  if (offline) return 'direct';
+  const normalizedProxy = mode === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : '';
+  log(mode === 'mirror'
+    ? `[network] 手动开启 GitHub 镜像：${normalizedProxy || '代理已禁用'}；文件仅校验格式和大小。`
+    : '[network] 原始源（未开启 GitHub 镜像）');
+  return mode;
+}
+
+/** Shared by indexes, files, Spine follow-ups and fonts; a tripped mirror stays off. */
+export class MirrorPolicy {
+  constructor({ source = 'direct', proxyPrefix, failureLimit = 3, timeoutMs = 8000, log = console.log } = {}) {
+    this.source = validateSource(source);
+    this.proxyPrefix = this.source === 'mirror' ? normalizeProxyPrefix(proxyPrefix) : '';
+    this.failureLimit = Math.max(1, Number(failureLimit) || 3);
+    this.timeoutMs = Math.max(1, Number(timeoutMs) || 8000);
+    this.log = log;
+    this.failures = 0;
+    this.disabled = false;
+    this.hintShown = false;
+    this.abortMirror = new AbortController();
+  }
+
+  urls(url) {
+    return downloadUrls(url, { source: this.disabled ? 'direct' : this.source, proxyPrefix: this.proxyPrefix });
+  }
+
+  isProxy(url) {
+    return this.source === 'mirror' && !!this.proxyPrefix && url.startsWith(this.proxyPrefix);
+  }
+
+  skip(url) { return this.isProxy(url) && this.disabled; }
+
+  async request(url, fetchImpl, options = {}, timeoutMs = 120000) {
+    if (!this.isProxy(url)) {
+      return fetchImpl(url, { ...options, signal: options.signal ?? AbortSignal.timeout(timeoutMs) });
+    }
+    const headerDeadline = new AbortController();
+    const signal = AbortSignal.any([headerDeadline.signal, this.abortMirror.signal]);
+    const timer = setTimeout(() => headerDeadline.abort(new Error('GitHub mirror response headers timed out')), Math.min(timeoutMs, this.timeoutMs));
+    try {
+      return await fetchImpl(url, { ...options, signal });
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async readBody(url, response) {
+    if (!this.isProxy(url)) return Buffer.from(await response.arrayBuffer());
+    if (!response.body) return Buffer.alloc(0);
+    const reader = response.body.getReader();
+    const chunks = [];
+    let size = 0;
+    try {
+      while (true) {
+        const { value, done } = await readChunk(reader, this.timeoutMs, this.abortMirror.signal);
+        if (done) break;
+        const chunk = Buffer.from(value);
+        chunks.push(chunk);
+        size += chunk.length;
+      }
+    } catch (error) {
+      try { await reader.cancel(error); } catch { /* ignore */ }
+      throw error;
+    } finally {
+      reader.releaseLock();
+    }
+    return Buffer.concat(chunks, size);
+  }
+
+  succeeded(url) {
+    // A success from a request already in flight must not reopen a tripped circuit.
+    if (this.isProxy(url) && !this.disabled) this.failures = 0;
+  }
+
+  failed(url) {
+    if (this.isProxy(url)) {
+      if (this.disabled || ++this.failures < this.failureLimit) return;
+      this.disabled = true;
+      this.log(`[network] GitHub 镜像连续失败 ${this.failures} 次：本次运行停用镜像，继续使用原始源 / jsDelivr。`);
+      this.abortMirror.abort(new Error('GitHub mirror disabled for this run'));
+    }
+  }
+
+  directFailureHint() {
+    if (this.source !== 'direct' || this.hintShown) return;
+    this.hintShown = true;
+    this.log('[network] GitHub 下载失败；如需使用第三方镜像，可重新运行并加 --asset-source=mirror（或 SP_ASSET_SOURCE=mirror）。');
+  }
+}
+
+function readChunk(reader, timeoutMs, signal) {
+  return new Promise((resolve, reject) => {
+    let timer;
+    const cleanup = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', onAbort);
+    };
+    const onAbort = () => { cleanup(); reject(signal.reason ?? new Error('GitHub mirror request aborted')); };
+    if (signal?.aborted) return onAbort();
+    timer = setTimeout(() => { cleanup(); reject(new Error(`GitHub mirror response body idle for ${timeoutMs} ms`)); }, timeoutMs);
+    signal?.addEventListener('abort', onAbort, { once: true });
+    reader.read().then((value) => { cleanup(); resolve(value); }, (error) => { cleanup(); reject(error); });
+  });
+}

tools/assets/sources.mjs

@@ -1,10 +1,59 @@
 // Upstream asset sources (GitHub dumps of the official client), URL helpers and
 // local-path helpers shared by the asset pipeline (tools/fetch-assets.mjs).
 //
-// Every URL we download from is a raw.githubusercontent.com URL; mirrorUrl()
-// maps it to the equivalent jsDelivr URL used as a fallback when GitHub raw
-// fails. jsDelivr 404s on the ArknightsAssets2 `voice` branch, so no mirror is
-// offered for audio. See docs/ASSETS.md for the full source list and credits.
+// GitHub URLs support a prefix proxy and a jsDelivr fallback. jsDelivr does
+// not serve the voice branch, but the prefix proxy can still be tried there.
+// See docs/ASSETS.md for the full source list and credits.
+
+export const DEFAULT_GITHUB_PROXY = 'https://gh-proxy.com/';
+
+export function normalizeProxyPrefix(prefix = DEFAULT_GITHUB_PROXY) {
+  if (prefix.trim() === '') return '';
+  const url = new URL(prefix);
+  if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) {
+    throw new Error('GitHub proxy prefix must be an HTTPS URL without credentials, query or fragment');
+  }
+  return url.href.endsWith('/') ? url.href : url.href + '/';
+}
+
+function isGithubDownloadUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === 'https:' && !parsed.username && !parsed.password &&
+      ['github.com', 'raw.githubusercontent.com', 'objects.githubusercontent.com'].includes(parsed.host);
+  } catch { return false; }
+}
+
+/**
+ * Legacy ledgers store the successful proxy URL. Spine page jobs derived from
+ * those URLs must use today's policy, not implicitly opt in to the old proxy.
+ * Recognize embedded public GitHub URLs even when the old custom prefix is no
+ * longer configured. An ordinary GitHub URL (including its path) stays intact.
+ */
+export function githubSourceUrl(url) {
+  let candidate = String(url);
+  while (candidate.startsWith('https://')) {
+    if (isGithubDownloadUrl(candidate)) return candidate;
+    const marker = candidate.indexOf('/https://', 8);
+    if (marker < 0) break;
+    candidate = candidate.slice(marker + 1);
+  }
+  return url;
+}
+
+/** Only public GitHub download URLs are sent to the proxy; never prefix twice. */
+export function githubProxyUrl(url, prefix = DEFAULT_GITHUB_PROXY) {
+  const normalized = normalizeProxyPrefix(prefix);
+  if (!normalized || !isGithubDownloadUrl(url)) return null;
+  return normalized + url;
+}
+
+/** Ordered transport alternatives; direct mode retains the existing jsDelivr order. */
+export function downloadUrls(url, { source = 'direct', proxyPrefix = DEFAULT_GITHUB_PROXY } = {}) {
+  url = githubSourceUrl(url);
+  const proxy = source === 'mirror' ? githubProxyUrl(url, proxyPrefix) : null;
+  return [...new Set([proxy, url, mirrorUrl(url)].filter(Boolean))];
+}
 
 /** Raw base URLs (always end with '/'). */
 export const RAW = Object.freeze({