ark-stronghold-protocol/tools/fetch-assets.mjs

#!/usr/bin/env node
// Downloads, post-processes and indexes every art/audio/font asset the game
// needs, from community dumps of the official client (research 07):
//   public/assets/**   images, Spine models, audio   (git-ignored)
//   public/fonts/**    Bender / Novecento (.otf/.ttf + .woff2) and fonts.css
//   data/assets.json   manifest used by the client (schema: docs/ASSETS.md)
//
// Enemy models no dump carries are taken from the local client when
// tools/local-extract/extract.py has extracted them (public/assets/local/spine/
// enemy/<id>/, ASSETS.md "Enemy aliases"); otherwise another enemy's model.
//
// Idempotent: existing files with the right size are skipped, so re-running is
// cheap. Downloads use ~16 parallel connections, 3 retries per source and a
// jsDelivr mirror fallback. Spine atlases get `size:` (and `pma: true` for
// enemies); every skeleton is parsed to resolve animation roles.
//
// Usage: node tools/fetch-assets.mjs [--concurrency=16] [--force] [--offline]
//                                    [--dry-run] [--refresh-index] [--prune] [--help]

import { readFile, writeFile, mkdir, rename, readdir, unlink } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { dirname, join, relative, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { Downloader } from './assets/downloader.mjs';
import { loadIndexes } from './assets/cache.mjs';
import { indexAudio } from './assets/audio.mjs';
import { buildPlan } from './assets/plan.mjs';
import { processModels, findLocalEnemyModels } from './assets/spine.mjs';
import { collectLeaves, downloadLeaves, resolveTemplate, totalBytes, contentHash, MANIFEST_VERSION } from './assets/manifest.mjs';
import { fontJobs, buildFonts } from './assets/fonts.mjs';
import { skelParserAvailable } from './assets/skel.mjs';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const ASSETS = join(ROOT, 'public', 'assets');
const FONTS = join(ROOT, 'public', 'fonts');
const CACHE = join(ROOT, '.cache');
const MANIFEST = join(ROOT, 'data', 'assets.json');
const REPORT = join(CACHE, 'assets-report.json');

const HELP = `Usage: node tools/fetch-assets.mjs [options]
  --concurrency=N   parallel downloads (default 16)
  --force           re-download files even when present
  --offline         no network: post-process what is on disk and rebuild data/assets.json
  --dry-run         print the plan and exit
  --refresh-index   re-download audio_data.json / models_data.json indexes
  --prune           delete files under public/assets that the manifest no longer references
  --help            this text`;

/**
 * Parse CLI flags.
 * @param {string[]} argv
 * @returns {{concurrency:number, force:boolean, offline:boolean, dryRun:boolean, refreshIndex:boolean, prune:boolean, help:boolean}}
 */
function parseArgs(argv) {
  const o = { concurrency: 16, force: false, offline: false, dryRun: false, refreshIndex: false, prune: false, help: false };
  for (const a of argv) {
    const [k, v] = a.split('=');
    if (k === '--concurrency') o.concurrency = Math.max(1, Math.min(64, parseInt(v, 10) || 16));
    else if (k === '--force') o.force = true;
    else if (k === '--offline') o.offline = true;
    else if (k === '--dry-run') o.dryRun = true;
    else if (k === '--refresh-index') o.refreshIndex = true;
    else if (k === '--prune') o.prune = true;
    else if (k === '--help' || k === '-h') o.help = true;
    else throw new Error(`unknown option ${a}\n${HELP}`);
  }
  return o;
}

const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;
const log = (m) => console.log(m);

async function readJson(rel) {
  const p = join(ROOT, rel);
  try { return JSON.parse(await readFile(p, 'utf8')); } catch (e) { throw new Error(`cannot read ${rel}: ${e.message}`); }
}

async function writeJsonAtomic(path, value, indent) {
  await mkdir(dirname(path), { recursive: true });
  await writeFile(path + '.tmp', JSON.stringify(value, null, indent) + '\n');
  await rename(path + '.tmp', path);
}

/** All files under a directory, as forward-slash paths relative to it. */
async function listFiles(dir, base = dir, out = []) {
  let entries = [];
  try { entries = await readdir(dir, { withFileTypes: true }); } catch { return out; }
  for (const e of entries) {
    const p = join(dir, e.name);
    if (e.isDirectory()) await listFiles(p, base, out);
    else if (e.isFile()) out.push(relative(base, p).split(sep).join('/'));
  }
  return out;
}

/** Remove fields that only make sense next to a resolved spine. */
function tidyManifest(m) {
  for (const e of Object.values(m.enemies || {})) if (!e.spine) delete e.spineAliasOf;
  for (const t of Object.values(m.tokens || {})) if (!t.spine) delete t.spineVariant;
  if (m.skillsById) {
    for (const [sid, iconId] of Object.entries(m.skillsById)) if (!m.skills?.[iconId]) delete m.skillsById[sid];
  }
  for (const c of Object.values(m.chars || {})) if (c.spine && !Object.keys(c.spine).length) delete c.spine;
}

function countStats(m, bytes, files) {
  const vals = (o) => Object.values(o || {});
  const spines = new Set();
  for (const c of vals(m.chars)) for (const s of vals(c.spine)) spines.add(s.skel);
  for (const e of vals(m.enemies)) if (e.spine) spines.add(e.spine.skel);
  for (const t of vals(m.tokens)) if (t.spine) spines.add(t.spine.skel);
  return {
    files,
    bytes,
    chars: Object.keys(m.chars || {}).length,
    charsWithBack: vals(m.chars).filter((c) => c.spine?.back).length,
    enemies: Object.keys(m.enemies || {}).length,
    enemiesWithSpine: vals(m.enemies).filter((e) => e.spine).length,
    tokens: Object.keys(m.tokens || {}).length,
    tokensWithSpine: vals(m.tokens).filter((t) => t.spine).length,
    spineModels: spines.size,
    bonds: Object.keys(m.bonds || {}).length,
    items: Object.keys(m.items || {}).length,
    bands: Object.keys(m.bands || {}).length,
    skills: Object.keys(m.skills || {}).length,
    ui: Object.keys(m.ui || {}).length,
    sfxUnits: Object.keys(m.audio?.sfx?.units || {}).length,
  };
}

/** Required assets whose absence makes the run fail (exit code 1). */
function requiredMisses(m, charIds) {
  const out = [];
  for (const id of charIds) {
    const c = m.chars?.[id];
    if (!c?.avatar) out.push(`${id}.avatar`);
    if (!c?.portrait) out.push(`${id}.portrait`);
    if (!c?.spine?.front) out.push(`${id}.spine.front`);
  }
  return out;
}

async function main() {
  const opts = parseArgs(process.argv.slice(2));
  if (opts.help) { log(HELP); return 0; }
  const t0 = Date.now();
  log(`[assets] root ${ROOT}`);
  if (!skelParserAvailable()) throw new Error('@pixi-spine/runtime-3.8 not found — run `npm install` first');

  const [assets07, ops03, enemies05, maps05] = await Promise.all([
    readJson('docs/research/07-assets.json'),
    readJson('docs/research/03-operators.json'),
    readJson('docs/research/05-enemies.json'),
    readJson('docs/research/05-maps.json'),
  ]);
  const { audioData, modelsData } = await loadIndexes(ROOT, { refresh: opts.refreshIndex && !opts.offline, offline: opts.offline, log });
  const audio = indexAudio(audioData);
  // The game data built by tools/build-data.mjs (when present) may reference more
  // spawnable enemies/tokens than research lists (e.g. 机变 enemy swaps): cover them too.
  const [dataEnemies, dataTokens, dataBosses] = await Promise.all(
    ['data/enemies.json', 'data/tokens.json', 'data/bosses.json'].map((f) => readJson(f).catch(() => null)));
  const extraHandbook = {};
  for (const b of Object.values(dataBosses || {})) if (b?.enemyKey && typeof b.handbookId === 'string') extraHandbook[b.enemyKey] = b.handbookId;
  const plan = buildPlan({
    assets07, ops03, enemies05, maps05, audio, modelsData,
    extraEnemyIds: Object.keys(dataEnemies || {}),
    extraTokenIds: Object.keys(dataTokens || {}),
    extraHandbook,
    localEnemyModels: await findLocalEnemyModels(ASSETS),
  });
  const leaves = collectLeaves(plan.template);
  log(`[plan] ${leaves.length} files + ${plan.models.size} Spine models ` +
    `(${Object.keys(plan.template.chars).length} chars, ${Object.keys(plan.template.enemies).length} enemies, ` +
    `${Object.keys(plan.template.tokens).length} tokens, ${Object.keys(plan.template.ui).length} UI sprites, ` +
    `${Object.keys(plan.template.audio.sfx.units).length} units with SFX)`);
  if (opts.dryRun) {
    for (const n of plan.notes) log(`  note: ${n}`);
    return 0;
  }

  const dl = new Downloader({
    root: ASSETS, ledgerPath: join(CACHE, 'assets-ledger.json'),
    concurrency: opts.concurrency, force: opts.force, log,
  });
  await dl.loadLedger();
  const downloadErrors = opts.offline ? [] : await downloadLeaves(leaves, dl, ASSETS, 'files');

  // Fonts
  let fontErrors = [];
  if (!opts.offline) {
    const fdl = new Downloader({ root: FONTS, ledgerPath: join(CACHE, 'fonts-ledger.json'), concurrency: 4, force: opts.force, log });
    await fdl.loadLedger();
    await fdl.run(fontJobs(), 'fonts');
    dl.totals.bytesDownloaded += fdl.totals.bytesDownloaded;
    for (const k of ['ok', 'skip', 'miss', 'error']) dl.totals[k] += fdl.totals[k];
  }
  const fonts = await buildFonts(FONTS, log);
  fontErrors = fonts.errors;

  // Spine
  const spine = await processModels(plan.models, {
    root: ASSETS, dl, cachePath: join(CACHE, 'spine-info.json'), download: !opts.offline, log,
  });

  // Manifest
  const resolved = resolveTemplate(plan.template, { root: ASSETS, spine: spine.entries, sourceOf: (rel) => dl.ledger.files[rel]?.url });
  const body = resolved.value;
  tidyManifest(body);
  const fontFaces = {};
  for (const [name, f] of Object.entries(fonts.files)) fontFaces[name] = f;
  body.fonts = existsSync(join(FONTS, 'fonts.css')) ? { css: '/fonts/fonts.css', faces: fontFaces } : { faces: fontFaces };
  const bytes = totalBytes(ASSETS, resolved.files);
  const manifest = {
    version: MANIFEST_VERSION,
    hash: contentHash(body),
    generator: 'tools/fetch-assets.mjs',
    stats: countStats(body, bytes, resolved.files.size),
    ...body,
  };
  await writeJsonAtomic(MANIFEST, manifest);

  // Orphans: files on disk that the manifest does not reference (e.g. after a mapping change). public/assets/local/**
  // belongs to tools/local-extract (data/local-assets.json) and is never an orphan: --prune used to delete it.
  const orphans = (await listFiles(ASSETS)).filter((r) => !resolved.files.has(r) && !r.startsWith('local/'));
  if (opts.prune) for (const r of orphans) { try { await unlink(join(ASSETS, r)); } catch { /* ignore */ } }

  const charIds = Object.keys(assets07.operators || {});
  const required = requiredMisses(manifest, charIds);
  const report = {
    downloadedBytes: dl.totals.bytesDownloaded,
    totals: dl.totals,
    stats: manifest.stats,
    requiredMisses: required,
    misses: resolved.misses,
    downloadErrors, // leaves whose primary failed transiently (fallbacks not tried; re-run to retry)
    fallbacks: resolved.fallbacks,
    spineProblems: spine.problems,
    fontErrors,
    orphans: opts.prune ? [] : orphans,
    pruned: opts.prune ? orphans : [],
    notes: plan.notes,
  };
  await writeJsonAtomic(REPORT, report, 1);

  // Summary
  const s = manifest.stats;
  log('');
  log('=== fetch-assets summary ===');
  log(`downloaded this run : ${mb(dl.totals.bytesDownloaded)} (ok ${dl.totals.ok}, skipped ${dl.totals.skip}, missing ${dl.totals.miss}, errors ${dl.totals.error}` +
    `${dl.totals.sizeChanged ? `, ${dl.totals.sizeChanged} changed upstream` : ''})`);
  log(`on disk (manifest)  : ${mb(s.bytes)} in ${s.files} files`);
  log(`chars ${s.chars} (Back model ${s.charsWithBack}) · enemies ${s.enemies} (Spine ${s.enemiesWithSpine}) · tokens ${s.tokens} (Spine ${s.tokensWithSpine}) · Spine models ${s.spineModels}`);
  log(`bonds ${s.bonds} · items ${s.items} · bands ${s.bands} · skill icons ${s.skills} · UI ${s.ui} · units with SFX ${s.sfxUnits}`);
  log(`fonts: ${Object.values(fonts.files).map((f) => f.woff2 || f.original).join(', ') || 'none'}`);
  if (resolved.fallbacks.length) { log(`fallbacks used (${resolved.fallbacks.length}):`); for (const f of resolved.fallbacks.slice(0, 20)) log(`  ${f}`); }
  if (downloadErrors.length) log(`download errors (${downloadErrors.length}, re-run to retry): ${downloadErrors.slice(0, 10).join(', ')}`);
  if (resolved.misses.length) {
    log(`missing (${resolved.misses.length}, omitted from manifest; client uses fallbacks):`);
    for (const m of resolved.misses.slice(0, 40)) log(`  ${m}`);
    if (resolved.misses.length > 40) log(`  … see ${REPORT}`);
  }
  if (spine.problems.length) { log(`spine notes (${spine.problems.length}):`); for (const p of spine.problems.slice(0, 20)) log(`  ${p}`); }
  for (const e of fontErrors) log(`font error: ${e}`);
  if (orphans.length) log(opts.prune ? `pruned ${orphans.length} unreferenced files` : `${orphans.length} unreferenced files on disk (run with --prune to delete)`);
  log(`manifest: ${MANIFEST} · report: ${REPORT} · ${((Date.now() - t0) / 1000).toFixed(0)} s`);
  if (required.length) {
    log(`ERROR: ${required.length} required assets missing: ${required.slice(0, 10).join(', ')}`);
    return 1;
  }
  return 0;
}

main().then((code) => { process.exitCode = code; }, (e) => {
  console.error(`[assets] FAILED: ${process.env.DEBUG ? e?.stack || e : e?.message || e}`);
  process.exitCode = 1;
});