From cefd1e7313c684a25dd85254284d485efb79cbd1 Mon Sep 17 00:00:00 2001 From: sganggs Date: Wed, 7 Oct 2026 11:23:55 +0800 Subject: [PATCH] =?UTF-8?q?fb5-patch-pkg:=20every=20release=20zip=20carrie?= =?UTF-8?q?s=20MANIFEST.json=20(size=20+=20sha256=20of=20each=20shipped=20?= =?UTF-8?q?non-art=20file,=20written=20after=20npm=20ci;=20the=20full,=20l?= =?UTF-8?q?ite=20and=20update=20zips=20of=20a=20version=20list=20the=20sam?= =?UTF-8?q?e),=20and=20`node=20tools/package.mjs=20--update=20--from=20[,=E2=80=A6]`=20builds=20Stronghold-Protocol-v-update.zip=20=E2=80=94=20the=20full=20stage=20built=20as=20be?= =?UTF-8?q?fore,=20compared=20file=20by=20file=20with=20every=20base=20(an?= =?UTF-8?q?=20earlier=20full=20zip=20or=20its=20extracted=20folder;=20tool?= =?UTF-8?q?s/package-update.mjs=20reads=20zips=20itself,=20zip64=20include?= =?UTF-8?q?d),=20shipping=20each=20file=20that=20is=20new=20or=20differs?= =?UTF-8?q?=20from=20at=20least=20one=20base=20plus=20MANIFEST.json=20and?= =?UTF-8?q?=20UPDATE.json=20(from,=20count,=20bytes,=20files,=20removed=20?= =?UTF-8?q?with=20each=20base's=20bytes);=20bases=20must=20be=20older,=20f?= =?UTF-8?q?ull=20(not=20lite=20or=20update)=20and=20distinct;=20the=20upda?= =?UTF-8?q?te=20stage=20gets=20the=20refusal=20and=20personal-info=20check?= =?UTF-8?q?s=20(sha256=20values=20left=20out=20of=20the=20name=20scan);=20?= =?UTF-8?q?test/update-package.test.js=20(8),=20test/package.test.js=20exp?= =?UTF-8?q?ects=20MANIFEST.json=20in=20the=20zip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/package.test.js | 3 +- test/update-package.test.js | 333 ++++++++++++++++++++++++++++++++++++ tools/package-update.mjs | 253 +++++++++++++++++++++++++++ tools/package.mjs | 290 +++++++++++++++++++++++++------ 4 files changed, 826 insertions(+), 53 deletions(-) create mode 100644 test/update-package.test.js create mode 100644 tools/package-update.mjs diff --git a/test/package.test.js b/test/package.test.js index d9177f7f..00409acf 100644 --- a/test/package.test.js +++ b/test/package.test.js @@ -247,7 +247,8 @@ test('a --no-install build of the temporary checkout zips exactly the plan in on assert.deepEqual(fs.readdirSync(out), [path.basename(zip)], 'the stage is removed after zipping'); if (hasUnzip) { const entries = spawnSync('unzip', ['-Z1', zip], { encoding: 'utf8' }).stdout.split('\n').filter((l) => l && !l.endsWith('/')); - assert.deepEqual(entries.sort(), [...SHIPPED_TRACKED, ...ART].map((f) => `${FOLDER}/${f}`).sort()); + // and MANIFEST.json, written after npm ci (server/update.js; test/update-package.test.js checks its content) + assert.deepEqual(entries.sort(), [...SHIPPED_TRACKED, ...ART, 'MANIFEST.json'].map((f) => `${FOLDER}/${f}`).sort()); } const again = runTool(['--root', dir, '--out', out, '--no-install', '--allow-dirty']); assert.equal(again.status, 1, 'an existing zip is not replaced without --force'); diff --git a/test/update-package.test.js b/test/update-package.test.js new file mode 100644 index 00000000..06554b52 --- /dev/null +++ b/test/update-package.test.js @@ -0,0 +1,333 @@ +// test/update-package.test.js — the update package's packaging side (tools/package.mjs --update, tools/package-update.mjs, +// docs/DEPLOY.md §7): the comparison with the bases, reading a base (full zip or folder) and refusing what cannot be one, +// the zip reader, MANIFEST.json / UPDATE.json in the personal-info scan, MANIFEST.json in every zip and a whole update +// round trip on a small temporary checkout (--no-install), finished by the boot step. No network, no npm install. +// Run: node --test test/update-package.test.js (the player's side: test/update-boot.test.js) + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import zlib from 'node:zlib'; +import { APPLIED_FILE, MANIFEST_FILE, UPDATE_FILE, applyPendingUpdate, checkInstall, parseUpdate } from '../server/update.js'; +import { compareVersions, diffBases, readBase, readZip } from '../tools/package-update.mjs'; +import { FOLDER, readBases, scanFiles, stageProblems } from '../tools/package.mjs'; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +const TOOL = path.join(ROOT, 'tools', 'package.mjs'); +const ENV = { SP_PACKAGE_SCAN_USER: '0', SP_PACKAGE_SCAN_NAMES: 'plantedname' }; + +const sha = (body) => crypto.createHash('sha256').update(body).digest('hex'); +const dg = (body) => ({ size: Buffer.byteLength(body), sha256: sha(body) }); +const tmp = (tag) => fs.mkdtempSync(path.join(os.tmpdir(), `sp-update-${tag}-`)); +const put = (dir, rel, body = `// ${rel}\n`) => { + fs.mkdirSync(path.join(dir, path.dirname(rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), body); +}; +const has = (dir, rel) => fs.existsSync(path.join(dir, rel)); +const read = (dir, rel) => fs.readFileSync(path.join(dir, rel), 'utf8'); +const quiet = () => { + const lines = []; + return { lines, log: { log: (m) => lines.push(['log', m]), warn: (m) => lines.push(['warn', m]), error: (m) => lines.push(['error', m]) } }; +}; +const files = (obj) => new Map(Object.entries(obj).map(([rel, body]) => [rel, dg(body)])); + +// --------------------------------------------------------------------------------------------------------------------- +// The comparison + +test('diff: a file ships when it is new or differs from the base; removed = what the base shipped and the new one does not', () => { + const next = files({ 'server/a.js': 'A2', 'server/b.js': 'B', 'server/new.js': 'N', 'public/assets/x.png': 'X' }); + const base = { version: '0.2.0', files: files({ 'server/a.js': 'A1', 'server/b.js': 'B', 'server/gone.js': 'G', 'public/assets/x.png': 'X' }) }; + const d = diffBases(next, [base]); + assert.deepEqual(d.ship, ['server/a.js', 'server/new.js'], 'changed + added; unchanged files and art stay out'); + assert.deepEqual(d.removed, [{ path: 'server/gone.js', sha256: [sha('G')] }]); + assert.deepEqual(d.perBase, [{ version: '0.2.0', added: 1, changed: 1, unchanged: 2, removed: 1 }]); + assert.deepEqual(d.caseOnly, []); + // same size, other bytes (a version string 0.2.0 → 0.2.1) still ships + const same = diffBases(files({ 'shared/constants.js': "v='0.2.1'" }), [{ version: '0.2.0', files: files({ 'shared/constants.js': "v='0.2.0'" }) }]); + assert.deepEqual(same.ship, ['shared/constants.js']); +}); + +test('diff with two bases: cumulative — one update works over each; a file changed in the later base and reverted ships', () => { + const v020 = { version: '0.2.0', files: files({ a: 'a0', b: 'b0', c: 'c0', d: 'd0', g: 'g0', 'P/x.png': 'p' }) }; + const v021 = { version: '0.2.1', files: files({ a: 'a1', b: 'b0', c: 'c1', e: 'e1', g: 'g1', 'P/x.png': 'p' }) }; + // 0.2.2: a reverted to 0.2.0's bytes, b unchanged everywhere, c new bytes, d re-added after 0.2.1 dropped it, + // e (only 0.2.1 had it) and g (both had it, with other bytes) dropped, P/X.png renamed by case only + const next = files({ a: 'a0', b: 'b0', c: 'c2', d: 'd0', 'P/X.png': 'p' }); + const d = diffBases(next, [v020, v021]); + assert.deepEqual(d.ship, ['P/X.png', 'a', 'c', 'd'], 'a differs from 0.2.1, d is missing in 0.2.1, c differs from both'); + assert.ok(!d.ship.includes('b'), 'the same in every base: not shipped'); + assert.deepEqual(d.removed, [ + { path: 'e', sha256: [sha('e1')] }, + { path: 'g', sha256: [sha('g0'), sha('g1')].sort() }, + ], 'each base\'s bytes, so the copy of either install is recognised'); + assert.deepEqual(d.caseOnly, ['P/x.png'], 'one file on Windows / macOS: never deleted'); + assert.deepEqual(d.perBase.map((b) => [b.version, b.added, b.changed, b.unchanged, b.removed]), [ + ['0.2.0', 1, 1, 3, 2], // P/X.png is new to both bases (compared case-sensitively); removed: g and P/x.png + ['0.2.1', 2, 2, 1, 3], // added: d and P/X.png; changed: a and c; removed: e, g and P/x.png + ]); +}); + +test('versions compare numerically, a pre-release before its release', () => { + assert.equal(compareVersions('0.2.0', '0.2.1'), -1); + assert.equal(compareVersions('0.10.0', '0.9.9'), 1); + assert.equal(compareVersions('0.2.1-test', '0.2.1'), -1); + assert.equal(compareVersions('0.2.0', '0.2.1-test'), -1); + assert.equal(compareVersions('1.0.0', '1.0.0'), 0); + assert.throws(() => compareVersions('x', '1.0.0')); +}); + +// --------------------------------------------------------------------------------------------------------------------- +// The checks MANIFEST.json and UPDATE.json pass + +test('MANIFEST.json and UPDATE.json pass the personal-info scan: paths are scanned, sha256 values (a hex-only name could match) are not', () => { + const dir = tmp('scan'); + try { + const hex = `${'0'.repeat(30)}beef${'0'.repeat(30)}`; + fs.writeFileSync(path.join(dir, MANIFEST_FILE), `{"format":1,"app":"9.9.9","files":{\n"a.js":{"size":1,"sha256":"${hex}"}\n}}\n`); + assert.deepEqual(scanFiles(dir, [MANIFEST_FILE], { names: ['beef'] }), [`${MANIFEST_FILE} (account name)`], 'the plain scan would refuse it'); + assert.deepEqual(scanFiles(dir, [MANIFEST_FILE], { names: ['beef'], strip: /[0-9a-f]{64}/g }), []); + assert.deepEqual(stageProblems(dir, new Set([MANIFEST_FILE]), { env: { SP_PACKAGE_SCAN_USER: '0', SP_PACKAGE_SCAN_NAMES: 'beef' } }).problems, []); + fs.writeFileSync(path.join(dir, UPDATE_FILE), `{"removed":[{"path":"${['', 'home', 'someone'].join('/')}/x"}]}`); + assert.deepEqual(stageProblems(dir, new Set([MANIFEST_FILE, UPDATE_FILE]), { env: ENV }).problems, [`personal info: ${UPDATE_FILE} (home-directory path)`]); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// --------------------------------------------------------------------------------------------------------------------- +// Bases and a whole round trip on a temporary checkout (tools/package.mjs, --no-install) + +const hasZipTool = !spawnSync('zip', ['-v'], { encoding: 'utf8' }).error + || /bsdtar/.test(spawnSync('tar', ['--version'], { encoding: 'utf8' }).stdout || ''); +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13]); + +function fakeCheckout(version) { + const dir = tmp('co'); + const scripts = { start: 'node server/index.js', setup: 'node tools/setup.mjs', doctor: 'node tools/doctor.mjs', launch: 'node scripts/launch.mjs', + postinstall: 'node tools/vendor.mjs', vendor: 'node tools/vendor.mjs', assets: 'node tools/vendor.mjs && node tools/fetch-assets.mjs' }; + put(dir, 'package.json', JSON.stringify({ name: 'sp-test', version, main: 'server/index.js', scripts, dependencies: {} })); + put(dir, 'package-lock.json', JSON.stringify({ name: 'sp-test', version, lockfileVersion: 3, packages: { '': { name: 'sp-test', version } } })); + put(dir, 'server/index.js', "import './sim/rng.js';\n"); + put(dir, 'server/sim/rng.js', 'export const rng = 1;\n'); + put(dir, 'server/old.js', 'export const old = 1;\n'); + put(dir, 'shared/constants.js', `export const APP_VERSION = '${version}';\n`); + put(dir, 'public/index.html', '\n'); + put(dir, 'data/chess.json', '{}\n'); + put(dir, 'data/assets.json', JSON.stringify({ chars: { a: { avatar: '/assets/char/a.png' } }, fonts: { css: '/fonts/fonts.css' } })); + for (const f of ['tools/setup.mjs', 'tools/vendor.mjs', 'tools/fetch-assets.mjs', 'tools/doctor.mjs', 'scripts/launch.mjs', 'tools/golden.mjs']) put(dir, f); + put(dir, 'README.md', '# readme\n'); + put(dir, '.gitignore', 'public/assets/\npublic/fonts/\ndata/local-assets.json\n'); + put(dir, 'public/assets/char/a.png', PNG); + put(dir, 'public/fonts/fonts.css', '@font-face{}\n'); + const git = (...args) => spawnSync('git', ['-C', dir, ...args], { encoding: 'utf8' }); + assert.equal(git('init', '-q').status, 0); + assert.equal(git('add', '-A').status, 0); + return { dir, git }; +} + +function bump({ dir }, version) { + for (const f of ['package.json', 'package-lock.json']) put(dir, f, read(dir, f).replaceAll(/"version":"[^"]+"/g, `"version":"${version}"`)); + put(dir, 'shared/constants.js', `export const APP_VERSION = '${version}';\n`); +} + +const runTool = (args, env = ENV) => spawnSync(process.execPath, [TOOL, ...args], { encoding: 'utf8', env: { ...process.env, ...env }, maxBuffer: 16 * 1024 * 1024 }); + +/** Extract a release zip into `dir` (over what is there), as a player would: readZip writes every file. */ +function extract(zipPath, dir) { + readZip(zipPath, (name, data) => { + const to = path.join(dir, ...name.split('/')); + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.writeFileSync(to, data); + }); +} + +test('a whole update: base zip v9.9.8 → --update v9.9.9 ships only what changed, the boot step finishes it over the extracted base', { skip: !hasZipTool && 'no zip / tar' }, () => { + const co = fakeCheckout('9.9.8'); + const out = tmp('out'); + const inst = tmp('player'); + try { + let r = runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty']); + assert.equal(r.status, 0, r.stdout + r.stderr); + const baseZip = path.join(out, `${FOLDER}-v9.9.8.zip`); + // the next version: a code change, a new module, a dropped one, other art, the version + bump(co, '9.9.9'); + put(co.dir, 'server/sim/rng.js', 'export const rng = 2;\n'); + put(co.dir, 'server/new.js', 'export const n = 1;\n'); + put(co.dir, 'public/assets/char/a.png', Buffer.concat([PNG, Buffer.from('v2')])); + fs.rmSync(path.join(co.dir, 'server/old.js')); + assert.equal(co.git('add', '-A').status, 0); + + r = runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty', '--update', '--from', baseZip]); + assert.equal(r.status, 0, r.stdout + r.stderr); + assert.match(r.stdout, /^base: v9\.9\.8 · \d+ files \(1 art\) · /m); + assert.match(r.stdout, /^update: v9\.9\.9 over v9\.9\.8$/m); + assert.match(r.stdout, /^ {2}vs v9\.9\.8: 5 changed, 1 added, \d+ unchanged, 1 removed$/m); + assert.match(r.stdout, /^removed: 1 file\(s\) \(server\/old\.js\)$/m); + const upZip = path.join(out, `${FOLDER}-v9.9.9-update.zip`); + assert.deepEqual(fs.readdirSync(out).sort(), [`${FOLDER}-v9.9.8.zip`, `${FOLDER}-v9.9.9-update.zip`], 'both stages removed'); + const got = new Map(); + readZip(upZip, (name, data) => got.set(name, data)); + const SHIP = ['package-lock.json', 'package.json', 'public/assets/char/a.png', 'server/new.js', 'server/sim/rng.js', 'shared/constants.js']; + assert.deepEqual([...got.keys()].sort(), [...SHIP, MANIFEST_FILE, UPDATE_FILE].map((f) => `${FOLDER}/${f}`).sort()); + const u = parseUpdate(got.get(`${FOLDER}/${UPDATE_FILE}`).toString('utf8')); + assert.deepEqual([u.app, u.from, u.count, [...u.files.keys()]], ['9.9.9', ['9.9.8'], 6, SHIP]); + assert.equal(u.bytes, SHIP.reduce((n, f) => n + got.get(`${FOLDER}/${f}`).length, 0)); + assert.deepEqual(u.removed, [{ path: 'server/old.js', sha256: [sha('export const old = 1;\n')] }]); + + // the full and lite zips of the same version carry the same MANIFEST.json, listing the non-art files + r = runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty']); + assert.equal(r.status, 0, r.stdout + r.stderr); + r = runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty', '--lite']); + assert.equal(r.status, 0, r.stdout + r.stderr); + const manifestOf = (zip) => { let m = null; readZip(path.join(out, zip), (n, d) => { if (n === `${FOLDER}/${MANIFEST_FILE}`) m = d.toString('utf8'); }); return m; }; + const m = manifestOf(`${FOLDER}-v9.9.9.zip`); + assert.equal(manifestOf(`${FOLDER}-v9.9.9-lite.zip`), m); + assert.equal(got.get(`${FOLDER}/${MANIFEST_FILE}`).toString('utf8'), m); + const listed = JSON.parse(m); + assert.equal(listed.app, '9.9.9'); + assert.deepEqual(Object.keys(listed.files).sort(), ['README.md', 'data/chess.json', 'package-lock.json', 'package.json', 'public/index.html', + 'scripts/launch.mjs', 'server/index.js', 'server/new.js', 'server/sim/rng.js', 'shared/constants.js', 'tools/doctor.mjs', + 'tools/fetch-assets.mjs', 'tools/setup.mjs', 'tools/vendor.mjs'], 'no art, no data/assets.json'); + assert.deepEqual(listed.files['server/new.js'], dg('export const n = 1;\n')); + + // the player: the base extracted, a pack of their own, the update extracted over it, the server started + extract(baseZip, inst); + const root = path.join(inst, FOLDER); + put(root, 'packs/mine/pack.json', '{"type":"lang"}'); + extract(upZip, inst); + assert.equal(checkInstall(root).state, 'ok', 'the extracted update already matches its manifest'); + const q = quiet(); + const done = applyPendingUpdate(root, { log: q.log }); + assert.equal(done.state, 'applied', JSON.stringify(q.lines)); + assert.deepEqual(done.deleted, ['server/old.js']); + assert.ok(!has(root, 'server/old.js') && has(root, APPLIED_FILE) && !has(root, UPDATE_FILE) && has(root, 'packs/mine/pack.json')); + assert.equal(read(root, 'shared/constants.js'), "export const APP_VERSION = '9.9.9';\n"); + // the bases a later update may name: the full zip and the folder it was extracted to read the same + const asZip = readBase(path.join(out, `${FOLDER}-v9.9.9.zip`)); + const fresh = tmp('fresh'); + try { + extract(path.join(out, `${FOLDER}-v9.9.9.zip`), fresh); + put(fresh, `${FOLDER}/.DS_Store`, 'finder'); + const asFolder = readBases([fresh], '9.9.10')[0]; + assert.deepEqual([...asFolder.files.keys()].sort(), [...asZip.files.keys()].filter((f) => f !== MANIFEST_FILE).sort(), 'MANIFEST.json and clutter left out'); + assert.equal(asFolder.version, '9.9.9'); + } finally { + fs.rmSync(fresh, { recursive: true, force: true }); + } + } finally { + for (const d of [co.dir, out, inst]) fs.rmSync(d, { recursive: true, force: true }); + } +}); + +test('bases are refused when they cannot be one: lite, an update zip, not older, twice, not a release; the options go together', { skip: !hasZipTool && 'no zip / tar' }, () => { + const co = fakeCheckout('9.9.8'); + const out = tmp('out'); + try { + assert.equal(runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty']).status, 0); + assert.equal(runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty', '--lite']).status, 0); + const full = path.join(out, `${FOLDER}-v9.9.8.zip`); + const lite = path.join(out, `${FOLDER}-v9.9.8-lite.zip`); + bump(co, '9.9.9'); + put(co.dir, 'server/sim/rng.js', 'export const rng = 2;\n'); + assert.equal(co.git('add', '-A').status, 0); + assert.equal(runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty', '--update', '--from', full]).status, 0); + const update = path.join(out, `${FOLDER}-v9.9.9-update.zip`); + const refuse = (from, re) => { + const r = runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty', '--update', '--force', '--dry-run', '--from', from]); + assert.equal(r.status, 1, r.stdout); + assert.match(r.stderr, re); + }; + refuse(lite, /has no game art \(a lite zip\?\): pass the full zip, Stronghold-Protocol-v9\.9\.8\.zip/); + refuse(update, /is an update package: pass the earlier release's full zip/); + refuse(`${full},${full}`, /two bases are v9\.9\.8/); + refuse(path.join(out, 'nothing.zip'), /not found/); + fs.writeFileSync(path.join(out, 'other.zip'), 'not a zip'); + refuse(path.join(out, 'other.zip'), /not a zip file/); + assert.throws(() => readBases([full], '9.9.8'), /is v9\.9\.8, not older than v9\.9\.8/); + const dry = runTool(['--root', co.dir, '--out', out, '--allow-dirty', '--update', '--dry-run', '--from', full]); + assert.equal(dry.status, 0, dry.stderr); + assert.match(dry.stdout, /^update: the bases are fine/m); + for (const args of [['--from', full], ['--update'], ['--update', '--lite', '--from', full]]) { + const r = runTool(['--root', co.dir, '--dry-run', ...args]); + assert.equal(r.status, 2, args.join(' ')); + } + } finally { + fs.rmSync(co.dir, { recursive: true, force: true }); + fs.rmSync(out, { recursive: true, force: true }); + } +}); + +test('a base folder: npm\'s files are compared like the stage\'s; a name an update never deletes stays out of removed, named', { skip: !hasZipTool && 'no zip / tar' }, () => { + const co = fakeCheckout('9.9.8'); + const out = tmp('out'); + const base = tmp('base'); + try { + assert.equal(runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty']).status, 0); + extract(path.join(out, `${FOLDER}-v9.9.8.zip`), base); + // as if 9.9.8's npm ci had installed a package 9.9.9 no longer has, one file of it with a per-machine name + put(base, `${FOLDER}/node_modules/gone/index.js`, 'module.exports = 1;\n'); + put(base, `${FOLDER}/node_modules/gone/.env.example`, 'X=1\n'); + put(base, `${FOLDER}/logs/server.log`, 'a log'); // per-machine: never part of a base + bump(co, '9.9.9'); + assert.equal(co.git('add', '-A').status, 0); + const r = runTool(['--root', co.dir, '--out', out, '--no-install', '--allow-dirty', '--update', '--from', path.join(base, FOLDER), '--keep-stage']); + assert.equal(r.status, 0, r.stdout + r.stderr); + assert.match(r.stdout, /^removed: 1 file\(s\) \(node_modules\/gone\/index\.js\)$/m); + assert.match(r.stdout, /^not removed \(an update never deletes these names\): node_modules\/gone\/\.env\.example$/m); + const u = parseUpdate(read(path.join(out, `${FOLDER}-v9.9.9-update`, FOLDER), UPDATE_FILE)); + assert.deepEqual(u.removed.map((x) => x.path), ['node_modules/gone/index.js']); + assert.ok(has(path.join(out, `${FOLDER}-v9.9.9-update`, '.full', FOLDER), MANIFEST_FILE), '--keep-stage keeps the full stage too'); + } finally { + for (const d of [co.dir, out, base]) fs.rmSync(d, { recursive: true, force: true }); + } +}); + +test('the zip reader: stored and deflated entries, zip64, UTF-8 names; damaged and foreign zips are refused', { skip: !hasZipTool && 'no zip / tar' }, () => { + const dir = tmp('zip'); + try { + put(dir, `${FOLDER}/package.json`, '{"version":"0.2.0"}'); + put(dir, `${FOLDER}/public/assets/卫戍 a.png`, Buffer.alloc(5000, 7)); + put(dir, `${FOLDER}/server/empty.js`, ''); + const zipped = []; + const make = (name, args) => { + const z = spawnSync('zip', ['-q', '-r', ...args, name, FOLDER], { cwd: dir }); + if (!z.error && z.status === 0) zipped.push(name); + }; + make('plain.zip', ['-X']); + make('stored.zip', ['-X', '-0']); + make('zip64.zip', ['-X', '-fz']); + if (!zipped.length) { + const t = spawnSync('tar', ['-c', '--format', 'zip', '-f', 'plain.zip', FOLDER], { cwd: dir }); + if (!t.error && t.status === 0) zipped.push('plain.zip'); + } + assert.ok(zipped.length > 0); + for (const z of zipped) { + const b = readBase(path.join(dir, z)); + assert.deepEqual([b.version, b.art, b.update], ['0.2.0', 1, false], z); + assert.deepEqual([...b.files.keys()].sort(), ['package.json', 'public/assets/卫戍 a.png', 'server/empty.js'], z); + assert.deepEqual(b.files.get('public/assets/卫戍 a.png'), dg(Buffer.alloc(5000, 7)), z); + assert.deepEqual(b.files.get('server/empty.js'), dg(''), z); + } + // one damaged byte in a stored entry: the CRC refuses it (zlib.crc32, Node ≥ 22.2) + if (zipped.includes('stored.zip') && typeof zlib.crc32 === 'function') { + const bytes = fs.readFileSync(path.join(dir, 'stored.zip')); + bytes[bytes.indexOf(Buffer.alloc(64, 7)) + 100] ^= 0xff; + fs.writeFileSync(path.join(dir, 'bad.zip'), bytes); + assert.throws(() => readBase(path.join(dir, 'bad.zip')), /bad\.zip: Stronghold-Protocol\/public\/assets\/卫戍 a\.png: CRC mismatch/); + } + const cut = fs.readFileSync(path.join(dir, zipped[0])); + fs.writeFileSync(path.join(dir, 'cut.zip'), cut.subarray(0, cut.length - 30)); + assert.throws(() => readBase(path.join(dir, 'cut.zip')), /cut\.zip: (?:not a zip file|damaged|unexpected end)/, 'a truncated download'); + // entries outside the one folder: not a release zip + put(dir, 'other/x.txt', 'x'); + const z = spawnSync('zip', ['-q', '-r', 'foreign.zip', 'other'], { cwd: dir }); + if (!z.error && z.status === 0) assert.throws(() => readBase(path.join(dir, 'foreign.zip')), /outside the Stronghold-Protocol\/ folder/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tools/package-update.mjs b/tools/package-update.mjs new file mode 100644 index 00000000..03b37cc2 --- /dev/null +++ b/tools/package-update.mjs @@ -0,0 +1,253 @@ +// tools/package-update.mjs — what `node tools/package.mjs --update --from [,…]` (docs/DEPLOY.md §7) needs +// besides the stage: read an earlier release — its full zip, or the folder that zip was extracted to — into +// {path → size + sha256}, compare the new full stage with every base, and write MANIFEST.json / UPDATE.json. +// tools/package.mjs builds the stage, copies what ships, checks and zips it; server/update.js finishes the update on +// the player's machine (and reads the two files back). +// +// A file ships when it is new or differs from at least one base, so one update zip works over each of them (0.2.0 and +// every later 0.2.x given as a base); `removed` lists the files some base shipped that the new version does not, with +// the bytes each base had (the player's copy is deleted only when it still holds them). A removed path that differs +// from a new one only in case is the same file on Windows / macOS: it is left out of `removed` (`caseOnly`). +// The zip reader handles what tools/package.mjs and GitHub hand out: stored and deflated entries, zip64, UTF-8 names. + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import { UPDATE_FILE, UPDATE_FORMAT, digestFile } from '../server/update.js'; + +/** Read `len` bytes at `pos` of an open file. */ +function readAt(fd, pos, len) { + const buf = Buffer.alloc(len); + let got = 0; + while (got < len) { + const n = fs.readSync(fd, buf, got, len - got, pos + got); + if (n <= 0) throw new Error('unexpected end of the zip file'); + got += n; + } + return buf; +} + +const SIG = { eocd: 0x06054b50, loc64: 0x07064b50, eocd64: 0x06064b50, central: 0x02014b50, local: 0x04034b50 }; +const U32 = 0xffffffff; + +/** + * Call `onFile(name, data)` for every file entry of a zip (directories skipped), in central-directory order. + * @param {string} zipPath + * @param {(name: string, data: Buffer) => void} onFile + */ +export function readZip(zipPath, onFile) { + const fd = fs.openSync(zipPath, 'r'); + try { + const size = fs.fstatSync(fd).size; + const tailLen = Math.min(size, 22 + 65535); + const tail = readAt(fd, size - tailLen, tailLen); + let e = -1; + for (let i = tailLen - 22; i >= 0; i--) if (tail.readUInt32LE(i) === SIG.eocd) { e = i; break; } + if (e < 0) throw new Error('not a zip file (no end of central directory)'); + let count = tail.readUInt16LE(e + 10); + let cdSize = tail.readUInt32LE(e + 12); + let cdOffset = tail.readUInt32LE(e + 16); + if (count === 0xffff || cdSize === U32 || cdOffset === U32) { + const locPos = size - tailLen + e - 20; + const loc = locPos >= 0 ? readAt(fd, locPos, 20) : null; + if (!loc || loc.readUInt32LE(0) !== SIG.loc64) throw new Error('zip64 locator missing'); + const rec = readAt(fd, Number(loc.readBigUInt64LE(8)), 56); + if (rec.readUInt32LE(0) !== SIG.eocd64) throw new Error('zip64 end record missing'); + count = Number(rec.readBigUInt64LE(32)); + cdSize = Number(rec.readBigUInt64LE(40)); + cdOffset = Number(rec.readBigUInt64LE(48)); + } + const cd = readAt(fd, cdOffset, cdSize); + let p = 0; + for (let n = 0; n < count; n++) { + if (p + 46 > cd.length || cd.readUInt32LE(p) !== SIG.central) throw new Error('damaged central directory'); + const madeBy = cd.readUInt16LE(p + 4); + const flags = cd.readUInt16LE(p + 8); + const method = cd.readUInt16LE(p + 10); + const crc = cd.readUInt32LE(p + 16); + let packedSize = cd.readUInt32LE(p + 20); + let rawSize = cd.readUInt32LE(p + 24); + const nameLen = cd.readUInt16LE(p + 28); + const extraLen = cd.readUInt16LE(p + 30); + const commentLen = cd.readUInt16LE(p + 32); + const external = cd.readUInt32LE(p + 38); + let offset = cd.readUInt32LE(p + 42); + const name = cd.toString('utf8', p + 46, p + 46 + nameLen); + if (packedSize === U32 || rawSize === U32 || offset === U32) { + // the zip64 extra field carries the values the header marks 0xFFFFFFFF, in this order + for (let q = p + 46 + nameLen, end = q + extraLen; q + 4 <= end;) { + const id = cd.readUInt16LE(q); + const len = cd.readUInt16LE(q + 2); + if (id === 1) { + let r = q + 4; + if (rawSize === U32) { rawSize = Number(cd.readBigUInt64LE(r)); r += 8; } + if (packedSize === U32) { packedSize = Number(cd.readBigUInt64LE(r)); r += 8; } + if (offset === U32) { offset = Number(cd.readBigUInt64LE(r)); r += 8; } + break; + } + q += 4 + len; + } + } + p += 46 + nameLen + extraLen + commentLen; + if (name.endsWith('/')) continue; + if ((madeBy >> 8) === 3 && ((external >>> 16) & 0o170000) === 0o120000) throw new Error(`${name}: a symbolic link (release zips have none)`); + if (flags & 1) throw new Error(`${name}: encrypted`); + const local = readAt(fd, offset, 30); + if (local.readUInt32LE(0) !== SIG.local) throw new Error(`${name}: damaged local header`); + const packed = readAt(fd, offset + 30 + local.readUInt16LE(26) + local.readUInt16LE(28), packedSize); + let data; + if (method === 0) data = packed; + else if (method === 8) { + try { data = zlib.inflateRawSync(packed); } catch (err) { throw new Error(`${name}: ${err.message}`, { cause: err }); } + } else throw new Error(`${name}: compression method ${method} (only stored and deflated entries)`); + if (data.length !== rawSize) throw new Error(`${name}: ${data.length} bytes, the directory says ${rawSize}`); + if (typeof zlib.crc32 === 'function' && (zlib.crc32(data) >>> 0) !== crc) throw new Error(`${name}: CRC mismatch (a damaged zip)`); + onFile(name, data); + } + } finally { + fs.closeSync(fd); + } +} + +/** -1 / 0 / 1 for two x.y.z[-pre] versions (a pre-release sorts before its release). */ +export function compareVersions(a, b) { + const parse = (v) => { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(String(v)); + if (!m) throw new Error(`not a version: ${v}`); + return { n: [Number(m[1]), Number(m[2]), Number(m[3])], pre: m[4] ?? null }; + }; + const x = parse(a); + const y = parse(b); + for (let i = 0; i < 3; i++) if (x.n[i] !== y.n[i]) return x.n[i] < y.n[i] ? -1 : 1; + if (x.pre === y.pre) return 0; + if (x.pre === null) return 1; + if (y.pre === null) return -1; + return x.pre < y.pre ? -1 : 1; +} + +const sha = (buf) => crypto.createHash('sha256').update(buf).digest('hex'); + +/** Files under `dir` (following symlinks) as posix paths relative to it. */ +function walk(dir, rel = '', out = []) { + for (const e of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) { + const child = rel ? `${rel}/${e.name}` : e.name; + let st; + try { st = fs.statSync(path.join(dir, child)); } catch { continue; } + if (st.isDirectory()) walk(dir, child, out); + else if (st.isFile()) out.push(child); + } + return out; +} + +/** + * An earlier release as a base: its full zip, or the folder it was extracted to (the Stronghold-Protocol folder or the + * one holding it — an unmodified extraction). `skip(rel)` leaves paths out (per-machine files, OS clutter, the update + * files themselves). + * @param {string} source + * @param {{ folder?: string, skip?: (rel: string) => boolean }} [opts] + * @returns {{ source: string, version: string, files: Map, art: number, update: boolean }} + */ +export function readBase(source, { folder = 'Stronghold-Protocol', skip = () => false } = {}) { + const abs = path.resolve(source); + let st; + try { st = fs.statSync(abs); } catch { throw new Error(`base ${source}: not found`); } + const files = new Map(); + let pkg = null; + let update = false; + const add = (rel, d, read) => { + if (rel === 'package.json') { try { pkg = JSON.parse(read().toString('utf8')); } catch { pkg = {}; } } + if (rel === UPDATE_FILE) update = true; + if (!skip(rel)) files.set(rel, d); + }; + if (st.isDirectory()) { + let root = abs; + if (!fs.existsSync(path.join(root, 'package.json')) && fs.existsSync(path.join(root, folder, 'package.json'))) root = path.join(root, folder); + if (!fs.existsSync(path.join(root, 'package.json'))) throw new Error(`base ${source}: no package.json (pass a release's full zip or the folder it was extracted to)`); + for (const rel of walk(root)) { + const p = path.join(root, ...rel.split('/')); + const d = digestFile(p); + if (!d) throw new Error(`base ${source}: cannot read ${rel}`); + add(rel, d, () => fs.readFileSync(p)); + } + } else { + const prefix = `${folder}/`; + try { + readZip(abs, (name, data) => { + if (!name.startsWith(prefix)) throw new Error(`${name} is outside the ${folder}/ folder (not a release zip)`); + add(name.slice(prefix.length), { size: data.length, sha256: sha(data) }, () => data); + }); + } catch (e) { + throw new Error(`base ${source}: ${e.message}`, { cause: e }); + } + if (!pkg) throw new Error(`base ${source}: no ${folder}/package.json (not a release zip)`); + } + const version = pkg?.version; + if (typeof version !== 'string' || !/^\d+\.\d+\.\d+/.test(version)) throw new Error(`base ${source}: package.json names no version`); + let art = 0; + for (const rel of files.keys()) if (rel.startsWith('public/assets/')) art++; + return { source: abs, version, files, art, update }; +} + +/** + * Compare the new full stage with the bases. + * @param {Map} next every file of the new version (the update files left out) + * @param {{ version: string, files: Map }[]} bases + * @returns {{ ship: string[], removed: { path: string, sha256: string[] }[], caseOnly: string[], + * perBase: { version: string, added: number, changed: number, unchanged: number, removed: number }[] }} + */ +export function diffBases(next, bases) { + const ship = new Set(); + const perBase = []; + for (const b of bases) { + const n = { version: b.version, added: 0, changed: 0, unchanged: 0, removed: 0 }; + for (const [rel, d] of next) { + const o = b.files.get(rel); + if (!o) { n.added++; ship.add(rel); } else if (o.size !== d.size || o.sha256 !== d.sha256) { n.changed++; ship.add(rel); } else n.unchanged++; + } + for (const rel of b.files.keys()) if (!next.has(rel)) n.removed++; + perBase.push(n); + } + const lower = new Set([...next.keys()].map((f) => f.toLowerCase())); + const removed = new Map(); + const caseOnly = new Set(); + for (const b of bases) { + for (const [rel, d] of b.files) { + if (next.has(rel)) continue; + if (lower.has(rel.toLowerCase())) { caseOnly.add(rel); continue; } + if (!removed.has(rel)) removed.set(rel, new Set()); + removed.get(rel).add(d.sha256); + } + } + return { + ship: [...ship].sort(), + removed: [...removed.keys()].sort().map((rel) => ({ path: rel, sha256: [...removed.get(rel)].sort() })), + caseOnly: [...caseOnly].sort(), + perBase, + }; +} + +const fileLine = ([rel, d]) => `${JSON.stringify(rel)}:{"size":${d.size},"sha256":"${d.sha256}"}`; +const block = (lines) => (lines.length ? `\n${lines.join(',\n')}\n` : ''); + +/** {path → {size, sha256}} as a JSON object, one file per line (sorted): readable, small diffs between releases. */ +function filesJson(files) { + return `{${block([...files].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)).map(fileLine))}}`; +} + +/** MANIFEST.json's text (server/update.js parseManifest reads it). @param {{ app: string, files: Map }} m */ +export function manifestText({ app, files }) { + return `{"format":${UPDATE_FORMAT},"app":${JSON.stringify(app)},"files":${filesJson(files)}}\n`; +} + +/** + * UPDATE.json's text (server/update.js parseUpdate reads it). + * @param {{ app: string, from: string[], files: Map, removed: { path: string, sha256: string[] }[] }} u + */ +export function updateText({ app, from, files, removed }) { + let bytes = 0; + for (const d of files.values()) bytes += d.size; + const gone = removed.map((r) => `{"path":${JSON.stringify(r.path)},"sha256":${JSON.stringify(r.sha256)}}`); + return `{"format":${UPDATE_FORMAT},"app":${JSON.stringify(app)},"from":${JSON.stringify(from)},"count":${files.size},"bytes":${bytes},` + + `\n"removed":[${block(gone)}],\n"files":${filesJson(files)}}\n`; +} diff --git a/tools/package.mjs b/tools/package.mjs index 77178c94..972ed4b8 100644 --- a/tools/package.mjs +++ b/tools/package.mjs @@ -1,13 +1,16 @@ #!/usr/bin/env node -// tools/package.mjs — the two player release zips (docs/DEPLOY.md §7; README「方式一」says which one to pick). +// tools/package.mjs — the player release zips (docs/DEPLOY.md §7; README「方式一」says which one to pick). // -// npm run package → Stronghold-Protocol-v.zip full: what runs the game + the game art -// npm run package:lite → Stronghold-Protocol-v-lite.zip the same without the art (`npm run setup` +// npm run package → Stronghold-Protocol-v.zip full: what runs the game + the game art +// npm run package:lite → Stronghold-Protocol-v-lite.zip the same without the art (`npm run setup` // downloads it on the first start) -// node tools/package.mjs [--lite] [--dry-run [--list]] [--out ] [--force] [--keep-stage] [--no-install] -// [--allow-dirty] [--root ] +// node tools/package.mjs --update --from [,…] +// → Stronghold-Protocol-v-update.zip only the files that differ from the earlier full +// zips given as bases, to extract over an install of one of them (server/update.js) +// node tools/package.mjs [--lite | --update --from ] [--dry-run [--list]] [--out ] [--force] [--keep-stage] +// [--no-install] [--allow-dirty] [--allow-dev] [--root ] // -// Both zips hold one folder, Stronghold-Protocol/, with only what a player runs — an allowlist over `git ls-files`, so +// The zips hold one folder, Stronghold-Protocol/, with only what a player runs — an allowlist over `git ls-files`, so // untracked work, caches, logs and per-machine config never get in: server/, shared/, data/, public/ (not public/dev/), // packs/ (the content packs that ship with the repository, docs/PACKS.md; a pack installed on this machine and not // committed stays out), @@ -34,6 +37,18 @@ // The account name comes from the OS at run time (never written in the repository): SP_PACKAGE_SCAN_USER=0 skips it // (a name that is a common word), SP_PACKAGE_SCAN_NAMES=a,b adds more names to refuse. // +// Every zip carries MANIFEST.json at its root (written after npm ci; server/update.js): the size and sha256 of each +// shipped file except the art setup manages (public/assets/, public/fonts/, data/assets.json, data/local-assets.json) — +// what npm run doctor and an update's boot step verify; the full, lite and update zips of one version list the same. +// +// --update builds the full stage as above (npm ci and all) in /…-update/.full/, reads every base — an earlier +// release's full zip or the folder it was extracted to (tools/package-update.mjs: older than this version, not a lite or +// update zip, each version once) — and ships each file that is new or differs (size + sha256) from at least one base, +// plus MANIFEST.json and UPDATE.json (the base versions, what ships, and `removed`: the files a base shipped that this +// version does not, with the bytes each base had). The update stage gets the same checks (the refusal list, the +// personal-info scan; it must hold exactly that) before it is zipped. --dry-run reads and checks the bases; the +// comparison needs the built stage. +// // --out defaults to /stronghold-protocol-release and must be outside the repository. --no-install skips npm ci // (no node_modules, no public/vendor: a test build). Zipping needs `zip` (or a bsdtar `tar`, e.g. Windows 10+). @@ -44,6 +59,11 @@ import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { findSpecifiers } from './check-imports.mjs'; import { writePackIndex } from './packs.mjs'; +import { compareVersions, diffBases, manifestText, readBase, updateText } from './package-update.mjs'; +import { + APPLIED_FILE, MANIFEST_FILE, UPDATE_FILE, affectsRuntime, digestFile, inManifest, isSetupArt, listedArtFiles, + parseManifest, parseUpdate, pathProblem, removalProblem, +} from '../server/update.js'; const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); /** The one folder inside both zips (DEPLOY §1.1: "解压后把里面的 Stronghold-Protocol 文件夹放到…"). */ @@ -180,16 +200,7 @@ export function listTree(root, rel, { junk = false } = {}) { } /** The '/assets/…' and '/fonts/…' URLs of a manifest, as public/… paths (the walk of tools/setup.mjs checkAssets). */ -export function manifestFiles(manifest) { - const out = new Set(); - const walk = (v) => { - if (typeof v === 'string') { - if (/^\/(?:assets|fonts)\//.test(v)) out.add(`public/${v.split('/').filter(Boolean).map(decodeURIComponent).join('/')}`); - } else if (v && typeof v === 'object') for (const x of Object.values(v)) walk(x); - }; - walk(manifest); - return out; -} +export const manifestFiles = listedArtFiles; /** * The art of a full package: every file data/assets.json lists, public/fonts, public/assets/local/ and @@ -284,14 +295,16 @@ const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); /** * Files whose bytes (read as latin1, so binaries are scanned too) carry a home-directory path (`home`) or one of - * `names` (case-insensitive). Returns "path (what)" lines, never the matched text. + * `names` (case-insensitive). Returns "path (what)" lines, never the matched text. `strip`: text removed before the + * scan (the sha256 values of MANIFEST.json / UPDATE.json, which a hex-only name would match). */ -export function scanFiles(root, files, { home = true, names = [] } = {}) { +export function scanFiles(root, files, { home = true, names = [], strip = null } = {}) { const nameRe = names.length ? new RegExp(names.map((n) => escapeRe(Buffer.from(n, 'utf8').toString('latin1'))).join('|'), 'i') : null; const hits = []; for (const rel of files) { let text; try { text = fs.readFileSync(path.join(root, rel)).toString('latin1'); } catch { continue; } + if (strip) text = text.replace(strip, ''); if (home && (HOME_POSIX.test(text) || HOME_WIN.test(text))) hits.push(`${rel} (home-directory path)`); else if (nameRe && nameRe.test(text)) hits.push(`${rel} (account name)`); } @@ -418,30 +431,64 @@ function zipFolder(cwd, zipPath) { if (t.error || t.status !== 0 || !isFile(zipPath)) throw new Error('zipping failed: install `zip` (or a bsdtar `tar`)'); } -/** Copy the plan into //Stronghold-Protocol, npm ci, check the stage, zip it. */ -export function build(root, p, { out, install = true, force = false, keepStage = false, env = process.env, log = console.log } = {}) { - if (p.problems.length) throw new Error('refusing to build: the plan has problems (see the summary)'); - const dest = path.resolve(out); - if (packageOutIsUnsafe(dest, root)) throw new Error(`refusing to write into the repository or a parent of it: ${dest}`); - const name = `${FOLDER}-v${p.version}${p.lite ? '-lite' : ''}`; // 0.1.x's release asset names - const work = path.join(dest, name); - const stage = path.join(work, FOLDER); - const zipPath = path.join(dest, `${name}.zip`); - for (const t of [work, zipPath]) { +const isLib = (f) => f.startsWith('node_modules/') || f.startsWith('public/vendor/'); +const META_FILES = new Set([MANIFEST_FILE, UPDATE_FILE]); +const HASHES = /[0-9a-f]{64}/g; + +/** + * What is wrong with a stage before it is zipped: a file not planned, a planned one missing (node_modules and + * public/vendor, npm ci's output, only count when `libsPlanned`), a refused path, personal info (home paths and names + * in every file, names only in the libraries; MANIFEST.json / UPDATE.json without their sha256 values). + * @param {string} stage + * @param {Set} planned + * @returns {{ staged: string[], problems: string[] }} + */ +export function stageProblems(stage, planned, { env = process.env, libsPlanned = false } = {}) { + const problems = []; + const staged = listTree(stage, '', { junk: true }); + const held = libsPlanned ? staged : staged.filter((f) => !isLib(f)); + for (const f of held) if (!planned.has(f)) problems.push(`in the stage but not planned: ${f}`); + const have = new Set(held); + for (const f of planned) if (!have.has(f)) problems.push(`planned but not staged: ${f}`); + for (const r of REFUSE) if (fs.existsSync(path.join(stage, r))) problems.push(`refused: ${r} is in the stage`); + const names = scanNames(env); + const rest = staged.filter((f) => !isLib(f)); + for (const h of scanFiles(stage, rest.filter((f) => !META_FILES.has(f)), { home: true, names })) problems.push(`personal info: ${h}`); + for (const h of scanFiles(stage, rest.filter((f) => META_FILES.has(f)), { home: true, names, strip: HASHES })) problems.push(`personal info: ${h}`); + for (const h of scanFiles(stage, staged.filter(isLib), { home: false, names })) problems.push(`personal info: ${h}`); + return { staged, problems }; +} + +function digestOf(root, rel) { + const d = digestFile(path.join(root, rel)); + if (!d) throw new Error(`cannot read ${path.join(root, rel)}`); + return d; +} + +function copyInto(from, to, rel) { + const src = path.join(from, rel); + const dst = path.join(to, rel); + fs.mkdirSync(path.dirname(dst), { recursive: true }); + fs.copyFileSync(src, dst, fs.constants.COPYFILE_FICLONE); + fs.chmodSync(dst, fs.statSync(src).mode & 0o777); +} + +function freshTargets(targets, force) { + for (const t of targets) { if (!fs.existsSync(t)) continue; if (!force) throw new Error(`${t} already exists (--force replaces it)`); fs.rmSync(t, { recursive: true, force: true }); } +} + +/** + * Copy the plan into `stage`, write the pack index, npm ci (node_modules, public/vendor), write MANIFEST.json, check the + * stage: what `build` zips and `buildUpdate` compares. Throws on a problem. Returns the staged files. + */ +function makeStage(root, p, stage, { install, env, log }) { const generated = new Set(p.generated || []); log(`copying ${p.files.length - generated.size} files → ${stage}`); - for (const rel of p.files) { - if (generated.has(rel)) continue; - const src = path.join(root, rel); - const to = path.join(stage, rel); - fs.mkdirSync(path.dirname(to), { recursive: true }); - fs.copyFileSync(src, to, fs.constants.COPYFILE_FICLONE); - fs.chmodSync(to, fs.statSync(src).mode & 0o777); - } + for (const rel of p.files) if (!generated.has(rel)) copyInto(root, stage, rel); if (generated.has(GENERATED_PACK_INDEX)) { const index = writePackIndex(stage, path.join(stage, GENERATED_PACK_INDEX)); log(`${GENERATED_PACK_INDEX}: ${index.packs.map((x) => `${x.type} ${x.id}`).join(', ') || 'no pack'}`); @@ -454,35 +501,152 @@ export function build(root, p, { out, install = true, force = false, keepStage = fs.rmSync(path.join(stage, 'node_modules', '.cache'), { recursive: true, force: true }); if (!isFile(path.join(stage, 'public', 'vendor', 'pixi.min.js'))) throw new Error('public/vendor/pixi.min.js missing after npm ci (postinstall tools/vendor.mjs)'); } - const problems = []; - const planned = new Set(p.files); - const staged = listTree(stage, '', { junk: true }); - const rest = staged.filter((f) => !f.startsWith('node_modules/') && !f.startsWith('public/vendor/')); - for (const f of rest) if (!planned.has(f)) problems.push(`in the stage but not planned: ${f}`); - const have = new Set(rest); - for (const f of p.files) if (!have.has(f)) problems.push(`planned but not staged: ${f}`); - for (const r of REFUSE) if (fs.existsSync(path.join(stage, r))) problems.push(`refused: ${r} is in the stage`); - const names = scanNames(env); - for (const h of scanFiles(stage, rest, { home: true, names })) problems.push(`personal info: ${h}`); - const libs = staged.filter((f) => f.startsWith('node_modules/') || f.startsWith('public/vendor/')); - for (const h of scanFiles(stage, libs, { home: false, names })) problems.push(`personal info: ${h}`); + const files = new Map(); + for (const f of listTree(stage, '', { junk: true })) if (inManifest(f)) files.set(f, digestOf(stage, f)); + fs.writeFileSync(path.join(stage, MANIFEST_FILE), manifestText({ app: p.version, files })); + parseManifest(fs.readFileSync(path.join(stage, MANIFEST_FILE), 'utf8')); // as doctor and an update read it back + log(`${MANIFEST_FILE}: ${files.size} files (the art is setup's)`); + const { staged, problems } = stageProblems(stage, new Set([...p.files, MANIFEST_FILE]), { env }); if (problems.length) throw new Error(`refusing to zip ${stage}:\n${problems.map((x) => ` ${x}`).join('\n')}`); + return staged; +} + +/** Copy the plan into //Stronghold-Protocol, npm ci, write MANIFEST.json, check the stage, zip it. */ +export function build(root, p, { out, install = true, force = false, keepStage = false, env = process.env, log = console.log } = {}) { + if (p.problems.length) throw new Error('refusing to build: the plan has problems (see the summary)'); + const dest = path.resolve(out); + if (packageOutIsUnsafe(dest, root)) throw new Error(`refusing to write into the repository or a parent of it: ${dest}`); + const name = `${FOLDER}-v${p.version}${p.lite ? '-lite' : ''}`; // 0.1.x's release asset names + const work = path.join(dest, name); + const stage = path.join(work, FOLDER); + const zipPath = path.join(dest, `${name}.zip`); + freshTargets([work, zipPath], force); + const staged = makeStage(root, p, stage, { install, env, log }); log(`zipping ${staged.length} files (${MB(sumBytes(stage, staged))}) → ${zipPath}`); zipFolder(work, zipPath); if (!keepStage) fs.rmSync(work, { recursive: true, force: true }); return { zipPath, zipBytes: fileBytes(zipPath), files: staged.length, stage: keepStage ? stage : null }; } -const USAGE = 'usage: node tools/package.mjs [--lite] [--dry-run [--list]] [--out ] [--force] [--keep-stage] [--no-install] [--allow-dirty] [--allow-dev] [--root ]'; +// --------------------------------------------------------------------------------------------------------------------- +// The update zip + +/** OS clutter a folder can gather (a release zip has none; the extension rules of isJunk would hit node_modules files). */ +const isClutter = (rel) => { + const parts = posixRel(rel).split('/'); + const base = parts[parts.length - 1]; + return parts.includes('__pycache__') || base === '.DS_Store' || base === 'Thumbs.db' || base === 'desktop.ini' || base.startsWith('._'); +}; +/** + * Never part of a base: the update files, and outside node_modules / public/vendor (npm's output, which the stage takes + * as npm writes it) per-machine files (the refusal list) and OS clutter. + */ +const baseSkip = (rel) => rel === MANIFEST_FILE || rel === UPDATE_FILE || rel === APPLIED_FILE || (!isLib(rel) && (isRefused(rel) || isClutter(rel))); + +/** + * Read and check the bases of an update to `version`: each an earlier release's full zip or its extracted folder, older + * than `version`, with art (not a lite zip), not an update zip, each version once. Sorted by version. + * @param {string[]} sources + * @param {string} version + */ +export function readBases(sources, version, { log = () => {} } = {}) { + const bases = []; + for (const src of sources) { + log(`reading base ${src} …`); + const b = readBase(src, { folder: FOLDER, skip: baseSkip }); + if (b.update) throw new Error(`base ${src} is an update package: pass the earlier release's full zip`); + if (!b.art) throw new Error(`base ${src} has no game art (a lite zip?): pass the full zip, ${FOLDER}-v${b.version}.zip`); + if (compareVersions(b.version, version) >= 0) throw new Error(`base ${src} is v${b.version}, not older than v${version}`); + const twin = bases.find((x) => x.version === b.version); + if (twin) throw new Error(`two bases are v${b.version}: ${twin.source} and ${b.source}`); + bases.push(b); + } + return bases.sort((a, b) => compareVersions(a.version, b.version)); +} + +/** + * The update zip: build the full stage (as `build` does) in //.full/, compare every file with each base, copy + * what is new or differs from at least one base into //Stronghold-Protocol with MANIFEST.json and + * UPDATE.json, check that stage and zip it as .zip ( = Stronghold-Protocol-v-update). + */ +export function buildUpdate(root, p, bases, { out, install = true, force = false, keepStage = false, env = process.env, log = console.log } = {}) { + if (p.problems.length) throw new Error('refusing to build: the plan has problems (see the summary)'); + if (p.lite) throw new Error('an update is compared from the full stage (no --lite)'); + if (!bases.length) throw new Error('an update needs at least one base (--from)'); + const dest = path.resolve(out); + if (packageOutIsUnsafe(dest, root)) throw new Error(`refusing to write into the repository or a parent of it: ${dest}`); + const name = `${FOLDER}-v${p.version}-update`; + const work = path.join(dest, name); + const full = path.join(work, '.full', FOLDER); + const stage = path.join(work, FOLDER); + const zipPath = path.join(dest, `${name}.zip`); + freshTargets([work, zipPath], force); + const staged = makeStage(root, p, full, { install, env, log }); + log(`comparing ${staged.length} files with ${bases.map((b) => `v${b.version}`).join(', ')} …`); + const next = new Map(); + for (const f of staged) if (f !== MANIFEST_FILE) next.set(f, digestOf(full, f)); + const diff = diffBases(next, bases); + // a path no update deletes (a per-machine name such as node_modules/x/.env.example) stays behind, named in the summary + diff.left = diff.removed.filter((r) => removalProblem(r.path)).map((r) => r.path); + for (const rel of diff.left) { + const bad = pathProblem(rel); + if (bad) throw new Error(`a base ships ${rel}, which is ${bad}`); + } + diff.removed = diff.removed.filter((r) => !removalProblem(r.path)); + for (const rel of diff.ship) copyInto(full, stage, rel); + copyInto(full, stage, MANIFEST_FILE); + const update = { app: p.version, from: bases.map((b) => b.version), files: new Map(diff.ship.map((f) => [f, next.get(f)])), removed: diff.removed }; + fs.writeFileSync(path.join(stage, UPDATE_FILE), updateText(update)); + // what the player's server will read back + parseUpdate(fs.readFileSync(path.join(stage, UPDATE_FILE), 'utf8')); + parseManifest(fs.readFileSync(path.join(stage, MANIFEST_FILE), 'utf8')); + const { staged: held, problems } = stageProblems(stage, new Set([...diff.ship, MANIFEST_FILE, UPDATE_FILE]), { env, libsPlanned: true }); + if (problems.length) throw new Error(`refusing to zip ${stage}:\n${problems.map((x) => ` ${x}`).join('\n')}`); + log(`zipping ${held.length} files (${MB(sumBytes(stage, held))}) → ${zipPath}`); + zipFolder(work, zipPath); + if (!keepStage) fs.rmSync(work, { recursive: true, force: true }); + return { + zipPath, zipBytes: fileBytes(zipPath), files: held.length, diff, bytes: diff.ship.reduce((n, f) => n + next.get(f).size, 0), + stage: keepStage ? stage : null, full: keepStage ? full : null, + }; +} + +/** The bases as the summary lists them. */ +export function formatBases(bases) { + return bases.map((b) => `base: v${b.version} · ${b.files.size} files (${b.art} art) · ${b.source}\n`).join(''); +} + +/** What an update ships, by kind, and what it removes. */ +export function formatUpdate(r, version) { + const kinds = { 'code / data': 0, node_modules: 0, 'public/vendor': 0, art: 0, 'docs / scripts / tools': 0 }; + for (const f of r.diff.ship) { + if (isSetupArt(f)) kinds.art++; + else if (f.startsWith('node_modules/')) kinds.node_modules++; + else if (f.startsWith('public/vendor/')) kinds['public/vendor']++; + else if (!affectsRuntime(f)) kinds['docs / scripts / tools']++; + else kinds['code / data']++; + } + const lines = [`update: v${version} over ${r.diff.perBase.map((b) => `v${b.version}`).join(', ')}`]; + for (const b of r.diff.perBase) lines.push(` vs v${b.version}: ${b.changed} changed, ${b.added} added, ${b.unchanged} unchanged, ${b.removed} removed`); + lines.push(`ships: ${r.diff.ship.length} files, ${MB(r.bytes)} — ${Object.entries(kinds).map(([k, n]) => `${k} ${n}`).join(', ')} (+ ${MANIFEST_FILE}, ${UPDATE_FILE})`); + const gone = r.diff.removed.map((x) => x.path); + lines.push(`removed: ${gone.length} file(s)${gone.length ? ` (${gone.slice(0, 8).join(', ')}${gone.length > 8 ? ' …' : ''})` : ''}`); + if (r.diff.caseOnly.length) lines.push(`not removed (a new file differs only in case): ${r.diff.caseOnly.join(', ')}`); + if (r.diff.left?.length) lines.push(`not removed (an update never deletes these names): ${r.diff.left.join(', ')}`); + return lines.join('\n') + '\n'; +} + +const USAGE = 'usage: node tools/package.mjs [--lite | --update --from [,<…>]] [--dry-run [--list]] [--out ] [--force] [--keep-stage] [--no-install] [--allow-dirty] [--allow-dev] [--root ]'; /** Whether `version` is a development (pre-release) version, e.g. 0.2.0-dev. */ export const isDevVersion = (version) => /-/.test(String(version || '')); export function parseArgs(argv) { - const o = { lite: false, dryRun: false, list: false, out: '', force: false, keepStage: false, install: true, allowDirty: false, allowDev: false, root: REPO, help: false }; + const o = { lite: false, update: false, from: [], dryRun: false, list: false, out: '', force: false, keepStage: false, install: true, allowDirty: false, allowDev: false, root: REPO, help: false }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--lite') o.lite = true; + else if (a === '--update') o.update = true; else if (a === '--dry-run') o.dryRun = true; else if (a === '--list') o.list = true; else if (a === '--force') o.force = true; @@ -491,12 +655,21 @@ export function parseArgs(argv) { else if (a === '--allow-dirty') o.allowDirty = true; else if (a === '--allow-dev') o.allowDev = true; else if (a === '--help' || a === '-h') o.help = true; - else if (a === '--out' || a === '--root') { + else if (a === '--from') { + const v = argv[++i]; + if (!v || v.startsWith('--')) throw new Error(`--from needs an earlier release's full zip or its folder\n${USAGE}`); + for (const one of v.split(',')) if (one.trim()) o.from.push(path.resolve(one.trim())); + } else if (a === '--out' || a === '--root') { const v = argv[++i]; if (!v || v.startsWith('--')) throw new Error(`${a} needs a directory\n${USAGE}`); o[a.slice(2)] = path.resolve(v); } else throw new Error(`unknown argument ${a}\n${USAGE}`); } + if (!o.help) { + if (o.from.length && !o.update) throw new Error(`--from goes with --update\n${USAGE}`); + if (o.update && !o.from.length) throw new Error(`--update needs --from [,<…>]\n${USAGE}`); + if (o.update && o.lite) throw new Error(`--update is compared from the full stage: no --lite\n${USAGE}`); + } return o; } @@ -507,8 +680,14 @@ function main(argv) { let p; try { p = plan(o.root, { lite: o.lite, allowDirty: o.allowDirty }); } catch (e) { console.error(`package: ${e.message}`); return 1; } process.stdout.write(formatSummary(p)); + let bases = []; + if (o.update) { + try { bases = readBases(o.from, p.version, { log: (m) => console.log(m) }); } catch (e) { console.error(`package: ${e.message}`); return 1; } + process.stdout.write(formatBases(bases)); + } if (o.dryRun) { if (o.list) for (const f of p.files) process.stdout.write(`file ${f}\n`); + if (o.update) console.log('update: the bases are fine; which files ship is known after the build (it compares the npm ci stage)'); return p.problems.length ? 1 : 0; } if (p.problems.length) return 1; @@ -518,8 +697,15 @@ function main(argv) { console.error(`package: v${p.version} is a development version — a release zip needs a release version; --allow-dev builds a test zip named after it`); return 1; } + const opts = { out: o.out || path.join(os.tmpdir(), 'stronghold-protocol-release'), install: o.install, force: o.force, keepStage: o.keepStage }; try { - const r = build(o.root, p, { out: o.out || path.join(os.tmpdir(), 'stronghold-protocol-release'), install: o.install, force: o.force, keepStage: o.keepStage }); + if (o.update) { + const r = buildUpdate(o.root, p, bases, opts); + process.stdout.write(formatUpdate(r, p.version)); + console.log(`zip: ${r.zipPath} (${MB(r.zipBytes)}, ${r.files} files)${r.stage ? ` · stages kept: ${r.stage}, ${r.full}` : ''}`); + return 0; + } + const r = build(o.root, p, opts); console.log(`zip: ${r.zipPath} (${MB(r.zipBytes)}, ${r.files} files)${r.stage ? ` · stage kept: ${r.stage}` : ''}`); return 0; } catch (e) {