diff --git a/test/update-package.test.js b/test/update-package.test.js index 9fcf81bc..762b3a43 100644 --- a/test/update-package.test.js +++ b/test/update-package.test.js @@ -15,7 +15,7 @@ import { fileURLToPath } from 'node:url'; import zlib from 'node:zlib'; import { APPLIED_FILE, MANIFEST_FILE, UPDATE_FILE, applyPendingUpdate, checkInstall, parseUpdate, removalProblem } from '../server/update.js'; import { compareVersions, diffBases, readBase, readZip } from '../tools/package-update.mjs'; -import { FOLDER, readBases, scanFiles, stageProblems } from '../tools/package.mjs'; +import { FOLDER, readBases, scanFiles, stageProblems, zipFolder } from '../tools/package.mjs'; const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const TOOL = path.join(ROOT, 'tools', 'package.mjs'); @@ -307,13 +307,11 @@ test('the zip reader: stored and deflated entries, zip64, UTF-8 names; damaged a const z = spawnSync('zip', ['-q', '-r', ...args, name, FOLDER], { cwd: dir }); if (!z.error && z.status === 0) zipped.push(name); }; - make('plain.zip', ['-X']); + // plain.zip is what tools/package.mjs writes (zip, else bsdtar); the other two need Info-ZIP + zipFolder(dir, path.join(dir, 'plain.zip')); + zipped.push('plain.zip'); make('stored.zip', ['-X', '-0']); make('zip64.zip', ['-X', '-fz']); - if (!zipped.length) { - const t = spawnSync('tar', ['-c', '--format', 'zip', '-f', 'plain.zip', FOLDER], { cwd: dir }); - if (!t.error && t.status === 0) zipped.push('plain.zip'); - } assert.ok(zipped.length > 0); for (const z of zipped) { const b = readBase(path.join(dir, z)); @@ -340,3 +338,63 @@ test('the zip reader: stored and deflated entries, zip64, UTF-8 names; damaged a fs.rmSync(dir, { recursive: true, force: true }); } }); + +/** + * A zip of stored entries built byte by byte: `{ name: Buffer (the header bytes), data, flags?, extra? }` each — the + * name encodings and extra fields a zip tool would write, independent of the zip tool and locale of the machine. + */ +function storedZip(entries) { + const locals = [], centrals = []; + let offset = 0; + for (const { name, data, flags = 0, extra = Buffer.alloc(0) } of entries) { + const crc = zlib.crc32(data) >>> 0; + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); local.writeUInt16LE(20, 4); local.writeUInt16LE(flags, 6); + local.writeUInt32LE(crc, 14); local.writeUInt32LE(data.length, 18); local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(name.length, 26); + locals.push(local, name, data); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); central.writeUInt16LE(20, 4); central.writeUInt16LE(20, 6); + central.writeUInt16LE(flags, 8); central.writeUInt32LE(crc, 16); central.writeUInt32LE(data.length, 20); + central.writeUInt32LE(data.length, 24); central.writeUInt16LE(name.length, 28); central.writeUInt16LE(extra.length, 30); + central.writeUInt32LE(offset, 42); + centrals.push(central, name, extra); + offset += 30 + name.length + data.length; + } + const cd = Buffer.concat(centrals); + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); eocd.writeUInt16LE(entries.length, 8); eocd.writeUInt16LE(entries.length, 10); + eocd.writeUInt32LE(cd.length, 12); eocd.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, cd, eocd]); +} + +/** Info-ZIP's Unicode Path extra field (0x7075, version 1): the CRC of the header name, then the UTF-8 name. */ +function unicodePath(utf8, headerName, crc = zlib.crc32(headerName) >>> 0) { + const name = Buffer.from(utf8, 'utf8'); + const f = Buffer.alloc(9); + f.writeUInt16LE(0x7075, 0); f.writeUInt16LE(5 + name.length, 2); f.writeUInt8(1, 4); f.writeUInt32LE(crc, 5); + return Buffer.concat([f, name]); +} + +test('the zip reader takes the UTF-8 name of Info-ZIP\'s Unicode Path field for a name stored in the system code page (zip on Windows)', { skip: typeof zlib.crc32 !== 'function' && 'zlib.crc32 needs Node ≥ 22.2' }, () => { + const dir = tmp('zipname'); + try { + // 卫戍 in GBK (code page 936), as Info-ZIP on a Chinese Windows writes it; on code page 1252 it is "??" + const gbk = Buffer.concat([Buffer.from('F/'), Buffer.from('cec0caf9', 'hex'), Buffer.from(' a.png')]); + const lossy = Buffer.from('F/?? b.png'); + const utf8 = Buffer.from('F/卫戍 c.png', 'utf8'); + const zip = path.join(dir, 'names.zip'); + fs.writeFileSync(zip, storedZip([ + { name: gbk, data: Buffer.from('a'), extra: unicodePath('F/卫戍 a.png', gbk) }, + { name: lossy, data: Buffer.from('b'), extra: unicodePath('F/卫戍 b.png', lossy) }, + { name: utf8, data: Buffer.from('c'), flags: 0x800 }, + // a field whose CRC does not match the header name (renamed by a tool that kept the field) is ignored + { name: Buffer.from('F/plain.txt'), data: Buffer.from('d'), extra: unicodePath('F/stale.txt', Buffer.from('F/old.txt')) }, + ])); + const names = []; + readZip(zip, (name) => names.push(name)); + assert.deepEqual(names, ['F/卫戍 a.png', 'F/卫戍 b.png', 'F/卫戍 c.png', 'F/plain.txt']); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tools/package-update.mjs b/tools/package-update.mjs index c500f46c..9578007a 100644 --- a/tools/package-update.mjs +++ b/tools/package-update.mjs @@ -8,7 +8,8 @@ // every later 0.2.x given as a base); `removed` lists the files some base shipped that the new version does not, with // the bytes each base had (the player's copy is deleted only when it still holds them). A removed path that differs // from a new one only in case is the same file on Windows / macOS: it is left out of `removed` (`caseOnly`). -// The zip reader handles what tools/package.mjs and GitHub hand out: stored and deflated entries, zip64, UTF-8 names. +// The zip reader handles what tools/package.mjs and GitHub hand out: stored and deflated entries, zip64, UTF-8 names — +// also a name zip on Windows (Info-ZIP) stored in the system code page, through its Unicode Path extra field. import crypto from 'node:crypto'; import fs from 'node:fs'; @@ -31,6 +32,28 @@ function readAt(fd, pos, len) { const SIG = { eocd: 0x06054b50, loc64: 0x07064b50, eocd64: 0x06064b50, central: 0x02014b50, local: 0x04034b50 }; const U32 = 0xffffffff; +/** + * The name of the central-directory entry at `p`: the header bytes when flag bit 11 marks them UTF-8; otherwise the + * Info-ZIP Unicode Path extra field (0x7075, APPNOTE 4.6.9) when its CRC matches the header name — Info-ZIP's zip on + * Windows stores a non-ASCII name in the system code page and the UTF-8 name only there; without that field the header + * bytes, read as UTF-8 (an ASCII name). + */ +function entryName(cd, p, flags, nameLen, extraLen) { + const raw = cd.subarray(p + 46, p + 46 + nameLen); + if (!(flags & 0x800)) { + for (let q = p + 46 + nameLen, end = q + extraLen; q + 4 <= end;) { + const id = cd.readUInt16LE(q); + const len = cd.readUInt16LE(q + 2); + if (id === 0x7075 && len >= 5 && cd[q + 4] === 1 && q + 4 + len <= end + && (typeof zlib.crc32 !== 'function' || (zlib.crc32(raw) >>> 0) === cd.readUInt32LE(q + 5))) { + return cd.toString('utf8', q + 9, q + 4 + len); + } + q += 4 + len; + } + } + return raw.toString('utf8'); +} + /** * Call `onFile(name, data)` for every file entry of a zip (directories skipped), in central-directory order. * @param {string} zipPath @@ -73,7 +96,7 @@ export function readZip(zipPath, onFile) { const commentLen = cd.readUInt16LE(p + 32); const external = cd.readUInt32LE(p + 38); let offset = cd.readUInt32LE(p + 42); - const name = cd.toString('utf8', p + 46, p + 46 + nameLen); + const name = entryName(cd, p, flags, nameLen, extraLen); if (packedSize === U32 || rawSize === U32 || offset === U32) { // the zip64 extra field carries the values the header marks 0xFFFFFFFF, in this order for (let q = p + 46 + nameLen, end = q + extraLen; q + 4 <= end;) { diff --git a/tools/package.mjs b/tools/package.mjs index b9fbea7e..1e788d6f 100644 --- a/tools/package.mjs +++ b/tools/package.mjs @@ -423,11 +423,16 @@ export function packageOutIsUnsafe(out, root = REPO) { return inside(r, o) || inside(o, r); } -function zipFolder(cwd, zipPath) { +/** + * Zip `cwd`/FOLDER into `zipPath`: Info-ZIP `zip`, else a bsdtar `tar --format zip` (Windows 10+, macOS). bsdtar writes + * the names in UTF-8 (flag bit 11) only when told: its default is the system code page, which loses a name it cannot + * spell (卫戍 becomes ?? on Windows code page 1252) and leaves no UTF-8 copy for a reader. + */ +export function zipFolder(cwd, zipPath) { // deflate everything: storing PNG / MP3 as they are made the full zip 8.5 MB larger and saved no real time const z = spawnSync('zip', ['-q', '-r', '-X', zipPath, FOLDER], { cwd, stdio: 'inherit' }); if (!z.error && z.status === 0) return; - const t = spawnSync('tar', ['-c', '--format', 'zip', '-f', zipPath, FOLDER], { cwd, stdio: 'inherit' }); // bsdtar (Windows 10+, macOS) + const t = spawnSync('tar', ['-c', '--format', 'zip', '--options', 'zip:hdrcharset=UTF-8', '-f', zipPath, FOLDER], { cwd, stdio: 'inherit' }); if (t.error || t.status !== 0 || !isFile(zipPath)) throw new Error('zipping failed: install `zip` (or a bsdtar `tar`)'); }