0.2.6-dev: brotli (quality 5) on the fly with the existing memory cache for clients that name br, gzip for the rest; separate ETags per encoding (measured against gzip-6: .skel −27 %, data JSON −26 %, vendor −14 %, scripts −5 %) — host bandwidth layer 1.4; golden: unchanged

server/http/files.js

@@ -1,8 +1,9 @@
 // server/http/files.js — one file on disk (serveFile) or one body in memory (serveBuffer, the served manifests of
 // assetViews.js) → an HTTP response (static.js and media.js decide which file):
 //     MIME types incl. .mjs/.js text/javascript, .skel application/octet-stream, .atlas text/plain;
-//     gzip for text-like types, .skel and uncompressed fonts when the client accepts it (small files are
-//     compressed once and cached in memory); strong ETag + Last-Modified with 304s — the ETag is the content hash
+//     brotli (quality 5) for text-like types, .skel and uncompressed fonts when the client accepts it, else gzip
+//     (level 6) — browsers offer brotli on HTTPS pages and localhost only (small files are compressed once per
+//     encoding and cached in memory); strong ETag + Last-Modified with 304s — the ETag is the content hash
 //     when the caller has one (assetIndex.js), else size + mtime; Cache-Control (html & code/data: no-cache +
 //     revalidate; public/assets|fonts|vendor: 1 day; a URL whose content-hash segment matches the file (static.js,
 //     media.js): immutable for a year; any `?v=` URL: immutable); single byte-range requests (206/416, used by <audio>).
@@ -55,28 +56,60 @@ export const MIME = Object.freeze({
   '.ttf': 'font/ttf',
 });
 
-/** Extensions worth gzipping (text-like, .skel, uncompressed fonts). */
+/** Extensions worth compressing (text-like, .skel, uncompressed fonts). */
 export const COMPRESSIBLE = new Set([
   '.html', '.htm', '.js', '.mjs', '.css', '.json', '.map', '.webmanifest', '.txt', '.md', '.csv', '.xml',
   '.atlas', '.skel', '.bin', '.wasm', '.svg', '.ico', '.otf', '.ttf', '.wav',
 ]);
 
+/** Types brotli compresses in its text mode (the rest of COMPRESSIBLE: generic mode). */
+const TEXT_LIKE = new Set(['.html', '.htm', '.js', '.mjs', '.css', '.json', '.map', '.webmanifest', '.txt', '.md', '.csv', '.xml', '.atlas', '.svg']);
+
 const GZIP_MIN_BYTES = 512;
-const GZIP_CACHE_MAX_FILE = 8 << 20;      // larger files are gzip-streamed on the fly
-const GZIP_CACHE_MAX_TOTAL = 96 << 20;
-// Asset URLs carry no content hash yet, and tools/fetch-assets.mjs / tools/vendor.mjs can rewrite files in
-// place (atlas + png + skel must stay consistent), so "long" is one day; revalidation after that is a cheap 304.
+const GZIP_CACHE_MAX_FILE = 8 << 20;      // larger files are compressed as a stream on the fly
+const GZIP_CACHE_MAX_TOTAL = 96 << 20;    // both encodings together
+/**
+ * Brotli quality of the on-the-fly compression: faster than gzip-6 and smaller — measured on this repository against
+ * gzip-6: .skel −27 %, data/*.json −26 %, vendor −14 %, public/js and server/sim −5 %, .atlas −7 %. Quality 11 would gain
+ * another ~10 % at ~100× the time, so it is not done per request.
+ */
+export const BROTLI_QUALITY = 5;
+// A plain asset URL (no content-hash segment) names a file tools/fetch-assets.mjs / tools/vendor.mjs can rewrite in
+// place (atlas + png + skel must stay consistent), so "long" is one day; revalidation after that is a cheap 304. A
+// content-hashed URL whose hash names the file is immutable (static.js, media.js).
 export const LONG_CACHE = 'public, max-age=86400';          // 1 day
 export const IMMUTABLE_CACHE = 'public, max-age=31536000, immutable';
 const LONG_CACHE_DIRS = ['assets', 'fonts', 'vendor']; // first path segment under public/
 
 const gzipAsync = promisify(zlib.gzip);
+const brotliAsync = promisify(zlib.brotliCompress);
+
+/** zlib options of an encoding for a body of `size` bytes of type `ext`. */
+function brotliParams(ext, size) {
+  return {
+    params: {
+      [zlib.constants.BROTLI_PARAM_QUALITY]: BROTLI_QUALITY,
+      [zlib.constants.BROTLI_PARAM_MODE]: TEXT_LIKE.has(ext) ? zlib.constants.BROTLI_MODE_TEXT : zlib.constants.BROTLI_MODE_GENERIC,
+      ...(size > 0 ? { [zlib.constants.BROTLI_PARAM_SIZE_HINT]: size } : {}),
+    },
+  };
+}
+
+/** `raw` compressed with `encoding` ('br' | 'gzip'). */
+function compress(raw, encoding, ext) {
+  return encoding === 'br' ? brotliAsync(raw, brotliParams(ext, raw.length)) : gzipAsync(raw, { level: 6 });
+}
+
+/** A streaming compressor for a file too large for the cache. */
+function compressStream(encoding, ext, size) {
+  return encoding === 'br' ? zlib.createBrotliCompress(brotliParams(ext, size)) : zlib.createGzip();
+}
 
 // ---------------------------------------------------------------------------------------------------
-// gzip cache (LRU by bytes)
+// compression cache (LRU by bytes; one entry per file and encoding)
 // ---------------------------------------------------------------------------------------------------
 
-export class GzipCache {
+export class CompressCache {
   constructor(maxTotal = GZIP_CACHE_MAX_TOTAL) {
     this.maxTotal = maxTotal;
     this.total = 0;
@@ -84,24 +117,27 @@ export class GzipCache {
     /** @type {Map<string, Promise<Buffer>>} */ this.inflight = new Map();
   }
 
-  /** @returns {Promise<Buffer>} gzip of the file identified by (path, size, mtime) */
-  get(absPath, stat) {
-    return this.compressed(`${absPath}\0${stat.size}\0${stat.mtimeMs}`, () => fsp.readFile(absPath));
+  /** @returns {Promise<Buffer>} the file identified by (path, size, mtime) compressed with `encoding` ('gzip' by default) */
+  get(absPath, stat, encoding = 'gzip') {
+    return this.compressed(`${absPath}\0${stat.size}\0${stat.mtimeMs}`, () => fsp.readFile(absPath), encoding, path.extname(absPath).toLowerCase());
   }
 
-  /** @returns {Promise<Buffer>} gzip of the body `read()` returns, cached under `key` (one compression per key at a time) */
-  compressed(key, read) {
-    const hit = this.map.get(key);
-    if (hit) { this.map.delete(key); this.map.set(key, hit); return Promise.resolve(hit); }
-    const pending = this.inflight.get(key);
+  /**
+   * @returns {Promise<Buffer>} the body `read()` returns compressed with `encoding`, cached under `key` + encoding (one
+   *   compression per entry at a time)
+   */
+  compressed(key, read, encoding = 'gzip', ext = '') {
+    const k = `${encoding}\0${key}`;
+    const hit = this.map.get(k);
+    if (hit) { this.map.delete(k); this.map.set(k, hit); return Promise.resolve(hit); }
+    const pending = this.inflight.get(k);
     if (pending) return pending;
     const p = (async () => {
-      const raw = await read();
-      const gz = await gzipAsync(raw, { level: 6 });
-      this.store(key, gz);
-      return gz;
-    })().finally(() => this.inflight.delete(key));
-    this.inflight.set(key, p);
+      const out = await compress(await read(), encoding, ext);
+      this.store(k, out);
+      return out;
+    })().finally(() => this.inflight.delete(k));
+    this.inflight.set(k, p);
     return p;
   }
 
@@ -117,10 +153,46 @@ export class GzipCache {
   }
 }
 
+/** The cache's former name (it held gzip only). */
+export const GzipCache = CompressCache;
+
 // ---------------------------------------------------------------------------------------------------
 // HTTP helpers
 // ---------------------------------------------------------------------------------------------------
 
+/** Accept-Encoding → { token: q } (lower-case tokens; a malformed q counts as 0). */
+function acceptedEncodings(header) {
+  const out = {};
+  if (!header || typeof header !== 'string') return out;
+  for (const part of header.split(',')) {
+    const [token, ...params] = part.trim().toLowerCase().split(';');
+    if (!token) continue;
+    let q = 1;
+    for (const p of params) {
+      const m = /^\s*q=([0-9.]+)\s*$/.exec(p);
+      if (m) q = Number(m[1]);
+    }
+    out[token.trim()] = Number.isFinite(q) ? q : 0;
+  }
+  return out;
+}
+
+/**
+ * The content coding to answer with: 'br' when the client names brotli (q > 0) at least as high as gzip, else 'gzip'
+ * when it accepts gzip (or `*`), else null (identity). `*` alone never picks brotli. Browsers name `br` on HTTPS pages
+ * and localhost only, so a LAN `http://` address gets gzip.
+ * @param {string | undefined} header
+ * @returns {'br' | 'gzip' | null}
+ */
+export function pickEncoding(header) {
+  const acc = acceptedEncodings(header);
+  const brQ = acc.br ?? 0;
+  const gzQ = acc.gzip ?? acc['x-gzip'] ?? acc['*'] ?? 0;
+  if (brQ > 0 && brQ >= gzQ) return 'br';
+  if (gzQ > 0) return 'gzip';
+  return null;
+}
+
 /** Does the Accept-Encoding header allow gzip (q > 0)? @param {string | undefined} header */
 export function acceptsGzip(header) {
   if (!header || typeof header !== 'string') return false;
@@ -206,9 +278,9 @@ export async function serveFile(req, res, absPath, stat, mountName, segments, qu
   const type = MIME[ext] || 'application/octet-stream';
   const compressible = COMPRESSIBLE.has(ext);
   const rangeHeader = req.headers.range;
-  const useGzip = compressible && stat.size >= GZIP_MIN_BYTES && !rangeHeader && acceptsGzip(req.headers['accept-encoding']);
+  const encoding = compressible && stat.size >= GZIP_MIN_BYTES && !rangeHeader ? pickEncoding(req.headers['accept-encoding']) : null;
   const baseTag = opts.contentHash || `${stat.size.toString(16)}-${Math.floor(stat.mtimeMs).toString(16)}`;
-  const etag = `"${baseTag}${useGzip ? '-gz' : ''}"`;
+  const etag = `"${baseTag}${encoding === 'br' ? '-br' : encoding === 'gzip' ? '-gz' : ''}"`;
   const lastModified = stat.mtime.toUTCString();
   const isHead = req.method === 'HEAD';
 
@@ -226,18 +298,18 @@ export async function serveFile(req, res, absPath, stat, mountName, segments, qu
     return;
   }
 
-  if (useGzip) {
-    headers['Content-Encoding'] = 'gzip';
+  if (encoding) {
+    headers['Content-Encoding'] = encoding;
     if (stat.size <= GZIP_CACHE_MAX_FILE) {
-      const gz = await gzipCache.get(absPath, stat);
-      headers['Content-Length'] = gz.length;
+      const body = await gzipCache.get(absPath, stat, encoding);
+      headers['Content-Length'] = body.length;
       res.writeHead(200, headers);
-      res.end(isHead ? undefined : gz);
+      res.end(isHead ? undefined : body);
       return;
     }
     res.writeHead(200, headers);
     if (isHead) { res.end(); return; }
-    await streamTo(fs.createReadStream(absPath), res, log, zlib.createGzip());
+    await streamTo(fs.createReadStream(absPath), res, log, compressStream(encoding, ext, stat.size));
     return;
   }
 
@@ -265,13 +337,13 @@ export async function serveFile(req, res, absPath, stat, mountName, segments, qu
 }
 
 /**
- * One body in memory → the response (the served manifests): its ETag from `hash`, gzip when accepted (cached under
- * `key`), 304 on a matching If-None-Match, no ranges.
- * @param {{ body: Buffer, hash: string, type: string, cacheControl: string, key: string, cache: GzipCache }} o
+ * One body in memory → the response (the served manifests): its ETag from `hash`, brotli or gzip when accepted (cached
+ * under `key` + `hash`), 304 on a matching If-None-Match, no ranges.
+ * @param {{ body: Buffer, hash: string, type: string, cacheControl: string, key: string, cache: CompressCache }} o
  */
 export async function serveBuffer(req, res, { body, hash, type, cacheControl, key, cache }) {
-  const useGzip = body.length >= GZIP_MIN_BYTES && acceptsGzip(req.headers['accept-encoding']);
-  const etag = `"${hash}${useGzip ? '-gz' : ''}"`;
+  const encoding = body.length >= GZIP_MIN_BYTES ? pickEncoding(req.headers['accept-encoding']) : null;
+  const etag = `"${hash}${encoding === 'br' ? '-br' : encoding === 'gzip' ? '-gz' : ''}"`;
   const headers = { 'Content-Type': type, 'Cache-Control': cacheControl, ETag: etag, Vary: 'Accept-Encoding' };
   const inm = req.headers['if-none-match'];
   if (typeof inm === 'string' && (inm.trim() === '*' || inm.split(',').some((t) => stripWeak(t) === etag))) {
@@ -280,9 +352,9 @@ export async function serveBuffer(req, res, { body, hash, type, cacheControl, ke
     return;
   }
   let out = body;
-  if (useGzip) {
-    out = await cache.compressed(`${key}\0${hash}`, async () => body);
-    headers['Content-Encoding'] = 'gzip';
+  if (encoding) {
+    out = await cache.compressed(`${key}\0${hash}`, async () => body, encoding, path.extname(key).toLowerCase());
+    headers['Content-Encoding'] = encoding;
   }
   headers['Content-Length'] = out.length;
   res.writeHead(200, headers);

server/http/static.js

@@ -34,7 +34,7 @@ import { MEDIA_PREFIX } from '../../shared/media.js';
 import { splitHashedPath } from '../../shared/assetPaths.js';
 import { ROOT, noopLog } from './config.js';
 import { sendError, sendJson } from './common.js';
-import { MIME, GzipCache, isNotModified, serveFile, serveBuffer } from './files.js';
+import { MIME, CompressCache, isNotModified, serveFile, serveBuffer } from './files.js';
 import { ASSET_INDEX_ROOTS, hashedFileOpts } from './assetIndex.js';
 import { serveMedia } from './media.js';
 import { createPackRegistry } from '../packs.js';
@@ -81,7 +81,7 @@ export function createStaticHandler({ publicDir, dataDir, sharedDir, simDir = pa
   ];
   const shimBody = Buffer.from(DATA_SHIM_JS);
   const shimTag = `"shim-${shimBody.length.toString(16)}"`;
-  const gzipCache = new GzipCache();
+  const gzipCache = new CompressCache(); // brotli and gzip bodies (files.js)
 
   return async function serveStatic(req, res, rawPath, query) {
     let decoded;