optimise zip

Author: sunnylqm

src/bundle-pack.ts

@@ -2,6 +2,7 @@ import path from 'path';
 import * as fs from 'fs-extra';
 import { ZipFile as YazlZipFile } from 'yazl';
 import { t } from './utils/i18n';
+import { zipOptionsForPayloadFile } from './utils/zip-options';
 
 const ignorePackingFileNames = [
   '.',
@@ -32,7 +33,11 @@ export async function packBundle(dir: string, output: string): Promise<void> {
         const fullPath = path.join(root, name);
         const stat = fs.statSync(fullPath);
         if (stat.isFile()) {
-          zipfile.addFile(fullPath, rel + name);
+          zipfile.addFile(
+            fullPath,
+            rel + name,
+            zipOptionsForPayloadFile(fullPath, rel + name),
+          );
         } else if (stat.isDirectory()) {
           addDirectory(fullPath, `${rel}${name}/`);
         }

src/diff.ts

@@ -6,9 +6,54 @@ import type { CommandContext } from './types';
 import { translateOptions } from './utils';
 import { isPPKBundleFileName, scriptName, tempDir } from './utils/constants';
 import { t } from './utils/i18n';
-import { enumZipEntries, readEntry } from './utils/zip-entries';
+import {
+  enumZipEntries,
+  readEntry,
+  readEntryPrefix,
+} from './utils/zip-entries';
+import {
+  ZIP_ENTRY_SNIFF_BYTES,
+  zipOptionsForManifestEntry,
+  zipOptionsForPatchEntry,
+  zipOptionsForPayloadEntry,
+} from './utils/zip-options';
 
 type Diff = (oldSource?: Buffer, newSource?: Buffer) => Buffer;
+type HpatchCover = {
+  oldPos: number | string | bigint;
+  newPos: number | string | bigint;
+  len: number | string | bigint;
+};
+type HpatchCompatiblePlan = {
+  covers?: HpatchCover[];
+};
+type HdiffWithCoversOptions = {
+  mode?: 'replace' | 'merge' | 'native-coalesce';
+};
+type HdiffModule = {
+  diff?: Diff;
+  diffWithCovers?: (
+    oldSource: Buffer,
+    newSource: Buffer,
+    covers: HpatchCover[],
+    options?: HdiffWithCoversOptions,
+  ) => { diff?: Buffer };
+};
+type BsdiffModule = {
+  diff?: Diff;
+};
+type ChiffModule = {
+  hpatchCompatiblePlanResult?: (
+    oldSource: Buffer,
+    newSource: Buffer,
+  ) => HpatchCompatiblePlan;
+  hpatchApproximatePlanResult?: (
+    oldSource: Buffer,
+    newSource: Buffer,
+  ) => HpatchCompatiblePlan;
+};
+type ChiffHpatchPolicy = 'off' | 'costed';
+type ChiffHpatchExactPolicy = 'off' | 'on';
 type EntryMap = Record<string, { crc32: number; fileName: string }>;
 type CrcMap = Record<number, string>;
 type CopyMap = Record<string, string>;
@@ -28,22 +73,134 @@ type DiffCommandConfig = {
 
 export { enumZipEntries, readEntry };
 
-const loadDiffModule = (pkgName: string): Diff | undefined => {
+const loadModule = <T>(pkgName: string): T | undefined => {
   const resolvePaths = ['.', npm.packages, yarn.packages];
 
   try {
     const resolved = require.resolve(pkgName, { paths: resolvePaths });
-    const mod = require(resolved);
-    if (mod?.diff) {
-      return mod.diff as Diff;
-    }
+    return require(resolved) as T;
   } catch {}
 
   return undefined;
 };
 
-const hdiff = loadDiffModule('node-hdiffpatch');
-const bsdiff = loadDiffModule('node-bsdiff');
+const hdiff = loadModule<HdiffModule>('node-hdiffpatch');
+const bsdiff = loadModule<BsdiffModule>('node-bsdiff');
+const chiff = loadModule<ChiffModule>('@chiff/node');
+
+// Structured covers are experimental and can be expensive on real Hermes input.
+// Keep native hdiff as the default unless the server explicitly opts in.
+function resolveChiffHpatchPolicy(policy?: unknown): ChiffHpatchPolicy {
+  const value = String(
+    policy ?? process.env.RNU_CHIFF_HPATCH_POLICY ?? 'off',
+  ).toLowerCase();
+  if (
+    value === 'costed' ||
+    value === 'on' ||
+    value === 'true' ||
+    value === '1'
+  ) {
+    return 'costed';
+  }
+  return 'off';
+}
+
+function resolveChiffHpatchMinNativeBytes(value?: unknown): number {
+  const raw = value ?? process.env.RNU_CHIFF_HPATCH_MIN_NATIVE_BYTES ?? 4096;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return 4096;
+  }
+  return Math.floor(parsed);
+}
+
+function resolveChiffHpatchExactPolicy(policy?: unknown): ChiffHpatchExactPolicy {
+  const value = String(
+    policy ?? process.env.RNU_CHIFF_HPATCH_EXACT_COVERS ?? 'off',
+  ).toLowerCase();
+  if (value === 'on' || value === 'true' || value === '1') {
+    return 'on';
+  }
+  return 'off';
+}
+
+function createChiffAwareHdiff(
+  hdiffModule: HdiffModule,
+  chiffModule: ChiffModule | undefined,
+  policy: ChiffHpatchPolicy,
+  minNativeBytes: number,
+  exactPolicy: ChiffHpatchExactPolicy,
+): Diff {
+  const baseDiff = hdiffModule.diff;
+  if (!baseDiff) {
+    throw new Error(t('nodeHdiffpatchRequired', { scriptName }));
+  }
+
+  if (policy === 'off') {
+    return baseDiff;
+  }
+
+  return (oldSource?: Buffer, newSource?: Buffer) => {
+    const nativeDiff = baseDiff(oldSource, newSource);
+    if (!oldSource || !newSource || !hdiffModule.diffWithCovers) {
+      return nativeDiff;
+    }
+
+    let bestDiff = nativeDiff;
+    const tryDiffWithCovers = (
+      covers: HpatchCover[],
+      mode: 'replace' | 'merge' | 'native-coalesce',
+    ) => {
+      try {
+        const result = hdiffModule.diffWithCovers?.(
+          oldSource,
+          newSource,
+          covers,
+          { mode },
+        );
+        if (
+          Buffer.isBuffer(result?.diff) &&
+          result.diff.length < bestDiff.length
+        ) {
+          bestDiff = result.diff;
+        }
+      } catch {}
+    };
+
+    tryDiffWithCovers([], 'native-coalesce');
+
+    if (nativeDiff.length < minNativeBytes) {
+      return bestDiff;
+    }
+
+    try {
+      const approximatePlan = chiffModule?.hpatchApproximatePlanResult?.(
+        oldSource,
+        newSource,
+      );
+      if (Array.isArray(approximatePlan?.covers)) {
+        tryDiffWithCovers(approximatePlan.covers, 'merge');
+      }
+    } catch {}
+
+    if (
+      exactPolicy === 'off' ||
+      !chiffModule?.hpatchCompatiblePlanResult
+    ) {
+      return bestDiff;
+    }
+
+    try {
+      const plan = chiffModule.hpatchCompatiblePlanResult(oldSource, newSource);
+      if (Array.isArray(plan.covers)) {
+        tryDiffWithCovers(plan.covers, 'replace');
+        tryDiffWithCovers(plan.covers, 'merge');
+      }
+    } catch {}
+
+    return bestDiff;
+  };
+}
 
 function basename(fn: string): string | undefined {
   const m = /^(.+\/)[^\/]+\/?$/.exec(fn);
@@ -123,6 +280,7 @@ async function diffFromPPK(
       zipfile.addBuffer(
         diffFn(originSource, newSource),
         `${entry.fileName}.patch`,
+        zipOptionsForPatchEntry(),
       );
       //console.log('End diff');
     } else {
@@ -150,6 +308,11 @@ async function diffFromPPK(
         addEntry(basePath);
       }
 
+      const entryPrefix = await readEntryPrefix(
+        entry,
+        nextZipfile,
+        ZIP_ENTRY_SNIFF_BYTES,
+      );
       await new Promise<void>((resolve, reject) => {
         nextZipfile.openReadStream(entry, (err, readStream) => {
           if (err) {
@@ -160,7 +323,11 @@ async function diffFromPPK(
               new Error(`Unable to read zip entry: ${entry.fileName}`),
             );
           }
-          zipfile.addReadStream(readStream, entry.fileName);
+          zipfile.addReadStream(
+            readStream,
+            entry.fileName,
+            zipOptionsForPayloadEntry(entry.fileName, entryPrefix),
+          );
           readStream.on('end', () => {
             //console.log('add finished');
             resolve(void 0);
@@ -183,6 +350,7 @@ async function diffFromPPK(
   zipfile.addBuffer(
     Buffer.from(JSON.stringify({ copies, deletes })),
     '__diff.json',
+    zipOptionsForManifestEntry(),
   );
   zipfile.end();
   await writePromise;
@@ -248,6 +416,7 @@ async function diffFromPackage(
       zipfile.addBuffer(
         diffFn(originSource, newSource),
         `${entry.fileName}.patch`,
+        zipOptionsForPatchEntry(),
       );
       //console.log('End diff');
     } else {
@@ -263,6 +432,11 @@ async function diffFromPackage(
         return;
       }
 
+      const entryPrefix = await readEntryPrefix(
+        entry,
+        nextZipfile,
+        ZIP_ENTRY_SNIFF_BYTES,
+      );
       await new Promise<void>((resolve, reject) => {
         nextZipfile.openReadStream(entry, (err, readStream) => {
           if (err) {
@@ -273,7 +447,11 @@ async function diffFromPackage(
               new Error(`Unable to read zip entry: ${entry.fileName}`),
             );
           }
-          zipfile.addReadStream(readStream, entry.fileName);
+          zipfile.addReadStream(
+            readStream,
+            entry.fileName,
+            zipOptionsForPayloadEntry(entry.fileName, entryPrefix),
+          );
           readStream.on('end', () => {
             //console.log('add finished');
             resolve(void 0);
@@ -283,13 +461,22 @@ async function diffFromPackage(
     }
   });
 
-  zipfile.addBuffer(Buffer.from(JSON.stringify({ copies })), '__diff.json');
+  zipfile.addBuffer(
+    Buffer.from(JSON.stringify({ copies })),
+    '__diff.json',
+    zipOptionsForManifestEntry(),
+  );
   zipfile.end();
   await writePromise;
 }
 
 type DiffCommandOptions = {
   customDiff?: Diff;
+  customHdiffModule?: HdiffModule;
+  customChiffModule?: ChiffModule;
+  chiffHpatchPolicy?: ChiffHpatchPolicy;
+  chiffHpatchMinNativeBytes?: number | string;
+  chiffHpatchExactCovers?: ChiffHpatchExactPolicy | boolean | string | number;
   [key: string]: any;
 };
 
@@ -302,16 +489,23 @@ function resolveDiffImplementation(
   }
 
   if (useHdiff) {
-    if (!hdiff) {
+    const hdiffModule = options.customHdiffModule ?? hdiff;
+    if (!hdiffModule?.diff) {
       throw new Error(t('nodeHdiffpatchRequired', { scriptName }));
     }
-    return hdiff;
+    return createChiffAwareHdiff(
+      hdiffModule,
+      options.customChiffModule ?? chiff,
+      resolveChiffHpatchPolicy(options.chiffHpatchPolicy),
+      resolveChiffHpatchMinNativeBytes(options.chiffHpatchMinNativeBytes),
+      resolveChiffHpatchExactPolicy(options.chiffHpatchExactCovers),
+    );
   }
 
-  if (!bsdiff) {
+  if (!bsdiff?.diff) {
     throw new Error(t('nodeBsdiffRequired', { scriptName }));
   }
-  return bsdiff;
+  return bsdiff.diff;
 }
 
 function diffArgsCheck(