📦 离线包打包深度剖析：压缩库 · 哈希算法 · 压缩策略#

async function createZip(outDir: string, packageName: string, version: string) {
  const zip = new AdmZip();
  zip.addLocalFile(manifestPath);        // 同步：读文件 + 压缩 + 入库
  zip.addLocalFolder(filesDir, 'files'); // 同步：递归遍历 + 批量压缩
  zip.writeZip(zipPath);                 // 同步：一次性 flush 到磁盘
  return zipPath;
}

import archiver from 'archiver';
import {createWriteStream} from 'node:fs';

const output = createWriteStream('pkg.zip');
const archive = archiver('zip', {
  zlib: {level: 9},           // 压缩等级 0-9
  store: false,                // 不压缩直接 store
  forceLocalTime: true
});

archive.pipe(output);
archive.directory('files/', 'files');   // 递归，流式
archive.file('manifest.json', {name: 'manifest.json'});

await archive.finalize();                // Promise

import compressing from 'compressing';

// 打 zip
await compressing.zip.compressDir('files/', 'pkg.zip');

// 打 tgz（游戏行业常见）
await compressing.tgz.compressDir('files/', 'pkg.tgz');

// 流式
const tarStream = new compressing.tar.Stream();
tarStream.addEntry('manifest.json');
tarStream.addEntry('files/');
tarStream.pipe(createWriteStream('pkg.tar'));

const zip = new JSZip();
zip.file('manifest.json', manifestStr);
zip.folder('files')!.file('home/index.html', htmlBuf);
const buf = await zip.generateAsync({
  type: 'nodebuffer',
  compression: 'DEFLATE',
  compressionOptions: {level: 6}
});
await writeFile('pkg.zip', buf);

import {zip, strToU8} from 'fflate';

zip({
  'manifest.json': strToU8(manifestJson),
  'files/home/index.html': htmlU8
}, {level: 6, mem: 8}, (err, data) => {
  writeFile('pkg.zip', data);
});

import CRC32 from 'crc-32';
const hex = (CRC32.buf(buf) >>> 0).toString(16).padStart(8, '0');

import {createHash} from 'node:crypto';
const md5 = createHash('md5').update(buf).digest('hex');

async function hashFile(filePath: string): Promise<string> {
  return new Promise((resolveHash, rejectHash) => {
    const hash = createHash('sha256');
    const stream = createReadStream(filePath);
    stream.on('error', rejectHash);
    stream.on('data', chunk => hash.update(chunk));
    stream.on('end', () => resolveHash(hash.digest('hex')));
  });
}

import {createHash} from 'blake3';
const hash = createHash().update(buf).digest('hex');

// 或多线程并行（Rust 绑定）
import {hash} from '@noble/hashes/blake3';

import {createGzip} from 'node:zlib';
createReadStream('app.js').pipe(createGzip({level: 9})).pipe(createWriteStream('app.js.gz'));

import {createBrotliCompress} from 'node:zlib';
createReadStream('app.js')
  .pipe(createBrotliCompress({
    params: {
      [zlib.constants.BROTLI_PARAM_QUALITY]: 11,  // 0-11，11 最极致
      [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT
    }
  }))
  .pipe(createWriteStream('app.js.br'));

import AdmZip from 'adm-zip';
const zip = new AdmZip();
zip.addLocalFile(manifestPath);
zip.addLocalFolder(filesDir, 'files');
zip.writeZip(zipPath);

async function hashFile(filePath: string): Promise<string> {
  const hash = createHash('sha256');
  const stream = createReadStream(filePath);
  stream.on('data', chunk => hash.update(chunk));
  ...
}

const blockNoUrls = block.replace(/"urls":\s*\[[^\]]*\]/g, '"urls":[]');
const blockNoTags = blockNoUrls.replace(/<script\s+src="https?:[^"]*"[^>]*><\/script>\s*/g, '');

for (const file of files) {
  const stats = await stat(file);
  const hash = await hashFile(file);
  ...
}

import pLimit from 'p-limit';

async function generateOfflineManifest(config: OfflineConfig, buildDir: string) {
  const files = await walkFiles(buildDir);
  const filter = buildOfflineFileFilter(config);
  const limit = pLimit(8); // 控制并发，避免打爆 IO

  const entries = await Promise.all(
    files
      .map(file => ({file, rel: posix.normalize(relative(buildDir, file).split('\\').join('/'))}))
      .filter(e => filter(e.rel))
      .map(e => limit(async () => {
        const [stats, hash] = await Promise.all([stat(e.file), hashFile(e.file)]);
        return {
          path: e.rel,
          size: stats.size,
          hash,
          contentType: contentTypeOf(e.rel)
        };
      }))
  );

  const assets = entries.sort((a, b) => a.path.localeCompare(b.path));
  return {
    /* ... */
    totalSize: assets.reduce((s, a) => s + a.size, 0),
    assets
  };
}

async function hashBuffer(buf: Buffer): Promise<string> {
  return createHash('sha256').update(buf).digest('hex');
}

export async function buildOfflinePackage(options: BuildOptions): Promise<BuildResult> {
  /* ...existing... */
  const zipPath = options.zip === false ? undefined : await createZip(...);

  // 新增：整包 hash
  let packageHash: string | undefined;
  if (zipPath) {
    const zipBuf = await readFile(zipPath);
    packageHash = await hashBuffer(zipBuf);
    // 写入 manifest 副本（打进 zip 的 manifest 不需要整包 hash，
    // 因为循环依赖；而是写到 dist-offline/manifest.json 里）
    const manifestWithHash = {...manifest, packageHash, packageSize: zipBuf.length};
    await writeFile(manifestPath, JSON.stringify(manifestWithHash, null, 2));
  }
  /* ... */
}

export const OfflineConfigSchema = z.object({
  /* ...existing... */
  zipEngine: z.enum(['adm-zip', 'archiver']).default('adm-zip'),
  zipLevel: z.number().int().min(0).max(9).default(6)
});

async function createZip(outDir, packageName, version, opts: {engine: string; level: number}) {
  if (opts.engine === 'archiver') {
    return createZipArchiver(outDir, packageName, version, opts.level);
  }
  return createZipAdm(outDir, packageName, version);
}

async function createZipArchiver(outDir, name, version, level) {
  const zipPath = join(outDir, `${name}-${version}.zip`);
  const output = createWriteStream(zipPath);
  const archive = archiver('zip', {zlib: {level}});

  return new Promise<string>((res, rej) => {
    output.on('close', () => res(zipPath));
    archive.on('error', rej);
    archive.pipe(output);
    archive.file(join(outDir, 'manifest.json'), {name: 'manifest.json'});
    archive.directory(join(outDir, 'files'), 'files');
    archive.finalize();
  });
}

import {brotliCompress} from 'node:zlib';
import {promisify} from 'node:util';
const brotli = promisify(brotliCompress);

const COMPRESSIBLE_EXT = new Set(['.html', '.js', '.css', '.json', '.svg', '.mjs']);

async function precompressFiles(filesDir: string, assets: OfflineManifestAsset[]) {
  const targets = assets.filter(a => {
    const ext = '.' + a.path.split('.').pop();
    return COMPRESSIBLE_EXT.has(ext) && a.size > 1024; // < 1KB 不压
  });

  await Promise.all(targets.map(async a => {
    const src = join(filesDir, a.path);
    const buf = await readFile(src);
    const compressed = await brotli(buf, {
      params: {
        [zlib.constants.BROTLI_PARAM_QUALITY]: 11,
        [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT
      }
    });
    if (compressed.length < buf.length * 0.9) { // 收益 >10% 才保留
      await writeFile(`${src}.br`, compressed);
    }
  }));
}

// 在 asset 里加 brotliSize 字段
{
  path: 'home/index.html',
  size: 676,
  brotliSize: 382,             // ← 新增
  hash: 'ab12cd34...',
  contentType: 'text/html'
}