feat: merge 4.1 dev into master

modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts

@@ -1,6 +1,6 @@
 import {FileProvider} from '@loaders.gl/loader-utils';
 import {
-  cdSignature as cdHeaderSignature,
+  CD_HEADER_SIGNATURE,
   makeHashTableFromZipHeaders,
   parseHashTable,
   parseZipCDFileHeader,
@@ -19,7 +19,7 @@ export const parse3DTilesArchive = async (
   fileProvider: FileProvider,
   cb?: (msg: string) => void
 ): Promise<Tiles3DArchive> => {
-  const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature);
+  const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE);
 
   const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
 

modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts

@@ -1,7 +1,7 @@
 import {FileProvider} from '@loaders.gl/loader-utils';
 import {
   ZipFileSystem,
-  cdSignature as cdHeaderSignature,
+  CD_HEADER_SIGNATURE,
   searchFromTheEnd,
   parseZipCDFileHeader,
   parseHashTable,
@@ -66,7 +66,7 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem {
       throw new Error('No data detected in the zip archive');
     }
 
-    const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature);
+    const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE);
 
     const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
 

modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -1,7 +1,7 @@
 import {FileProvider} from '@loaders.gl/loader-utils';
 import {
   parseZipCDFileHeader,
-  cdSignature as cdHeaderSignature,
+  CD_HEADER_SIGNATURE,
   parseZipLocalFileHeader,
   searchFromTheEnd,
   parseHashTable,
@@ -19,7 +19,7 @@ export async function parseSLPKArchive(
   fileProvider: FileProvider,
   cb?: (msg: string) => void
 ): Promise<SLPKArchive> {
-  const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature);
+  const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE);
 
   const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
 

modules/zip/src/index.ts

@@ -8,7 +8,8 @@ export {TarBuilder} from './tar-builder';
 export {
   parseZipCDFileHeader,
   makeZipCDHeaderIterator,
-  signature as cdSignature
+  signature as CD_HEADER_SIGNATURE,
+  generateCDHeader
 } from './parse-zip/cd-file-header';
 export {
   parseZipLocalFileHeader,

modules/zip/src/parse-zip/cd-file-header.ts

@@ -1,9 +1,10 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
 import {parseEoCDRecord} from './end-of-central-directory';
 import {ZipSignature} from './search-from-the-end';
+import {createZip64Info, NUMBER_SETTERS} from './zip64-info-generation';
 
 /**
  * zip central directory file header info
@@ -188,3 +189,183 @@ const findExpectedData = (zip64data: Zip64Data): {length: number; name: string}[
 
   return zip64dataList;
 };
+
+/** info that can be placed into cd header */
+type GenerateCDOptions = {
+  /** CRC-32 of uncompressed data */
+  crc32: number;
+  /** File name */
+  fileName: string;
+  /** File size */
+  length: number;
+  /** Relative offset of local file header */
+  offset: number;
+};
+
+/**
+ * generates cd header for the file
+ * @param options info that can be placed into cd header
+ * @returns buffer with header
+ */
+export function generateCDHeader(options: GenerateCDOptions): ArrayBuffer {
+  const optionsToUse = {
+    ...options,
+    fnlength: options.fileName.length,
+    extraLength: 0
+  };
+
+  let zip64header: ArrayBuffer = new ArrayBuffer(0);
+
+  const optionsToZip64: any = {};
+  if (optionsToUse.offset >= 0xffffffff) {
+    optionsToZip64.offset = optionsToUse.offset;
+    optionsToUse.offset = 0xffffffff;
+  }
+  if (optionsToUse.length >= 0xffffffff) {
+    optionsToZip64.size = optionsToUse.length;
+    optionsToUse.length = 0xffffffff;
+  }
+
+  if (Object.keys(optionsToZip64).length) {
+    zip64header = createZip64Info(optionsToZip64);
+    optionsToUse.extraLength = zip64header.byteLength;
+  }
+  const header = new DataView(new ArrayBuffer(46));
+
+  for (const field of ZIP_HEADER_FIELDS) {
+    NUMBER_SETTERS[field.size](
+      header,
+      field.offset,
+      optionsToUse[field.name ?? ''] ?? field.default ?? 0
+    );
+  }
+
+  const encodedName = new TextEncoder().encode(optionsToUse.fileName);
+
+  const resHeader = concatenateArrayBuffers(header.buffer, encodedName, zip64header);
+
+  return resHeader;
+}
+
+/** Fields map */
+const ZIP_HEADER_FIELDS = [
+  // Central directory file header signature = 0x02014b50
+  {
+    offset: 0,
+    size: 4,
+    default: new DataView(signature.buffer).getUint32(0, true)
+  },
+
+  // Version made by
+  {
+    offset: 4,
+    size: 2,
+    default: 45
+  },
+
+  // Version needed to extract (minimum)
+  {
+    offset: 6,
+    size: 2,
+    default: 45
+  },
+
+  // General purpose bit flag
+  {
+    offset: 8,
+    size: 2,
+    default: 0
+  },
+
+  // Compression method
+  {
+    offset: 10,
+    size: 2,
+    default: 0
+  },
+
+  // File last modification time
+  {
+    offset: 12,
+    size: 2,
+    default: 0
+  },
+
+  // File last modification date
+  {
+    offset: 14,
+    size: 2,
+    default: 0
+  },
+
+  // CRC-32 of uncompressed data
+  {
+    offset: 16,
+    size: 4,
+    name: 'crc32'
+  },
+
+  // Compressed size (or 0xffffffff for ZIP64)
+  {
+    offset: 20,
+    size: 4,
+    name: 'length'
+  },
+
+  // Uncompressed size (or 0xffffffff for ZIP64)
+  {
+    offset: 24,
+    size: 4,
+    name: 'length'
+  },
+
+  // File name length (n)
+  {
+    offset: 28,
+    size: 2,
+    name: 'fnlength'
+  },
+
+  // Extra field length (m)
+  {
+    offset: 30,
+    size: 2,
+    default: 0,
+    name: 'extraLength'
+  },
+
+  // File comment length (k)
+  {
+    offset: 32,
+    size: 2,
+    default: 0
+  },
+
+  // Disk number where file starts (or 0xffff for ZIP64)
+  {
+    offset: 34,
+    size: 2,
+    default: 0
+  },
+
+  // Internal file attributes
+  {
+    offset: 36,
+    size: 2,
+    default: 0
+  },
+
+  // External file attributes
+  {
+    offset: 38,
+    size: 4,
+    default: 0
+  },
+
+  // Relative offset of local file header
+  {
+    offset: 42,
+    size: 4,
+    name: 'offset'
+  }
+];

modules/zip/src/parse-zip/zip64-info-generation.ts

@@ -0,0 +1,90 @@
+import {concatenateArrayBuffers} from '@loaders.gl/loader-utils';
+
+export const signature = new Uint8Array([0x01, 0x00]);
+
+/** info that can be placed into zip64 field, doc: https://en.wikipedia.org/wiki/ZIP_(file_format)#ZIP64 */
+type Zip64Options = {
+  /** Original uncompressed file size and Size of compressed data */
+  size?: number;
+  /** Offset of local header record */
+  offset?: number;
+};
+
+/**
+ * creates zip64 extra field
+ * @param options info that can be placed into zip64 field
+ * @returns buffer with field
+ */
+export function createZip64Info(options: Zip64Options): ArrayBuffer {
+  const optionsToUse = {
+    ...options,
+    zip64Length: (options.offset ? 1 : 0) * 8 + (options.size ? 1 : 0) * 16
+  };
+
+  const arraysToConcat: ArrayBuffer[] = [];
+
+  for (const field of ZIP64_FIELDS) {
+    if (!optionsToUse[field.name ?? ''] && !field.default) {
+      continue;
+    }
+    const newValue = new DataView(new ArrayBuffer(field.size));
+    NUMBER_SETTERS[field.size](newValue, 0, optionsToUse[field.name ?? ''] ?? field.default);
+    arraysToConcat.push(newValue.buffer);
+  }
+
+  return concatenateArrayBuffers(...arraysToConcat);
+}
+
+/**
+ * Function to write values into buffer
+ * @param header buffer where to write a value
+ * @param offset offset of the writing start
+ * @param value value to be written
+ */
+type NumberSetter = (header: DataView, offset: number, value: number) => void;
+
+/** functions to write values into buffer according to the bytes amount */
+export const NUMBER_SETTERS: {[key: number]: NumberSetter} = {
+  2: (header, offset, value) => {
+    header.setUint16(offset, value, true);
+  },
+  4: (header, offset, value) => {
+    header.setUint32(offset, value, true);
+  },
+  8: (header, offset, value) => {
+    header.setBigUint64(offset, BigInt(value), true);
+  }
+};
+
+/** zip64 info fields description, we need it as a pattern to build a zip64 info */
+const ZIP64_FIELDS = [
+  // Header ID 0x0001
+  {
+    size: 2,
+    default: new DataView(signature.buffer).getUint16(0, true)
+  },
+
+  // Size of the extra field chunk (8, 16, 24 or 28)
+  {
+    size: 2,
+    name: 'zip64Length'
+  },
+
+  // Original uncompressed file size
+  {
+    size: 8,
+    name: 'size'
+  },
+
+  // Size of compressed data
+  {
+    size: 8,
+    name: 'size'
+  },
+
+  // Offset of local header record
+  {
+    size: 8,
+    name: 'offset'
+  }
+];

modules/zip/test/zip-utils/cd-file-header.spec.ts

@@ -5,7 +5,8 @@ import test from 'tape-promise/tape';
 import {DATA_ARRAY} from '@loaders.gl/i3s/test/data/test.zip';
 
 import {DataViewFile} from '@loaders.gl/loader-utils';
-import {parseZipCDFileHeader} from '../../src//parse-zip/cd-file-header';
+import {generateCDHeader, parseZipCDFileHeader} from '../../src/parse-zip/cd-file-header';
+import {createZip64Info} from '../../src/parse-zip/zip64-info-generation';
 
 test('SLPKLoader#central directory file header parse', async (t) => {
   const cdFileHeader = await parseZipCDFileHeader(
@@ -18,3 +19,22 @@ test('SLPKLoader#central directory file header parse', async (t) => {
   t.deepEqual(cdFileHeader?.localHeaderOffset, 0n);
   t.end();
 });
+
+test('SLPKLoader#central directory file header generation', async (t) => {
+  const header = generateCDHeader({
+    crc32: 0,
+    fileName: '@specialIndexFileHASH128@1',
+    offset: 0xffffffffff,
+    length: 0
+  });
+  t.equal(header.byteLength, 84);
+  t.end();
+});
+
+test('SLPKLoader#zip64 info generation', async (t) => {
+  const header = createZip64Info({
+    size: 0xffffffffff
+  });
+  t.equal(header.byteLength, 20);
+  t.end();
+});