diff --git a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts index 49906d8497..4bb16e14c0 100644 --- a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts +++ b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts @@ -1,6 +1,6 @@ import {FileProvider} from '@loaders.gl/loader-utils'; import { - cdSignature as cdHeaderSignature, + CD_HEADER_SIGNATURE, makeHashTableFromZipHeaders, parseHashTable, parseZipCDFileHeader, @@ -19,7 +19,7 @@ export const parse3DTilesArchive = async ( fileProvider: FileProvider, cb?: (msg: string) => void ): Promise => { - const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature); + const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE); const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); diff --git a/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts b/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts index 4756f289f2..2865ac8ce0 100644 --- a/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts +++ b/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts @@ -1,7 +1,7 @@ import {FileProvider} from '@loaders.gl/loader-utils'; import { ZipFileSystem, - cdSignature as cdHeaderSignature, + CD_HEADER_SIGNATURE, searchFromTheEnd, parseZipCDFileHeader, parseHashTable, @@ -66,7 +66,7 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem { throw new Error('No data detected in the zip archive'); } - const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature); + const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE); const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); diff --git a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts index 51150ec5f8..0751026c4f 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts @@ -1,7 +1,7 @@ import {FileProvider} from '@loaders.gl/loader-utils'; import { parseZipCDFileHeader, - cdSignature as cdHeaderSignature, + CD_HEADER_SIGNATURE, parseZipLocalFileHeader, searchFromTheEnd, parseHashTable, @@ -19,7 +19,7 @@ export async function parseSLPKArchive( fileProvider: FileProvider, cb?: (msg: string) => void ): Promise { - const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature); + const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE); const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); diff --git a/modules/zip/src/index.ts b/modules/zip/src/index.ts index a7954a0a67..7f6635098b 100644 --- a/modules/zip/src/index.ts +++ b/modules/zip/src/index.ts @@ -8,7 +8,8 @@ export {TarBuilder} from './tar-builder'; export { parseZipCDFileHeader, makeZipCDHeaderIterator, - signature as cdSignature + signature as CD_HEADER_SIGNATURE, + generateCDHeader } from './parse-zip/cd-file-header'; export { parseZipLocalFileHeader, diff --git a/modules/zip/src/parse-zip/cd-file-header.ts b/modules/zip/src/parse-zip/cd-file-header.ts index a9a86bec4b..ec75d9cfaf 100644 --- a/modules/zip/src/parse-zip/cd-file-header.ts +++ b/modules/zip/src/parse-zip/cd-file-header.ts @@ -1,9 +1,10 @@ // loaders.gl, MIT license // Copyright (c) vis.gl contributors -import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils'; import {parseEoCDRecord} from './end-of-central-directory'; import {ZipSignature} from './search-from-the-end'; +import {createZip64Info, NUMBER_SETTERS} from './zip64-info-generation'; /** * zip central directory file header info @@ -188,3 +189,183 @@ const findExpectedData = (zip64data: Zip64Data): {length: number; name: string}[ return zip64dataList; }; + +/** info that can be placed into cd header */ +type GenerateCDOptions = { + /** CRC-32 of uncompressed data */ + crc32: number; + /** File name */ + fileName: string; + /** File size */ + length: number; + /** Relative offset of local file header */ + offset: number; +}; + +/** + * generates cd header for the file + * @param options info that can be placed into cd header + * @returns buffer with header + */ +export function generateCDHeader(options: GenerateCDOptions): ArrayBuffer { + const optionsToUse = { + ...options, + fnlength: options.fileName.length, + extraLength: 0 + }; + + let zip64header: ArrayBuffer = new ArrayBuffer(0); + + const optionsToZip64: any = {}; + if (optionsToUse.offset >= 0xffffffff) { + optionsToZip64.offset = optionsToUse.offset; + optionsToUse.offset = 0xffffffff; + } + if (optionsToUse.length >= 0xffffffff) { + optionsToZip64.size = optionsToUse.length; + optionsToUse.length = 0xffffffff; + } + + if (Object.keys(optionsToZip64).length) { + zip64header = createZip64Info(optionsToZip64); + optionsToUse.extraLength = zip64header.byteLength; + } + const header = new DataView(new ArrayBuffer(46)); + + for (const field of ZIP_HEADER_FIELDS) { + NUMBER_SETTERS[field.size]( + header, + field.offset, + optionsToUse[field.name ?? ''] ?? field.default ?? 0 + ); + } + + const encodedName = new TextEncoder().encode(optionsToUse.fileName); + + const resHeader = concatenateArrayBuffers(header.buffer, encodedName, zip64header); + + return resHeader; +} + +/** Fields map */ +const ZIP_HEADER_FIELDS = [ + // Central directory file header signature = 0x02014b50 + { + offset: 0, + size: 4, + default: new DataView(signature.buffer).getUint32(0, true) + }, + + // Version made by + { + offset: 4, + size: 2, + default: 45 + }, + + // Version needed to extract (minimum) + { + offset: 6, + size: 2, + default: 45 + }, + + // General purpose bit flag + { + offset: 8, + size: 2, + default: 0 + }, + + // Compression method + { + offset: 10, + size: 2, + default: 0 + }, + + // File last modification time + { + offset: 12, + size: 2, + default: 0 + }, + + // File last modification date + { + offset: 14, + size: 2, + default: 0 + }, + + // CRC-32 of uncompressed data + { + offset: 16, + size: 4, + name: 'crc32' + }, + + // Compressed size (or 0xffffffff for ZIP64) + { + offset: 20, + size: 4, + name: 'length' + }, + + // Uncompressed size (or 0xffffffff for ZIP64) + { + offset: 24, + size: 4, + name: 'length' + }, + + // File name length (n) + { + offset: 28, + size: 2, + name: 'fnlength' + }, + + // Extra field length (m) + { + offset: 30, + size: 2, + default: 0, + name: 'extraLength' + }, + + // File comment length (k) + { + offset: 32, + size: 2, + default: 0 + }, + + // Disk number where file starts (or 0xffff for ZIP64) + { + offset: 34, + size: 2, + default: 0 + }, + + // Internal file attributes + { + offset: 36, + size: 2, + default: 0 + }, + + // External file attributes + { + offset: 38, + size: 4, + default: 0 + }, + + // Relative offset of local file header + { + offset: 42, + size: 4, + name: 'offset' + } +]; diff --git a/modules/zip/src/parse-zip/zip64-info-generation.ts b/modules/zip/src/parse-zip/zip64-info-generation.ts new file mode 100644 index 0000000000..aa197f51ac --- /dev/null +++ b/modules/zip/src/parse-zip/zip64-info-generation.ts @@ -0,0 +1,90 @@ +import {concatenateArrayBuffers} from '@loaders.gl/loader-utils'; + +export const signature = new Uint8Array([0x01, 0x00]); + +/** info that can be placed into zip64 field, doc: https://en.wikipedia.org/wiki/ZIP_(file_format)#ZIP64 */ +type Zip64Options = { + /** Original uncompressed file size and Size of compressed data */ + size?: number; + /** Offset of local header record */ + offset?: number; +}; + +/** + * creates zip64 extra field + * @param options info that can be placed into zip64 field + * @returns buffer with field + */ +export function createZip64Info(options: Zip64Options): ArrayBuffer { + const optionsToUse = { + ...options, + zip64Length: (options.offset ? 1 : 0) * 8 + (options.size ? 1 : 0) * 16 + }; + + const arraysToConcat: ArrayBuffer[] = []; + + for (const field of ZIP64_FIELDS) { + if (!optionsToUse[field.name ?? ''] && !field.default) { + continue; + } + const newValue = new DataView(new ArrayBuffer(field.size)); + NUMBER_SETTERS[field.size](newValue, 0, optionsToUse[field.name ?? ''] ?? field.default); + arraysToConcat.push(newValue.buffer); + } + + return concatenateArrayBuffers(...arraysToConcat); +} + +/** + * Function to write values into buffer + * @param header buffer where to write a value + * @param offset offset of the writing start + * @param value value to be written + */ +type NumberSetter = (header: DataView, offset: number, value: number) => void; + +/** functions to write values into buffer according to the bytes amount */ +export const NUMBER_SETTERS: {[key: number]: NumberSetter} = { + 2: (header, offset, value) => { + header.setUint16(offset, value, true); + }, + 4: (header, offset, value) => { + header.setUint32(offset, value, true); + }, + 8: (header, offset, value) => { + header.setBigUint64(offset, BigInt(value), true); + } +}; + +/** zip64 info fields description, we need it as a pattern to build a zip64 info */ +const ZIP64_FIELDS = [ + // Header ID 0x0001 + { + size: 2, + default: new DataView(signature.buffer).getUint16(0, true) + }, + + // Size of the extra field chunk (8, 16, 24 or 28) + { + size: 2, + name: 'zip64Length' + }, + + // Original uncompressed file size + { + size: 8, + name: 'size' + }, + + // Size of compressed data + { + size: 8, + name: 'size' + }, + + // Offset of local header record + { + size: 8, + name: 'offset' + } +]; diff --git a/modules/zip/test/zip-utils/cd-file-header.spec.ts b/modules/zip/test/zip-utils/cd-file-header.spec.ts index e59e61eec9..91ef05549d 100644 --- a/modules/zip/test/zip-utils/cd-file-header.spec.ts +++ b/modules/zip/test/zip-utils/cd-file-header.spec.ts @@ -5,7 +5,8 @@ import test from 'tape-promise/tape'; import {DATA_ARRAY} from '@loaders.gl/i3s/test/data/test.zip'; import {DataViewFile} from '@loaders.gl/loader-utils'; -import {parseZipCDFileHeader} from '../../src//parse-zip/cd-file-header'; +import {generateCDHeader, parseZipCDFileHeader} from '../../src/parse-zip/cd-file-header'; +import {createZip64Info} from '../../src/parse-zip/zip64-info-generation'; test('SLPKLoader#central directory file header parse', async (t) => { const cdFileHeader = await parseZipCDFileHeader( @@ -18,3 +19,22 @@ test('SLPKLoader#central directory file header parse', async (t) => { t.deepEqual(cdFileHeader?.localHeaderOffset, 0n); t.end(); }); + +test('SLPKLoader#central directory file header generation', async (t) => { + const header = generateCDHeader({ + crc32: 0, + fileName: '@specialIndexFileHASH128@1', + offset: 0xffffffffff, + length: 0 + }); + t.equal(header.byteLength, 84); + t.end(); +}); + +test('SLPKLoader#zip64 info generation', async (t) => { + const header = createZip64Info({ + size: 0xffffffffff + }); + t.equal(header.byteLength, 20); + t.end(); +});