From c5ec864d4bc8798dae26cedb063a5a8c2bb062bd Mon Sep 17 00:00:00 2001 From: Daria Terekhova <98411986+dariaterekhova-actionengine@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:50:59 +0200 Subject: [PATCH] feat(zip): Added append and truncate functionality (#2800) --- .../src/lib/file-provider/file-handle-file.ts | 26 +++++++++++----- .../src/lib/files/node-file-facade.ts | 12 +++++++- .../polyfills/src/filesystems/node-file.ts | 30 ++++++++++++++++++- .../test/filesystems/node-file.spec.ts | 29 +++++++++++++++--- 4 files changed, 84 insertions(+), 13 deletions(-) diff --git a/modules/loader-utils/src/lib/file-provider/file-handle-file.ts b/modules/loader-utils/src/lib/file-provider/file-handle-file.ts index 73b2c4f32b..99683ca414 100644 --- a/modules/loader-utils/src/lib/file-provider/file-handle-file.ts +++ b/modules/loader-utils/src/lib/file-provider/file-handle-file.ts @@ -12,13 +12,25 @@ export class FileHandleFile implements FileProvider { /** The FileHandle from which data is provided */ private file: NodeFile; - /** The file length in bytes */ - private size: bigint; - /** Create a new FileHandleFile */ - constructor(path: string) { - this.file = new NodeFile(path, 'r'); - this.size = this.file.bigsize; + constructor(path: string, append: boolean = false) { + this.file = new NodeFile(path, append ? 'a+' : 'r'); + } + + /** + * Truncates the file descriptor. + * @param length desired file lenght + */ + async truncate(length: number): Promise { + await this.file.truncate(length); + } + + /** + * Append data to a file. + * @param buffer data to append + */ + async append(buffer: Uint8Array): Promise { + await this.file.append(buffer); } /** Close file */ @@ -96,6 +108,6 @@ export class FileHandleFile implements FileProvider { * the length (in bytes) of the data. */ get length(): bigint { - return this.size; + return this.file.bigsize; } } diff --git a/modules/loader-utils/src/lib/files/node-file-facade.ts b/modules/loader-utils/src/lib/files/node-file-facade.ts index cc841f76bb..43032d7a4a 100644 --- a/modules/loader-utils/src/lib/files/node-file-facade.ts +++ b/modules/loader-utils/src/lib/files/node-file-facade.ts @@ -13,7 +13,7 @@ export class NodeFileFacade implements ReadableFile, WritableFile { bigsize: bigint = 0n; url: string = ''; - constructor(url: string, flags?: 'r' | 'w' | 'wx', mode?: number) { + constructor(url: string, flags?: 'r' | 'w' | 'wx' | 'a+', mode?: number) { // Return the actual implementation instance if (globalThis.loaders?.NodeFile) { return new globalThis.loaders.NodeFile(url, flags, mode); @@ -35,6 +35,16 @@ export class NodeFileFacade implements ReadableFile, WritableFile { async stat(): Promise { throw NOT_IMPLEMENTED; } + + /** Truncates the file descriptor. Only available on NodeFile. */ + async truncate(length: number): Promise { + throw NOT_IMPLEMENTED; + } + + /** Append data to a file. Only available on NodeFile. */ + async append(data: Uint8Array): Promise { + throw NOT_IMPLEMENTED; + } /** Close the file */ async close(): Promise {} } diff --git a/modules/polyfills/src/filesystems/node-file.ts b/modules/polyfills/src/filesystems/node-file.ts index 6d7e0e3f0d..3fd25f40dd 100644 --- a/modules/polyfills/src/filesystems/node-file.ts +++ b/modules/polyfills/src/filesystems/node-file.ts @@ -8,7 +8,7 @@ export class NodeFile implements ReadableFile, WritableFile { bigsize: bigint; url: string; - constructor(path: string, flags: 'r' | 'w' | 'wx', mode?: number) { + constructor(path: string, flags: 'r' | 'w' | 'wx' | 'a+', mode?: number) { path = resolvePath(path); this.handle = fs.openSync(path, flags, mode); const stats = fs.fstatSync(this.handle, {bigint: true}); @@ -23,6 +23,34 @@ export class NodeFile implements ReadableFile, WritableFile { }); } + async truncate(length: number): Promise { + return new Promise((resolve, reject) => { + fs.ftruncate(this.handle, length, (err) => { + if (err) { + reject(err); + } else { + this.bigsize = BigInt(length); + this.size = Number(this.bigsize); + resolve(); + } + }); + }); + } + + async append(data: Uint8Array): Promise { + return new Promise((resolve, reject) => { + fs.appendFile(this.handle, data, (err) => { + if (err) { + reject(err); + } else { + this.bigsize = this.bigsize + BigInt(data.length); + this.size = Number(this.bigsize); + resolve(); + } + }); + }); + } + async stat(): Promise { return await new Promise((resolve, reject) => fs.fstat(this.handle, {bigint: true}, (err, info) => { diff --git a/modules/polyfills/test/filesystems/node-file.spec.ts b/modules/polyfills/test/filesystems/node-file.spec.ts index 84c5ce6e55..a9cfb29e89 100644 --- a/modules/polyfills/test/filesystems/node-file.spec.ts +++ b/modules/polyfills/test/filesystems/node-file.spec.ts @@ -2,16 +2,37 @@ import test from 'tape-promise/tape'; import {isBrowser} from '@loaders.gl/core'; import {NodeFile} from '@loaders.gl/loader-utils'; -const SLPK_URL = '@loaders.gl/i3s/test/data/DA12_subset.slpk'; +const SLPK_URL = 'modules/i3s/test/data/DA12_subset.slpk'; +const TEST_OFFSET = 100n; -// TODO v4.0 restore this test -test.skip('NodeFile#open and read', async (t) => { +const getSize = async (provider: NodeFile): Promise => { + const stat = await provider.stat(); + return stat.bigsize; +}; + +test('NodeFile#open and read', async (t) => { if (!isBrowser) { const provider = new NodeFile(SLPK_URL); const arrayBuffer = await provider.read(4, 1); - const reference = new Buffer(new Uint8Array([0])); + const reference = Buffer.from(new Uint8Array([0x2d])); t.equals(reference.compare(Buffer.from(arrayBuffer)), 0); } t.end(); }); + +test('NodeFile#truncate and append', async (t) => { + if (!isBrowser) { + const provider = new NodeFile(SLPK_URL, 'a+'); + const initialSize = await getSize(provider); + + const ending = await provider.read(TEST_OFFSET, Number(initialSize - TEST_OFFSET)); + + await provider.truncate(Number(TEST_OFFSET)); + t.equals(await getSize(provider), TEST_OFFSET); + + await provider.append(new Uint8Array(ending)); + t.equals(await getSize(provider), initialSize); + } + t.end(); +});