refactor: expose internal chunk and fetchable as npm packages (#641)

* refactor: expose `@cogeotiff/chunk`

* refactor: use stronger lint settings

* refactor: remove more unused code

* refactor: remove `@cogeotiff/core` from sources

* refactor: dead code removal

* refactor: dead code removal

* refactor: ensure the logger is passed arround

* refactor: cleanup names, update docs

* refactor: remove unused files

* refactor: add git location and author to packages

* refactor: correct naming of fetchBytes

.eslintrc.js

@@ -1,16 +1,3 @@
 module.exports = {
-    parser: "@typescript-eslint/parser",
-    extends: [
-        "plugin:@typescript-eslint/recommended",
-        "prettier/@typescript-eslint",
-        "plugin:prettier/recommended"
-    ],
-    parserOptions: {
-        ecmaVersion: 2018,
-        sourceType: "module"
-    },
-    rules: {
-        "@typescript-eslint/explicit-member-accessibility" : "off",
-        "@typescript-eslint/explicit-function-return-type": "off"
-    }
-};
+    ...require('@linzjs/style/.eslintrc.js'),
+};

.prettierrc.js

@@ -1,7 +1,4 @@
 module.exports = {
-    semi: true,
-    trailingComma: "all",
-    singleQuote: true,
-    printWidth: 120,
+    ...require('@linzjs/style/.prettierrc.js'),
     tabWidth: 4
-};
+};

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Blayne Chard
+Copyright (c) 2021 Blayne Chard
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -5,11 +5,12 @@
 
 Tools to work with [Cloud optimized GEOTiff](https://www.cogeo.org/)
 
--   Completely javascript based, works in the browser and nodejs
--   Lazy load COG images and metadata
--   Supports huge 100GB+ COGs
--   Uses GDAL COG optimizations, generally only one read per tile!
--   Loads COGs from URL, File or AWS S3
+-  Completely javascript based, works in the browser and nodejs
+-  Lazy load COG images and metadata
+-  Supports huge 100GB+ COGs
+-  Uses GDAL COG optimizations, generally only one or two reads per tile!
+-  Loads COGs from URL, File or AWS S3
+-  Used in production for [Basemaps](https://github.com/basemaps) Over 1 billion tiles fetched from COGs!
 
 ## Usage
 
@@ -18,8 +19,12 @@ Load a COG from a URL using `fetch`
 ```javascript
 import { CogSourceUrl } from '@cogeotiff/source-url';
 
-const cog = await CogSourceUrl.create('https://example.com/cog.tif');
-const tile = await cog.getTile(2, 2, 5);
+const source = new CogSourceUrl('https://example.com/cog.tif');
+const cog = await CogTiff.create(source);
+
+const img = cog.getImage(0);
+if (img.isTiled()) throw new Error('Tiff is not tiled');
+const tile = await img.getTile(2, 2); // Fetch a tile from a tiff x:2, y:2
 ```
 
 ## Scripts

package.json

@@ -1,32 +1,26 @@
 {
-  "name": "@cogeotiff/base",
-  "version": "0.1.0",
-  "scripts": {
-    "clean": "tsc -b --clean && rimraf 'packages/*/build'",
-    "build": "tsc -b --pretty",
-    "build-watch": "tsc -b --pretty --watch",
-    "version": "lerna version --conventional-commits --no-push --sign-git-commit --sign-git-tag",
-    "lint": "eslint 'packages/*/{src,test}/**/*.{js,ts,tsx}' --quiet --fix",
-    "test": "ospec packages/*/build/**/*.test.js"
-  },
-  "private": true,
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "description": "",
-  "devDependencies": {
-    "@typescript-eslint/eslint-plugin": "^4.0.1",
-    "@typescript-eslint/parser": "^4.0.1",
-    "eslint": "^7.8.1",
-    "eslint-config-prettier": "^7.1.0",
-    "eslint-plugin-prettier": "^3.1.0",
-    "lerna": "^3.16.4",
-    "ospec": "^4.0.0",
-    "prettier": "^2.1.1",
-    "rimraf": "^3.0.0",
-    "typescript": "^4.0.2"
-  },
-  "workspaces": [
-    "packages/*"
-  ]
-}
+    "name": "@cogeotiff/base",
+    "version": "0.1.0",
+    "scripts": {
+        "clean": "tsc -b --clean && rimraf 'packages/*/build'",
+        "build": "tsc -b --pretty",
+        "build-watch": "tsc -b --pretty --watch",
+        "version": "lerna version --conventional-commits --no-push --sign-git-commit --sign-git-tag",
+        "lint": "eslint 'packages/*/{src,test}/**/*.{js,ts,tsx}' --quiet --fix",
+        "test": "ospec packages/*/build/**/*.test.js"
+    },
+    "private": true,
+    "keywords": [],
+    "author": "Blayne Chard",
+    "license": "ISC",
+    "description": "",
+    "devDependencies": {
+        "@linzjs/style": "^0.4.3",
+        "lerna": "^3.16.4",
+        "ospec": "^4.0.0",
+        "rimraf": "^3.0.0"
+    },
+    "workspaces": [
+        "packages/*"
+    ]
+}

packages/chunk/README.md

@@ -0,0 +1,51 @@
+# Chunked object reading
+
+Split up a file into chunks and then read the chunks as needed.
+
+## Example Sources
+- File - [@cogeotiff/source-file](https://www.npmjs.com/package/@cogeotiff/source-file)
+- AWS - [@cogeotiff/source-aws](https://www.npmjs.com/package/@cogeotiff/source-aws)
+- Url - [@cogeotiff/source-url](https://www.npmjs.com/package/@cogeotiff/source-url)
+
+### Example Usage
+Fetching Data
+```typescript
+const source = new SourceUrl('https://example.com/foo')
+// Read 1KB chunks
+source.chunkSize = 1024;
+
+// Read the first 2KB of the file, or two chunks of data, this will be one HTTP Range requests
+if (!source.hasBytes(0, 2048)) await source.loadBytes(0, 2048)
+
+const bytes = source.bytes(0, 2048);
+```
+
+Fetching multiple ranges at the same time
+
+```typescript
+const source = new SourceUrl('https://example.com/foo')
+// Read 1KB chunks
+source.chunkSize = 1024;
+
+// Read in the first two KB and 1KB starting at 4KB
+// This will do one HTTP Range request for all of the data even though 2048-4096 has not been requested
+// Chunks 0 (0-1024), 1 (1024-2048), 2 (2048-3096) 3 (3096-4096) and 4 (4096 - 5120) will be fetched
+await Promise.all([
+    source.loadBytes(0, 2048), 
+    source.loadBytes(4096, 1024)
+]) 
+
+const bytes = source.bytes(0, 5120);
+```
+
+
+Reading raw bytes
+```typescript
+const source = new SourceUrl('https://example.com/foo')
+
+if (!source.hasBytes(0, 1024)) await source.loadBytes(0, 1024)
+// Read a UInt8 starting at offset 0
+const firstNumber = source.uint8(0);
+// Read a buffer from offset 10, with length of 100
+const firstBuffer = source.bytes(10, 100)
+```

packages/chunk/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "@cogeotiff/chunk",
+    "version": "3.1.0",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/blacha/cogeotiff.git",
+        "directory": "packages/chunk"
+    },
+    "author": "Blayne Chard",
+    "main": "./build/index.js",
+    "types": "./build/index.d.ts",
+    "license": "MIT",
+    "scripts": {},
+    "dependencies": {},
+    "devDependencies": {},
+    "publishConfig": {
+        "access": "public"
+    }
+}

packages/core/src/__test__/fake.source.ts → packages/chunk/src/__test__/chunk.source.fake.ts

@@ -1,12 +1,12 @@
 import * as fs from 'fs';
-import { CogSource } from '../source/cog.source';
+import { ChunkSource } from '../chunk.source';
 
-export class FakeCogSource extends CogSource {
+export class FakeChunkSource extends ChunkSource {
     type = 'fake';
     uri = 'fake';
     chunkSize = 100;
 
-    fetchBytes(offset: number, length: number): Promise<ArrayBuffer> {
+    async fetchBytes(offset: number, length: number): Promise<ArrayBuffer> {
         const bytes = new Uint8Array(length);
         for (let i = 0; i < length; i++) {
             bytes[i] = offset + i;
@@ -18,7 +18,7 @@ export class FakeCogSource extends CogSource {
 }
 
 let Id = 0;
-export class TestFileCogSource extends CogSource {
+export class TestFileChunkSource extends ChunkSource {
     id = Id++;
     type = 'test-file';
     uri = '/test/file';
@@ -33,6 +33,6 @@ export class TestFileCogSource extends CogSource {
     }
     async fetchBytes(offset: number, length: number): Promise<ArrayBuffer> {
         const fileData = await fs.promises.readFile(this.fileName);
-        return fileData.buffer.slice(fileData.byteOffset + offset, fileData.byteOffset + fileData.byteLength);
+        return fileData.buffer.slice(fileData.byteOffset + offset, fileData.byteOffset + length);
     }
 }

packages/core/src/source/__test__/cog.source.chunk.test.ts → packages/chunk/src/__test__/chunk.source.test.ts

@@ -1,12 +1,12 @@
 import * as o from 'ospec';
 import 'source-map-support/register';
-import { ByteSize } from '../../const/byte.size';
-import { TiffEndian } from '../../const/tiff.endian';
-import { FakeCogSource } from '../../__test__/fake.source';
+import { ByteSize } from '../bytes';
+import { ChunkId } from '../chunk.source';
+import { FakeChunkSource } from './chunk.source.fake';
 
 // Reference uin64 from MDN
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView
-function getUint64(dataview: DataView, byteOffset: number, isLittleEndian: boolean) {
+function getUint64(dataview: DataView, byteOffset: number, isLittleEndian: boolean): number {
     // split 64-bit number into two 32-bit (4-byte) parts
     const left = dataview.getUint32(byteOffset, isLittleEndian);
     const right = dataview.getUint32(byteOffset + 4, isLittleEndian);
@@ -17,20 +17,17 @@ function getUint64(dataview: DataView, byteOffset: number, isLittleEndian: boole
 
 o.spec('CogSourceChunk', () => {
     const CHUNK_SIZE = 10;
-    let source: FakeCogSource;
+    let source: FakeChunkSource;
 
     o.beforeEach(() => {
-        source = new FakeCogSource();
+        source = new FakeChunkSource();
         source.isLittleEndian = true;
         source.chunkSize = CHUNK_SIZE;
     });
 
-    async function Chunk(id: number) {
-        const cnk = source.chunk(id).fetch();
-        if (cnk == null) {
-            throw new Error('Failed to fetch');
-        }
-        return cnk;
+    async function Chunk(chunkId: number): Promise<DataView> {
+        await source.loadBytes(chunkId * source.chunkSize, source.chunkSize, undefined);
+        return source.getView(chunkId as ChunkId);
     }
 
     o('should get unit8', async () => {
@@ -41,14 +38,12 @@ o.spec('CogSourceChunk', () => {
         o(source.uint32(0)).equals(50462976);
         o(source.uint32(4)).equals(117835012);
 
-        o(source.uint64(0)).equals(getUint64(chunk.view, 0, true));
+        o(source.uint64(0)).equals(getUint64(chunk, 0, true));
     });
 
     o('should get unit8 from range', async () => {
         await Chunk(0);
-        for (let i = 0; i < 10; i++) {
-            o(source.uint8(i)).equals(i);
-        }
+        for (let i = 0; i < 10; i++) o(source.uint8(i)).equals(i);
     });
 
     o('should use chunk offset', async () => {
@@ -70,19 +65,18 @@ o.spec('CogSourceChunk', () => {
         source.isLittleEndian = false;
         const chunk = await Chunk(0);
         for (let i = 0; i < source.chunkSize - 1; i++) {
-            o(chunk.view.getUint16(i, source.isLittleEndian)).equals(source.uint16(i));
+            o(chunk.getUint16(i, source.isLittleEndian)).equals(source.uint16(i));
         }
     });
 
-    for (const endian of [TiffEndian.BIG, TiffEndian.LITTLE]) {
-        const isLittleEndian = endian === TiffEndian.LITTLE;
+    for (const isLittleEndian of [true, false]) {
         const word = isLittleEndian ? 'LE' : 'BE';
         o(`should fetch uint16 (${word})`, async () => {
             source.isLittleEndian = isLittleEndian;
 
             const chunk = await Chunk(0);
             for (let i = 0; i < source.chunkSize - ByteSize.UInt16; i++) {
-                o(chunk.view.getUint16(i, source.isLittleEndian)).equals(source.uint16(i));
+                o(chunk.getUint16(i, source.isLittleEndian)).equals(source.uint16(i));
             }
         });
 
@@ -91,7 +85,7 @@ o.spec('CogSourceChunk', () => {
 
             const chunk = await Chunk(0);
             for (let i = 0; i < source.chunkSize - ByteSize.UInt32; i++) {
-                o(chunk.view.getUint32(i, source.isLittleEndian)).equals(source.uint32(i));
+                o(chunk.getUint32(i, source.isLittleEndian)).equals(source.uint32(i));
             }
         });
 
@@ -100,7 +94,7 @@ o.spec('CogSourceChunk', () => {
 
             const chunk = await Chunk(0);
             for (let i = 0; i < source.chunkSize - ByteSize.UInt64; i++) {
-                o(getUint64(chunk.view, i, source.isLittleEndian)).equals(source.uint64(i));
+                o(getUint64(chunk, i, source.isLittleEndian)).equals(source.uint64(i));
             }
         });
     }