refactor: monorepo

.gitignore

@@ -3,3 +3,4 @@ build/
 .cache
 dist/
 output
+tsconfig.tsbuildinfo

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+
+# Contributing
+
+This repository uses [Conventional Commits](https://www.conventionalcommits.org/)
+
+Example options:
+- **build**: Changes that affect the build system or external dependencies
+- **ci**: Changes to our CI configuration files and scripts
+- **docs**: Documentation only changes
+- **feat**: A new feature
+- **fix**: A bug fix
+- **perf**: A code change that improves performance
+- **refactor**: A code change that neither fixes a bug nor adds a feature
+- **style**: Changes that do not affect the meaning of the code
+- **test**: Adding missing tests or correcting existing tests

README.md

@@ -3,20 +3,29 @@
 
 Tools to work with [Cloud optimized GEOTiff](https://www.cogeo.org/)
 
-* Completely javascript based, works in the browser!
-* Dynamically loads tif files 1-3 requests per tile
+* Completely javascript based, works in the browser and nodejs
+* Lazy load COG images and metadata
+* Supports huge 100GB+ COGs
+* Uses GDAL COG optimizations, generally only one read per tile!
+* Loads COGs from URL, File or AWS S3
 
 ## Usage
 
+
 Load a COG from a URL using `fetch`
 ```javascript
-import { CogTif, CogSourceUrl } from 'coginfo';
+import { CogSourceUrl } from '@coginfo/source-url';
 
-const cog = await new CogTif(new CogSourceUrl(url)).init();
+const cog = await CogSourceUrl.create('https://example.com/cog.tif');
 const tile = await cog.getTileRaw(2, 2, 5);
 ```
 
 ## Scripts
+
+```bash
+npm i -g @coginfo/cli
+```
+
 ### coginfo info
 
 Display basic information about COG
@@ -67,7 +76,6 @@ Load and dump a individual tile
 coginfo tile --file webp.cog.tif --xyz 1,1,1
 ```
 
-
 # Building
 This requires [NodeJs](https://nodejs.org/en/) > 12 & [Yarn](https://yarnpkg.com/en/)
 
@@ -87,18 +95,3 @@ yarn run build
 # Run the unit tests
 yarn run test
 ```
-
-# Contribuiting
-
-This repoistory uses [Conventional Commits](https://www.conventionalcommits.org/)
-
-Example options:
-- **build**: Changes that affect the build system or external dependencies
-- **ci**: Changes to our CI configuration files and scripts
-- **docs**: Documentation only changes
-- **feat**: A new feature
-- **fix**: A bug fix
-- **perf**: A code change that improves performance
-- **refactor**: A code change that neither fixes a bug nor adds a feature
-- **style**: Changes that do not affect the meaning of the code
-- **test**: Adding missing tests or correcting existing tests

lefthook.yml

@@ -0,0 +1,4 @@
+commit-msg:
+  commands:
+    commitlint:
+      run: commitlint -e

package.json

@@ -1,47 +1,31 @@
 {
-  "name": "coginfo",
-  "version": "0.0.12",
-  "main": "build/src/index.js",
+  "name": "@coginfo/base",
+  "version": "0.1.0",
   "scripts": {
-    "build": "tsc --pretty",
-    "build-watch": "tsc --pretty --watch",
-    "lint": "eslint '{src,test}/**/*.{js,ts,tsx}' --quiet --fix",
-    "test": "ospec build/**/*.spec.js",
-    "webpack": "NODE_ENV=production webpack"
-  },
-  "bin": {
-    "coginfo": "bin/coginfo"
+    "build": "tsc -b --pretty",
+    "build-watch": "tsc -b --pretty --watch",
+    "lint": "eslint 'packages/*/{src,test}/**/*.{js,ts,tsx}' --quiet --fix",
+    "test": "ospec packages/*/build/**/*.test.js"
   },
+  "private": true,
   "keywords": [],
   "author": "",
   "license": "ISC",
   "description": "",
   "devDependencies": {
+    "@arkweid/lefthook": "^0.6.3",
     "@commitlint/cli": "^8.0.0",
     "@commitlint/config-conventional": "^8.0.0",
-    "@types/leaflet": "^1.4.4",
-    "@types/node": "^12.0.7",
     "@typescript-eslint/eslint-plugin": "^1.10.2",
     "@typescript-eslint/parser": "^1.10.2",
     "eslint": "^5.16.0",
     "eslint-config-prettier": "^6.0.0",
     "eslint-plugin-prettier": "^3.1.0",
-    "html-webpack-plugin": "^3.2.0",
-    "husky": "^3.0.0",
-    "leaflet": "^1.5.1",
     "ospec": "^4.0.0",
     "prettier": "^1.18.2",
-    "typescript": "^3.5.1",
-    "webpack": "^4.33.0",
-    "webpack-cli": "^3.3.3",
-    "webpack-dev-server": "^3.7.1"
+    "typescript": "^3.5.1"
   },
-  "dependencies": {
-    "@microsoft/ts-command-line": "^4.2.6",
-    "bblog": "^2.0.0",
-    "chalk": "^2.4.2",
-    "ieee754": "^1.1.13",
-    "node-fetch": "^2.6.0",
-    "source-map-support": "^0.5.12"
-  }
+  "workspaces": [
+    "packages/*"
+  ]
 }

packages/cli/package.json

@@ -0,0 +1,24 @@
+{
+    "name": "@coginfo/cli",
+    "version": "0.1.0",
+    "repository": "[email protected]:blacha/coginfo.git",
+    "author": "",
+    "license": "MIT",
+    "bin": {
+        "coginfo": "bin/coginfo"
+    },
+    "scripts": {},
+    "dependencies": {
+        "@coginfo/core": "^0.1.0",
+        "@coginfo/source-url": "^0.1.0",
+        "@coginfo/source-file": "^0.1.0",
+        "@microsoft/ts-command-line": "^4.2.6",
+        "bblog": "^2.0.0",
+        "chalk": "^2.4.2",
+        "source-map-support": "^0.5.12"
+    },
+    "devDependencies": {
+        "@types/node": "^12.7.2",
+        "@types/node-fetch": "^2.5.0"
+    }
+}

src/cli/action.dump.tile.ts → packages/cli/src/action.dump.tile.ts

@@ -1,13 +1,10 @@
+import { CogTif, Log, TiffVersion, TileUtil, toByteSizeString, CogLogger } from '@coginfo/core';
 import { CommandLineAction, CommandLineIntegerParameter, CommandLineStringParameter } from '@microsoft/ts-command-line';
 import chalk from 'chalk';
 import { promises as fs } from 'fs';
 import * as path from 'path';
-import { CogTif } from '../cog.tif';
-import { TiffVersion } from '../read/tif';
-import { toByteSizeString } from '../util/util.bytes';
-import { Logger } from '../util/util.log';
-import { TileUtil } from '../util/util.tile';
 import { ActionUtil, CliResultMap } from './action.util';
+import { writeTile } from './util.tile';
 
 const Rad2Deg = 180 / Math.PI;
 const A = 6378137.0; // 900913 properties.
@@ -43,18 +40,20 @@ export class ActionDumpTile extends CommandLineAction {
     private zoom: CommandLineIntegerParameter | null = null;
     private output: CommandLineStringParameter | null = null;
     private outputCount: number = 0;
+    private logger: CogLogger;
 
     public constructor() {
         super({
             actionName: 'dump',
             summary: 'Dump tiles from a COG',
             documentation: 'Stuff',
         });
+        this.logger = Log.get().child({ action: 'tile' });
     }
 
     // TODO this only works for WSG84
     async dumpBounds(tif: CogTif, output: string, zoom: number) {
-        Logger.info({ zoom }, 'CreateTileBounds');
+        this.logger.info({ zoom }, 'CreateTileBounds');
         const img = tif.getImage(zoom);
         if (!img.isTiled()) {
             return;
@@ -92,7 +91,7 @@ export class ActionDumpTile extends CommandLineAction {
     }
 
     async dumpIndex(tif: CogTif, output: string, zoom: number) {
-        Logger.info({ zoom }, 'CreateIndexHtml');
+        this.logger.info({ zoom }, 'CreateIndexHtml');
         const img = tif.getImage(zoom);
         if (!img.isTiled()) {
             return;
@@ -125,7 +124,7 @@ export class ActionDumpTile extends CommandLineAction {
             return;
         }
 
-        Logger.info({ ...img.tileInfo, ...img.tileCount }, 'TileInfo');
+        this.logger.info({ ...img.tileInfo, ...img.tileCount }, 'TileInfo');
         const tileCount = img.tileCount;
 
         // Load all offsets in
@@ -134,7 +133,7 @@ export class ActionDumpTile extends CommandLineAction {
         for (let x = 0; x < tileCount.nx; x++) {
             for (let y = 0; y < tileCount.ny; y++) {
                 // TODO should limit how many of these we run at a time
-                promises.push(TileUtil.write(tif, x, y, zoom, output));
+                promises.push(writeTile(tif, x, y, zoom, output, this.logger));
                 this.outputCount++;
             }
         }

src/cli/action.info.ts → packages/cli/src/action.info.ts

@@ -1,7 +1,6 @@
+import { TiffVersion, toByteSizeString } from '@coginfo/core';
 import { CommandLineAction, CommandLineStringParameter } from '@microsoft/ts-command-line';
 import chalk from 'chalk';
-import { TiffVersion } from '../read/tif';
-import { toByteSizeString } from '../util/util.bytes';
 import { ActionUtil, CliResultMap } from './action.util';
 
 export class ActionCogInfo extends CommandLineAction {

src/cli/action.tile.ts → packages/cli/src/action.tile.ts

@@ -1,10 +1,8 @@
+import { TiffVersion, toByteSizeString, Log } from '@coginfo/core';
 import { CommandLineAction, CommandLineStringParameter } from '@microsoft/ts-command-line';
-import { ActionUtil, CliResultMap, CLiResultMapLine } from './action.util';
-import { TiffVersion, TiffTag } from '../read/tif';
-import { toByteSizeString } from '../util/util.bytes';
 import chalk from 'chalk';
-import { Logger } from '../util/util.log';
-import { TileUtil } from '../util/util.tile';
+import { ActionUtil, CliResultMap } from './action.util';
+import { writeTile } from './util.tile';
 
 export class ActionTile extends CommandLineAction {
     private file: CommandLineStringParameter | null = null;
@@ -19,6 +17,7 @@ export class ActionTile extends CommandLineAction {
     }
 
     async onExecute(): Promise<void> {
+        const logger = Log.get().child({ action: 'tile' });
         // abstract
         const { tif } = await ActionUtil.getCogSource(this.file);
         if (this.xyz == null || this.xyz.value == null) {
@@ -27,15 +26,15 @@ export class ActionTile extends CommandLineAction {
 
         const isCogOptimized = tif.options.isCogOptimized;
         if (!isCogOptimized) {
-            Logger.warn('COG is not optimized, fetching specific tiles will be slow.');
+            logger.warn('COG is not optimized, fetching specific tiles will be slow.');
         }
 
         const [x, y, z] = this.xyz.value.split(',').map(c => parseInt(c));
         if (isNaN(x) || isNaN(y) || isNaN(z)) {
             throw new Error('Invalid XYZ, format: "X,Y,Z"');
         }
 
-        await TileUtil.write(tif, x, y, z, '.');
+        await writeTile(tif, x, y, z, '.', logger);
 
         const chunkIds = Object.keys(tif.source.chunks).filter(f => tif.source.chunk(parseInt(f, 10)).isReady());
 

src/cli/action.util.ts → packages/cli/src/action.util.ts

@@ -1,9 +1,8 @@
+import { CogSource, CogTif } from '@coginfo/core';
+import { CogSourceUrl } from '@coginfo/source-url';
+import { CogSourceFile } from '@coginfo/source-file';
 import { CommandLineStringParameter } from '@microsoft/ts-command-line';
 import chalk from 'chalk';
-import { CogSource } from '../cog.source';
-import { CogTif } from '../cog.tif';
-import { CogSourceFile } from '../source/cog.source.file';
-import { CogSourceUrl } from '../source/cog.source.web';
 
 export interface CLiResultMapLine {
     key: string;

src/cli/cli.cog.info.ts → packages/cli/src/cli.cog.info.ts

@@ -1,11 +1,11 @@
 import { CommandLineParser, CommandLineFlagParameter } from '@microsoft/ts-command-line';
 import { ActionDumpTile } from './action.dump.tile';
-import { Logger, LoggerConfig } from '../util/util.log';
 import chalk from 'chalk';
-import { Log } from 'bblog';
 import { ChalkLogStream } from './cli.log';
 import { ActionCogInfo } from './action.info';
 import { ActionTile } from './action.tile';
+import { Log } from 'bblog';
+import * as Core from '@coginfo/core';
 
 export class CogInfoCommandLine extends CommandLineParser {
     verbose = this.defineFlagParameter({
@@ -31,18 +31,24 @@ export class CogInfoCommandLine extends CommandLineParser {
     }
 
     protected onExecute(): Promise<void> {
-        // override
+        if (!chalk.supportsColor) {
+            return super.onExecute();
+        }
+        const logger = Log.createLogger({
+            name: 'coginfo',
+            hostname: '',
+            streams: [ChalkLogStream],
+        });
+        Core.Log.set(logger);
+
         if (this.verbose.value) {
-            LoggerConfig.level = Log.INFO;
+            ChalkLogStream.level = Log.INFO;
         } else if (this.extraVerbose.value) {
-            LoggerConfig.level = Log.TRACE;
+            ChalkLogStream.level = Log.TRACE;
         } else {
-            LoggerConfig.level = Log.ERROR;
+            ChalkLogStream.level = Log.ERROR;
         }
 
-        if (chalk.supportsColor) {
-            Logger['streams'] = [ChalkLogStream];
-        }
         return super.onExecute();
     }
     protected onDefineParameters(): void {}

src/cli/cli.log.ts → packages/cli/src/cli.log.ts

@@ -1,6 +1,6 @@
+import { LoggerConfig } from '@coginfo/core';
 import { Log, LogLevel, LogMessage } from 'bblog';
 import chalk from 'chalk';
-import { LoggerConfig } from '../util/util.log';
 
 function getLogStatus(level: LogLevel | number): string {
     if (level <= Log.TRACE) {
@@ -48,10 +48,10 @@ function isHex(s: any): s is string {
 export const ChalkLogStream = {
     level: Log.ERROR,
     setLevel(level: LogLevel): void {
-        LoggerConfig.level = level;
+        this.level = level;
     },
     write(msg: LogMessage): void {
-        if (msg.level < LoggerConfig.level) {
+        if (msg.level < this.level) {
             return;
         }
         const output = `[${msg.time.toISOString()}] ${getLogStatus(msg.level)} ${chalk.blue(msg.msg)}`;

src/cli/index.ts → packages/cli/src/index.ts

@@ -1,10 +1,6 @@
 import 'source-map-support/register';
 
 import { CogInfoCommandLine } from './cli.cog.info';
-import { CogSourceUrl } from '../source/cog.source.web';
-import * as fetch from 'node-fetch';
-
-CogSourceUrl.fetch = (a, b) => fetch(a, b);
 
 const cogInfo: CogInfoCommandLine = new CogInfoCommandLine();
 cogInfo.execute();

packages/cli/src/util.tile.ts

@@ -0,0 +1,14 @@
+import { CogLogger, CogTif, TileUtil } from '@coginfo/core';
+import { promises as fs } from 'fs';
+import * as path from 'path';
+
+export async function writeTile(tif: CogTif, x: number, y: number, zoom: number, outputPath: string, logger: CogLogger) {
+    const tile = await tif.getTileRaw(x, y, zoom);
+    if (tile == null) {
+        logger.error('Unable to write file, missing data..');
+        return;
+    }
+    const fileName = TileUtil.name(tile.mimeType, zoom, x, y);
+    fs.writeFile(path.join(outputPath, fileName), tile.bytes);
+    logger.debug({ fileName }, 'WriteFile');
+}