diff --git a/.eslintrc.json b/.eslintrc.json index 393989e..6744b3b 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -6,9 +6,7 @@ "project": "tsconfig.json", "sourceType": "module" }, - "plugins": [ - "@typescript-eslint" - ], + "plugins": ["@typescript-eslint"], "rules": { "@typescript-eslint/no-floating-promises": "warn", "semi": "warn" diff --git a/README.md b/README.md index 2ce5658..ae50487 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -jify -==== +# jify jify is an experimental library/tool for querying large (GBs) JSON files. It does this by first indexing the required fields. It can also be used as an @@ -8,75 +7,72 @@ append-only database. When a JSON file is indexed (eg. `data.json`) an index file is created in the same directory with a `.index.json` extension (eg. `data.index.json`). -Install -------- +## Install npm install jify -Usage ------ +## Usage ```javascript -const { Database, predicate: p } = require('jify'); +const { Database, predicate: p } = require("jify"); async function main() { - const db = new Database('books.json'); + const db = new Database("books.json"); // Create await db.create(); // Insert - Single await db.insert({ - title: 'Robinson Crusoe', + title: "Robinson Crusoe", year: 1719, - author: { name: 'Daniel Defoe' } + author: { name: "Daniel Defoe" }, }); // Insert - Batch await db.insert([ { - title: 'Great Expectations', + title: "Great Expectations", year: 1861, - author: { name: 'Charles Dickens' } + author: { name: "Charles Dickens" }, }, { - title: 'Oliver Twist', + title: "Oliver Twist", year: 1838, - author: { name: 'Charles Dickens' } + author: { name: "Charles Dickens" }, }, { - title: 'Pride and Prejudice', + title: "Pride and Prejudice", year: 1813, - author: { name: 'Jane Austen' } + author: { name: "Jane Austen" }, }, { - title: 'Nineteen Eighty-Four', + title: "Nineteen Eighty-Four", year: 1949, - author: { name: 'George Orwell' } - } + author: { name: "George Orwell" }, + }, ]); // Index - creates books.index.json file - await db.index('title', 'year', 'author.name'); + await db.index("title", "year", "author.name"); // Query - console.log('author.name = Charles Dickens, year > 1840'); - const query = { 'author.name': 'Charles Dickens', year: p`> ${1840}` }; - for await (const record of db.find(query)) - console.log(record); + console.log("author.name = Charles Dickens, year > 1840"); + const query = { "author.name": "Charles Dickens", year: p`> ${1840}` }; + for await (const record of db.find(query)) console.log(record); let records; // Range query - console.log('1800 <= year < 1900'); + console.log("1800 <= year < 1900"); records = await db.find({ year: p`>= ${1800} < ${1900}` }).toArray(); console.log(records); // Multiple queries - console.log('year < 1800 or year > 1900'); - records = await db.find( - { year: p`< ${1800}` }, { year: p`> ${1900}` } - ).toArray(); + console.log("year < 1800 or year > 1900"); + records = await db + .find({ year: p`< ${1800}` }, { year: p`> ${1900}` }) + .toArray(); console.log(records); } @@ -92,8 +88,7 @@ $ jify find --query "year>=1800<1900" books.json $ jify find --query "year<1800" --query "year>1900" books.json ``` -Implementation --------------- +## Implementation The index is implemented as a JSON array of skip list entries. The entries are encoded as strings and all numbers embedded in the string are encoded using @@ -102,8 +97,7 @@ its simplicity and to allow for using a single JSON file as an index. Better performance might be achieved by using a different data structure, a binary format, or multiple index files. -Performance ------------ +## Performance jify is reasonably fast. It can index about 1M records (~700 MB) per minute and supports parallel indexing of fields. Inserting (with indexes) has similar diff --git a/src/bin.ts b/src/bin.ts index 90b9c74..e514e93 100644 --- a/src/bin.ts +++ b/src/bin.ts @@ -1,115 +1,118 @@ #!/usr/bin/env node -import { ArgumentParser } from 'argparse'; -import { Database, predicate, Query } from './main'; +import { ArgumentParser } from "argparse"; +import { Database, predicate, Query } from "./main"; async function index(file: string, fields: string[]) { const db = new Database(file); - await db.index(...fields.map(f => { - const [name, type] = f.split(':'); - return { name, type }; - })); + await db.index( + ...fields.map((f) => { + const [name, type] = f.split(":"); + return { name, type }; + }) + ); } async function find(file: string, queries: string[]) { const db = new Database(file); - const iter = db.find(...queries.map(q => { - const query: Query = {}; - let name = ''; - let ops: string[] = []; - let values: any[] = []; - let start = 0; - let i = 0; - const getValue = () => { - const value = q.slice(start, i).trim(); - if (!value) - throw new Error('Empty value'); - start = i; - return value; - }; - const addValue = () => { - const value = getValue(); - values.push( - value == 'null' ? null : - value == 'false' ? false : - value == 'true' ? true : - Number.isFinite(Number(value)) ? Number(value) : value - ); - }; - const addPredicate = () => { - query[name] = ops[0] == '=' ? - values[0] : predicate(ops as any, ...values); - name = ''; - ops = []; - values = []; - }; - for (; i < q.length; i++) { - const c = q[i]; - if (['<', '>', '='].includes(c)) { - if (!name) - name = getValue(); - else if (ops.length) + const iter = db.find( + ...queries.map((q) => { + const query: Query = {}; + let name = ""; + let ops: string[] = []; + let values: any[] = []; + let start = 0; + let i = 0; + const getValue = () => { + const value = q.slice(start, i).trim(); + if (!value) throw new Error("Empty value"); + start = i; + return value; + }; + const addValue = () => { + const value = getValue(); + values.push( + value == "null" + ? null + : value == "false" + ? false + : value == "true" + ? true + : Number.isFinite(Number(value)) + ? Number(value) + : value + ); + }; + const addPredicate = () => { + query[name] = + ops[0] == "=" ? values[0] : predicate(ops as any, ...values); + name = ""; + ops = []; + values = []; + }; + for (; i < q.length; i++) { + const c = q[i]; + if (["<", ">", "="].includes(c)) { + if (!name) name = getValue(); + else if (ops.length) addValue(); + const op = q.slice(start, i + 1 + Number(q[i + 1] == "=")); + ops.push(op); + i += op.length - 1; + start = i + 1; + } else if (c == ",") { addValue(); - const op = q.slice(start, i + 1 + Number(q[i + 1] == '=')); - ops.push(op); - i += op.length - 1; - start = i + 1; - } else if (c == ',') { - addValue(); - start += 1; - addPredicate(); + start += 1; + addPredicate(); + } } - } - addValue(); - addPredicate(); - return query; - })); + addValue(); + addPredicate(); + return query; + }) + ); - for await (const record of iter) - console.log(JSON.stringify(record)); + for await (const record of iter) console.log(JSON.stringify(record)); } async function main() { const parser = new ArgumentParser({ add_help: true, - description: 'query JSON files' + description: "query JSON files", }); - parser.add_argument('--version', { - action: 'version', - version: require('../package.json').version, + parser.add_argument("--version", { + action: "version", + version: require("../package.json").version, }); - const subparsers = parser.add_subparsers({ dest: 'command' }); + const subparsers = parser.add_subparsers({ dest: "command" }); const commands = { - index: subparsers.add_parser( - 'index', { help: 'index JSON file' }, - ), - find: subparsers.add_parser( - 'find', { help: 'query JSON file' }, - ) + index: subparsers.add_parser("index", { help: "index JSON file" }), + find: subparsers.add_parser("find", { help: "query JSON file" }), }; // Index - commands.index.add_argument('file'); - commands.index.add_argument('--field', { action: 'append' }); + commands.index.add_argument("file"); + commands.index.add_argument("--field", { action: "append" }); // Find - commands.find.add_argument('file'); - commands.find.add_argument('--query', { action: 'append' }); + commands.find.add_argument("file"); + commands.find.add_argument("--query", { action: "append" }); const args = parser.parse_args(); switch (args.command) { - case 'index': + case "index": await index(args.file, args.field); break; - case 'find': + case "find": await find(args.file, args.query); break; } } -process.once('unhandledRejection', err => { throw err; }); +process.once("unhandledRejection", (err) => { + throw err; +}); main(); diff --git a/src/database.ts b/src/database.ts index 82c6d31..3db8879 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,23 +1,21 @@ -import * as path from 'path'; -import * as child_process from 'child_process'; +import * as path from "path"; +import * as child_process from "child_process"; -import Index, { IndexField, IndexFieldInfo, ObjectField } from './index'; -import JSONStore from './json-store'; -import { Query } from './query'; -import { logger } from './utils'; +import Index, { IndexField, IndexFieldInfo, ObjectField } from "./index"; +import JSONStore from "./json-store"; +import { Query } from "./query"; +import { logger } from "./utils"; class DatabaseIterableIterator implements AsyncIterableIterator { - constructor(protected iterator: AsyncIterableIterator<[number, T]>) { } + constructor(protected iterator: AsyncIterableIterator<[number, T]>) {} async next() { const res = (await this.iterator.next()) as IteratorResult; - if (!res.done) - res.value = res.value[1]; + if (!res.done) res.value = res.value[1]; return res as IteratorResult; } async toArray() { const array = []; - for await (const i of this) - array.push(i); + for await (const i of this) array.push(i); return array; } [Symbol.asyncIterator]() { @@ -28,7 +26,7 @@ class DatabaseIterableIterator implements AsyncIterableIterator { class Database { protected store: JSONStore; protected _index: Index; - protected logger = logger('database'); + protected logger = logger("database"); constructor(filename: string) { this.store = new JSONStore(filename); @@ -55,60 +53,54 @@ class Database { try { await this._index.drop(); } catch (e) { - if ((e as NodeJS.ErrnoException).code != 'ENOENT') - throw e; + if ((e as NodeJS.ErrnoException).code != "ENOENT") throw e; } } find(...queries: Query[]) { - return new DatabaseIterableIterator(async function* (this: Database) { - let positions: Set | undefined; + return new DatabaseIterableIterator( + async function* (this: Database) { + let positions: Set | undefined; - let indexAlreadyOpen = this._index.isOpen; + let indexAlreadyOpen = this._index.isOpen; - if (!indexAlreadyOpen) - await this._index.open(); + if (!indexAlreadyOpen) await this._index.open(); - for (const query of queries) { - const queryPositions = await this.findQuery(query); - if (!positions) { - positions = queryPositions; - continue; + for (const query of queries) { + const queryPositions = await this.findQuery(query); + if (!positions) { + positions = queryPositions; + continue; + } + for (const position of queryPositions) positions.add(position); } - for (const position of queryPositions) - positions.add(position); - } - if (!indexAlreadyOpen) - await this._index.close(); + if (!indexAlreadyOpen) await this._index.close(); - if (!positions) - return; + if (!positions) return; - const alreadyOpen = this.store.isOpen; - if (!alreadyOpen) - await this.store.open(); - try { - for (const position of positions) { - const res = await this.store.get(position); - yield [res.start, res.value] as [number, T]; + const alreadyOpen = this.store.isOpen; + if (!alreadyOpen) await this.store.open(); + try { + for (const position of positions) { + const res = await this.store.get(position); + yield [res.start, res.value] as [number, T]; + } + } finally { + if (!alreadyOpen) await this.store.close(); } - } finally { - if (!alreadyOpen) - await this.store.close(); - } - }.bind(this)()); + }.bind(this)() + ); } protected async findQuery(query: Query) { - this.logger.time('find'); + this.logger.time("find"); let positions: Set | undefined; for (const field in query) { - if (positions && !positions.size) - break; + if (positions && !positions.size) break; let predicate = query[field]; - if (typeof predicate != 'function') { + if (typeof predicate != "function") { let start = predicate; let converted = false; predicate = (value: any) => { @@ -118,7 +110,7 @@ class Database { } return { seek: value < start ? -1 : value > start ? 1 : 0, - match: value == start + match: value == start, }; }; } @@ -133,23 +125,20 @@ class Database { const intersection = new Set(); for (const position of fieldPositions) - if (positions.has(position)) - intersection.add(position); + if (positions.has(position)) intersection.add(position); positions = intersection; } positions = positions || new Set(); - this.logger.timeEnd('find'); + this.logger.timeEnd("find"); return positions; } async insert(objects: T | T[]) { - if (!Array.isArray(objects)) - objects = [objects]; + if (!Array.isArray(objects)) objects = [objects]; - if (!objects.length) - return; + if (!objects.length) return; let indexAlreadyOpen = this._index.isOpen; let indexExists = true; @@ -158,15 +147,13 @@ class Database { try { await this._index.open(); } catch (e) { - if ((e as NodeJS.ErrnoException).code != 'ENOENT') - throw e; + if ((e as NodeJS.ErrnoException).code != "ENOENT") throw e; indexExists = false; } } const alreadyOpen = this.store.isOpen; - if (!alreadyOpen) - await this.store.open(); + if (!alreadyOpen) await this.store.open(); await this.store.lock(Number.MAX_SAFE_INTEGER, { exclusive: true }); try { @@ -188,7 +175,7 @@ class Database { let joiner = first ? this.store.joiner.slice(1) : this.store.joiner; const offset = this.store.joiner.length; - this.logger.time('inserts'); + this.logger.time("inserts"); for (const object of objects) { const start = insertPosition + offset; @@ -206,35 +193,33 @@ class Database { if (indexExists) { for (const o of this.getObjectFields(object, start, indexFields)) { const objectFields = objectFieldsMap[o.name]; - if (objectFields) - objectFields.push(o); - else - objectFieldsMap[o.name] = [o]; + if (objectFields) objectFields.push(o); + else objectFieldsMap[o.name] = [o]; } } } - this.logger.timeEnd('inserts'); + this.logger.timeEnd("inserts"); pendingRaw.push(Buffer.from(this.store.trail)); await this.store.write(Buffer.concat(pendingRaw), startPosition); if (indexExists) { - this.logger.time('indexing'); + this.logger.time("indexing"); if (indexFields.length) { - await Promise.all(Object.values(objectFieldsMap).map( - objectFields => this._index.insert(objectFields)) + await Promise.all( + Object.values(objectFieldsMap).map((objectFields) => + this._index.insert(objectFields) + ) ); } - this.logger.timeEnd('indexing'); + this.logger.timeEnd("indexing"); for (const { name } of indexFields) await this._index.endTransaction(name); } } finally { await this.store.unlock(Number.MAX_SAFE_INTEGER); - if (!alreadyOpen) - await this.store.close(); - if (indexExists && !indexAlreadyOpen) - await this._index.close(); + if (!alreadyOpen) await this.store.close(); + if (indexExists && !indexAlreadyOpen) await this._index.close(); } } @@ -247,8 +232,7 @@ class Database { try { await this._index.open(); } catch (e) { - if ((e as NodeJS.ErrnoException).code != 'ENOENT') - throw e; + if ((e as NodeJS.ErrnoException).code != "ENOENT") throw e; indexExists = false; } } @@ -259,7 +243,7 @@ class Database { indexOutdated = await this.isIndexOutdated(); currentIndexFields = new Map( (await this._index.getFields()).map( - f => [f.name, f] as [string, IndexFieldInfo] + (f) => [f.name, f] as [string, IndexFieldInfo] ) ); for (const field of currentIndexFields.values()) { @@ -285,12 +269,13 @@ class Database { newIndexFields.set(field.name, field); const indexFields = Array.from( - (indexOutdated ? new Map([...currentIndexFields, ...newIndexFields]) - : newIndexFields).values() + (indexOutdated + ? new Map([...currentIndexFields, ...newIndexFields]) + : newIndexFields + ).values() ); - if (!indexFields.length) - return; + if (!indexFields.length) return; await this._index.addFields(indexFields); for (const { name } of indexFields) @@ -300,17 +285,19 @@ class Database { const batches: { [field: string]: ObjectField[] } = {}; for (const { name } of indexFields) { - subprocesses[name] = child_process.fork( - `${__dirname}/indexer`, [this._index.filename] - ); - subprocesses[name].once('error', err => { throw err; }); - subprocesses[name].once('exit', code => { + subprocesses[name] = child_process.fork(`${__dirname}/indexer`, [ + this._index.filename, + ]); + subprocesses[name].once("error", (err) => { + throw err; + }); + subprocesses[name].once("exit", (code) => { // Can be null if (code != 0) { delete subprocesses[name]; for (const subprocess of Object.values(subprocesses)) subprocess.kill(); - throw new Error('Error in subprocess'); + throw new Error("Error in subprocess"); } }); batches[name] = []; @@ -319,9 +306,8 @@ class Database { await Promise.all(Object.values(subprocesses).map(Database.waitForReady)); const alreadyOpen = this.store.isOpen; - if (!alreadyOpen) - await this.store.open(); - this.logger.time('read records'); + if (!alreadyOpen) await this.store.open(); + this.logger.time("read records"); for await (const [pos, object] of this.store.getAll()) { for (const o of this.getObjectFields(object, pos, indexFields)) { const batch = batches[o.name]; @@ -332,9 +318,8 @@ class Database { } } } - this.logger.timeEnd('read records'); - if (!alreadyOpen) - await this.store.close(); + this.logger.timeEnd("read records"); + if (!alreadyOpen) await this.store.close(); for (const [name, subprocess] of Object.entries(subprocesses)) { subprocess.send(batches[name]); @@ -343,18 +328,16 @@ class Database { await Promise.all(Object.values(subprocesses).map(Database.waitForClose)); - for (const { name } of indexFields) - await this._index.endTransaction(name); + for (const { name } of indexFields) await this._index.endTransaction(name); - if (!indexAlreadyOpen) - await this._index.close(); + if (!indexAlreadyOpen) await this._index.close(); } private static async waitForReady(subprocess: child_process.ChildProcess) { - const timeout = setInterval(() => { }, ~0 >>> 1); - await new Promise(resolve => { - subprocess.once('message', message => { - if (message == 'ready') { + const timeout = setInterval(() => {}, ~0 >>> 1); + await new Promise((resolve) => { + subprocess.once("message", (message) => { + if (message == "ready") { clearInterval(timeout); resolve(); } @@ -363,9 +346,9 @@ class Database { } private static async waitForClose(subprocess: child_process.ChildProcess) { - const timeout = setInterval(() => { }, ~0 >>> 1); - await new Promise(resolve => { - subprocess.once('close', () => { + const timeout = setInterval(() => {}, ~0 >>> 1); + await new Promise((resolve) => { + subprocess.once("close", () => { clearInterval(timeout); resolve(); }); @@ -375,19 +358,20 @@ class Database { protected async isIndexOutdated() { const [dbModified, indexModified] = await Promise.all([ this.store.lastModified(), - this._index.lastModified() + this._index.lastModified(), ]); return indexModified < dbModified; } protected getObjectFields( - object: Record, position: number, fields: IndexField[] + object: Record, + position: number, + fields: IndexField[] ) { const objectFields: ObjectField[] = []; for (const { name } of fields) { const value = Database.getField(object, name); - if (value == undefined) - continue; + if (value == undefined) continue; objectFields.push({ name, value, position }); } return objectFields; @@ -398,19 +382,16 @@ class Database { ): IndexField[] { const map = new Map(); for (const f of indexFields) { - if (typeof f == 'string') - map.set(f, { name: f }); - else - map.set(f.name, f); + if (typeof f == "string") map.set(f, { name: f }); + else map.set(f.name, f); } return Array.from(map.values()); } protected static getField(object: Record, field: string) { let value: any = object; - for (const f of field.split('.')) { - if (!value) - return; + for (const f of field.split(".")) { + if (!value) return; value = value[f]; } return value; diff --git a/src/declarations.d.ts b/src/declarations.d.ts index 2e91723..e259e74 100644 --- a/src/declarations.d.ts +++ b/src/declarations.d.ts @@ -1,4 +1,4 @@ -declare module 'z85' { - export function encode(data: Buffer): string - export function decode(string: string): Buffer +declare module "z85" { + export function encode(data: Buffer): string; + export function decode(string: string): Buffer; } diff --git a/src/file.ts b/src/file.ts index 90c7da4..ad868c8 100644 --- a/src/file.ts +++ b/src/file.ts @@ -1,16 +1,18 @@ -import { EventEmitter } from 'events'; -import { promises as fs } from 'fs'; -import { FileHandle } from 'fs/promises'; +import { EventEmitter } from "events"; +import { promises as fs } from "fs"; +import { FileHandle } from "fs/promises"; -import { lock, unlock } from 'os-lock'; +import { lock, unlock } from "os-lock"; -import { logger, read } from './utils'; +import { logger, read } from "./utils"; class File extends EventEmitter { protected file: FileHandle | null = null; - protected lockedPositions = - new Map(); - protected logger = logger('file'); + protected lockedPositions = new Map< + number, + { exclusive: boolean; count: number } + >(); + protected logger = logger("file"); protected reads = 0; protected writes = 0; @@ -25,8 +27,7 @@ class File extends EventEmitter { read(position: number, reverse = false, buffer?: Buffer) { ++this.reads; - if (!this.file) - throw new Error('Need to call open() before read()'); + if (!this.file) throw new Error("Need to call open() before read()"); return read(this.file, position, reverse, buffer); } @@ -35,7 +36,7 @@ class File extends EventEmitter { await this.file!.write(buffer, 0, buffer.length, position); } - async clear(position: number, length: number, char = ' ') { + async clear(position: number, length: number, char = " ") { const buffer = Buffer.alloc(length, char); await this.file!.write(buffer, 0, length, position); } @@ -56,19 +57,17 @@ class File extends EventEmitter { return await fs.stat(this.filename); } - async open(mode = 'r+') { - this.logger.log('opening', this.filename); - if (this.isOpen) - throw new Error('File already open'); + async open(mode = "r+") { + this.logger.log("opening", this.filename); + if (this.isOpen) throw new Error("File already open"); this.file = await fs.open(this.filename, mode); } async close() { - this.logger.log('closing', this.filename); - this.logger.log('reads', this.reads); - this.logger.log('writes', this.writes); - if (!this.file) - throw new Error('No open file to close'); + this.logger.log("closing", this.filename); + this.logger.log("reads", this.reads); + this.logger.log("writes", this.writes); + if (!this.file) throw new Error("No open file to close"); await this.file.close(); this.file = null; this.reads = 0; @@ -76,11 +75,14 @@ class File extends EventEmitter { } async lock(pos = 0, options = { exclusive: false }) { - const lockedPosition = this.lockedPositions.get(pos) || - { count: 0, exclusive: false }; + const lockedPosition = this.lockedPositions.get(pos) || { + count: 0, + exclusive: false, + }; - const canGetLock = options.exclusive ? - !lockedPosition.count : !lockedPosition.exclusive; + const canGetLock = options.exclusive + ? !lockedPosition.count + : !lockedPosition.exclusive; if (canGetLock) { ++lockedPosition.count; @@ -89,9 +91,9 @@ class File extends EventEmitter { await lock(this.file!.fd, pos, 1, options); return; } else { - const timeout = setInterval(() => { }, ~0 >>> 1); - await new Promise(resolve => { - this.once('unlock', async () => { + const timeout = setInterval(() => {}, ~0 >>> 1); + await new Promise((resolve) => { + this.once("unlock", async () => { clearInterval(timeout); await this.lock(pos, options); resolve(); @@ -107,7 +109,7 @@ class File extends EventEmitter { --lockedPosition.count; if (!lockedPosition.count) { this.lockedPositions.delete(pos); - this.emit('unlock', pos); + this.emit("unlock", pos); } } } diff --git a/src/index.ts b/src/index.ts index 69ed1e5..337653c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,15 +1,17 @@ -import JSONStore from './json-store'; -import { Predicate } from './query'; +import JSONStore from "./json-store"; +import { Predicate } from "./query"; import { logger, - z85DecodeAsUInt, z85EncodeAsUInt, - z85DecodeAsDouble, z85EncodeAsDouble -} from './utils'; + z85DecodeAsUInt, + z85EncodeAsUInt, + z85DecodeAsDouble, + z85EncodeAsDouble, +} from "./utils"; class Index { protected store: JSONStore; protected maxHeight = 32; - protected logger = logger('index'); + protected logger = logger("index"); constructor(public filename: string) { this.store = new JSONStore(filename, 0); @@ -54,8 +56,7 @@ class Index { async endTransaction(field: string) { const head = await this.lockHead(field, true); const value = JSON.parse(head.node.value as string); - if (!value.tx) - throw new IndexError(`Field "${field}" not in transaction`); + if (!value.tx) throw new IndexError(`Field "${field}" not in transaction`); value.tx = 0; head.node.value = JSON.stringify(value); await this.updateEntry(head); @@ -63,21 +64,20 @@ class Index { } async insert( - objectFields: ObjectField | ObjectField[], cache: IndexCache = new Map() + objectFields: ObjectField | ObjectField[], + cache: IndexCache = new Map() ) { - if (!Array.isArray(objectFields)) - objectFields = [objectFields]; + if (!Array.isArray(objectFields)) objectFields = [objectFields]; - if (!objectFields.length) - return; + if (!objectFields.length) return; const fieldName = objectFields[0].name; const head = await this.lockHead(fieldName, true, cache); const info: IndexFieldInfo = JSON.parse(head.node.value as string); - const isDateTime = info.type == 'date-time'; + const isDateTime = info.type == "date-time"; const transform = (o: ObjectField) => { - if (isDateTime && typeof o.value == 'string') + if (isDateTime && typeof o.value == "string") o.value = Date.parse(o.value); }; @@ -98,11 +98,13 @@ class Index { this.logger.time(`traversing index entries - ${fieldName}`); for (const objectField of objectFields) { const entries = await this.indexObjectField( - objectField, head!, cache, inserts + objectField, + head!, + cache, + inserts ); for (const entry of entries) - if (entry.position > 0) - updates.set(entry.position, entry); + if (entry.position > 0) updates.set(entry.position, entry); } this.logger.timeEnd(`traversing index entries - ${fieldName}`); @@ -122,8 +124,7 @@ class Index { entry.position = position; for (let i = 0; i < entry.node.levels.length; i++) { const pos = entry.node.levels[i]; - if (pos >= 0) - continue; + if (pos >= 0) continue; const next = inserts[-pos - 1]; entry.node.levels[i] = next.position; } @@ -147,15 +148,13 @@ class Index { prev = entry; } while ((dupe = inserts[i + 1]) && dupe.node.value == value) { - if (prev) - dupe.link = prev.position; + if (prev) dupe.link = prev.position; process(dupe); prev = dupe; ++i; } if (!entry.node.isDuplicate) { - if (prev) - entry.link = prev.position; + if (prev) entry.link = prev.position; process(entry); } } @@ -170,11 +169,9 @@ class Index { for (const entry of updates.values()) { for (let i = 0; i < entry.node.levels.length; i++) { const p = entry.node.levels[i]; - if (p < 0) - entry.node.levels[i] = inserts[-p - 1].position; + if (p < 0) entry.node.levels[i] = inserts[-p - 1].position; } - if (entry.link < 0) - entry.link = inserts[-entry.link - 1].position; + if (entry.link < 0) entry.link = inserts[-entry.link - 1].position; await this.updateEntry(entry); } this.logger.timeEnd(`updating index entries - ${fieldName}`); @@ -183,18 +180,19 @@ class Index { } protected async indexObjectField( - objectField: ObjectField, head: IndexEntry, - cache: IndexCache = new Map(), inserts: IndexEntry[] + objectField: ObjectField, + head: IndexEntry, + cache: IndexCache = new Map(), + inserts: IndexEntry[] ) { const { value, position } = objectField; - let height = head.node.levels.filter(p => p != 0).length; + let height = head.node.levels.filter((p) => p != 0).length; const maxLevel = Math.min(height, this.maxHeight - 1); let level = 0; - while (level < maxLevel && Math.round(Math.random())) - ++level; + while (level < maxLevel && Math.round(Math.random())) ++level; height = Math.max(height, level + 1); @@ -204,18 +202,18 @@ class Index { for (let i = height - 1; i >= 0; i--) { let nextNodePos: number; - while (nextNodePos = current.node.next(i)) { + while ((nextNodePos = current.node.next(i))) { // Check cache ourselves to avoid promise overhead - const next = nextNodePos < 0 ? inserts[-nextNodePos - 1] : - cache.get(nextNodePos) || await this.getEntry(nextNodePos, cache); - if (next.node.value! <= value) - current = next; - if (next.node.value! >= value) - break; + const next = + nextNodePos < 0 + ? inserts[-nextNodePos - 1] + : cache.get(nextNodePos) || + (await this.getEntry(nextNodePos, cache)); + if (next.node.value! <= value) current = next; + if (next.node.value! >= value) break; } - if (i > level) - continue; + if (i > level) continue; updates.push(current); } @@ -223,11 +221,12 @@ class Index { const prev = updates[updates.length - 1]; const isDuplicate = prev.node.value == value; - const entry = isDuplicate ? - new IndexEntry(position, new SkipListNode([], value)) : - new IndexEntry(position, - new SkipListNode(Array(level + 1).fill(0), value) - ); + const entry = isDuplicate + ? new IndexEntry(position, new SkipListNode([], value)) + : new IndexEntry( + position, + new SkipListNode(Array(level + 1).fill(0), value) + ); inserts.push(entry); entry.position = -inserts.length; // placeholder position @@ -264,8 +263,7 @@ class Index { } async addFields(fields: IndexField[]) { - if (!fields.length) - return; + if (!fields.length) return; const root = await this.lockRootEntry(true); let head = root; @@ -274,21 +272,19 @@ class Index { while (head.link) { head = await this.getEntry(head.link); --lock; - const name = - (JSON.parse(head.node.value as string) as IndexFieldInfo).name; - fields = fields.filter(f => f.name != name); + const name = (JSON.parse(head.node.value as string) as IndexFieldInfo) + .name; + fields = fields.filter((f) => f.name != name); } let position: number | undefined; for (const field of fields) { const prevHeadPos = head.position; const info: IndexFieldInfo = { name: field.name, tx: 0 }; - if (field.type == 'date-time') - info.type = field.type; + if (field.type == "date-time") info.type = field.type; head = new IndexEntry( - 0, new SkipListNode( - Array(this.maxHeight).fill(0), JSON.stringify(info) - ) + 0, + new SkipListNode(Array(this.maxHeight).fill(0), JSON.stringify(info)) ); position = await this.insertEntry(head, undefined, position); // Lock and update the previous head @@ -297,8 +293,7 @@ class Index { const prevHead = await this.getEntry(prevHeadPos); prevHead.link = head.position; await this.updateEntry(prevHead); - if (prevHeadPos != root.position) - await this.store.unlock(lock); + if (prevHeadPos != root.position) await this.store.unlock(lock); --lock; } @@ -306,12 +301,14 @@ class Index { } protected async lockHead( - field: string, exclusive = false, cache?: IndexCache + field: string, + exclusive = false, + cache?: IndexCache ) { let head = await this.lockRootEntry(); let lock = (head as any).lock; - let name = ''; + let name = ""; while (name != field && head.link) { head = await this.getEntry(head.link, cache); @@ -353,28 +350,30 @@ class Index { } protected async insertEntry( - entry: IndexEntry, cache?: IndexCache, position?: number + entry: IndexEntry, + cache?: IndexCache, + position?: number ) { const { start, length } = await this.store.append( - entry.encoded(), position + entry.encoded(), + position ); entry.position = start; - if (cache) - cache.set(entry.position, entry); + if (cache) cache.set(entry.position, entry); return start + length; } protected async getEntry( - position: number, cache?: IndexCache, update = false + position: number, + cache?: IndexCache, + update = false ) { const cached = cache && !update ? cache.get(position) : undefined; - if (cached) - return cached; + if (cached) return cached; const { start, value } = await this.store.get(position); const entry = new IndexEntry(value); entry.position = start; - if (cache) - cache.set(entry.position, entry); + if (cache) cache.set(entry.position, entry); return entry; } @@ -386,14 +385,13 @@ class Index { const cache: IndexCache = new Map(); const head = await this.lockHead(field, false, cache); - const height = head.node.levels.filter(p => p != 0).length; + const height = head.node.levels.filter((p) => p != 0).length; const info: IndexFieldInfo = JSON.parse(head.node.value as string); - if (info.tx) - throw new IndexError(`Field "${field}" in transaction`); + if (info.tx) throw new IndexError(`Field "${field}" in transaction`); - if (info.type == 'date-time') + if (info.type == "date-time") predicate.key = Date.parse as (s: SkipListValue) => number; let found = false; @@ -401,35 +399,31 @@ class Index { let current: IndexEntry | null = head; for (let i = height - 1; i >= 0; i--) { let nextNodePos: number; - while (nextNodePos = current.node.next(i)) { + while ((nextNodePos = current.node.next(i))) { const next = await this.getEntry(nextNodePos, cache); const { seek } = predicate(next.node.value); - if (seek <= 0) - current = next; - if (seek == 0) - found = true; - if (seek >= 0) - break; + if (seek <= 0) current = next; + if (seek == 0) found = true; + if (seek >= 0) break; } - if (found) - break; + if (found) break; } if (current == head) - current = current.node.next(0) ? - await this.getEntry(current.node.next(0), cache) : null; + current = current.node.next(0) + ? await this.getEntry(current.node.next(0), cache) + : null; const pointers = new Set(); while (current) { let entry = current; - current = current.node.next(0) ? - await this.getEntry(current.node.next(0), cache) : null; + current = current.node.next(0) + ? await this.getEntry(current.node.next(0), cache) + : null; const { seek, match } = predicate(entry.node.value); - if (seek <= 0 && !match) - continue; - if (!match) - break; + if (seek <= 0 && !match) continue; + if (!match) break; pointers.add(entry.pointer); while (entry.link) { const link = await this.getEntry(entry.link, cache); @@ -444,14 +438,14 @@ class Index { } } -export class IndexError extends Error { } -IndexError.prototype.name = 'IndexError'; +export class IndexError extends Error {} +IndexError.prototype.name = "IndexError"; const enum SkipListValueType { Null, Boolean, Number, - String + String, } type SkipListValue = null | boolean | number | string; @@ -461,10 +455,13 @@ class SkipListNode { public value: SkipListValue = null; get type() { - return typeof this.value == 'boolean' ? SkipListValueType.Boolean : - typeof this.value == 'number' ? SkipListValueType.Number : - typeof this.value == 'string' ? SkipListValueType.String : - SkipListValueType.Null; + return typeof this.value == "boolean" + ? SkipListValueType.Boolean + : typeof this.value == "number" + ? SkipListValueType.Number + : typeof this.value == "string" + ? SkipListValueType.String + : SkipListValueType.Null; } constructor(encodedNode: string); @@ -472,22 +469,21 @@ class SkipListNode { constructor(obj: any, value?: SkipListValue) { if (Array.isArray(obj)) { const levels: number[] = obj; - if (!levels) - throw new TypeError('levels is required'); - if (typeof value == 'number' && !Number.isFinite(value)) - throw new TypeError('Number value must be finite'); + if (!levels) throw new TypeError("levels is required"); + if (typeof value == "number" && !Number.isFinite(value)) + throw new TypeError("Number value must be finite"); this.value = value == null ? null : value; this.levels = levels; } else { const encodedNode = obj as string; - if (!encodedNode) - return; - const parts = encodedNode.split(';'); + if (!encodedNode) return; + const parts = encodedNode.split(";"); const [encodedLevels, encodedType] = parts.slice(0, 2); const type = z85DecodeAsUInt(encodedType, true); - this.value = SkipListNode.decodeValue(type, parts.slice(2).join(';')); - this.levels = encodedLevels ? - encodedLevels.split(',').map(l => z85DecodeAsUInt(l)) : []; + this.value = SkipListNode.decodeValue(type, parts.slice(2).join(";")); + this.levels = encodedLevels + ? encodedLevels.split(",").map((l) => z85DecodeAsUInt(l)) + : []; } } @@ -500,20 +496,18 @@ class SkipListNode { } encoded() { - if (this.isDuplicate) - return ''; + if (this.isDuplicate) return ""; - const encodedLevels = this.levels.map(l => z85EncodeAsUInt(l)).join(','); + const encodedLevels = this.levels.map((l) => z85EncodeAsUInt(l)).join(","); const encodedType = z85EncodeAsUInt(this.type, true); - let encodedValue = ''; - if (typeof this.value == 'boolean') + let encodedValue = ""; + if (typeof this.value == "boolean") encodedValue = z85EncodeAsUInt(Number(this.value), true); - else if (typeof this.value == 'number') + else if (typeof this.value == "number") encodedValue = z85EncodeAsDouble(this.value, true); - else if (typeof this.value == 'string') - encodedValue = this.value; + else if (typeof this.value == "string") encodedValue = this.value; return `${encodedLevels};${encodedType};${encodedValue}`; } @@ -542,18 +536,18 @@ class IndexEntry { constructor(encodedEntry: string); constructor(pointer: number, node: SkipListNode); constructor(obj: any, node?: SkipListNode) { - if (typeof obj == 'string') { + if (typeof obj == "string") { const encodedEntry = obj as string; - const encodedParts = encodedEntry.split(';'); + const encodedParts = encodedEntry.split(";"); const encodedPointer = encodedParts[0]; const encodedLink = encodedParts[1]; - const encodedNode = encodedParts.slice(2).join(';'); + const encodedNode = encodedParts.slice(2).join(";"); this.pointer = z85DecodeAsUInt(encodedPointer); this.link = z85DecodeAsUInt(encodedLink); this.node = new SkipListNode(encodedNode); } else { if (obj == null || !node) - throw new TypeError('pointer and node are required'); + throw new TypeError("pointer and node are required"); this.pointer = obj; this.node = node; } diff --git a/src/indexer.ts b/src/indexer.ts index a1d6f58..56028a3 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -1,6 +1,6 @@ -import lru from 'tiny-lru'; +import lru from "tiny-lru"; -import Index, { ObjectField, IndexCache } from './index'; +import Index, { ObjectField, IndexCache } from "./index"; const size = 1_000_000; let batch: ObjectField[] = []; @@ -15,8 +15,7 @@ async function main(filename: string) { function insertBatch() { const b = batch; batch = []; - if (b.length) - index.insert(b, cache); + if (b.length) index.insert(b, cache); } const handler = async (objectFields?: ObjectField[]) => { @@ -24,22 +23,23 @@ async function main(filename: string) { // Insert remaining fields insertBatch(); // Cleanup - process.off('message', handler); - process.once('beforeExit', async () => { + process.off("message", handler); + process.once("beforeExit", async () => { await index.close(); }); return; } batch.push(...objectFields); - if (batch.length >= size) - insertBatch(); + if (batch.length >= size) insertBatch(); }; - process.on('message', handler); - process.send!('ready'); + process.on("message", handler); + process.send!("ready"); } -process.once('unhandledRejection', err => { throw err; }); +process.once("unhandledRejection", (err) => { + throw err; +}); main(process.argv[2]); diff --git a/src/json-store.ts b/src/json-store.ts index 592b3ac..5b25ea0 100644 --- a/src/json-store.ts +++ b/src/json-store.ts @@ -1,11 +1,11 @@ -import File from './file'; -import Store from './store'; -import { Char, readJSON } from './utils'; +import File from "./file"; +import Store from "./store"; +import { Char, readJSON } from "./utils"; class JSONStore implements Store { protected file: File; - trail = '\n]\n'; + trail = "\n]\n"; constructor(filename: string, protected indent = 2) { this.file = new File(filename); @@ -24,11 +24,12 @@ class JSONStore implements Store { } async create(objects: T[] = []) { - const content = objects.length ? - `${this.joiner.slice(1)}${ - objects.map(this.stringify.bind(this)).join(this.joiner) - }` : ''; - await this.file.open('wx'); + const content = objects.length + ? `${this.joiner.slice(1)}${objects + .map(this.stringify.bind(this)) + .join(this.joiner)}` + : ""; + await this.file.open("wx"); await this.file.write(0, Buffer.from(`[${content}${this.trail}`)); await this.file.close(); } @@ -46,8 +47,9 @@ class JSONStore implements Store { } async get(position: number) { - const { value, start, length } = - (await readJSON(this.file.read(position)).next()).value!; + const { value, start, length } = ( + await readJSON(this.file.read(position)).next() + ).value!; return { value: value as T, start, length }; } @@ -77,8 +79,7 @@ class JSONStore implements Store { for await (const chars of this.file.read(-1, true)) { for (let i = 0; i < chars.length; i += 2) { const codePoint = chars[i + 1]; - if (codePoint == Char.Space || codePoint == Char.Newline) - continue; + if (codePoint == Char.Space || codePoint == Char.Newline) continue; if (!position) { if (codePoint != Char.RightBracket) { done = true; @@ -86,18 +87,15 @@ class JSONStore implements Store { } position = chars[i] - 1; } else { - if (codePoint == Char.LeftBracket) - first = true; + if (codePoint == Char.LeftBracket) first = true; done = true; break; } } - if (done) - break; + if (done) break; } - if (!position) - throw new Error('Invalid JSON file'); + if (!position) throw new Error("Invalid JSON file"); return { position, first }; } @@ -105,13 +103,11 @@ class JSONStore implements Store { async append(data: T, position?: number) { const dataString = this.stringify(data); - if (!dataString) - throw new Error('Cannot append empty string'); + if (!dataString) throw new Error("Cannot append empty string"); let first = false; - if (!position) - ({ position, first } = await this.getAppendPosition()); + if (!position) ({ position, first } = await this.getAppendPosition()); const joiner = first ? this.joiner.slice(1) : this.joiner; @@ -122,7 +118,7 @@ class JSONStore implements Store { return { start: position + joiner.length, length: buffer.length - joiner.length - this.trail.length, - raw: buffer + raw: buffer, }; } @@ -140,14 +136,14 @@ class JSONStore implements Store { } stringify(data: T) { - const str = this.indent ? JSON.stringify([data], null, this.indent).slice( - 2 + this.indent, -2 - ) : JSON.stringify(data); + const str = this.indent + ? JSON.stringify([data], null, this.indent).slice(2 + this.indent, -2) + : JSON.stringify(data); return str; } get joiner() { - return `,\n${' '.repeat(this.indent)}`; + return `,\n${" ".repeat(this.indent)}`; } } diff --git a/src/main.ts b/src/main.ts index 0a7539e..64e8159 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,5 +1,5 @@ -import Database from './database'; -import { predicate, Query } from './query'; +import Database from "./database"; +import { predicate, Query } from "./query"; export default Database; export { Database, predicate, Query }; diff --git a/src/query.ts b/src/query.ts index e7aa6c9..eb5c670 100644 --- a/src/query.ts +++ b/src/query.ts @@ -1,10 +1,11 @@ export function predicate( - strings: TemplateStringsArray, ...values: T[] + strings: TemplateStringsArray, + ...values: T[] ): Predicate { - let ops = strings.filter(o => o); + let ops = strings.filter((o) => o); if (!ops.length || ops.length > 2 || ops.length != values.length) - throw new Error('Invalid predicate'); + throw new Error("Invalid predicate"); let start: T | undefined; let end: T | undefined; @@ -15,27 +16,25 @@ export function predicate( ops = ops.map((o, i) => { const parts = o.trim().split(/\s+/); - if (parts.length > 2) - throw new Error('Invalid operator'); - if (parts.length > 1 && parts[0] != '&&') - throw new Error('Only `&&` is supported between conditions'); + if (parts.length > 2) throw new Error("Invalid operator"); + if (parts.length > 1 && parts[0] != "&&") + throw new Error("Only `&&` is supported between conditions"); const op = parts[parts.length - 1]; - if (!['<', '>', '<=', '>='].includes(op)) + if (!["<", ">", "<=", ">="].includes(op)) throw new Error(`Unsupported operator "${op}" in predicate`); - if (op.startsWith('>')) { + if (op.startsWith(">")) { if (start != undefined) - throw new Error('Redundant operator in predicate'); - excludeStart = op == '>'; + throw new Error("Redundant operator in predicate"); + excludeStart = op == ">"; start = values[i]; } - if (op.startsWith('<')) { - if (end != undefined) - throw new Error('Redundant operator in predicate'); - excludeEnd = op == '<'; + if (op.startsWith("<")) { + if (end != undefined) throw new Error("Redundant operator in predicate"); + excludeEnd = op == "<"; end = values[i]; } @@ -46,14 +45,12 @@ export function predicate( const test: Predicate = function (value: T) { if (test.key && !converted) { - if (start != undefined) - start = test.key(start); - if (end != undefined) - end = test.key(end); + if (start != undefined) start = test.key(start); + if (end != undefined) end = test.key(end); converted = true; } - const seek = start == undefined ? 1 : - (value < start ? -1 : value > start ? 1 : 0); + const seek = + start == undefined ? 1 : value < start ? -1 : value > start ? 1 : 0; const match = (start == undefined || (excludeStart ? value > start : value >= start)) && (end == undefined || (excludeEnd ? value < end : value <= end)); @@ -65,7 +62,7 @@ export function predicate( export interface Predicate { key?: (value: T) => any; - (value: T): { seek: number, match: boolean }; + (value: T): { seek: number; match: boolean }; } export interface Query { diff --git a/src/store.ts b/src/store.ts index b909e4c..fe88935 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,6 +1,8 @@ export default interface Store { get(position: number): Promise<{ - value: T, start: number, length: number + value: T; + start: number; + length: number; }>; create(): Promise; destroy(): Promise; diff --git a/src/utils.ts b/src/utils.ts index 91ecaa5..9df0e8b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,11 +1,14 @@ -import { Console } from 'console'; -import { FileHandle } from 'fs/promises'; -import { Writable } from 'stream'; +import { Console } from "console"; +import { FileHandle } from "fs/promises"; +import { Writable } from "stream"; -import * as z85 from 'z85'; +import * as z85 from "z85"; const enum JSONType { - Unknown, Array, Object, String + Unknown, + Array, + Object, + String, } export const enum Char { @@ -17,12 +20,10 @@ export const enum Char { LeftBrace = 123, RightBrace = 125, LeftBracket = 91, - RightBracket = 93 + RightBracket = 93, } -export async function* readJSON( - stream: AsyncIterator, parse = true -) { +export async function* readJSON(stream: AsyncIterator, parse = true) { let charCodes: number[] = []; let type = JSONType.Unknown; let start = -1; @@ -37,82 +38,70 @@ export async function* readJSON( const codePoint = res.value[i + 1]; if (start == -1) { if ( - codePoint == Char.Space || codePoint == Char.Newline || + codePoint == Char.Space || + codePoint == Char.Newline || codePoint == Char.Comma ) continue; else { start = res.value[i]; - if (codePoint == Char.LeftBrace) - type = JSONType.Object; - else if (codePoint == Char.Quote) - type = JSONType.String; - else if (codePoint == Char.LeftBracket) - type = JSONType.Array; + if (codePoint == Char.LeftBrace) type = JSONType.Object; + else if (codePoint == Char.Quote) type = JSONType.String; + else if (codePoint == Char.LeftBracket) type = JSONType.Array; } } if (parse) { - if (codePoint > 0xFFFF) { + if (codePoint > 0xffff) { const code = codePoint - 0x10000; - charCodes.push(0xD800 | (code >> 10), 0xDC00 | (code & 0x3FF)); - } else - charCodes.push(codePoint); + charCodes.push(0xd800 | (code >> 10), 0xdc00 | (code & 0x3ff)); + } else charCodes.push(codePoint); } const isStringQuote = codePoint == Char.Quote && !escaping; - if (escaping) - escaping = false; - else if (codePoint == Char.Backslash) - escaping = true; + if (escaping) escaping = false; + else if (codePoint == Char.Backslash) escaping = true; - if (isStringQuote) - inString = !inString; + if (isStringQuote) inString = !inString; - if (inString && type != JSONType.String) - continue; + if (inString && type != JSONType.String) continue; switch (type) { case JSONType.Array: - if (codePoint == Char.LeftBracket) - ++depth; - else if (codePoint == Char.RightBracket) - --depth; + if (codePoint == Char.LeftBracket) ++depth; + else if (codePoint == Char.RightBracket) --depth; break; case JSONType.Object: - if (codePoint == Char.LeftBrace) - ++depth; - else if (codePoint == Char.RightBrace) - --depth; + if (codePoint == Char.LeftBrace) ++depth; + else if (codePoint == Char.RightBrace) --depth; break; case JSONType.String: - if (isStringQuote) - depth = Number(!depth); + if (isStringQuote) depth = Number(!depth); break; default: if ( - codePoint == Char.Space || codePoint == Char.Newline || - codePoint == Char.Comma || codePoint == Char.RightBrace || + codePoint == Char.Space || + codePoint == Char.Newline || + codePoint == Char.Comma || + codePoint == Char.RightBrace || codePoint == Char.RightBracket ) { --depth; - if (parse) - charCodes.pop(); - } else if (!depth) - ++depth; + if (parse) charCodes.pop(); + } else if (!depth) ++depth; } if (!depth) { const length = res.value[i] - start + Number(type != JSONType.Unknown); const result: { - start: number, length: number, value?: any + start: number; + length: number; + value?: any; } = { start, length }; if (parse) - result.value = JSON.parse( - String.fromCharCode.apply(null, charCodes) - ); + result.value = JSON.parse(String.fromCharCode.apply(null, charCodes)); yield result; @@ -130,42 +119,35 @@ export function findJSONfield(text: string, field: string) { let prevChar: string | undefined; let inString = false; let depth = 0; - let str = ''; + let str = ""; // TODO - review this code especially depth for (let i = 0; i < text.length; i++) { const char = text[i]; - if (char == ' ' || char == '\n') - continue; + if (char == " " || char == "\n") continue; - const isStringQuote = char == '"' && prevChar != '\\'; + const isStringQuote = char == '"' && prevChar != "\\"; prevChar = char; if (isStringQuote) { inString = !inString; - if (inString) - str = ''; + if (inString) str = ""; } if (inString) { - if (!isStringQuote) - str += char; + if (!isStringQuote) str += char; continue; } - if (char == '}') - --depth; - else if (char == '{') - ++depth; + if (char == "}") --depth; + else if (char == "{") ++depth; - if (depth > 1) - continue; + if (depth > 1) continue; // TODO - ensure only one object is searched - if (char == ':' && str == field) - return i + 1; + if (char == ":" && str == field) return i + 1; } return; @@ -174,38 +156,46 @@ export function findJSONfield(text: string, field: string) { function utf8codepoint(buf: Array | Buffer, i = 0) { const c = buf[i]; switch (c >> 4) { - case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7: + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: // 0xxxxxxx return c; - case 12: case 13: + case 12: + case 13: // 110xxxxx 10xxxxxx - return ((c & 0x1F) << 6) | (buf[i + 1] & 0x3F); + return ((c & 0x1f) << 6) | (buf[i + 1] & 0x3f); case 14: // 1110xxxx 10xxxxxx 10xxxxxx return ( - ((c & 0x0F) << 12) | - ((buf[i + 1] & 0x3F) << 6) | - (buf[i + 2] & 0x3F) + ((c & 0x0f) << 12) | ((buf[i + 1] & 0x3f) << 6) | (buf[i + 2] & 0x3f) ); case 15: if (!(c & 0x8)) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx return ( ((c & 0x07) << 18) | - ((buf[i + 1] & 0x3F) << 12) | - ((buf[i + 2] & 0x3F) << 6) | - (buf[i + 3] & 0x3F) + ((buf[i + 1] & 0x3f) << 12) | + ((buf[i + 2] & 0x3f) << 6) | + (buf[i + 3] & 0x3f) ); default: - throw new Error('Invalid UTF-8'); + throw new Error("Invalid UTF-8"); } } export async function* read( - file: FileHandle, position = 0, reverse = false, buffer?: Buffer + file: FileHandle, + position = 0, + reverse = false, + buffer?: Buffer ) { - if (position < 0) - position += (await file.stat()).size; + if (position < 0) position += (await file.stat()).size; buffer = buffer || Buffer.alloc(1 << 12); const size = buffer.length; @@ -229,17 +219,15 @@ export async function* read( ({ bytesRead } = await file.read(buffer, 0, length, pos)); - for (let i = 0; i < bytesRead;) { - let index = reverse ? (bytesRead - i - 1) : i; + for (let i = 0; i < bytesRead; ) { + let index = reverse ? bytesRead - i - 1 : i; let count = 0; if (reverse) { count = 1; - while ((buffer[index] & 0xc0) == 0x80) - --index, ++count; + while ((buffer[index] & 0xc0) == 0x80) --index, ++count; } else { - for (let b = 7; (buffer[index] >> b) & 1; b--) - ++count; + for (let b = 7; (buffer[index] >> b) & 1; b--) ++count; count = count || 1; } @@ -258,15 +246,12 @@ export async function* read( continuing = null; try { - if (charCount == chars.length) - yield chars; - else - yield chars.slice(0, charCount); + if (charCount == chars.length) yield chars; + else yield chars.slice(0, charCount); // If we reached here, it means we yielded successfully and can update // the state for the next iteration. continuing = false; - if (bytesRead < size || (reverse && !pos)) - break; + if (bytesRead < size || (reverse && !pos)) break; pos += reverse ? -length : length; } finally { if (continuing == null) { @@ -285,13 +270,13 @@ const uint32Buffer = Buffer.alloc(4); export function z85EncodeAsUInt32(number: number, compact = false) { uint32Buffer.writeUInt32BE(number, 0); const encoded = z85.encode(uint32Buffer); - return compact ? encoded.replace(/^0+/, '') : encoded; + return compact ? encoded.replace(/^0+/, "") : encoded; } export function z85DecodeAsUInt32(string: string, compact = false) { if (string.length > 5) - throw new Error('Cannot decode string longer than 5 characters'); - const decoded = z85.decode(compact ? string.padStart(5, '0') : string); + throw new Error("Cannot decode string longer than 5 characters"); + const decoded = z85.decode(compact ? string.padStart(5, "0") : string); return decoded.readUInt32BE(0); } @@ -299,15 +284,15 @@ const uintBuffer = Buffer.alloc(8); export function z85EncodeAsUInt(number: number, compact = false) { uintBuffer.writeUIntBE(number, 2, 6); const encoded = z85.encode(uintBuffer); - return compact ? encoded.replace(/^0+/, '') : encoded.slice(2); + return compact ? encoded.replace(/^0+/, "") : encoded.slice(2); } export function z85DecodeAsUInt(string: string, compact = false) { if (!compact && string.length != 8) - throw new Error('String must be 8 characters long'); + throw new Error("String must be 8 characters long"); else if (string.length > 8) - throw new Error('Cannot decode string longer than 8 characters'); - const decoded = z85.decode(string.padStart(10, '0')); + throw new Error("Cannot decode string longer than 8 characters"); + const decoded = z85.decode(string.padStart(10, "0")); return decoded.readUIntBE(2, 6); } @@ -315,37 +300,38 @@ const doubleBuffer = Buffer.alloc(8); export function z85EncodeAsDouble(number: number, compact = false) { doubleBuffer.writeDoubleBE(number, 0); const encoded = z85.encode(doubleBuffer); - return compact ? encoded.replace(/0+$/, '') : encoded; + return compact ? encoded.replace(/0+$/, "") : encoded; } export function z85DecodeAsDouble(string: string, compact = false) { if (string.length > 10) - throw new Error('Cannot decode string longer than 5 characters'); - const decoded = z85.decode(compact ? string.padEnd(10, '0') : string); + throw new Error("Cannot decode string longer than 5 characters"); + const decoded = z85.decode(compact ? string.padEnd(10, "0") : string); return decoded.readDoubleBE(0); } class Out extends Writable { - constructor(protected label: string) { super(); } + constructor(protected label: string) { + super(); + } _write(chunk: string, _: string, callback: (error?: Error | null) => void) { process.stdout.write(`${this.label}: ${chunk}`); - if (callback) - callback(null); + if (callback) callback(null); } } class Err extends Writable { - constructor(protected label: string) { super(); } + constructor(protected label: string) { + super(); + } _write(chunk: string, _: string, callback: (error?: Error | null) => void) { process.stderr.write(`${this.label}: ${chunk}`); - if (callback) - callback(null); + if (callback) callback(null); } } -const dummyConsole = new Console(new Writable); +const dummyConsole = new Console(new Writable()); export function logger(name: string, log?: boolean) { - if (log == null) - log = Boolean(process.env.DEBUG); + if (log == null) log = Boolean(process.env.DEBUG); return log ? new Console(new Out(name), new Err(name)) : dummyConsole; } diff --git a/test/index.ts b/test/index.ts index 756714c..2b84eef 100644 --- a/test/index.ts +++ b/test/index.ts @@ -1,13 +1,13 @@ -import * as assert from 'assert'; -import { promises as fs } from 'fs'; -import * as path from 'path'; +import * as assert from "assert"; +import { promises as fs } from "fs"; +import * as path from "path"; -import Database, { predicate as p } from '..'; -import { Record } from '../lib/database'; -import { IndexField } from '../lib'; -import * as utils from '../lib/utils'; +import Database, { predicate as p } from ".."; +import { Record } from "../lib/database"; +import { IndexField } from "../lib"; +import * as utils from "../lib/utils"; -const logger = utils.logger('test'); +const logger = utils.logger("test"); /* Helpers */ @@ -17,8 +17,7 @@ function getFilename(filename: string) { function getField(object: Record, field: string) { let value: any = object; - for (const f of field.split('.')) - value = value[f]; + for (const f of field.split(".")) value = value[f]; return value; } @@ -38,15 +37,13 @@ function fillArray(n: number, value: (i: number) => any) { } function compareObjects(a: object, b: object) { - if (a == b) - return 0; - const keys = - Array.from(new Set(Object.keys(a).concat(Object.keys(b)))).sort(); + if (a == b) return 0; + const keys = Array.from( + new Set(Object.keys(a).concat(Object.keys(b))) + ).sort(); for (const key of keys) - if ((a as any)[key] < (b as any)[key]) - return -1; - else if ((a as any)[key] > (b as any)[key]) - return 1; + if ((a as any)[key] < (b as any)[key]) return -1; + else if ((a as any)[key] > (b as any)[key]) return 1; return 0; } @@ -55,14 +52,16 @@ function sortObjectArray(arr: T[]) { } async function testInserts( - db: Database, fields: (string | IndexField)[], - n = 1000, size = 100_000, value: (i: number) => Record + db: Database, + fields: (string | IndexField)[], + n = 1000, + size = 100_000, + value: (i: number) => Record ) { try { await db.drop(); } catch (e) { - if ((e as NodeJS.ErrnoException).code != 'ENOENT') - throw e; + if ((e as NodeJS.ErrnoException).code != "ENOENT") throw e; } await db.create(fields); @@ -73,17 +72,18 @@ async function testInserts( logger.time(`insert ${n} objects`); for (let i = 0; i < n; i++) { objects[i % size] = value(i); - if ((i + 1) % size == 0) - await db.insert(objects); + if ((i + 1) % size == 0) await db.insert(objects); } - if (n % size != 0) - await db.insert(objects.slice(0, n % size)); + if (n % size != 0) await db.insert(objects.slice(0, n % size)); logger.timeEnd(`insert ${n} objects`); } async function testFind( - db: Database, field: string, n = 20, - value: (i: number) => any, count: (i: number) => number + db: Database, + field: string, + n = 20, + value: (i: number) => any, + count: (i: number) => number ) { for (let i = 0; i < n; i++) { const val = value(i); @@ -92,8 +92,7 @@ async function testFind( logger.timeEnd(`find ${field}=${val}`); logger.log(`${objects.length} results`); assert.equal(objects.length, count(val)); - for (const obj of objects) - assert.equal(getField(obj, field), val); + for (const obj of objects) assert.equal(getField(obj, field), val); } } @@ -102,38 +101,48 @@ async function testFind( async function testInsertAndFind(n = 10_000, size = 100_000, count = 20) { count = Math.min(n, count); - const fields = [ - 'id', 'person.age', { name: 'created', type: 'date-time' } - ]; + const fields = ["id", "person.age", { name: "created", type: "date-time" }]; const db = new Database(getFilename(`data-insert-${n}.json`)); - const { array: ids, count: idsCount } = - fillArray(n, _ => - Math.random().toString(36) + '😋'.repeat(Math.random() * 10) - ); - const { array: ages, count: agesCount } = - fillArray(n, _ => Math.round(Math.random() * 100)); - const { array: dates, count: datesCount } = - fillArray(n, _ => - new Date(+(new Date()) - Math.floor(Math.random() * 1e10)).toISOString() - ); + const { array: ids, count: idsCount } = fillArray( + n, + (_) => Math.random().toString(36) + "😋".repeat(Math.random() * 10) + ); + const { array: ages, count: agesCount } = fillArray(n, (_) => + Math.round(Math.random() * 100) + ); + const { array: dates, count: datesCount } = fillArray(n, (_) => + new Date(+new Date() - Math.floor(Math.random() * 1e10)).toISOString() + ); const checkFind = async () => { await testFind( - db, 'id', count, _ => ids[getRandomInt(0, n - 1)], val => idsCount[val] + db, + "id", + count, + (_) => ids[getRandomInt(0, n - 1)], + (val) => idsCount[val] ); await testFind( - db, 'person.age', count, _ => ages[getRandomInt(0, n - 1)], - val => agesCount[val] + db, + "person.age", + count, + (_) => ages[getRandomInt(0, n - 1)], + (val) => agesCount[val] ); await testFind( - db, 'created', count, _ => dates[getRandomInt(0, n - 1)], - val => datesCount[val] + db, + "created", + count, + (_) => dates[getRandomInt(0, n - 1)], + (val) => datesCount[val] ); }; - await testInserts(db, fields, n, size, i => ({ - id: ids[i], person: { age: ages[i] }, created: dates[i] + await testInserts(db, fields, n, size, (i) => ({ + id: ids[i], + person: { age: ages[i] }, + created: dates[i], })); await checkFind(); await (db as any)._index.drop(); @@ -142,72 +151,80 @@ async function testInsertAndFind(n = 10_000, size = 100_000, count = 20) { } async function testQueries() { - const db = new Database(getFilename('people.json')); + const db = new Database(getFilename("people.json")); try { await db.drop(); } catch (e) { - if ((e as NodeJS.ErrnoException).code != 'ENOENT') - throw e; + if ((e as NodeJS.ErrnoException).code != "ENOENT") throw e; } await db.create(); - await db.insert({ name: 'John', age: 42 }); - await db.insert({ name: 'John', age: 43 }); - await db.insert({ name: 'John', age: 17 }); - await db.insert({ name: 'John', age: 18 }); - await db.insert({ name: 'John', age: 20 }); - await db.insert({ name: 'John', age: 35 }); - await db.insert({ name: 'John', age: 50 }); + await db.insert({ name: "John", age: 42 }); + await db.insert({ name: "John", age: 43 }); + await db.insert({ name: "John", age: 17 }); + await db.insert({ name: "John", age: 18 }); + await db.insert({ name: "John", age: 20 }); + await db.insert({ name: "John", age: 35 }); + await db.insert({ name: "John", age: 50 }); - await db.index('name', 'age'); + await db.index("name", "age"); - let results = await db.find({ name: 'John', age: 42 }).toArray(); + let results = await db.find({ name: "John", age: 42 }).toArray(); assert.deepEqual( - sortObjectArray(results), sortObjectArray([{ name: 'John', age: 42 }]) + sortObjectArray(results), + sortObjectArray([{ name: "John", age: 42 }]) ); // age < 50 results = await db.find({ age: p`< ${50}` }).toArray(); - assert.deepEqual(sortObjectArray(results), sortObjectArray([ - { name: 'John', age: 42 }, - { name: 'John', age: 43 }, - { name: 'John', age: 17 }, - { name: 'John', age: 18 }, - { name: 'John', age: 20 }, - { name: 'John', age: 35 } - ])); + assert.deepEqual( + sortObjectArray(results), + sortObjectArray([ + { name: "John", age: 42 }, + { name: "John", age: 43 }, + { name: "John", age: 17 }, + { name: "John", age: 18 }, + { name: "John", age: 20 }, + { name: "John", age: 35 }, + ]) + ); // 18 <= age < 35 results = await db.find({ age: p`>= ${18} < ${35}` }).toArray(); - assert.deepEqual(sortObjectArray(results), sortObjectArray([ - { name: 'John', age: 18 }, - { name: 'John', age: 20 } - ])); + assert.deepEqual( + sortObjectArray(results), + sortObjectArray([ + { name: "John", age: 18 }, + { name: "John", age: 20 }, + ]) + ); // age < 18 or age > 35 results = await db.find({ age: p`< ${18}` }, { age: p`> ${35}` }).toArray(); - assert.deepEqual(sortObjectArray(results), sortObjectArray([ - { name: 'John', age: 42 }, - { name: 'John', age: 43 }, - { name: 'John', age: 17 }, - { name: 'John', age: 50 } - ])); + assert.deepEqual( + sortObjectArray(results), + sortObjectArray([ + { name: "John", age: 42 }, + { name: "John", age: 43 }, + { name: "John", age: 17 }, + { name: "John", age: 50 }, + ]) + ); } async function testInvalid() { - const filename = getFilename('invalid.json'); + const filename = getFilename("invalid.json"); try { await fs.unlink(filename); } catch (e) { - if ((e as NodeJS.ErrnoException).code != 'ENOENT') - throw e; + if ((e as NodeJS.ErrnoException).code != "ENOENT") throw e; } - const file = await fs.open(filename, 'wx'); + const file = await fs.open(filename, "wx"); await file.close(); const db = new Database(filename); await assert.rejects(db.insert({})); - await fs.writeFile(filename, 'invalid'); + await fs.writeFile(filename, "invalid"); await assert.rejects(db.insert({})); } @@ -216,9 +233,9 @@ async function main() { const n = Number(args.shift()) || undefined; const size = Number(args.shift()) || undefined; const count = Number(args.shift()) || undefined; - const debug = process.env.DEBUG || ''; + const debug = process.env.DEBUG || ""; - process.env.DEBUG = ''; + process.env.DEBUG = ""; await testInsertAndFind(1); await testInsertAndFind(200, 20); await testQueries(); @@ -228,6 +245,8 @@ async function main() { await testInsertAndFind(n, size, count); } -process.once('unhandledRejection', err => { throw err; }); +process.once("unhandledRejection", (err) => { + throw err; +}); main(); diff --git a/tsconfig.json b/tsconfig.json index 9f61d6c..4c0b4e1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,5 @@ { - "include": [ - "src" - ], + "include": ["src"], "compilerOptions": { "esModuleInterop": true, "module": "commonjs",