From 6ad08a456a2a460c58f2b8ad8815c03600ef9c9c Mon Sep 17 00:00:00 2001 From: Subramanian Elavathur Date: Fri, 5 Nov 2021 22:25:54 +0530 Subject: [PATCH] Code reorganization --- package-lock.json | 4 +- package.json | 2 +- src/GlitchBitemporalPartition.ts | 19 +++ src/GlitchDB.ts | 15 +- src/GlitchPartition.ts | 224 ++++++------------------------ src/GlitchUnitemporalPartition.ts | 179 ++++++++++++++++++++++++ src/index.ts | 7 +- test/versioned.test.ts | 4 +- 8 files changed, 253 insertions(+), 201 deletions(-) create mode 100644 src/GlitchBitemporalPartition.ts create mode 100644 src/GlitchUnitemporalPartition.ts diff --git a/package-lock.json b/package-lock.json index 094c9b4..69995d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { "name": "glitch-db", - "version": "0.12.1", + "version": "0.13.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "version": "0.12.1", + "version": "0.13.0", "license": "MIT", "dependencies": { "lodash.get": "^4.4.2", diff --git a/package.json b/package.json index e8e12b9..6a70d41 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "glitch-db", - "version": "0.12.1", + "version": "0.13.0", "description": "A simple, file based key-value database", "main": "./lib/src/index.js", "exports": "./lib/src/index.js", diff --git a/src/GlitchBitemporalPartition.ts b/src/GlitchBitemporalPartition.ts new file mode 100644 index 0000000..1666d1c --- /dev/null +++ b/src/GlitchBitemporalPartition.ts @@ -0,0 +1,19 @@ +import { UnitemporalVersion } from "./GlitchUnitemporalPartition"; + +interface BitemporalVersion extends UnitemporalVersion { + validFrom: number; + validTo: number; +} + +interface BitemporallyVersionedData extends BitemporalVersion { + data: Type; +} + +interface BitemporallyVersioned { + rangeMap: { + [validFrom: number]: number; // validFrom to version number map + }; + data: { + [key: number]: BitemporallyVersionedData; + }; +} diff --git a/src/GlitchDB.ts b/src/GlitchDB.ts index e157fe7..8375dda 100644 --- a/src/GlitchDB.ts +++ b/src/GlitchDB.ts @@ -1,9 +1,9 @@ import tar = require("tar"); import { DEFAULT_CACHE_SIZE } from "./constants"; -import GlitchPartitionImpl, { - GlitchPartition, - GlitchUnitemporallyVersionedPartition, -} from "./GlitchPartition"; +import GlitchPartitionImpl, { GlitchPartition } from "./GlitchPartition"; +import GlitchUniTemporalPartitionImpl, { + GlitchUnitemporalPartition, +} from "./GlitchUnitemporalPartition"; export default class GlitchDB { #baseDir: string; @@ -69,19 +69,18 @@ export default class GlitchDB { name: string, indices?: string[], cacheSize?: number - ): GlitchUnitemporallyVersionedPartition { + ): GlitchUnitemporalPartition { const cacheSizeWithDefault = cacheSize ?? this.#defaultCacheSize; this.#partitions[name] = { name, cache: cacheSizeWithDefault, versioned: true, }; - return new GlitchPartitionImpl( + return new GlitchUniTemporalPartitionImpl( this, `${this.#baseDir}/${name}`, cacheSizeWithDefault, - indices, - true + indices ); } } diff --git a/src/GlitchPartition.ts b/src/GlitchPartition.ts index ad95175..f2e3d63 100644 --- a/src/GlitchPartition.ts +++ b/src/GlitchPartition.ts @@ -1,7 +1,7 @@ import LRUCache = require("lru-cache"); import lget = require("lodash.get"); import fs = require("fs/promises"); -import { INDEX_FILE, INFINITY_TIME } from "./constants"; +import { INDEX_FILE } from "./constants"; import GlitchDB from "./GlitchDB"; const getIndexFilePath = (dir: string) => `${dir}/${INDEX_FILE}`; @@ -13,54 +13,12 @@ interface Joiner { joinName: string; } -interface UnitemporalVersion { - metadata?: { - [key: string]: string; - }; - version: number; - createdAt: number; - deletedAt: number; -} - -interface UnitemporallyVersionedData extends UnitemporalVersion { - data: Type; -} - -interface UnitemporallyVersioned { - latestVersion: number; - data: { - [key: number]: UnitemporallyVersionedData; - }; -} - -interface BitemporalVersion extends UnitemporalVersion { - validFrom: number; - validTo: number; -} - -interface BitemporallyVersionedData extends BitemporalVersion { - data: Type; -} - -interface BitemporallyVersioned { - rangeMap: { - [validFrom: number]: number; // validFrom to version number map - }; - data: { - [key: number]: BitemporallyVersionedData; - }; -} - export interface GlitchPartition { exists: (key: string, version?: number) => Promise; - get: (key: string, version?: number) => Promise; + get: (key: string) => Promise; keys: () => Promise; data: () => Promise<{ [key: string]: Type }>; - set: ( - key: string, - value: Type, - metadata?: { [key: string]: string } - ) => Promise; + set: (key: string, value: Type) => Promise; delete: (key: string) => Promise; createJoin: ( db: string, @@ -72,17 +30,8 @@ export interface GlitchPartition { getWithJoins: (key: string) => Promise; } -export interface GlitchUnitemporallyVersionedPartition - extends GlitchPartition { - getVersion: ( - key: string, - version?: number - ) => Promise>; - getAllVersions: (key: string) => Promise[]>; -} - export default class GlitchPartitionImpl - implements GlitchUnitemporallyVersionedPartition + implements GlitchPartition { #localDir: string; #initComplete: boolean; @@ -93,26 +42,23 @@ export default class GlitchPartitionImpl #joins: { [joinName: string]: Joiner; }; - #cache: LRUCache; // cache always maintains the latest data + protected cache: LRUCache; // cache always maintains the latest data #master: GlitchDB; - #versioned: boolean; constructor( master: GlitchDB, localDir: string, cacheSize?: number, - indices?: string[], - versioned?: boolean + indices?: string[] ) { this.#master = master; this.#localDir = localDir; this.#joins = {}; if (cacheSize > 0) { - this.#cache = new LRUCache(cacheSize); + this.cache = new LRUCache(cacheSize); } this.#indices = indices; this.#indexMap = {}; - this.#versioned = versioned; } async #loadIndex(): Promise { @@ -150,7 +96,7 @@ export default class GlitchPartitionImpl } } - async #init() { + protected async init() { if (this.#initComplete) { return; // no need to re-init } @@ -178,28 +124,28 @@ export default class GlitchPartitionImpl } async keys(): Promise { - await this.#init(); + await this.init(); const files = await fs.readdir(this.#localDir); return files .filter((each) => each !== INDEX_FILE) .map(this.#getKeyFromFile); } - #getKeyPath(key: string): string { + protected getKeyPath(key: string): string { return `${this.#localDir}/${key}.json`; } - #resolveKey(key: string) { + protected resolveKey(key: string) { return this.#indexMap[key] ?? key; } async exists(key: string): Promise { - await this.#init(); - const resolvedKey = this.#resolveKey(key); - if (this.#cache?.has(resolvedKey)) { + await this.init(); + const resolvedKey = this.resolveKey(key); + if (this.cache?.has(resolvedKey)) { return Promise.resolve(true); } - const keyPath = this.#getKeyPath(resolvedKey); + const keyPath = this.getKeyPath(resolvedKey); try { const stat = await fs.stat(keyPath); if (stat && stat.isFile()) { @@ -215,39 +161,21 @@ export default class GlitchPartitionImpl } } - #getVersionFromFile( - file: UnitemporallyVersioned, - version?: number - ): UnitemporallyVersionedData { - return file?.data[version ?? file?.latestVersion]; - } - - async get(key: string, version?: number): Promise { - await this.#init(); - const resolvedKey = this.#resolveKey(key); - if (!version) { - const cachedData = this.#cache?.get(resolvedKey); - if (cachedData) { - return Promise.resolve(cachedData); - } + async get(key: string): Promise { + await this.init(); + const resolvedKey = this.resolveKey(key); + const cachedData = this.cache?.get(resolvedKey); + if (cachedData) { + return Promise.resolve(cachedData); } - const keyPath = this.#getKeyPath(resolvedKey); + const keyPath = this.getKeyPath(resolvedKey); try { const fileData = await fs.readFile(keyPath, { encoding: "utf8", }); const parsed = JSON.parse(fileData); - if (this.#versioned) { - const data = parsed as UnitemporallyVersioned; - const result = this.#getVersionFromFile(data, version); - if (!version) { - this.#cache?.set(resolvedKey, result?.data); // do not set old versions to cache - } - return Promise.resolve(result?.data); - } else { - this.#cache?.set(resolvedKey, parsed); - return Promise.resolve(parsed); - } + this.cache?.set(resolvedKey, parsed); + return Promise.resolve(parsed); } catch (e) { // console.log( // `Could not read file at ${keyPath} due to error ${e}. Its likely that this key does not exist.` @@ -257,7 +185,7 @@ export default class GlitchPartitionImpl } async data(): Promise<{ [key: string]: Type }> { - await this.#init(); + await this.init(); const keys = await this.keys(); const data = {}; for (const key of keys) { @@ -266,42 +194,7 @@ export default class GlitchPartitionImpl return data; } - async #getVersionedData(key: string): Promise> { - const resolvedKey = this.#resolveKey(key); - const keyPath = this.#getKeyPath(resolvedKey); - try { - const fileData = await fs.readFile(keyPath, { - encoding: "utf8", - }); - const parsed = JSON.parse(fileData) as UnitemporallyVersioned; - return Promise.resolve(parsed); - } catch (e) { - // console.log( - // `Could not read file at ${keyPath} due to error ${e}. Its likely that this key does not exist.` - // ); - return Promise.resolve(undefined); - } - } - - async getVersion( - key: string, - version?: number - ): Promise> { - await this.#init(); - return Promise.resolve( - this.#getVersionFromFile(await this.#getVersionedData(key), version) - ); - } - - async getAllVersions( - key: string - ): Promise[]> { - await this.#init(); - const data = await this.#getVersionedData(key); - return Promise.resolve(data?.data ? Object.values(data.data) : undefined); - } - - async #setIndices(key: string, value: Type): Promise { + protected async setIndices(key: string, value: Type): Promise { if (this.#indices?.length) { for (const indexPattern of this.#indices) { const index = lget(value, indexPattern); @@ -315,7 +208,7 @@ export default class GlitchPartitionImpl return Promise.resolve(false); } - async #deleteIndices(value: Type) { + protected async deleteIndices(value: Type) { if (this.#indices?.length) { for (const indexPattern of this.#indices) { const index = lget(value, indexPattern); @@ -327,50 +220,15 @@ export default class GlitchPartitionImpl return Promise.resolve(false); } - async set( - key: string, - value: Type, - metadata?: { [key: string]: string } - ): Promise { - await this.#init(); + async set(key: string, value: Type): Promise { + await this.init(); try { - if (this.#versioned) { - let data = await this.#getVersionedData(key); - if (data) { - await this.#deleteIndices(this.#getVersionFromFile(data)?.data); - data.latestVersion = data.latestVersion + 1; - } else { - data = { - latestVersion: 1, - data: {}, - }; - } - const currentTime = new Date().valueOf(); - if (data.latestVersion !== 1) { - data.data[data.latestVersion - 1] = { - ...data.data[data.latestVersion - 1], - deletedAt: currentTime, - }; - } - data.data[data.latestVersion] = { - data: value, - createdAt: currentTime, - deletedAt: INFINITY_TIME, - version: data.latestVersion, - metadata, - }; - await fs.writeFile(this.#getKeyPath(key), JSON.stringify(data)); - } else { - if (await this.exists(key)) { - await this.#deleteIndices(await this.get(key)); - } - await fs.writeFile(this.#getKeyPath(key), JSON.stringify(value)); - } - await this.#setIndices(key, value); - this.#cache?.set(key, value); - if (this.#versioned) { - this.#cache?.set(key, value); + if (await this.exists(key)) { + await this.deleteIndices(await this.get(key)); } + await fs.writeFile(this.getKeyPath(key), JSON.stringify(value)); + await this.setIndices(key, value); + this.cache?.set(key, value); return Promise.resolve(true); } catch (error) { console.log(`Error setting value for key: ${key}, due to error ${error}`); @@ -379,14 +237,14 @@ export default class GlitchPartitionImpl } async delete(key: string): Promise { - await this.#init(); - const resolvedKey = this.#resolveKey(key); + await this.init(); + const resolvedKey = this.resolveKey(key); const value = await this.get(resolvedKey); - this.#cache?.del(resolvedKey); + this.cache?.del(resolvedKey); if (value) { try { - await fs.rm(this.#getKeyPath(resolvedKey)); - await this.#deleteIndices(value); + await fs.rm(this.getKeyPath(resolvedKey)); + await this.deleteIndices(value); return Promise.resolve(true); } catch (e) { console.log( @@ -420,8 +278,8 @@ export default class GlitchPartitionImpl // eslint-disable-next-line @typescript-eslint/no-explicit-any async getWithJoins(key: string): Promise { - await this.#init(); - const resolvedKey = this.#resolveKey(key); + await this.init(); + const resolvedKey = this.resolveKey(key); if (!Object.keys(this.#joins)?.length) { throw new Error( `No joins defined. Please create a join using 'createJoin' api.` diff --git a/src/GlitchUnitemporalPartition.ts b/src/GlitchUnitemporalPartition.ts new file mode 100644 index 0000000..b3a79e8 --- /dev/null +++ b/src/GlitchUnitemporalPartition.ts @@ -0,0 +1,179 @@ +import fs = require("fs/promises"); +import GlitchDB from "."; +import { INFINITY_TIME } from "./constants"; +import GlitchPartitionImpl, { GlitchPartition } from "./GlitchPartition"; + +export interface UnitemporalVersion { + metadata?: { + [key: string]: string; + }; + version: number; + createdAt: number; + deletedAt: number; +} + +interface UnitemporallyVersionedData extends UnitemporalVersion { + data: Type; +} + +interface UnitemporallyVersioned { + latestVersion: number; + data: { + [key: number]: UnitemporallyVersionedData; + }; +} + +export interface GlitchUnitemporalPartition + extends GlitchPartition { + get: (key: string, version?: number) => Promise; + set: ( + key: string, + value: Type, + metadata?: { [key: string]: string } + ) => Promise; + getVersion: ( + key: string, + version?: number + ) => Promise>; + getAllVersions: (key: string) => Promise[]>; +} + +export default class GlitchUniTemporalPartitionImpl + extends GlitchPartitionImpl + implements GlitchUnitemporalPartition +{ + #versioned: boolean; + constructor( + master: GlitchDB, + localDir: string, + cacheSize?: number, + indices?: string[] + ) { + super(master, localDir, cacheSize, indices); + this.#versioned = true; + } + + #getVersionFromFile( + file: UnitemporallyVersioned, + version?: number + ): UnitemporallyVersionedData { + return file?.data[version ?? file?.latestVersion]; + } + + async get(key: string, version?: number): Promise { + await this.init(); + const resolvedKey = this.resolveKey(key); + if (!version) { + const cachedData = this.cache?.get(resolvedKey); + if (cachedData) { + return Promise.resolve(cachedData); + } + } + const keyPath = this.getKeyPath(resolvedKey); + try { + const fileData = await fs.readFile(keyPath, { + encoding: "utf8", + }); + const parsed = JSON.parse(fileData); + if (this.#versioned) { + const data = parsed as UnitemporallyVersioned; + const result = this.#getVersionFromFile(data, version); + if (!version) { + this.cache?.set(resolvedKey, result?.data); // do not set old versions to cache + } + return Promise.resolve(result?.data); + } else { + this.cache?.set(resolvedKey, parsed); + return Promise.resolve(parsed); + } + } catch (e) { + // console.log( + // `Could not read file at ${keyPath} due to error ${e}. Its likely that this key does not exist.` + // ); + return Promise.resolve(undefined); + } + } + + async #getVersionedData(key: string): Promise> { + const resolvedKey = this.resolveKey(key); + const keyPath = this.getKeyPath(resolvedKey); + try { + const fileData = await fs.readFile(keyPath, { + encoding: "utf8", + }); + const parsed = JSON.parse(fileData) as UnitemporallyVersioned; + return Promise.resolve(parsed); + } catch (e) { + // console.log( + // `Could not read file at ${keyPath} due to error ${e}. Its likely that this key does not exist.` + // ); + return Promise.resolve(undefined); + } + } + + async getVersion( + key: string, + version?: number + ): Promise> { + await this.init(); + return Promise.resolve( + this.#getVersionFromFile(await this.#getVersionedData(key), version) + ); + } + + async getAllVersions( + key: string + ): Promise[]> { + await this.init(); + const data = await this.#getVersionedData(key); + return Promise.resolve(data?.data ? Object.values(data.data) : undefined); + } + + async set( + key: string, + value: Type, + metadata?: { [key: string]: string } + ): Promise { + await this.init(); + try { + if (this.#versioned) { + let data = await this.#getVersionedData(key); + if (data) { + await this.deleteIndices(this.#getVersionFromFile(data)?.data); + data.latestVersion = data.latestVersion + 1; + } else { + data = { + latestVersion: 1, + data: {}, + }; + } + const currentTime = new Date().valueOf(); + if (data.latestVersion !== 1) { + data.data[data.latestVersion - 1] = { + ...data.data[data.latestVersion - 1], + deletedAt: currentTime, + }; + } + data.data[data.latestVersion] = { + data: value, + createdAt: currentTime, + deletedAt: INFINITY_TIME, + version: data.latestVersion, + metadata, + }; + await fs.writeFile(this.getKeyPath(key), JSON.stringify(data)); + } else { + if (await this.exists(key)) { + await this.deleteIndices(await this.get(key)); + } + await fs.writeFile(this.getKeyPath(key), JSON.stringify(value)); + } + await this.setIndices(key, value); + this.cache?.set(key, value); + return Promise.resolve(true); + } catch (error) { + console.log(`Error setting value for key: ${key}, due to error ${error}`); + return Promise.resolve(false); + } + } +} diff --git a/src/index.ts b/src/index.ts index 6dac886..e9b3962 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,5 @@ import GlitchDB from "./GlitchDB"; -export { - GlitchPartition, - GlitchUnitemporallyVersionedPartition, -} from "./GlitchPartition"; - +export { GlitchPartition } from "./GlitchPartition"; +export { GlitchUnitemporalPartition } from "./GlitchUnitemporalPartition"; export default GlitchDB; diff --git a/test/versioned.test.ts b/test/versioned.test.ts index 02b6ad7..45bef0f 100644 --- a/test/versioned.test.ts +++ b/test/versioned.test.ts @@ -3,7 +3,7 @@ import * as path from "path"; import * as os from "os"; import { group } from "good-vibes"; -import GlitchDB, { GlitchUnitemporallyVersionedPartition } from "../src"; +import GlitchDB, { GlitchUnitemporalPartition } from "../src"; const { before, test, after, sync } = group("versioned"); @@ -15,7 +15,7 @@ interface TestData { lengthInSeconds: number; } -let glitchDB: GlitchUnitemporallyVersionedPartition; +let glitchDB: GlitchUnitemporalPartition; let tempDirectory: string; before(async (context) => {