From ab760b37f789c4ecf75b9a06c6fcd7fd9a4c3bc2 Mon Sep 17 00:00:00 2001 From: Vadim Dalecky Date: Mon, 4 Mar 2024 23:09:07 +0100 Subject: [PATCH 1/9] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20Com?= =?UTF-8?q?pressionTable=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/CompressionTable.ts | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/json-pack/util/CompressionTable.ts diff --git a/src/json-pack/util/CompressionTable.ts b/src/json-pack/util/CompressionTable.ts new file mode 100644 index 0000000000..4c899f0fbe --- /dev/null +++ b/src/json-pack/util/CompressionTable.ts @@ -0,0 +1,96 @@ +import {JsonPackExtension} from "../JsonPackExtension"; + +const isSafeInteger = Number.isSafeInteger; + +export class CompressionTable { + public static from(value: unknown): CompressionTable { + const table = new CompressionTable(); + table.from(value); + return table; + } + + protected integers = new Set(); + protected nonIntegers = new Set(); + + protected table: unknown[] = []; + protected map: Map = new Map(); + + public addInteger(int: number): void { + this.integers.add(int); + } + + public addLiteral(value: number | string | unknown): void { + if (isSafeInteger(value)) { + this.addInteger(value as number); + return; + } + this.nonIntegers.add(value); + } + + public from(value: unknown): void { + switch (typeof value) { + case 'object': { + if (!value) return this.addLiteral(null); + const constructor = value.constructor; + switch (constructor) { + case Object: { + const obj = value as Record; + for (const key in obj) { + this.addLiteral(key); + this.from(obj[key]); + } + break; + } + case Array: { + const arr = value as unknown[]; + const len = arr.length; + for (let i = 0; i < len; i++) this.from(arr[i]); + break; + } + case Map: { + const map = value as Map; + map.forEach((value, key) => { + this.from(key); + this.from(value); + }); + break; + } + case Set: { + const set = value as Set; + set.forEach((value) => { + this.from(value); + }); + break; + } + case JsonPackExtension: { + const ext = value as JsonPackExtension; + this.addInteger(ext.tag); + this.from(ext.val); + } + } + return; + } + default: + return this.addLiteral(value); + } + } + + public finalize(): void { + const integers = Array.from(this.integers).sort(); + const len = integers.length; + if (len > 0) { + const first = integers[0]; + this.table[0] = first; + this.map.set(first, 0); + let last = first; + for (let i = 1; i < len; i++) { + const int = integers[i]; + this.table.push(int - last); + this.map.set(int, i); + last = int; + } + } + this.integers.clear(); + this.nonIntegers.clear(); + } +} From f548b1603d0651baf83dfa4f054c7dd9a6e5a7c8 Mon Sep 17 00:00:00 2001 From: Vadim Dalecky Date: Mon, 4 Mar 2024 23:41:45 +0100 Subject: [PATCH 2/9] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20improve?= =?UTF-8?q?=20CompressionTable=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/CompressionTable.ts | 50 ++++++++++----- .../util/__tests__/CompressionTable.spec.ts | 62 +++++++++++++++++++ 2 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 src/json-pack/util/__tests__/CompressionTable.spec.ts diff --git a/src/json-pack/util/CompressionTable.ts b/src/json-pack/util/CompressionTable.ts index 4c899f0fbe..39d74168e9 100644 --- a/src/json-pack/util/CompressionTable.ts +++ b/src/json-pack/util/CompressionTable.ts @@ -3,9 +3,10 @@ import {JsonPackExtension} from "../JsonPackExtension"; const isSafeInteger = Number.isSafeInteger; export class CompressionTable { - public static from(value: unknown): CompressionTable { + public static create(value: unknown): CompressionTable { const table = new CompressionTable(); - table.from(value); + table.walk(value); + table.finalize(); return table; } @@ -27,7 +28,7 @@ export class CompressionTable { this.nonIntegers.add(value); } - public from(value: unknown): void { + public walk(value: unknown): void { switch (typeof value) { case 'object': { if (!value) return this.addLiteral(null); @@ -37,35 +38,35 @@ export class CompressionTable { const obj = value as Record; for (const key in obj) { this.addLiteral(key); - this.from(obj[key]); + this.walk(obj[key]); } break; } case Array: { const arr = value as unknown[]; const len = arr.length; - for (let i = 0; i < len; i++) this.from(arr[i]); + for (let i = 0; i < len; i++) this.walk(arr[i]); break; } case Map: { const map = value as Map; map.forEach((value, key) => { - this.from(key); - this.from(value); + this.walk(key); + this.walk(value); }); break; } case Set: { const set = value as Set; set.forEach((value) => { - this.from(value); + this.walk(value); }); break; } case JsonPackExtension: { const ext = value as JsonPackExtension; this.addInteger(ext.tag); - this.from(ext.val); + this.walk(ext.val); } } return; @@ -76,21 +77,42 @@ export class CompressionTable { } public finalize(): void { - const integers = Array.from(this.integers).sort(); + const integers = Array.from(this.integers); + integers.sort((a, b) => a - b); const len = integers.length; + const table = this.table; + const map = this.map; if (len > 0) { const first = integers[0]; - this.table[0] = first; - this.map.set(first, 0); + table.push(first); + map.set(first, 0); let last = first; for (let i = 1; i < len; i++) { const int = integers[i]; - this.table.push(int - last); - this.map.set(int, i); + table.push(int - last); + map.set(int, i); last = int; } } + const nonIntegers = Array.from(this.nonIntegers) + nonIntegers.sort(); + const lenNonIntegers = nonIntegers.length; + for (let i = 0; i < lenNonIntegers; i++) { + const value = nonIntegers[i]; + table.push(value); + map.set(value, len + i); + } this.integers.clear(); this.nonIntegers.clear(); } + + public getIndex(value: unknown): number { + const index = this.map.get(value); + if (index === undefined) throw new Error(`Value [${value}] not found in compression table.`); + return index; + } + + public getTable(): unknown[] { + return this.table; + } } diff --git a/src/json-pack/util/__tests__/CompressionTable.spec.ts b/src/json-pack/util/__tests__/CompressionTable.spec.ts new file mode 100644 index 0000000000..9335ccee23 --- /dev/null +++ b/src/json-pack/util/__tests__/CompressionTable.spec.ts @@ -0,0 +1,62 @@ +import {CompressionTable} from '../CompressionTable'; + +test('create a compression table from a primitive value', () => { + const table = CompressionTable.create(42).getTable(); + expect(table).toEqual([42]); +}); + +test('collects literals from object', () => { + const json = { + foo: 'bar', + baz: 42, + gg: 'foo', + true: false, + }; + const table = CompressionTable.create(json).getTable(); + expect(table).toEqual([ + 42, + 'bar', + 'baz', + false, + 'foo', + 'gg', + 'true', + ]); +}); + +test('run-length encodes integers', () => { + const json = { + foo: [-3, 12, 42, 12345], + baz: 42, + }; + const table = CompressionTable.create(json).getTable(); + expect(table).toEqual([ + -3, + 15, + 30, + 12303, + 'baz', + 'foo', + ]); +}); + +test('run-length encodes integers - 2', () => { + const json = { + foo: [5, 1, 2, 4, 8, 16, 17, 22], + baz: -1.5, + }; + const table = CompressionTable.create(json).getTable(); + expect(table).toEqual([ + 1, + 1, + 2, + 1, + 3, + 8, + 1, + 5, + -1.5, + 'baz', + 'foo', + ]); +}); From 748b4fae3c4243d691165c9cafcabdcea6afab9e Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 09:59:02 +0100 Subject: [PATCH 3/9] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20implement?= =?UTF-8?q?=20DecompressionTable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/DecompressionTable.ts | 35 +++++++++++++++++++ .../util/__tests__/DecompressionTable.spec.ts | 23 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 src/json-pack/util/DecompressionTable.ts create mode 100644 src/json-pack/util/__tests__/DecompressionTable.spec.ts diff --git a/src/json-pack/util/DecompressionTable.ts b/src/json-pack/util/DecompressionTable.ts new file mode 100644 index 0000000000..bd95ab8d1a --- /dev/null +++ b/src/json-pack/util/DecompressionTable.ts @@ -0,0 +1,35 @@ +const isSafeInteger = Number.isSafeInteger; + +export class DecompressionTable { + protected readonly table: unknown[] = []; + + public importTable(rleTable: unknown[]) { + const length = rleTable.length; + if (!length) return; + const table = this.table; + const first = rleTable[0]; + table.push(first); + let i = 1; + if (isSafeInteger(first)) { + let prev: number = first; + let value: unknown; + while (i < length) { + value = rleTable[i]; + if (isSafeInteger(value)) { + prev = prev + value; + table.push(prev); + i++; + } else { + break; + } + } + } + while (i < length) table.push(rleTable[i++]); + } + + public getLiteral(index: number): unknown { + const table = this.table; + // if (index < 0 || index >= table.length) throw new Error('OUT_OF_BOUNDS'); + return table[index]; + } +} diff --git a/src/json-pack/util/__tests__/DecompressionTable.spec.ts b/src/json-pack/util/__tests__/DecompressionTable.spec.ts new file mode 100644 index 0000000000..dcebe0cce4 --- /dev/null +++ b/src/json-pack/util/__tests__/DecompressionTable.spec.ts @@ -0,0 +1,23 @@ +import {CompressionTable} from '../CompressionTable'; +import {DecompressionTable} from '../DecompressionTable'; + +test('can import back compression table', () => { + const json = { + a: [-10, -5, 5, 100], + b: [true, false, null, null], + c: 'c', + }; + const table = CompressionTable.create(json); + const decompressionTable = new DecompressionTable(); + decompressionTable.importTable(table.getTable()); + expect(decompressionTable.getLiteral(0)).toBe(-10); + expect(decompressionTable.getLiteral(1)).toBe(-5); + expect(decompressionTable.getLiteral(2)).toBe(5); + expect(decompressionTable.getLiteral(3)).toBe(100); + expect(decompressionTable.getLiteral(table.getIndex(true))).toBe(true); + expect(decompressionTable.getLiteral(table.getIndex(false))).toBe(false); + expect(decompressionTable.getLiteral(table.getIndex(null))).toBe(null); + expect(decompressionTable.getLiteral(table.getIndex('a'))).toBe('a'); + expect(decompressionTable.getLiteral(table.getIndex('b'))).toBe('b'); + expect(decompressionTable.getLiteral(table.getIndex('c'))).toBe('c'); +}); From 827c03d7abbd1b90078a45853f76bc7f286fbc06 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 10:02:18 +0100 Subject: [PATCH 4/9] =?UTF-8?q?style(json-pack):=20=F0=9F=92=84=20run=20Pr?= =?UTF-8?q?ettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/CompressionTable.ts | 4 +-- src/json-pack/util/DecompressionTable.ts | 2 +- .../util/__tests__/CompressionTable.spec.ts | 33 ++----------------- 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/src/json-pack/util/CompressionTable.ts b/src/json-pack/util/CompressionTable.ts index 39d74168e9..54c3cd4613 100644 --- a/src/json-pack/util/CompressionTable.ts +++ b/src/json-pack/util/CompressionTable.ts @@ -1,4 +1,4 @@ -import {JsonPackExtension} from "../JsonPackExtension"; +import {JsonPackExtension} from '../JsonPackExtension'; const isSafeInteger = Number.isSafeInteger; @@ -94,7 +94,7 @@ export class CompressionTable { last = int; } } - const nonIntegers = Array.from(this.nonIntegers) + const nonIntegers = Array.from(this.nonIntegers); nonIntegers.sort(); const lenNonIntegers = nonIntegers.length; for (let i = 0; i < lenNonIntegers; i++) { diff --git a/src/json-pack/util/DecompressionTable.ts b/src/json-pack/util/DecompressionTable.ts index bd95ab8d1a..ed51fdd113 100644 --- a/src/json-pack/util/DecompressionTable.ts +++ b/src/json-pack/util/DecompressionTable.ts @@ -2,7 +2,7 @@ const isSafeInteger = Number.isSafeInteger; export class DecompressionTable { protected readonly table: unknown[] = []; - + public importTable(rleTable: unknown[]) { const length = rleTable.length; if (!length) return; diff --git a/src/json-pack/util/__tests__/CompressionTable.spec.ts b/src/json-pack/util/__tests__/CompressionTable.spec.ts index 9335ccee23..8bf5a539de 100644 --- a/src/json-pack/util/__tests__/CompressionTable.spec.ts +++ b/src/json-pack/util/__tests__/CompressionTable.spec.ts @@ -13,15 +13,7 @@ test('collects literals from object', () => { true: false, }; const table = CompressionTable.create(json).getTable(); - expect(table).toEqual([ - 42, - 'bar', - 'baz', - false, - 'foo', - 'gg', - 'true', - ]); + expect(table).toEqual([42, 'bar', 'baz', false, 'foo', 'gg', 'true']); }); test('run-length encodes integers', () => { @@ -30,14 +22,7 @@ test('run-length encodes integers', () => { baz: 42, }; const table = CompressionTable.create(json).getTable(); - expect(table).toEqual([ - -3, - 15, - 30, - 12303, - 'baz', - 'foo', - ]); + expect(table).toEqual([-3, 15, 30, 12303, 'baz', 'foo']); }); test('run-length encodes integers - 2', () => { @@ -46,17 +31,5 @@ test('run-length encodes integers - 2', () => { baz: -1.5, }; const table = CompressionTable.create(json).getTable(); - expect(table).toEqual([ - 1, - 1, - 2, - 1, - 3, - 8, - 1, - 5, - -1.5, - 'baz', - 'foo', - ]); + expect(table).toEqual([1, 1, 2, 1, 3, 8, 1, 5, -1.5, 'baz', 'foo']); }); From 0e3746588c70f8792ee4d1cbea4d8d788a951080 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 10:39:56 +0100 Subject: [PATCH 5/9] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20implement?= =?UTF-8?q?=20pojo=20compression=20function?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/CompressionTable.ts | 49 ++++++++++ .../util/__tests__/CompressionTable.spec.ts | 92 +++++++++++++------ 2 files changed, 113 insertions(+), 28 deletions(-) diff --git a/src/json-pack/util/CompressionTable.ts b/src/json-pack/util/CompressionTable.ts index 54c3cd4613..d8218cb83c 100644 --- a/src/json-pack/util/CompressionTable.ts +++ b/src/json-pack/util/CompressionTable.ts @@ -115,4 +115,53 @@ export class CompressionTable { public getTable(): unknown[] { return this.table; } + + public compress(value: unknown): unknown { + switch (typeof value) { + case 'object': { + if (!value) return this.getIndex(null); + const constructor = value.constructor; + switch (constructor) { + case Object: { + const obj = value as Record; + const newObj: Record = {}; + for (const key in obj) newObj[this.getIndex(key)] = this.compress(obj[key]); + return newObj; + } + case Array: { + const arr = value as unknown[]; + const newArr: unknown[] = []; + const len = arr.length; + for (let i = 0; i < len; i++) newArr.push(this.compress(arr[i])); + return newArr; + } + case Map: { + const map = value as Map; + const newMap = new Map(); + map.forEach((value, key) => { + newMap.set(this.compress(key), this.compress(value)); + }); + return newMap; + } + case Set: { + const set = value as Set; + const newSet = new Set(); + set.forEach((value) => { + newSet.add(this.compress(value)); + }); + break; + } + case JsonPackExtension: { + const ext = value as JsonPackExtension; + const newExt = new JsonPackExtension(this.getIndex(ext.tag), this.compress(ext.val)); + return newExt; + } + } + throw new Error('UNEXPECTED_OBJECT'); + } + default: { + return this.getIndex(value); + } + } + } } diff --git a/src/json-pack/util/__tests__/CompressionTable.spec.ts b/src/json-pack/util/__tests__/CompressionTable.spec.ts index 8bf5a539de..a55b98a004 100644 --- a/src/json-pack/util/__tests__/CompressionTable.spec.ts +++ b/src/json-pack/util/__tests__/CompressionTable.spec.ts @@ -1,35 +1,71 @@ import {CompressionTable} from '../CompressionTable'; -test('create a compression table from a primitive value', () => { - const table = CompressionTable.create(42).getTable(); - expect(table).toEqual([42]); -}); +describe('.walk()', () => { + test('create a compression table from a primitive value', () => { + const table = CompressionTable.create(42).getTable(); + expect(table).toEqual([42]); + }); -test('collects literals from object', () => { - const json = { - foo: 'bar', - baz: 42, - gg: 'foo', - true: false, - }; - const table = CompressionTable.create(json).getTable(); - expect(table).toEqual([42, 'bar', 'baz', false, 'foo', 'gg', 'true']); -}); + test('collects literals from object', () => { + const json = { + foo: 'bar', + baz: 42, + gg: 'foo', + true: false, + }; + const table = CompressionTable.create(json).getTable(); + expect(table).toEqual([42, 'bar', 'baz', false, 'foo', 'gg', 'true']); + }); -test('run-length encodes integers', () => { - const json = { - foo: [-3, 12, 42, 12345], - baz: 42, - }; - const table = CompressionTable.create(json).getTable(); - expect(table).toEqual([-3, 15, 30, 12303, 'baz', 'foo']); + test('run-length encodes integers', () => { + const json = { + foo: [-3, 12, 42, 12345], + baz: 42, + }; + const table = CompressionTable.create(json).getTable(); + expect(table).toEqual([-3, 15, 30, 12303, 'baz', 'foo']); + }); + + test('run-length encodes integers - 2', () => { + const json = { + foo: [5, 1, 2, 4, 8, 16, 17, 22], + baz: -1.5, + }; + const table = CompressionTable.create(json).getTable(); + expect(table).toEqual([1, 1, 2, 1, 3, 8, 1, 5, -1.5, 'baz', 'foo']); + }); }); -test('run-length encodes integers - 2', () => { - const json = { - foo: [5, 1, 2, 4, 8, 16, 17, 22], - baz: -1.5, - }; - const table = CompressionTable.create(json).getTable(); - expect(table).toEqual([1, 1, 2, 1, 3, 8, 1, 5, -1.5, 'baz', 'foo']); +describe('.compress()', () => { + test('replaces literals with indices', () => { + const json = { + foo: 'bar', + baz: 42, + gg: 'foo', + true: false, + }; + const table = CompressionTable.create(json); + const compressed = table.compress(json); + expect(compressed).toEqual({ '2': 0, '4': 1, '5': 4, '6': 3 }); + }); + + test('can share compression table across two documents', () => { + const json1 = { + foo: 'bar', + }; + const json2 = { + foo: [0, 0, 5, 5], + }; + const table = new CompressionTable(); + table.walk(json1); + table.walk(json2); + table.finalize(); + const compressed1 = table.compress(json1); + const compressed2 = table.compress(json2); + expect(table.getTable()).toEqual([ + 0, 5, 'bar', 'foo', + ]); + expect(compressed1).toEqual({ '3': 2 }); + expect(compressed2).toEqual({ '3': [0, 0, 1, 1] }); + }); }); From 3473c5b6e9e6fa3ef59041a640f6b9376a6cb0f8 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 10:50:07 +0100 Subject: [PATCH 6/9] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20implement?= =?UTF-8?q?=20.decompress()=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/DecompressionTable.ts | 54 ++++++++++++++++++ .../util/__tests__/CompressionTable.spec.ts | 10 ++-- .../util/__tests__/DecompressionTable.spec.ts | 56 ++++++++++++------- 3 files changed, 95 insertions(+), 25 deletions(-) diff --git a/src/json-pack/util/DecompressionTable.ts b/src/json-pack/util/DecompressionTable.ts index ed51fdd113..dadfab2e92 100644 --- a/src/json-pack/util/DecompressionTable.ts +++ b/src/json-pack/util/DecompressionTable.ts @@ -1,3 +1,5 @@ +import {JsonPackExtension} from "../JsonPackExtension"; + const isSafeInteger = Number.isSafeInteger; export class DecompressionTable { @@ -32,4 +34,56 @@ export class DecompressionTable { // if (index < 0 || index >= table.length) throw new Error('OUT_OF_BOUNDS'); return table[index]; } + + public decompress(value: unknown): unknown { + switch (typeof value) { + case 'number': { + return this.getLiteral(value); + } + case 'object': { + if (!value) return null; + const constructor = value.constructor; + switch (constructor) { + case Object: { + const obj = value as Record; + const newObj: Record = {}; + for (const key in obj) newObj[String(this.getLiteral(Number(key)))] = this.decompress(obj[key]); + return newObj; + } + case Array: { + const arr = value as unknown[]; + const newArr: unknown[] = []; + const len = arr.length; + for (let i = 0; i < len; i++) newArr.push(this.decompress(arr[i])); + return newArr; + } + case Map: { + const map = value as Map; + const newMap = new Map(); + map.forEach((value, key) => { + newMap.set(this.decompress(key), this.decompress(value)); + }); + return newMap; + } + case Set: { + const set = value as Set; + const newSet = new Set(); + set.forEach((value) => { + newSet.add(this.decompress(value)); + }); + break; + } + case JsonPackExtension: { + const ext = value as JsonPackExtension; + const newExt = new JsonPackExtension(Number(this.getLiteral(ext.tag)), this.decompress(ext.val)); + return newExt; + } + } + return value; + } + default: { + return value; + } + } + } } diff --git a/src/json-pack/util/__tests__/CompressionTable.spec.ts b/src/json-pack/util/__tests__/CompressionTable.spec.ts index a55b98a004..6d3e5231fc 100644 --- a/src/json-pack/util/__tests__/CompressionTable.spec.ts +++ b/src/json-pack/util/__tests__/CompressionTable.spec.ts @@ -46,7 +46,7 @@ describe('.compress()', () => { }; const table = CompressionTable.create(json); const compressed = table.compress(json); - expect(compressed).toEqual({ '2': 0, '4': 1, '5': 4, '6': 3 }); + expect(compressed).toEqual({'2': 0, '4': 1, '5': 4, '6': 3}); }); test('can share compression table across two documents', () => { @@ -62,10 +62,8 @@ describe('.compress()', () => { table.finalize(); const compressed1 = table.compress(json1); const compressed2 = table.compress(json2); - expect(table.getTable()).toEqual([ - 0, 5, 'bar', 'foo', - ]); - expect(compressed1).toEqual({ '3': 2 }); - expect(compressed2).toEqual({ '3': [0, 0, 1, 1] }); + expect(table.getTable()).toEqual([0, 5, 'bar', 'foo']); + expect(compressed1).toEqual({'3': 2}); + expect(compressed2).toEqual({'3': [0, 0, 1, 1]}); }); }); diff --git a/src/json-pack/util/__tests__/DecompressionTable.spec.ts b/src/json-pack/util/__tests__/DecompressionTable.spec.ts index dcebe0cce4..86b24bbea4 100644 --- a/src/json-pack/util/__tests__/DecompressionTable.spec.ts +++ b/src/json-pack/util/__tests__/DecompressionTable.spec.ts @@ -1,23 +1,41 @@ import {CompressionTable} from '../CompressionTable'; import {DecompressionTable} from '../DecompressionTable'; -test('can import back compression table', () => { - const json = { - a: [-10, -5, 5, 100], - b: [true, false, null, null], - c: 'c', - }; - const table = CompressionTable.create(json); - const decompressionTable = new DecompressionTable(); - decompressionTable.importTable(table.getTable()); - expect(decompressionTable.getLiteral(0)).toBe(-10); - expect(decompressionTable.getLiteral(1)).toBe(-5); - expect(decompressionTable.getLiteral(2)).toBe(5); - expect(decompressionTable.getLiteral(3)).toBe(100); - expect(decompressionTable.getLiteral(table.getIndex(true))).toBe(true); - expect(decompressionTable.getLiteral(table.getIndex(false))).toBe(false); - expect(decompressionTable.getLiteral(table.getIndex(null))).toBe(null); - expect(decompressionTable.getLiteral(table.getIndex('a'))).toBe('a'); - expect(decompressionTable.getLiteral(table.getIndex('b'))).toBe('b'); - expect(decompressionTable.getLiteral(table.getIndex('c'))).toBe('c'); +describe('.importTable()', () => { + test('can import back compression table', () => { + const json = { + a: [-10, -5, 5, 100], + b: [true, false, null, null], + c: 'c', + }; + const table = CompressionTable.create(json); + const decompressionTable = new DecompressionTable(); + decompressionTable.importTable(table.getTable()); + expect(decompressionTable.getLiteral(0)).toBe(-10); + expect(decompressionTable.getLiteral(1)).toBe(-5); + expect(decompressionTable.getLiteral(2)).toBe(5); + expect(decompressionTable.getLiteral(3)).toBe(100); + expect(decompressionTable.getLiteral(table.getIndex(true))).toBe(true); + expect(decompressionTable.getLiteral(table.getIndex(false))).toBe(false); + expect(decompressionTable.getLiteral(table.getIndex(null))).toBe(null); + expect(decompressionTable.getLiteral(table.getIndex('a'))).toBe('a'); + expect(decompressionTable.getLiteral(table.getIndex('b'))).toBe('b'); + expect(decompressionTable.getLiteral(table.getIndex('c'))).toBe('c'); + }); +}); + +describe('.decompress()', () => { + test('can decompress a document', () => { + const json = { + a: [-10, -5, 5, 100], + b: [true, false, null, null], + c: 'c', + }; + const table = CompressionTable.create(json); + const compressed = table.compress(json); + const decompressionTable = new DecompressionTable(); + decompressionTable.importTable(table.getTable()); + const decompressed = decompressionTable.decompress(compressed); + expect(decompressed).toEqual(json); + }); }); From 36cf3a987025a9d1e9791be6879304640c556d09 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 10:50:29 +0100 Subject: [PATCH 7/9] =?UTF-8?q?style(json-pack):=20=F0=9F=92=84=20run=20Pr?= =?UTF-8?q?ettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/util/DecompressionTable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json-pack/util/DecompressionTable.ts b/src/json-pack/util/DecompressionTable.ts index dadfab2e92..c1b40cf9b1 100644 --- a/src/json-pack/util/DecompressionTable.ts +++ b/src/json-pack/util/DecompressionTable.ts @@ -1,4 +1,4 @@ -import {JsonPackExtension} from "../JsonPackExtension"; +import {JsonPackExtension} from '../JsonPackExtension'; const isSafeInteger = Number.isSafeInteger; From 047585972ada61f655d3f9465cb137fc0cd7050e Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 13:58:41 +0100 Subject: [PATCH 8/9] =?UTF-8?q?refactor(json-crdt):=20=F0=9F=92=A1=20do=20?= =?UTF-8?q?not=20load=20all=20codecs=20in=20File=20unnecessarily?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 88 +++++++++++++++-------- src/json-crdt/file/__tests__/File.spec.ts | 7 +- src/json-crdt/file/fileEncoders.ts | 23 ++++++ 3 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 src/json-crdt/file/fileEncoders.ts diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 6dcbd44ef9..bc5fd3e112 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -1,30 +1,41 @@ import {Model} from '../model'; import {PatchLog} from './PatchLog'; -import {FileModelEncoding} from './constants'; -import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder'; -import {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder'; -import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder'; -import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; -import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; -import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode'; -import {Writer} from '../../util/buffers/Writer'; -import {CborEncoder} from '../../json-pack/cbor/CborEncoder'; -import {JsonEncoder} from '../../json-pack/json/JsonEncoder'; import {printTree} from '../../util/print/printTree'; import {decodeModel, decodeNdjsonComponents, decodePatch, decodeSeqCborComponents} from './util'; import {Patch} from '../../json-crdt-patch'; +import {FileModelEncoding} from './constants'; +import type {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; +import type {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode'; +import type {CborEncoder} from '../../json-pack/cbor/CborEncoder'; +import type {JsonEncoder} from '../../json-pack/json/JsonEncoder'; +import type {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder'; +import type {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; +import type {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder'; +import type {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder'; import type * as types from './types'; import type {Printable} from '../../util/print/types'; +export interface FileOptions { + jsonEncoder?: JsonEncoder; + cborEncoder?: CborEncoder; + structuralCompactEncoder?: StructuralEncoderCompact; + structuralVerboseEncoder?: StructuralEncoderVerbose; + sidecarEncoder?: SidecarEncoder; + sidecarDecoder?: SidecarDecoder; + patchCompactEncoder?: typeof encodeCompact; + patchVerboseEncoder?: typeof encodeVerbose; +} + export class File implements Printable { - public static unserialize(components: types.FileReadSequence): File { + public static unserialize(components: types.FileReadSequence, options: FileOptions = {}): File { const [view, metadata, model, history, ...frontier] = components; const modelFormat = metadata[1]; let decodedModel: Model | null = null; if (model) { const isSidecar = modelFormat === FileModelEncoding.SidecarBinary; if (isSidecar) { - const decoder = new SidecarDecoder(); + const decoder = options.sidecarDecoder; + if (!decoder) throw new Error('NO_SIDECAR_DECODER'); if (!(model instanceof Uint8Array)) throw new Error('NOT_BLOB'); decodedModel = decoder.decode(view, model); } else { @@ -53,23 +64,24 @@ export class File implements Printable { return file; } - public static fromNdjson(blob: Uint8Array): File { + public static fromNdjson(blob: Uint8Array, options: FileOptions = {}): File { const components = decodeNdjsonComponents(blob); - return File.unserialize(components as types.FileReadSequence); + return File.unserialize(components as types.FileReadSequence, options); } - public static fromSeqCbor(blob: Uint8Array): File { + public static fromSeqCbor(blob: Uint8Array, options: FileOptions = {}): File { const components = decodeSeqCborComponents(blob); - return File.unserialize(components as types.FileReadSequence); + return File.unserialize(components as types.FileReadSequence, options); } - public static fromModel(model: Model): File { - return new File(model, PatchLog.fromModel(model)); + public static fromModel(model: Model, options: FileOptions = {}): File { + return new File(model, PatchLog.fromModel(model), options); } constructor( public readonly model: Model, public readonly log: PatchLog, + protected readonly options: FileOptions = {}, ) {} public apply(patch: Patch): void { @@ -79,6 +91,10 @@ export class File implements Printable { this.log.push(patch); } + /** + * @todo Remove synchronization from here. Make `File` just responsible for + * serialization and deserialization. + */ public sync(): () => void { const {model, log} = this; const api = model.api; @@ -104,7 +120,8 @@ export class File implements Printable { switch (modelFormat) { case 'sidecar': { metadata[1] = FileModelEncoding.SidecarBinary; - const encoder = new SidecarEncoder(); + const encoder = this.options.sidecarEncoder; + if (!encoder) throw new Error('NO_SIDECAR_ENCODER'); const [, uint8] = encoder.encode(this.model); model = uint8; break; @@ -114,11 +131,15 @@ export class File implements Printable { break; } case 'compact': { - model = new StructuralEncoderCompact().encode(this.model); + const encoder = this.options.structuralCompactEncoder; + if (!encoder) throw new Error('NO_COMPACT_ENCODER'); + model = encoder.encode(this.model); break; } case 'verbose': { - model = new StructuralEncoderVerbose().encode(this.model); + const encoder = this.options.structuralVerboseEncoder; + if (!encoder) throw new Error('NO_VERBOSE_ENCODER'); + model = encoder.encode(this.model); break; } case 'none': { @@ -139,16 +160,26 @@ export class File implements Printable { break; } case 'compact': { - history[0] = new StructuralEncoderCompact().encode(this.log.start); + const encoder = this.options.structuralCompactEncoder; + if (!encoder) throw new Error('NO_COMPACT_ENCODER'); + history[0] = encoder.encode(this.log.start); + const encodeCompact = this.options.patchCompactEncoder; + if (!encodeCompact) throw new Error('NO_COMPACT_PATCH_ENCODER'); + const list = history[1]; this.log.patches.forEach(({v}) => { - history[1].push(encodeCompact(v)); + list.push(encodeCompact(v)); }); break; } case 'verbose': { - history[0] = new StructuralEncoderVerbose().encode(this.log.start); + const encoder = this.options.structuralVerboseEncoder; + if (!encoder) throw new Error('NO_VERBOSE_ENCODER'); + history[0] = encoder.encode(this.log.start); + const encodeVerbose = this.options.patchVerboseEncoder; + if (!encodeVerbose) throw new Error('NO_VERBOSE_PATCH_ENCODER'); + const list = history[1]; this.log.patches.forEach(({v}) => { - history[1].push(encodeVerbose(v)); + list.push(encodeVerbose(v)); }); break; } @@ -163,10 +194,10 @@ export class File implements Printable { public toBinary(params: types.FileEncodingParams): Uint8Array { const sequence = this.serialize(params); - const writer = new Writer(16 * 1024); switch (params.format) { case 'ndjson': { - const json = new JsonEncoder(writer); + const json = this.options.jsonEncoder; + if (!json) throw new Error('NO_JSON_ENCODER'); for (const component of sequence) { json.writeAny(component); json.writer.u8('\n'.charCodeAt(0)); @@ -174,7 +205,8 @@ export class File implements Printable { return json.writer.flush(); } case 'seq.cbor': { - const cbor = new CborEncoder(writer); + const cbor = this.options.cborEncoder; + if (!cbor) throw new Error('NO_CBOR_ENCODER'); for (const component of sequence) cbor.writeAny(component); return cbor.writer.flush(); } diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 66222b0eda..b25262f8de 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -4,11 +4,12 @@ import {File} from '../File'; import {JsonDecoder} from '../../../json-pack/json/JsonDecoder'; import {CborDecoder} from '../../../json-pack/cbor/CborDecoder'; import {FileEncodingParams} from '../types'; +import {fileEncoders} from '../fileEncoders'; const setup = (view: unknown) => { const model = Model.withServerClock(); model.api.root(view); - const file = File.fromModel(model); + const file = File.fromModel(model, fileEncoders); return {model, file}; }; @@ -73,7 +74,7 @@ describe('.toBinary()', () => { const assertEncoding = (file: File, params: FileEncodingParams) => { const blob = file.toBinary(params); // if (params.format === 'ndjson') console.log(Buffer.from(blob).toString('utf8')) - const file2 = params.format === 'seq.cbor' ? File.fromSeqCbor(blob) : File.fromNdjson(blob); + const file2 = params.format === 'seq.cbor' ? File.fromSeqCbor(blob, fileEncoders) : File.fromNdjson(blob, fileEncoders); expect(file2.model.view()).toEqual(file.model.view()); expect(file2.model !== file.model).toBe(true); expect(file2.log.start.view()).toEqual(undefined); @@ -115,7 +116,7 @@ describe('.unserialize()', () => { }); serialized.push(clone.api.flush().toBinary()); expect(file.model.view()).toEqual({foo: 'bar'}); - const file2 = File.unserialize(serialized); + const file2 = File.unserialize(serialized, fileEncoders); expect(file2.model.view()).toEqual({foo: 'bar', xyz: 123}); }); }); diff --git a/src/json-crdt/file/fileEncoders.ts b/src/json-crdt/file/fileEncoders.ts new file mode 100644 index 0000000000..7520b51999 --- /dev/null +++ b/src/json-crdt/file/fileEncoders.ts @@ -0,0 +1,23 @@ +import {Writer} from '../../util/buffers/Writer'; +import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder'; +import {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder'; +import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder'; +import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; +import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; +import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode'; +import {CborEncoder} from '../../json-pack/cbor/CborEncoder'; +import {JsonEncoder} from '../../json-pack/json/JsonEncoder'; +import type {FileOptions} from './File'; + +const writer = new Writer(4096); + +export const fileEncoders: FileOptions = { + jsonEncoder: new JsonEncoder(writer), + cborEncoder: new CborEncoder(writer), + structuralCompactEncoder: new StructuralEncoderCompact(), + structuralVerboseEncoder: new StructuralEncoderVerbose(), + sidecarEncoder: new SidecarEncoder(), + sidecarDecoder: new SidecarDecoder(), + patchCompactEncoder: encodeCompact, + patchVerboseEncoder: encodeVerbose, +}; From 5d6500acc5eb9f0faae03bd7290e6e28362adcba Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Mar 2024 17:08:52 +0100 Subject: [PATCH 9/9] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20run=20Pr?= =?UTF-8?q?ettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/__tests__/File.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index b25262f8de..c97bcc7a51 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -74,7 +74,8 @@ describe('.toBinary()', () => { const assertEncoding = (file: File, params: FileEncodingParams) => { const blob = file.toBinary(params); // if (params.format === 'ndjson') console.log(Buffer.from(blob).toString('utf8')) - const file2 = params.format === 'seq.cbor' ? File.fromSeqCbor(blob, fileEncoders) : File.fromNdjson(blob, fileEncoders); + const file2 = + params.format === 'seq.cbor' ? File.fromSeqCbor(blob, fileEncoders) : File.fromNdjson(blob, fileEncoders); expect(file2.model.view()).toEqual(file.model.view()); expect(file2.model !== file.model).toBe(true); expect(file2.log.start.view()).toEqual(undefined);