Skip to content

Commit

Permalink
Merge pull request #533 from streamich/json-pack-compression
Browse files Browse the repository at this point in the history 
json-pack compression
  • Loading branch information
@streamich
streamich authored Mar 5, 2024
2 parents cd57c74 + 5d6500a commit 035f86a
Show file tree
Hide file tree
Showing 7 changed files with 454 additions and 31 deletions.
88 changes: 60 additions & 28 deletions src/json-crdt/file/File.ts
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,41 @@
import {Model} from '../model';
import {PatchLog} from './PatchLog';
import {FileModelEncoding} from './constants';
import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder';
import {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder';
import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder';
import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder';
import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode';
import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode';
import {Writer} from '../../util/buffers/Writer';
import {CborEncoder} from '../../json-pack/cbor/CborEncoder';
import {JsonEncoder} from '../../json-pack/json/JsonEncoder';
import {printTree} from '../../util/print/printTree';
import {decodeModel, decodeNdjsonComponents, decodePatch, decodeSeqCborComponents} from './util';
import {Patch} from '../../json-crdt-patch';
import {FileModelEncoding} from './constants';
import type {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode';
import type {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode';
import type {CborEncoder} from '../../json-pack/cbor/CborEncoder';
import type {JsonEncoder} from '../../json-pack/json/JsonEncoder';
import type {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder';
import type {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder';
import type {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder';
import type {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder';
import type * as types from './types';
import type {Printable} from '../../util/print/types';

export interface FileOptions {
jsonEncoder?: JsonEncoder;
cborEncoder?: CborEncoder;
structuralCompactEncoder?: StructuralEncoderCompact;
structuralVerboseEncoder?: StructuralEncoderVerbose;
sidecarEncoder?: SidecarEncoder;
sidecarDecoder?: SidecarDecoder;
patchCompactEncoder?: typeof encodeCompact;
patchVerboseEncoder?: typeof encodeVerbose;
}

export class File implements Printable {
public static unserialize(components: types.FileReadSequence): File {
public static unserialize(components: types.FileReadSequence, options: FileOptions = {}): File {
const [view, metadata, model, history, ...frontier] = components;
const modelFormat = metadata[1];
let decodedModel: Model<any> | null = null;
if (model) {
const isSidecar = modelFormat === FileModelEncoding.SidecarBinary;
if (isSidecar) {
const decoder = new SidecarDecoder();
const decoder = options.sidecarDecoder;
if (!decoder) throw new Error('NO_SIDECAR_DECODER');
if (!(model instanceof Uint8Array)) throw new Error('NOT_BLOB');
decodedModel = decoder.decode(view, model);
} else {
Expand Down Expand Up @@ -53,23 +64,24 @@ export class File implements Printable {
return file;
}

public static fromNdjson(blob: Uint8Array): File {
public static fromNdjson(blob: Uint8Array, options: FileOptions = {}): File {
const components = decodeNdjsonComponents(blob);
return File.unserialize(components as types.FileReadSequence);
return File.unserialize(components as types.FileReadSequence, options);
}

public static fromSeqCbor(blob: Uint8Array): File {
public static fromSeqCbor(blob: Uint8Array, options: FileOptions = {}): File {
const components = decodeSeqCborComponents(blob);
return File.unserialize(components as types.FileReadSequence);
return File.unserialize(components as types.FileReadSequence, options);
}

public static fromModel(model: Model<any>): File {
return new File(model, PatchLog.fromModel(model));
public static fromModel(model: Model<any>, options: FileOptions = {}): File {
return new File(model, PatchLog.fromModel(model), options);
}

constructor(
public readonly model: Model,
public readonly log: PatchLog,
protected readonly options: FileOptions = {},
) {}

public apply(patch: Patch): void {
Expand All @@ -79,6 +91,10 @@ export class File implements Printable {
this.log.push(patch);
}

/**
* @todo Remove synchronization from here. Make `File` just responsible for
* serialization and deserialization.
*/
public sync(): () => void {
const {model, log} = this;
const api = model.api;
Expand All @@ -104,7 +120,8 @@ export class File implements Printable {
switch (modelFormat) {
case 'sidecar': {
metadata[1] = FileModelEncoding.SidecarBinary;
const encoder = new SidecarEncoder();
const encoder = this.options.sidecarEncoder;
if (!encoder) throw new Error('NO_SIDECAR_ENCODER');
const [, uint8] = encoder.encode(this.model);
model = uint8;
break;
Expand All @@ -114,11 +131,15 @@ export class File implements Printable {
break;
}
case 'compact': {
model = new StructuralEncoderCompact().encode(this.model);
const encoder = this.options.structuralCompactEncoder;
if (!encoder) throw new Error('NO_COMPACT_ENCODER');
model = encoder.encode(this.model);
break;
}
case 'verbose': {
model = new StructuralEncoderVerbose().encode(this.model);
const encoder = this.options.structuralVerboseEncoder;
if (!encoder) throw new Error('NO_VERBOSE_ENCODER');
model = encoder.encode(this.model);
break;
}
case 'none': {
Expand All @@ -139,16 +160,26 @@ export class File implements Printable {
break;
}
case 'compact': {
history[0] = new StructuralEncoderCompact().encode(this.log.start);
const encoder = this.options.structuralCompactEncoder;
if (!encoder) throw new Error('NO_COMPACT_ENCODER');
history[0] = encoder.encode(this.log.start);
const encodeCompact = this.options.patchCompactEncoder;
if (!encodeCompact) throw new Error('NO_COMPACT_PATCH_ENCODER');
const list = history[1];
this.log.patches.forEach(({v}) => {
history[1].push(encodeCompact(v));
list.push(encodeCompact(v));
});
break;
}
case 'verbose': {
history[0] = new StructuralEncoderVerbose().encode(this.log.start);
const encoder = this.options.structuralVerboseEncoder;
if (!encoder) throw new Error('NO_VERBOSE_ENCODER');
history[0] = encoder.encode(this.log.start);
const encodeVerbose = this.options.patchVerboseEncoder;
if (!encodeVerbose) throw new Error('NO_VERBOSE_PATCH_ENCODER');
const list = history[1];
this.log.patches.forEach(({v}) => {
history[1].push(encodeVerbose(v));
list.push(encodeVerbose(v));
});
break;
}
Expand All @@ -163,18 +194,19 @@ export class File implements Printable {

public toBinary(params: types.FileEncodingParams): Uint8Array {
const sequence = this.serialize(params);
const writer = new Writer(16 * 1024);
switch (params.format) {
case 'ndjson': {
const json = new JsonEncoder(writer);
const json = this.options.jsonEncoder;
if (!json) throw new Error('NO_JSON_ENCODER');
for (const component of sequence) {
json.writeAny(component);
json.writer.u8('\n'.charCodeAt(0));
}
return json.writer.flush();
}
case 'seq.cbor': {
const cbor = new CborEncoder(writer);
const cbor = this.options.cborEncoder;
if (!cbor) throw new Error('NO_CBOR_ENCODER');
for (const component of sequence) cbor.writeAny(component);
return cbor.writer.flush();
}
Expand Down
8 changes: 5 additions & 3 deletions src/json-crdt/file/__tests__/File.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import {File} from '../File';
import {JsonDecoder} from '../../../json-pack/json/JsonDecoder';
import {CborDecoder} from '../../../json-pack/cbor/CborDecoder';
import {FileEncodingParams} from '../types';
import {fileEncoders} from '../fileEncoders';

const setup = (view: unknown) => {
const model = Model.withServerClock();
model.api.root(view);
const file = File.fromModel(model);
const file = File.fromModel(model, fileEncoders);
return {model, file};
};

Expand Down Expand Up @@ -73,7 +74,8 @@ describe('.toBinary()', () => {
const assertEncoding = (file: File, params: FileEncodingParams) => {
const blob = file.toBinary(params);
// if (params.format === 'ndjson') console.log(Buffer.from(blob).toString('utf8'))
const file2 = params.format === 'seq.cbor' ? File.fromSeqCbor(blob) : File.fromNdjson(blob);
const file2 =
params.format === 'seq.cbor' ? File.fromSeqCbor(blob, fileEncoders) : File.fromNdjson(blob, fileEncoders);
expect(file2.model.view()).toEqual(file.model.view());
expect(file2.model !== file.model).toBe(true);
expect(file2.log.start.view()).toEqual(undefined);
Expand Down Expand Up @@ -115,7 +117,7 @@ describe('.unserialize()', () => {
});
serialized.push(clone.api.flush().toBinary());
expect(file.model.view()).toEqual({foo: 'bar'});
const file2 = File.unserialize(serialized);
const file2 = File.unserialize(serialized, fileEncoders);
expect(file2.model.view()).toEqual({foo: 'bar', xyz: 123});
});
});
Expand Down
23 changes: 23 additions & 0 deletions src/json-crdt/file/fileEncoders.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import {Writer} from '../../util/buffers/Writer';
import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder';
import {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder';
import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder';
import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder';
import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode';
import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode';
import {CborEncoder} from '../../json-pack/cbor/CborEncoder';
import {JsonEncoder} from '../../json-pack/json/JsonEncoder';
import type {FileOptions} from './File';

const writer = new Writer(4096);

export const fileEncoders: FileOptions = {
jsonEncoder: new JsonEncoder(writer),
cborEncoder: new CborEncoder(writer),
structuralCompactEncoder: new StructuralEncoderCompact(),
structuralVerboseEncoder: new StructuralEncoderVerbose(),
sidecarEncoder: new SidecarEncoder(),
sidecarDecoder: new SidecarDecoder(),
patchCompactEncoder: encodeCompact,
patchVerboseEncoder: encodeVerbose,
};
Loading

0 comments on commit 035f86a

Please sign in to comment.