diff --git a/package.json b/package.json index e019f0a196..82ae264ad0 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,6 @@ "concurrently": "^8.0.1", "diamond-types-node": "1.0.2", "editing-traces": "https://github.com/streamich/editing-traces#6494020428530a6e382378b98d1d7e31334e2d7b", - "json-crdt-traces": "https://github.com/streamich/json-crdt-traces#5f68a13baf798897d39b87d7028b0b3cd5a50a6c", "eventsource": "^2.0.2", "fast-deep-equal": "^3.1.3", "fast-equals": "^5.0.1", @@ -118,10 +117,12 @@ "js-base64": "^3.7.2", "js-sdsl": "^4.4.0", "jsbi": "^4.3.0", + "json-crdt-traces": "https://github.com/streamich/json-crdt-traces#5f68a13baf798897d39b87d7028b0b3cd5a50a6c", "json-logic-js": "^2.0.1", "json-pack-napi": "^0.0.2", "load-script": "^2.0.0", "lodash": "^4.17.21", + "loro-crdt": "^0.4.1", "markdown-it": "^13.0.1", "messagepack": "^1.1.12", "msgpack-lite": "^0.1.26", diff --git a/src/json-crdt-patch/codec/clock/ClockEncoder.ts b/src/json-crdt-patch/codec/clock/ClockEncoder.ts index bd24cdcb38..c07567de7b 100644 --- a/src/json-crdt-patch/codec/clock/ClockEncoder.ts +++ b/src/json-crdt-patch/codec/clock/ClockEncoder.ts @@ -1,4 +1,4 @@ -import {ITimestampStruct, IClockVector, tick} from '../../clock'; +import {ITimestampStruct, IClockVector, tick, Timestamp} from '../../clock'; import {RelativeTimestamp} from './RelativeTimestamp'; class ClockTableEntry { @@ -24,12 +24,14 @@ export class ClockEncoder { const sid = ts.sid; let entry = this.table.get(sid); if (!entry) { - const clock = this.clock!.peers.get(sid)!; + let clock = this.clock!.peers.get(sid); + if (!clock) clock = new Timestamp(sid, this.clock!.time - 1); entry = new ClockTableEntry(this.index++, clock); this.table.set(sid, entry); } const clock = entry.clock; const timeDiff = clock.time - time; + if (timeDiff < 0) throw new Error('TIME_TRAVEL'); return new RelativeTimestamp(entry.index, timeDiff); } diff --git a/src/json-crdt-patch/codec/clock/__tests__/ClockEncoder.spec.ts b/src/json-crdt-patch/codec/clock/__tests__/ClockEncoder.spec.ts index f2ee6a99a9..cb0ef67f90 100644 --- a/src/json-crdt-patch/codec/clock/__tests__/ClockEncoder.spec.ts +++ b/src/json-crdt-patch/codec/clock/__tests__/ClockEncoder.spec.ts @@ -50,16 +50,3 @@ test('encodes each clock only once', () => { const encoded = encoder.toJson(); expect(encoded).toEqual([100, 100, 50, 50, 10, 10]); }); - -test('throws when unknown clock is being encoded', () => { - const clock = new ClockVector(100, 100); - const ts1 = ts(50, 50); - const ts2 = ts(10, 10); - clock.observe(ts1, 1); - clock.observe(ts2, 1); - const encoder = new ClockEncoder(); - encoder.reset(clock); - encoder.append(ts1); - encoder.append(ts2); - expect(() => encoder.append(ts(77, 77))).toThrow(); -}); diff --git a/src/json-crdt/__bench__/bench.traces.crdt-libs.ts b/src/json-crdt/__bench__/bench.traces.crdt-libs.ts index fa4e877dce..49d68e02f4 100644 --- a/src/json-crdt/__bench__/bench.traces.crdt-libs.ts +++ b/src/json-crdt/__bench__/bench.traces.crdt-libs.ts @@ -22,6 +22,7 @@ runTraceMatrix({ // 'Y.rs', // 'Automerge', // 'collabs', + // 'loro', ], iterationsPerEditor: 50, }); diff --git a/src/json-crdt/__bench__/util/editors.ts b/src/json-crdt/__bench__/util/editors.ts index aff18fed18..933aebd478 100644 --- a/src/json-crdt/__bench__/util/editors.ts +++ b/src/json-crdt/__bench__/util/editors.ts @@ -6,6 +6,7 @@ import * as Y from 'yjs'; import Yrs from 'ywasm'; import * as Automerge from '@automerge/automerge'; import {CRuntime, CText} from '@collabs/collabs'; +import {Loro} from 'loro-crdt'; import type {SequentialTraceEditor} from './types'; const Rope = require('rope.js'); @@ -203,6 +204,25 @@ const editorRopeJs: SequentialTraceEditor = { }, }; +const editorLoro: SequentialTraceEditor = { + name: 'loro', + factory: () => { + const doc = new Loro(); + const text = doc.getText('text'); + return { + ins: (pos: number, insert: string) => { + text.insert(pos, insert); + }, + del: (pos: number, len: number) => { + text.delete(pos, len); + }, + get: () => text.toString(), + len: () => text.toString().length, + chunks: () => 0, + }; + }, +}; + export const editors = { 'StrNode (json-joy)': editorStrNode, 'json-joy': editorJsonJoy, @@ -213,6 +233,7 @@ export const editors = { 'diamond-types-node': editorDiamondTypesNode, 'V8 strings': editorV8Strings, 'rope.js': editorRopeJs, + loro: editorLoro, }; export type SequentialEditorName = keyof typeof editors; diff --git a/src/json-crdt/__tests__/fuzzer/Picker.ts b/src/json-crdt/__tests__/fuzzer/Picker.ts index 4e3a130c60..37a9370b5d 100644 --- a/src/json-crdt/__tests__/fuzzer/Picker.ts +++ b/src/json-crdt/__tests__/fuzzer/Picker.ts @@ -71,7 +71,9 @@ export class Picker { public generateObjectKey(): string { const useCommonKey = Math.random() < 0.25; if (useCommonKey) { - return Fuzzer.pick(commonKeys); + const str = Fuzzer.pick(commonKeys); + if (this.opts.noProtoString && str === '__proto__') return this.generateObjectKey(); + return str; } else { const length = Math.floor(Math.random() * 20) + 1; return RandomJson.genString(length); diff --git a/src/json-crdt/__tests__/fuzzer/types.ts b/src/json-crdt/__tests__/fuzzer/types.ts index 6d8ceca255..a11ee39446 100644 --- a/src/json-crdt/__tests__/fuzzer/types.ts +++ b/src/json-crdt/__tests__/fuzzer/types.ts @@ -31,4 +31,7 @@ export interface FuzzerOptions { /** Whether to serialize/deserialize models and patches. */ testCodecs: boolean; + + /** Do not generate "__proto__" as a string, so it does not appear as object key. */ + noProtoString?: boolean; } diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 7c809f3f6f..0d9fe1c01f 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -133,14 +133,9 @@ export class Decoder { const decoder = this.dec; const reader = decoder.reader; const id = this.ts(); - const isTombstone = reader.uint8[reader.x] === 0; - if (isTombstone) { - reader.x++; - const length = reader.vu57(); - return new nodes.StrChunk(id, length, ''); - } - const text: string = decoder.readAsStr() as string; - return new nodes.StrChunk(id, text.length, text); + const val = decoder.val(); + if (typeof val === 'string') return new nodes.StrChunk(id, val.length, val); + return new nodes.StrChunk(id, ~~(val), ''); }; protected decodeBin(id: ITimestampStruct, length: number): nodes.BinNode { @@ -154,7 +149,7 @@ export class Decoder { const reader = this.dec.reader; const [deleted, length] = reader.b1vu56(); if (deleted) return new nodes.BinChunk(id, length, undefined); - else return new nodes.BinChunk(id, length, reader.buf(length)); + return new nodes.BinChunk(id, length, reader.buf(length)); }; protected decodeArr(id: ITimestampStruct, length: number): nodes.ArrNode { diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 2989d05086..e9cf90a6eb 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -129,10 +129,8 @@ export class Encoder { this.writeTL(CRDT_MAJOR_OVERLAY.STR, node.count); for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { this.ts(chunk.id); - if (chunk.del) { - writer.u8(0); - writer.vu57(chunk.span); - } else encoder.writeStr(chunk.data!); + if (chunk.del) encoder.writeUInteger(chunk.span); + else encoder.writeStr(chunk.data!); } return writer.flush(); } diff --git a/src/json-crdt/codec/sidecar/binary/Decoder.ts b/src/json-crdt/codec/sidecar/binary/Decoder.ts index 66d0070b25..25b6e7689b 100644 --- a/src/json-crdt/codec/sidecar/binary/Decoder.ts +++ b/src/json-crdt/codec/sidecar/binary/Decoder.ts @@ -6,6 +6,7 @@ import {CborDecoderBase} from '../../../../json-pack/cbor/CborDecoderBase'; import * as nodes from '../../../nodes'; import {CRDT_MAJOR} from '../../structural/binary/constants'; import {sort} from '../../../../util/sort/insertion'; +import {SESSION} from '../../../../json-crdt-patch/constants'; export class Decoder { protected doc!: Model; @@ -63,7 +64,7 @@ export class Decoder { const length = minor < 24 ? minor : minor === 24 ? reader.u8() : minor === 25 ? reader.u16() : reader.u32(); switch (major) { case CRDT_MAJOR.CON: - return this.cCon(view, id); + return this.cCon(view, id, length); case CRDT_MAJOR.VAL: return this.cVal(view, id); case CRDT_MAJOR.OBJ: @@ -80,9 +81,9 @@ export class Decoder { throw new Error('UNKNOWN_NODE'); } - protected cCon(view: unknown, id: ITimestampStruct): nodes.ConNode { + protected cCon(view: unknown, id: ITimestampStruct, length: number): nodes.ConNode { const doc = this.doc; - const node = new nodes.ConNode(id, view); + const node = new nodes.ConNode(id, length ? this.ts() : view); doc.index.set(id, node); return node; } @@ -116,11 +117,10 @@ export class Decoder { const elements = obj.elements; const reader = this.decoder.reader; for (let i = 0; i < length; i++) { - const octet = reader.peak(); - if (octet === 0xff) { - reader.x++; - elements.push(undefined); - } else elements.push(this.cNode(view[i]).id); + const child = this.cNode(view[i]); + const childId = child.id; + if (childId.sid === SESSION.SYSTEM) elements.push(undefined); + else elements.push(childId); } this.doc.index.set(id, obj); return obj; @@ -133,8 +133,8 @@ export class Decoder { let offset = 0; node.ingest(length, (): nodes.StrChunk => { const id = this.ts(); - const span = reader.vu57(); - if (!span) return new nodes.StrChunk(id, length, ''); + const [deleted, span] = reader.b1vu56(); + if (deleted) return new nodes.StrChunk(id, span, ''); const text = view.slice(offset, offset + span); offset += span; return new nodes.StrChunk(id, text.length, text); @@ -150,8 +150,8 @@ export class Decoder { let offset = 0; node.ingest(length, (): nodes.BinChunk => { const id = this.ts(); - const span = reader.vu57(); - if (!span) return new nodes.BinChunk(id, length, undefined); + const [deleted, span] = reader.b1vu56(); + if (deleted) return new nodes.BinChunk(id, span, undefined); const slice = view.slice(offset, offset + span); offset += span; return new nodes.BinChunk(id, slice.length, slice); diff --git a/src/json-crdt/codec/sidecar/binary/Encoder.ts b/src/json-crdt/codec/sidecar/binary/Encoder.ts index cfe83a7cff..f937ae7ef5 100644 --- a/src/json-crdt/codec/sidecar/binary/Encoder.ts +++ b/src/json-crdt/codec/sidecar/binary/Encoder.ts @@ -6,6 +6,7 @@ import {CborEncoder} from '../../../../json-pack/cbor/CborEncoder'; import {SESSION} from '../../../../json-crdt-patch/constants'; import {CRDT_MAJOR_OVERLAY} from '../../structural/binary/constants'; import {sort} from '../../../../util/sort/insertion'; +import {UNDEFINED} from '../../../model/Model'; import type {Model} from '../../../model'; export class Encoder { @@ -86,18 +87,20 @@ export class Encoder { protected cCon(node: nodes.ConNode): void { const val = node.val; this.ts(node.id); + const metaEncoder = this.metaEncoder; if (val instanceof Timestamp) { this.viewEncoder.writeNull(); - this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1); + metaEncoder.writer.u8(1); // this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1); + this.ts(val); } else { this.viewEncoder.writeAny(val); - this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0); + metaEncoder.writer.u8(0); // this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0); } } protected cVal(node: nodes.ValNode): void { this.ts(node.id); - this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0); + this.metaEncoder.writer.u8(0b00100000); // this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0); this.cNode(node.node()); } @@ -124,7 +127,7 @@ export class Encoder { const index = this.doc.index; for (let i = 0; i < length; i++) { const elementId = elements[i]; - if (!elementId) this.metaEncoder.writer.u8(0xff); + if (!elementId) this.cCon(UNDEFINED); else this.cNode(index.get(elementId)!); } } @@ -137,7 +140,7 @@ export class Encoder { const writer = this.metaEncoder.writer; for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { ts(chunk.id); - writer.vu57(chunk.span); + writer.b1vu56(~~chunk.del as 0 | 1, chunk.span); } } @@ -149,7 +152,7 @@ export class Encoder { const writer = this.metaEncoder.writer; for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { ts(chunk.id); - writer.vu57(chunk.span); + writer.b1vu56(~~chunk.del as 0 | 1, chunk.span); } } @@ -165,7 +168,7 @@ export class Encoder { const deleted = chunk.del; const span = chunk.span; writer.b1vu56(~~deleted as 0 | 1, span); - if (span) { + if (!deleted) { const elements = chunk.data!; const elementsLength = elements.length; for (let i = 0; i < elementsLength; i++) this.cNode(index.get(elements[i])!); diff --git a/src/json-crdt/codec/sidecar/binary/__tests__/Encoder.spec.ts b/src/json-crdt/codec/sidecar/binary/__tests__/Encoder.spec.ts index 105ba3fed6..d04720cb45 100644 --- a/src/json-crdt/codec/sidecar/binary/__tests__/Encoder.spec.ts +++ b/src/json-crdt/codec/sidecar/binary/__tests__/Encoder.spec.ts @@ -3,6 +3,7 @@ import {CborDecoder} from '../../../../../json-pack/cbor/CborDecoder'; import {Model} from '../../../../model'; import {Encoder} from '../Encoder'; import {Decoder} from '../Decoder'; +import {Timestamp} from '../../../../../json-crdt-patch/clock'; test('con', () => { const model = Model.withLogicalClock(); @@ -19,6 +20,21 @@ test('con', () => { expect(decoded.clock.time).toBe(model.clock.time); }); +test('con - timestamp', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root(s.con(new Timestamp(666, 1))); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.decode(view); + const decoded = decoder.decode(viewDecoded, meta); + expect(model.view()).toEqual(decoded.view()); + expect(viewDecoded).toEqual(null); + expect(decoded.clock.sid).toBe(model.clock.sid); + expect(decoded.clock.time).toBe(model.clock.time); +}); + test('val', () => { const model = Model.withLogicalClock(); const encoder = new Encoder(); @@ -68,6 +84,44 @@ test('obj - 2', () => { expect(decoded.clock.time).toBe(model.clock.time); }); +test('obj - with deleted keys', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root({ + a: 1, + b: 2, + c: 3, + d: 4, + }); + model.api.obj([]).del(['b', 'd']); + expect(model.view()).toEqual({a: 1, c: 3}); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.decode(view); + const decoded = decoder.decode(viewDecoded, meta); + expect(decoded.view()).toEqual({a: 1, c: 3}); +}); + +test('obj - supports "__proto__" key', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root({ + a: 1, + b: 2, + __proto__: 3, + d: 4, + }); + model.api.obj([]).del(['b', 'd']); + expect(model.view()).toEqual({a: 1, __proto__: 3}); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.decode(view); + const decoded = decoder.decode(viewDecoded, meta); + expect(decoded.view()).toEqual({a: 1, __proto__: 3}); +}); + test('vec', () => { const model = Model.withLogicalClock(); const encoder = new Encoder(); @@ -90,7 +144,7 @@ test('vec - 2', () => { const cborDecoder = new CborDecoder(); model.api.root(s.vec(s.con(false), s.con(1), s.con(null))); const [view, meta] = encoder.encode(model); - const viewDecoded = cborDecoder.decode(view); + const viewDecoded = cborDecoder.read(view); const decoded = decoder.decode(viewDecoded, meta); expect(model.view()).toEqual(decoded.view()); expect(model.view()).toEqual(viewDecoded); @@ -98,6 +152,22 @@ test('vec - 2', () => { expect(decoded.clock.time).toBe(model.clock.time); }); +test('vec - with gaps', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root(s.vec(s.con(1))); + model.api.vec([]).set([[2, s.con(3)]]); + expect(model.view()).toStrictEqual([1, undefined, 3]); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.read(view); + const decoded = decoder.decode(viewDecoded, meta); + expect(decoded.view()).toStrictEqual([1, undefined, 3]); + expect(decoded.clock.sid).toBe(model.clock.sid); + expect(decoded.clock.time).toBe(model.clock.time); +}); + test('str', () => { const model = Model.withLogicalClock(); const encoder = new Encoder(); @@ -106,7 +176,7 @@ test('str', () => { model.api.root('Hello'); model.api.str([]).ins(5, ' World'); const [view, meta] = encoder.encode(model); - const viewDecoded = cborDecoder.decode(view); + const viewDecoded = cborDecoder.read(view); const decoded = decoder.decode(viewDecoded, meta); expect(model.view()).toEqual(decoded.view()); expect(model.view()).toEqual(viewDecoded); @@ -114,6 +184,23 @@ test('str', () => { expect(decoded.clock.time).toBe(model.clock.time); }); +test('str - with deleted chunks', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root('Hello'); + model.api.str([]).ins(5, ' World'); + const model2 = model.fork(); + model2.api.str([]).ins(3, '~'); + model.api.str([]).del(2, 2); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.read(view); + const decoded = decoder.decode(viewDecoded, meta); + decoded.applyPatch(model2.api.flush()); + expect(decoded.view()).toBe('He~o World'); +}); + test('bin', () => { const model = Model.withLogicalClock(); const encoder = new Encoder(); @@ -130,6 +217,22 @@ test('bin', () => { expect(decoded.clock.time).toBe(model.clock.time); }); +test('bin - with deleted chunks', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root(new Uint8Array([1, 2, 3, 4, 5])); + const model2 = model.fork(); + model.api.bin([]).del(1, 2); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.read(view); + const decoded = decoder.decode(viewDecoded, meta); + model2.api.bin([]).ins(2, new Uint8Array([6, 7])); + decoded.applyPatch(model2.api.flush()); + expect(decoded.view()).toEqual(new Uint8Array([1, 6, 7, 4, 5])); +}); + test('arr', () => { const model = Model.withLogicalClock(); const encoder = new Encoder(); @@ -138,10 +241,26 @@ test('arr', () => { model.api.root([1, 2, 3]); model.api.arr([]).ins(3, [4, 5, 6]); const [view, meta] = encoder.encode(model); - const viewDecoded = cborDecoder.decode(view); + const viewDecoded = cborDecoder.read(view); const decoded = decoder.decode(viewDecoded, meta); expect(model.view()).toEqual(decoded.view()); expect(model.view()).toEqual(viewDecoded); expect(decoded.clock.sid).toBe(model.clock.sid); expect(decoded.clock.time).toBe(model.clock.time); }); + +test('arr - with deleted chunks', () => { + const model = Model.withLogicalClock(); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + model.api.root([1, 2, 3, 4, 5]); + const model2 = model.fork(); + model.api.arr([]).del(1, 2); + const [view, meta] = encoder.encode(model); + const viewDecoded = cborDecoder.read(view); + const decoded = decoder.decode(viewDecoded, meta); + model2.api.arr([]).ins(2, [6, 7]); + decoded.applyPatch(model2.api.flush()); + expect(decoded.view()).toEqual([1, 6, 7, 4, 5]); +}); diff --git a/src/json-crdt/codec/sidecar/binary/__tests__/all-types-smoketest.spec.ts b/src/json-crdt/codec/sidecar/binary/__tests__/all-types-smoketest.spec.ts new file mode 100644 index 0000000000..f679991c9d --- /dev/null +++ b/src/json-crdt/codec/sidecar/binary/__tests__/all-types-smoketest.spec.ts @@ -0,0 +1,17 @@ +import {Model} from '../../../../model'; +import {runCodecAllTypesSmokeTests} from '../../../structural/verbose/__tests__/runCodecAllTypesSmokeTests'; +import {Encoder} from '../Encoder'; +import {Decoder} from '../Decoder'; +import {CborDecoder} from '../../../../../json-pack/cbor/CborDecoder'; + +const encoder = new Encoder(); +const decoder = new Decoder(); +const cborDecoder = new CborDecoder(); + +const assertCodec = (doc: Model) => { + const [view, sidecar] = encoder.encode(doc); + const decoded = decoder.decode(cborDecoder.read(view), sidecar); + expect(doc.view()).toEqual(decoded.view()); +}; + +runCodecAllTypesSmokeTests(assertCodec); diff --git a/src/json-crdt/codec/sidecar/binary/__tests__/automated-logical.spec.ts b/src/json-crdt/codec/sidecar/binary/__tests__/automated-logical.spec.ts new file mode 100644 index 0000000000..e7e07cfa9b --- /dev/null +++ b/src/json-crdt/codec/sidecar/binary/__tests__/automated-logical.spec.ts @@ -0,0 +1,38 @@ +import {ClockVector} from '../../../../../json-crdt-patch/clock'; +import {Model} from '../../../../model'; +import {Encoder} from '../Encoder'; +import {Decoder} from '../Decoder'; +import {documents} from '../../../../../__tests__/json-documents'; +import {binaryDocuments} from '../../../../../__tests__/binary-documents'; +import {CborDecoder} from '../../../../../json-pack/cbor/CborDecoder'; + +for (const {name, json} of [...documents, ...binaryDocuments]) { + describe('fresh encoder and decoder', () => { + test(name, () => { + const doc1 = Model.withLogicalClock(new ClockVector(222, 0)); + doc1.api.root(json); + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + const [view, sidecar] = encoder.encode(doc1); + const doc2 = decoder.decode(cborDecoder.read(view), sidecar); + expect(doc1.view()).toEqual(json); + expect(doc2.view()).toEqual(json); + }); + }); + + describe('shared encoder and decoder', () => { + const encoder = new Encoder(); + const decoder = new Decoder(); + const cborDecoder = new CborDecoder(); + + test(name, () => { + const doc1 = Model.withLogicalClock(new ClockVector(222, 0)); + doc1.api.root(json); + const [view, sidecar] = encoder.encode(doc1); + const doc2 = decoder.decode(cborDecoder.read(view), sidecar); + expect(doc1.view()).toEqual(json); + expect(doc2.view()).toEqual(json); + }); + }); +} diff --git a/src/json-crdt/codec/sidecar/binary/__tests__/fuzzer-logical.spec.ts b/src/json-crdt/codec/sidecar/binary/__tests__/fuzzer-logical.spec.ts new file mode 100644 index 0000000000..315b63d0bd --- /dev/null +++ b/src/json-crdt/codec/sidecar/binary/__tests__/fuzzer-logical.spec.ts @@ -0,0 +1,30 @@ +import {JsonCrdtFuzzer} from '../../../../__tests__/fuzzer/JsonCrdtFuzzer'; +import {Encoder} from '../Encoder'; +import {Decoder} from '../Decoder'; +import {CborDecoder} from '../../../../../json-pack/cbor/CborDecoder'; + +const encoder = new Encoder(); +const decoder = new Decoder(); +const cborDecoder = new CborDecoder(); + +const runs = 10; +const sessionNum = 5; + +test('serialization fuzzing tests', () => { + for (let r = 0; r < runs; r++) { + const fuzzer = new JsonCrdtFuzzer({ + testCodecs: false, + noProtoString: true, + }); + fuzzer.setupModel(); + for (let ses = 0; ses < sessionNum; ses++) { + fuzzer.executeConcurrentSession(); + const model = fuzzer.model; + const json = model.view(); + const [view, sidecar] = encoder.encode(model); + const model2 = decoder.decode(cborDecoder.read(view), sidecar); + expect(model.view()).toEqual(json); + expect(model2.view()).toEqual(json); + } + } +}); diff --git a/src/json-crdt/codec/structural/binary/Decoder.ts b/src/json-crdt/codec/structural/binary/Decoder.ts index 0347ca9c21..1de5d32c76 100644 --- a/src/json-crdt/codec/structural/binary/Decoder.ts +++ b/src/json-crdt/codec/structural/binary/Decoder.ts @@ -149,16 +149,10 @@ export class Decoder extends CborDecoderBase { } private cStrChunk = (): nodes.StrChunk => { - const reader = this.reader; const id = this.ts(); - const isTombstone = reader.uint8[reader.x] === 0; - if (isTombstone) { - reader.x++; - const length = reader.vu57(); - return new nodes.StrChunk(id, length, ''); - } - const text: string = this.readAsStr() as string; - return new nodes.StrChunk(id, text.length, text); + const val = this.val(); + if (typeof val === 'string') return new nodes.StrChunk(id, val.length, val); + return new nodes.StrChunk(id, ~~(val), ''); }; protected cBin(id: ITimestampStruct, length: number): nodes.BinNode { @@ -173,7 +167,7 @@ export class Decoder extends CborDecoderBase { const reader = this.reader; const [deleted, length] = reader.b1vu56(); if (deleted) return new nodes.BinChunk(id, length, undefined); - else return new nodes.BinChunk(id, length, reader.buf(length)); + return new nodes.BinChunk(id, length, reader.buf(length)); }; protected cArr(id: ITimestampStruct, length: number): nodes.ArrNode { diff --git a/src/json-crdt/codec/structural/binary/Encoder.ts b/src/json-crdt/codec/structural/binary/Encoder.ts index 356260046f..9734da685e 100644 --- a/src/json-crdt/codec/structural/binary/Encoder.ts +++ b/src/json-crdt/codec/structural/binary/Encoder.ts @@ -145,15 +145,12 @@ export class Encoder extends CborEncoder { protected cStr(node: nodes.StrNode): void { const ts = this.ts; - const writer = this.writer; ts(node.id); this.writeTL(CRDT_MAJOR_OVERLAY.STR, node.count); for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { ts(chunk.id); - if (chunk.del) { - writer.u8(0); - writer.vu57(chunk.span); - } else this.writeStr(chunk.data!); + if (chunk.del) this.writeUInteger(chunk.span); + else this.writeStr(chunk.data!); } } @@ -166,7 +163,7 @@ export class Encoder extends CborEncoder { ts(chunk.id); const length = chunk.span; const deleted = chunk.del; - writer.b1vu56(~~chunk.del as 0 | 1, length); + writer.b1vu56(~~deleted as 0 | 1, length); if (deleted) continue; writer.buf(chunk.data!, length); } diff --git a/src/json-crdt/codec/structural/binary/ViewDecoder.ts b/src/json-crdt/codec/structural/binary/ViewDecoder.ts index 1992ed9968..187a447ede 100644 --- a/src/json-crdt/codec/structural/binary/ViewDecoder.ts +++ b/src/json-crdt/codec/structural/binary/ViewDecoder.ts @@ -88,18 +88,11 @@ export class ViewDecoder extends CborDecoderBase { } protected cStr(length: number): string { - const reader = this.reader; let str = ''; for (let i = 0; i < length; i++) { this.ts(); - const isTombstone = reader.uint8[reader.x] === 0; - if (isTombstone) { - reader.x++; - reader.vu57Skip(); - continue; - } - const text: string = this.val() as string; - str += text; + const val = this.val(); + if (typeof val === 'string') str += val; } return str; } diff --git a/src/json-crdt/codec/structural/binary/__tests__/all-types-smoketest.spec.ts b/src/json-crdt/codec/structural/binary/__tests__/all-types-smoketest.spec.ts index 9d0e6c0d21..6cce9985a5 100644 --- a/src/json-crdt/codec/structural/binary/__tests__/all-types-smoketest.spec.ts +++ b/src/json-crdt/codec/structural/binary/__tests__/all-types-smoketest.spec.ts @@ -6,6 +6,7 @@ const assertCodec = (doc: Model) => { const encoded = doc.toBinary(); const decoded = Model.fromBinary(encoded); expect(doc.view()).toEqual(decoded.view()); + expect(doc.toString()).toEqual(decoded.toString()); }; const view = new ViewDecoder(); diff --git a/src/json-crdt/codec/structural/binary/__tests__/codec-logical.spec.ts b/src/json-crdt/codec/structural/binary/__tests__/codec-logical.spec.ts index d3cc68c3a7..b27333ebb0 100644 --- a/src/json-crdt/codec/structural/binary/__tests__/codec-logical.spec.ts +++ b/src/json-crdt/codec/structural/binary/__tests__/codec-logical.spec.ts @@ -3,6 +3,7 @@ import {Encoder} from '../Encoder'; import {Decoder} from '../Decoder'; import {compare, equal, Timestamp, ClockVector} from '../../../../../json-crdt-patch/clock'; import {konst} from '../../../../../json-crdt-patch/builder/Konst'; +import {s} from '../../../../../json-crdt-patch'; const encoder = new Encoder(); const decoder = new Decoder(); @@ -117,3 +118,13 @@ test('can encode ID as const value', () => { expect(ts).toBeInstanceOf(Timestamp); expect(equal(ts, new Timestamp(model.clock.sid, 2))).toBe(true); }); + +test('can encode timestamp in "con" node', () => { + const model = Model.withLogicalClock(); + model.api.root(s.con(new Timestamp(666, 1))); + const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(model.view()).toEqual(decoded.view()); + expect(decoded.clock.sid).toBe(model.clock.sid); + expect(decoded.clock.time).toBe(model.clock.time); +}); diff --git a/src/json-crdt/codec/structural/verbose/__tests__/runCodecAllTypesSmokeTests.ts b/src/json-crdt/codec/structural/verbose/__tests__/runCodecAllTypesSmokeTests.ts index e183cc8aea..37754a7e23 100644 --- a/src/json-crdt/codec/structural/verbose/__tests__/runCodecAllTypesSmokeTests.ts +++ b/src/json-crdt/codec/structural/verbose/__tests__/runCodecAllTypesSmokeTests.ts @@ -1,3 +1,4 @@ +import {s} from '../../../../../json-crdt-patch'; import {konst} from '../../../../../json-crdt-patch/builder/Konst'; import {vec} from '../../../../../json-crdt-patch/builder/Tuple'; import {Model} from '../../../../model'; @@ -50,4 +51,17 @@ export const runCodecAllTypesSmokeTests = (assertCodec: (doc: Model) => void) => model.api.root({foo: 'bar', empty: {}}); assertCodec(model); }); + + test('vector', () => { + const model = Model.withLogicalClock(); + model.api.root(s.vec(s.con(1), s.con(2), s.con(3))); + assertCodec(model); + }); + + test('vector - with gaps', () => { + const model = Model.withLogicalClock(); + model.api.root(s.vec(s.con(1))); + model.api.vec([]).set([[2, s.con(3)]]); + assertCodec(model); + }); }; diff --git a/yarn.lock b/yarn.lock index 42d03311a9..8d331fc535 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3813,6 +3813,18 @@ loose-envify@^1.1.0: dependencies: js-tokens "^3.0.0 || ^4.0.0" +loro-crdt@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/loro-crdt/-/loro-crdt-0.4.1.tgz#4a8cf0602c7232a66f20b4d9a05d236b14d0192c" + integrity sha512-VoGJ6dPu38QqMOztU2q5GQgbV5cl0oZ4U0Trgjfz7l/u5S0RHpMhiZhSqSeYO4sieDtTA97GMirt0jYrJO3TGw== + dependencies: + loro-wasm "0.4.1" + +loro-wasm@0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/loro-wasm/-/loro-wasm-0.4.1.tgz#93e2aeb35d7518c9ad482388bcc35b838582c50a" + integrity sha512-tE+cH1AM8/IoWeA3gwq0DalsXVzqjH1uVL390xyjTvrmAtiNXfPxR8PSQDZQ+z6vTreJToLih2Hc9ajHC99JZA== + lower-case@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.2.tgz#6fa237c63dbdc4a82ca0fd882e4722dc5e634e28"