Adjust JSON CRDT binary encodings according to specs

package.json

@@ -99,7 +99,6 @@
     "concurrently": "^8.0.1",
     "diamond-types-node": "1.0.2",
     "editing-traces": "https://github.com/streamich/editing-traces#6494020428530a6e382378b98d1d7e31334e2d7b",
-    "json-crdt-traces": "https://github.com/streamich/json-crdt-traces#5f68a13baf798897d39b87d7028b0b3cd5a50a6c",
     "eventsource": "^2.0.2",
     "fast-deep-equal": "^3.1.3",
     "fast-equals": "^5.0.1",
@@ -118,10 +117,12 @@
     "js-base64": "^3.7.2",
     "js-sdsl": "^4.4.0",
     "jsbi": "^4.3.0",
+    "json-crdt-traces": "https://github.com/streamich/json-crdt-traces#5f68a13baf798897d39b87d7028b0b3cd5a50a6c",
     "json-logic-js": "^2.0.1",
     "json-pack-napi": "^0.0.2",
     "load-script": "^2.0.0",
     "lodash": "^4.17.21",
+    "loro-crdt": "^0.4.1",
     "markdown-it": "^13.0.1",
     "messagepack": "^1.1.12",
     "msgpack-lite": "^0.1.26",

src/json-crdt-patch/codec/clock/ClockEncoder.ts

@@ -1,4 +1,4 @@
-import {ITimestampStruct, IClockVector, tick} from '../../clock';
+import {ITimestampStruct, IClockVector, tick, Timestamp} from '../../clock';
 import {RelativeTimestamp} from './RelativeTimestamp';
 
 class ClockTableEntry {
@@ -24,12 +24,14 @@ export class ClockEncoder {
     const sid = ts.sid;
     let entry = this.table.get(sid);
     if (!entry) {
-      const clock = this.clock!.peers.get(sid)!;
+      let clock = this.clock!.peers.get(sid);
+      if (!clock) clock = new Timestamp(sid, this.clock!.time - 1);
       entry = new ClockTableEntry(this.index++, clock);
       this.table.set(sid, entry);
     }
     const clock = entry.clock;
     const timeDiff = clock.time - time;
+    if (timeDiff < 0) throw new Error('TIME_TRAVEL');
     return new RelativeTimestamp(entry.index, timeDiff);
   }
 

src/json-crdt-patch/codec/clock/__tests__/ClockEncoder.spec.ts

@@ -50,16 +50,3 @@ test('encodes each clock only once', () => {
   const encoded = encoder.toJson();
   expect(encoded).toEqual([100, 100, 50, 50, 10, 10]);
 });
-
-test('throws when unknown clock is being encoded', () => {
-  const clock = new ClockVector(100, 100);
-  const ts1 = ts(50, 50);
-  const ts2 = ts(10, 10);
-  clock.observe(ts1, 1);
-  clock.observe(ts2, 1);
-  const encoder = new ClockEncoder();
-  encoder.reset(clock);
-  encoder.append(ts1);
-  encoder.append(ts2);
-  expect(() => encoder.append(ts(77, 77))).toThrow();
-});

src/json-crdt/__bench__/bench.traces.crdt-libs.ts

@@ -22,6 +22,7 @@ runTraceMatrix({
     // 'Y.rs',
     // 'Automerge',
     // 'collabs',
+    // 'loro',
   ],
   iterationsPerEditor: 50,
 });

src/json-crdt/__bench__/util/editors.ts

@@ -6,6 +6,7 @@ import * as Y from 'yjs';
 import Yrs from 'ywasm';
 import * as Automerge from '@automerge/automerge';
 import {CRuntime, CText} from '@collabs/collabs';
+import {Loro} from 'loro-crdt';
 import type {SequentialTraceEditor} from './types';
 const Rope = require('rope.js');
 
@@ -203,6 +204,25 @@ const editorRopeJs: SequentialTraceEditor = {
   },
 };
 
+const editorLoro: SequentialTraceEditor = {
+  name: 'loro',
+  factory: () => {
+    const doc = new Loro();
+    const text = doc.getText('text');
+    return {
+      ins: (pos: number, insert: string) => {
+        text.insert(pos, insert);
+      },
+      del: (pos: number, len: number) => {
+        text.delete(pos, len);
+      },
+      get: () => text.toString(),
+      len: () => text.toString().length,
+      chunks: () => 0,
+    };
+  },
+};
+
 export const editors = {
   'StrNode (json-joy)': editorStrNode,
   'json-joy': editorJsonJoy,
@@ -213,6 +233,7 @@ export const editors = {
   'diamond-types-node': editorDiamondTypesNode,
   'V8 strings': editorV8Strings,
   'rope.js': editorRopeJs,
+  loro: editorLoro,
 };
 
 export type SequentialEditorName = keyof typeof editors;

src/json-crdt/__tests__/fuzzer/Picker.ts

@@ -71,7 +71,9 @@ export class Picker {
   public generateObjectKey(): string {
     const useCommonKey = Math.random() < 0.25;
     if (useCommonKey) {
-      return Fuzzer.pick(commonKeys);
+      const str = Fuzzer.pick(commonKeys);
+      if (this.opts.noProtoString && str === '__proto__') return this.generateObjectKey();
+      return str;
     } else {
       const length = Math.floor(Math.random() * 20) + 1;
       return RandomJson.genString(length);

src/json-crdt/__tests__/fuzzer/types.ts

@@ -31,4 +31,7 @@ export interface FuzzerOptions {
 
   /** Whether to serialize/deserialize models and patches. */
   testCodecs: boolean;
+
+  /** Do not generate "__proto__" as a string, so it does not appear as object key. */
+  noProtoString?: boolean;
 }

src/json-crdt/codec/indexed/binary/Decoder.ts

@@ -133,14 +133,9 @@ export class Decoder {
     const decoder = this.dec;
     const reader = decoder.reader;
     const id = this.ts();
-    const isTombstone = reader.uint8[reader.x] === 0;
-    if (isTombstone) {
-      reader.x++;
-      const length = reader.vu57();
-      return new nodes.StrChunk(id, length, '');
-    }
-    const text: string = decoder.readAsStr() as string;
-    return new nodes.StrChunk(id, text.length, text);
+    const val = decoder.val();
+    if (typeof val === 'string') return new nodes.StrChunk(id, val.length, val);
+    return new nodes.StrChunk(id, ~~(<number>val), '');
   };
 
   protected decodeBin(id: ITimestampStruct, length: number): nodes.BinNode {
@@ -154,7 +149,7 @@ export class Decoder {
     const reader = this.dec.reader;
     const [deleted, length] = reader.b1vu56();
     if (deleted) return new nodes.BinChunk(id, length, undefined);
-    else return new nodes.BinChunk(id, length, reader.buf(length));
+    return new nodes.BinChunk(id, length, reader.buf(length));
   };
 
   protected decodeArr(id: ITimestampStruct, length: number): nodes.ArrNode {

src/json-crdt/codec/indexed/binary/Encoder.ts

@@ -129,10 +129,8 @@ export class Encoder {
     this.writeTL(CRDT_MAJOR_OVERLAY.STR, node.count);
     for (let chunk = node.first(); chunk; chunk = node.next(chunk)) {
       this.ts(chunk.id);
-      if (chunk.del) {
-        writer.u8(0);
-        writer.vu57(chunk.span);
-      } else encoder.writeStr(chunk.data!);
+      if (chunk.del) encoder.writeUInteger(chunk.span);
+      else encoder.writeStr(chunk.data!);
     }
     return writer.flush();
   }

src/json-crdt/codec/sidecar/binary/Decoder.ts

@@ -6,6 +6,7 @@ import {CborDecoderBase} from '../../../../json-pack/cbor/CborDecoderBase';
 import * as nodes from '../../../nodes';
 import {CRDT_MAJOR} from '../../structural/binary/constants';
 import {sort} from '../../../../util/sort/insertion';
+import {SESSION} from '../../../../json-crdt-patch/constants';
 
 export class Decoder {
   protected doc!: Model;
@@ -63,7 +64,7 @@ export class Decoder {
     const length = minor < 24 ? minor : minor === 24 ? reader.u8() : minor === 25 ? reader.u16() : reader.u32();
     switch (major) {
       case CRDT_MAJOR.CON:
-        return this.cCon(view, id);
+        return this.cCon(view, id, length);
       case CRDT_MAJOR.VAL:
         return this.cVal(view, id);
       case CRDT_MAJOR.OBJ:
@@ -80,9 +81,9 @@ export class Decoder {
     throw new Error('UNKNOWN_NODE');
   }
 
-  protected cCon(view: unknown, id: ITimestampStruct): nodes.ConNode {
+  protected cCon(view: unknown, id: ITimestampStruct, length: number): nodes.ConNode {
     const doc = this.doc;
-    const node = new nodes.ConNode(id, view);
+    const node = new nodes.ConNode(id, length ? this.ts() : view);
     doc.index.set(id, node);
     return node;
   }
@@ -116,11 +117,10 @@ export class Decoder {
     const elements = obj.elements;
     const reader = this.decoder.reader;
     for (let i = 0; i < length; i++) {
-      const octet = reader.peak();
-      if (octet === 0xff) {
-        reader.x++;
-        elements.push(undefined);
-      } else elements.push(this.cNode(view[i]).id);
+      const child = this.cNode(view[i]);
+      const childId = child.id;
+      if (childId.sid === SESSION.SYSTEM) elements.push(undefined);
+      else elements.push(childId);
     }
     this.doc.index.set(id, obj);
     return obj;
@@ -133,8 +133,8 @@ export class Decoder {
     let offset = 0;
     node.ingest(length, (): nodes.StrChunk => {
       const id = this.ts();
-      const span = reader.vu57();
-      if (!span) return new nodes.StrChunk(id, length, '');
+      const [deleted, span] = reader.b1vu56();
+      if (deleted) return new nodes.StrChunk(id, span, '');
       const text = view.slice(offset, offset + span);
       offset += span;
       return new nodes.StrChunk(id, text.length, text);
@@ -150,8 +150,8 @@ export class Decoder {
     let offset = 0;
     node.ingest(length, (): nodes.BinChunk => {
       const id = this.ts();
-      const span = reader.vu57();
-      if (!span) return new nodes.BinChunk(id, length, undefined);
+      const [deleted, span] = reader.b1vu56();
+      if (deleted) return new nodes.BinChunk(id, span, undefined);
       const slice = view.slice(offset, offset + span);
       offset += span;
       return new nodes.BinChunk(id, slice.length, slice);

src/json-crdt/codec/sidecar/binary/Encoder.ts

@@ -6,6 +6,7 @@ import {CborEncoder} from '../../../../json-pack/cbor/CborEncoder';
 import {SESSION} from '../../../../json-crdt-patch/constants';
 import {CRDT_MAJOR_OVERLAY} from '../../structural/binary/constants';
 import {sort} from '../../../../util/sort/insertion';
+import {UNDEFINED} from '../../../model/Model';
 import type {Model} from '../../../model';
 
 export class Encoder {
@@ -86,18 +87,20 @@ export class Encoder {
   protected cCon(node: nodes.ConNode): void {
     const val = node.val;
     this.ts(node.id);
+    const metaEncoder = this.metaEncoder;
     if (val instanceof Timestamp) {
       this.viewEncoder.writeNull();
-      this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1);
+      metaEncoder.writer.u8(1); // this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1);
+      this.ts(val);
     } else {
       this.viewEncoder.writeAny(val);
-      this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0);
+      metaEncoder.writer.u8(0); // this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0);
     }
   }
 
   protected cVal(node: nodes.ValNode): void {
     this.ts(node.id);
-    this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0);
+    this.metaEncoder.writer.u8(0b00100000); // this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0);
     this.cNode(node.node());
   }
 
@@ -124,7 +127,7 @@ export class Encoder {
     const index = this.doc.index;
     for (let i = 0; i < length; i++) {
       const elementId = elements[i];
-      if (!elementId) this.metaEncoder.writer.u8(0xff);
+      if (!elementId) this.cCon(UNDEFINED);
       else this.cNode(index.get(elementId)!);
     }
   }
@@ -137,7 +140,7 @@ export class Encoder {
     const writer = this.metaEncoder.writer;
     for (let chunk = node.first(); chunk; chunk = node.next(chunk)) {
       ts(chunk.id);
-      writer.vu57(chunk.span);
+      writer.b1vu56(~~chunk.del as 0 | 1, chunk.span);
     }
   }
 
@@ -149,7 +152,7 @@ export class Encoder {
     const writer = this.metaEncoder.writer;
     for (let chunk = node.first(); chunk; chunk = node.next(chunk)) {
       ts(chunk.id);
-      writer.vu57(chunk.span);
+      writer.b1vu56(~~chunk.del as 0 | 1, chunk.span);
     }
   }
 
@@ -165,7 +168,7 @@ export class Encoder {
       const deleted = chunk.del;
       const span = chunk.span;
       writer.b1vu56(~~deleted as 0 | 1, span);
-      if (span) {
+      if (!deleted) {
         const elements = chunk.data!;
         const elementsLength = elements.length;
         for (let i = 0; i < elementsLength; i++) this.cNode(index.get(elements[i])!);