RESP improvements

src/json-pack/resp/RespDecoder.ts

@@ -3,10 +3,21 @@ import {RESP} from './constants';
 import {RespAttributes, RespPush} from './extensions';
 import type {IReader, IReaderResettable} from '../../util/buffers';
 import type {BinaryJsonDecoder, PackValue} from '../types';
+import {isUtf8} from '../../util/buffers/utf8/isUtf8';
 
 export class RespDecoder<R extends IReader & IReaderResettable = IReader & IReaderResettable>
   implements BinaryJsonDecoder
 {
+  /**
+   * When set to true, the decoder will attempt to decode RESP Bulk strings
+   * (which are binary strings, i.e. Uint8Array) as UTF-8 strings. If the
+   * string is not valid UTF-8, it will be returned as a Uint8Array.
+   *
+   * You can toggle this setting at any time, before each call to `decode()`
+   * or `read()`, or other methods.
+   */
+  public tryUtf8 = false;
+
   public constructor(public reader: R = new Reader() as any) {}
 
   public read(uint8: Uint8Array): PackValue {
@@ -32,20 +43,20 @@ export class RespDecoder<R extends IReader & IReaderResettable = IReader & IRead
         return this.readFloat();
       case RESP.STR_SIMPLE:
         return this.readStrSimple();
-      case RESP.STR_VERBATIM:
-        return this.readStrVerbatim();
+      case RESP.STR_BULK:
+        return this.readStrBulk();
       case RESP.BOOL:
         return this.readBool();
       case RESP.NULL:
         return reader.skip(2), null;
-      case RESP.STR_BULK:
-        return this.readStrBulk();
       case RESP.OBJ:
         return this.readObj();
       case RESP.ARR:
         return this.readArr();
+      case RESP.STR_VERBATIM:
+        return this.readStrVerbatim();
       case RESP.PUSH:
-        return new RespPush(this.readArr());
+        return new RespPush(this.readArr() || []);
       case RESP.BIG:
         return this.readBigint();
       case RESP.SET:
@@ -158,6 +169,29 @@ export class RespDecoder<R extends IReader & IReaderResettable = IReader & IRead
     }
   }
 
+  public readStrBulk(): Uint8Array | string | null {
+    const reader = this.reader;
+    if (reader.peak() === RESP.MINUS) {
+      reader.skip(4); // Skip "-1\r\n".
+      return null;
+    }
+    const length = this.readLength();
+    let res: Uint8Array | string;
+    if (this.tryUtf8 && isUtf8(reader.uint8, reader.x, length)) res = reader.utf8(length);
+    else res = reader.buf(length);
+    reader.skip(2); // Skip "\r\n".
+    return res;
+  }
+
+  public readAsciiAsStrBulk(): string {
+    const reader = this.reader;
+    reader.skip(1); // Skip "$".
+    const length = this.readLength();
+    const buf = reader.ascii(length);
+    reader.skip(2); // Skip "\r\n".
+    return buf;
+  }
+
   public readStrVerbatim(): string | Uint8Array {
     const reader = this.reader;
     const length = this.readLength();
@@ -173,14 +207,6 @@ export class RespDecoder<R extends IReader & IReaderResettable = IReader & IRead
     return buf;
   }
 
-  public readStrBulk(): Uint8Array {
-    const reader = this.reader;
-    const length = this.readLength();
-    const buf = reader.buf(length);
-    reader.skip(2); // Skip "\r\n".
-    return buf;
-  }
-
   // ------------------------------------------------------------ Error reading
 
   public readErrSimple(): Error {
@@ -207,7 +233,13 @@ export class RespDecoder<R extends IReader & IReaderResettable = IReader & IRead
 
   // ------------------------------------------------------------ Array reading
 
-  public readArr(): unknown[] {
+  public readArr(): unknown[] | null {
+    const reader = this.reader;
+    const c = reader.peak();
+    if (c === RESP.MINUS) {
+      reader.skip(4); // Skip "-1\r\n".
+      return null;
+    }
     const length = this.readLength();
     const arr: unknown[] = [];
     for (let i = 0; i < length; i++) arr.push(this.val());

src/json-pack/resp/RespEncoder.ts

@@ -51,6 +51,38 @@ export class RespEncoder<W extends IWriter & IWriterGrowable = IWriter & IWriter
     }
   }
 
+  public encodeCmd(args: unknown[]): Uint8Array {
+    this.writeCmd(args);
+    return this.writer.flush();
+  }
+
+  public writeCmd(args: unknown[]): void {
+    const length = args.length;
+    this.writeArrHdr(length);
+    for (let i = 0; i < length; i++) this.writeArg(args[i]);
+  }
+
+  public writeArg(arg: unknown): void {
+    if (arg instanceof Uint8Array) return this.writeBin(arg);
+    else this.writeBulkStrAscii(arg + '');
+  }
+
+  public encodeCmdUtf8(args: unknown[]): Uint8Array {
+    this.writeCmdUtf8(args);
+    return this.writer.flush();
+  }
+
+  public writeCmdUtf8(args: unknown[]): void {
+    const length = args.length;
+    this.writeArrHdr(length);
+    for (let i = 0; i < length; i++) this.writeArgUtf8(args[i]);
+  }
+
+  public writeArgUtf8(arg: unknown): void {
+    if (arg instanceof Uint8Array) return this.writeBin(arg);
+    else this.writeBulkStr(arg + '');
+  }
+
   public writeNull(): void {
     this.writer.u8u16(
       RESP.NULL, // _
@@ -179,6 +211,17 @@ export class RespEncoder<W extends IWriter & IWriterGrowable = IWriter & IWriter
   }
 
   public writeBulkStr(str: string): void {
+    const writer = this.writer;
+    const size = utf8Size(str);
+    writer.u8(RESP.STR_BULK); // $
+    writer.ascii(size + '');
+    writer.u16(RESP.RN); // \r\n
+    writer.ensureCapacity(size);
+    writer.utf8(str);
+    writer.u16(RESP.RN); // \r\n
+  }
+
+  public writeBulkStrAscii(str: string): void {
     const writer = this.writer;
     writer.u8(RESP.STR_BULK); // $
     writer.ascii(str.length + '');
@@ -190,7 +233,7 @@ export class RespEncoder<W extends IWriter & IWriterGrowable = IWriter & IWriter
   public writeAsciiStr(str: string): void {
     const isSimple = !REG_RN.test(str);
     if (isSimple) this.writeSimpleStr(str);
-    else this.writeBulkStr(str);
+    else this.writeBulkStrAscii(str);
   }
 
   public writeVerbatimStr(encoding: string, str: string): void {

src/json-pack/resp/RespStreamingDecoder.ts

@@ -22,6 +22,18 @@ export class RespStreamingDecoder {
   protected readonly reader = new StreamingReader();
   protected readonly decoder = new RespDecoder(this.reader);
 
+  /**
+   * When set to true, the decoder will attempt to decode RESP Bulk strings
+   * (which are binary strings, i.e. Uint8Array) as UTF-8 strings. If the
+   * string is not valid UTF-8, it will be returned as a Uint8Array.
+   */
+  public get tryUtf8(): boolean {
+    return this.decoder.tryUtf8;
+  }
+  public set tryUtf8(value: boolean) {
+    this.decoder.tryUtf8 = value;
+  }
+
   /**
    * Add a chunk of data to be decoded.
    * @param uint8 `Uint8Array` chunk of data to be decoded.

src/json-pack/resp/__tests__/RespDecoder.spec.ts

@@ -3,6 +3,7 @@ import {RespDecoder} from '../RespDecoder';
 import {bufferToUint8Array} from '../../../util/buffers/bufferToUint8Array';
 import {RespAttributes, RespPush} from '../extensions';
 import {Writer} from '../../../util/buffers/Writer';
+import {Uint} from '@automerge/automerge';
 
 const decode = (encoded: string | Uint8Array): unknown => {
   const decoder = new RespDecoder();
@@ -171,8 +172,38 @@ const maps: [string, Record<string, unknown>][] = [
 
 describe('objects', () => {
   for (const [name, value] of maps) test(name, () => assertCodec(value));
+
+  describe('when .tryUtf8 = true', () => {
+    test('parses bulk strings as UTF8 strings', () => {
+      const encoded = '%1\r\n$3\r\nfoo\r\n$3\r\nbar\r\n';
+      const decoder = new RespDecoder();
+      decoder.tryUtf8 = true;
+      const decoded = decoder.read(Buffer.from(encoded));
+      expect(decoded).toStrictEqual({foo: 'bar'});
+    });
+
+    test('parses invalid UTF8 as Uint8Array', () => {
+      const encoded = encoder.encode({foo: new Uint8Array([0xc3, 0x28])});
+      const decoder = new RespDecoder();
+      decoder.tryUtf8 = true;
+      const decoded = decoder.read(encoded);
+      expect(decoded).toStrictEqual({foo: new Uint8Array([0xc3, 0x28])});
+    });
+  });
 });
 
 describe('attributes', () => {
   for (const [name, value] of maps) test(name, () => assertCodec(new RespAttributes(value)));
 });
+
+describe('nulls', () => {
+  test('can decode string null', () => {
+    const decoded = decode('$-1\r\n');
+    expect(decoded).toBe(null);
+  });
+
+  test('can decode array null', () => {
+    const decoded = decode('*-1\r\n');
+    expect(decoded).toBe(null);
+  });
+});

src/json-pack/resp/__tests__/RespEncoder.spec.ts

@@ -312,3 +312,36 @@ describe('streaming data', () => {
     });
   });
 });
+
+describe('commands', () => {
+  describe('.writeCmd()', () => {
+    test('can encode a simple command', () => {
+      const encoder = new RespEncoder();
+      encoder.writeCmd(['SET', 'foo', 'bar']);
+      const encoded = encoder.writer.flush();
+      expect(toStr(encoded)).toBe('*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n');
+    });
+
+    test('casts numbers to strings', () => {
+      const encoder = new RespEncoder();
+      encoder.writeCmd(['SET', 'foo', 123]);
+      const encoded = encoder.writer.flush();
+      expect(toStr(encoded)).toBe('*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\n123\r\n');
+    });
+
+    test('can encode Uint8Array', () => {
+      const encoder = new RespEncoder();
+      const encoded = encoder.encodeCmd([Buffer.from('SET'), 'foo', 123]);
+      expect(toStr(encoded)).toBe('*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\n123\r\n');
+    });
+  });
+
+  describe('.can encode emojis()', () => {
+    test('can encode a simple command', () => {
+      const encoder = new RespEncoder();
+      encoder.writeCmdUtf8(['SET', 'foo 👍', 'bar']);
+      const encoded = encoder.writer.flush();
+      expect(toStr(encoded)).toBe('*3\r\n$3\r\nSET\r\n$8\r\nfoo 👍\r\n$3\r\nbar\r\n');
+    });
+  });
+});

src/util/buffers/utf8/__tests__/isUtf8.spec.ts

@@ -0,0 +1,54 @@
+import {isUtf8} from '../isUtf8';
+
+describe('returns true for valid UTF8', () => {
+  const strings = [
+    '',
+    'hello',
+    'hello world',
+    'emoji: 🤔',
+    'russian: Привет',
+    'chinese: 你好',
+    'japanese: こんにちは',
+    'korean: 안녕하세요',
+    'arabic: مرحبا',
+    'hebrew: שלום',
+    'greek: γεια σας',
+    'bulgarian: Здравейте',
+    'hindi: नमस्ते',
+    'thai: สวัสดี',
+    'special chars: !@#$%^&*()_+{}|:"<>?`-=[]\\;\',./',
+  ];
+  for (const str of strings) {
+    test(str, () => {
+      const buf = Buffer.from(str);
+      expect(isUtf8(buf, 0, buf.length)).toBe(true);
+    });
+  }
+});
+
+describe('returns false for non-UTF8 sequences', () => {
+  const strings: [name: string, Uint8Array][] = [
+    ['two octets', Buffer.from([0xc3, 0x28])],
+    ['three octets', Buffer.from([0xe2, 0x82, 0x28])],
+    ['four octets', Buffer.from([0xf0, 0x90, 0x82, 0x28])],
+    ['five octets', Buffer.from([0xf8, 0x88, 0x82, 0x82, 0x28])],
+    ['six octets', Buffer.from([0xfc, 0x84, 0x82, 0x82, 0x82, 0x28])],
+  ];
+  for (const [name, str] of strings) {
+    test(name, () => {
+      expect(isUtf8(str, 0, str.length)).toBe(false);
+    });
+  }
+});
+
+describe('returns true for valid non-UTF8 sequences in the middle of buffer', () => {
+  const strings: [name: string, str: Uint8Array, from: number, length: number][] = [
+    ['mid char valid', Buffer.from([0xc3, 0x28, 64, 0xc3, 0x28]), 2, 1],
+  ];
+  for (const [name, str, from, length] of strings) {
+    test(name, () => {
+      expect(isUtf8(str, from, length)).toBe(true);
+      expect(isUtf8(str, 0, from + length)).toBe(false);
+    });
+  }
+});

src/util/buffers/utf8/isUtf8.ts

@@ -0,0 +1,45 @@
+/**
+ * Validates that the given data is valid UTF-8 text.
+ * @param buf Data to check.
+ * @returns True if the data is valid UTF-8.
+ */
+export const isUtf8 = (buf: Uint8Array, from: number, length: number): boolean => {
+  const to = from + length;
+  while (from < to) {
+    const c = buf[from];
+    if (c <= 0x7f) {
+      from++;
+      continue;
+    }
+    if (c >= 0xc2 && c <= 0xdf) {
+      if (buf[from + 1] >> 6 === 2) {
+        from += 2;
+        continue;
+      } else return false;
+    }
+    const c1 = buf[from + 1];
+    if (
+      ((c === 0xe0 && c1 >= 0xa0 && c1 <= 0xbf) || (c === 0xed && c1 >= 0x80 && c1 <= 0x9f)) &&
+      buf[from + 2] >> 6 === 2
+    ) {
+      from += 3;
+      continue;
+    }
+    if (((c >= 0xe1 && c <= 0xec) || (c >= 0xee && c <= 0xef)) && c1 >> 6 === 2 && buf[from + 2] >> 6 === 2) {
+      from += 3;
+      continue;
+    }
+    if (
+      ((c === 0xf0 && c1 >= 0x90 && c1 <= 0xbf) ||
+        (c >= 0xf1 && c <= 0xf3 && c1 >> 6 === 2) ||
+        (c === 0xf4 && c1 >= 0x80 && c1 <= 0x8f)) &&
+      buf[from + 2] >> 6 === 2 &&
+      buf[from + 3] >> 6 === 2
+    ) {
+      from += 4;
+      continue;
+    }
+    return false;
+  }
+  return true;
+};

src/util/trees/avl/AvlMap.ts

@@ -68,6 +68,10 @@ export class AvlMap<K, V> implements Printable {
     this.root = remove(this.root, node as IAvlTreeNode<K, V>);
   }
 
+  public clear(): void {
+    this.root = undefined;
+  }
+
   public has(k: K): boolean {
     return !!this.find(k);
   }
@@ -81,6 +85,10 @@ export class AvlMap<K, V> implements Printable {
     return size;
   }
 
+  public isEmpty(): boolean {
+    return !this.root;
+  }
+
   public getOrNextLower(k: K): AvlNode<K, V> | undefined {
     return (findOrNextLower(this.root, k, this.comparator) as AvlNode<K, V>) || undefined;
   }