Adding basic version of DUMP and RESTORE commands (microsoft#899)

* Implement basic version of DUMP and RESTORE redis commands

* refactor and add dump, restore to cluster slot verification test

* Update comments to use 'RESP' encoding terminology

* fix formating

* fix tests

* add acl and tests and update default config to include the skip checksum
validation flag

* rm accidentally commited dump.rdb file

* Remove trailing whitespace in RespCommandTests.cs

* fix comments

* run CommandInfoUpdater and replace docs / info files

* Remove trailing whitespace in Options.cs

* fix RestoreACLsAsync test

* fix comments

* optimize RespLengthEncodingUtils

* implement suggestions

* fix comments

* use SET_Conditional directly

* rename SkipChecksumValidation

* fix cluster restore test

* directly write to the output buffer for non-large objects

* Refactor WriteDirect call in KeyAdminCommands

* Mark multiple commands as deprecated in JSON

* Mark commands as deprecated in documentation

---------

Co-authored-by: Badrish Chandramouli <[email protected]>

libs/common/Crc64.cs

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Port of redis crc64 from https://github.com/redis/redis/blob/7.2/src/crc64.c
+/// </summary>
+public static class Crc64
+{
+    /// <summary>
+    /// Polynomial (same as redis)
+    /// </summary>
+    private const ulong POLY = 0xad93d23594c935a9UL;
+
+    /// <summary>
+    /// Reverse all bits in a 64-bit value (bit reflection).
+    /// Only used for data_len == 64 in this code.
+    /// </summary>
+    private static ulong Reflect64(ulong data)
+    {
+        // swap odd/even bits
+        data = ((data >> 1) & 0x5555555555555555UL) | ((data & 0x5555555555555555UL) << 1);
+        // swap consecutive pairs
+        data = ((data >> 2) & 0x3333333333333333UL) | ((data & 0x3333333333333333UL) << 2);
+        // swap nibbles
+        data = ((data >> 4) & 0x0F0F0F0F0F0F0F0FUL) | ((data & 0x0F0F0F0F0F0F0F0FUL) << 4);
+        // swap bytes, then 2-byte pairs, then 4-byte pairs
+        data = System.Buffers.Binary.BinaryPrimitives.ReverseEndianness(data);
+        return data;
+    }
+
+    /// <summary>
+    /// A direct bit-by-bit CRC64 calculation (like _crc64 in C).
+    /// </summary>
+    private static ulong Crc64Bitwise(ReadOnlySpan<byte> data)
+    {
+        ulong crc = 0;
+
+        foreach (var c in data)
+        {
+            for (byte i = 1; i != 0; i <<= 1)
+            {
+                // interpret the top bit of 'crc' and current bit of 'c'
+                var bitSet = (crc & 0x8000000000000000UL) != 0;
+                var cbit = (c & i) != 0;
+
+                // if cbit flips the sense, invert bitSet
+                if (cbit)
+                    bitSet = !bitSet;
+
+                // shift
+                crc <<= 1;
+
+                // apply polynomial if needed
+                if (bitSet)
+                    crc ^= POLY;
+            }
+
+            // ensure it stays in 64 bits
+            crc &= 0xffffffffffffffffUL;
+        }
+
+        // reflect and XOR, per standard
+        crc &= 0xffffffffffffffffUL;
+        crc = Reflect64(crc) ^ 0x0000000000000000UL;
+        return crc;
+    }
+
+    /// <summary>
+    /// Computes crc64
+    /// </summary>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    public static byte[] Hash(ReadOnlySpan<byte> data)
+    {
+        var bitwiseCrc = Crc64Bitwise(data);
+        return BitConverter.GetBytes(bitwiseCrc);
+    }
+}

libs/common/RespLengthEncodingUtils.cs

@@ -0,0 +1,111 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Buffers.Binary;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Utils for working with RESP length encoding
+/// </summary>
+public static class RespLengthEncodingUtils
+{
+    /// <summary>
+    /// Maximum length that can be encoded
+    /// </summary>
+    private const int MaxLength = 0xFFFFFF;
+
+    /// <summary>
+    /// Try read RESP-encoded length
+    /// </summary>
+    /// <param name="input"></param>
+    /// <param name="length"></param>
+    /// <param name="bytesRead"></param>
+    /// <returns></returns>
+    public static bool TryReadLength(ReadOnlySpan<byte> input, out int length, out int bytesRead)
+    {
+        length = 0;
+        bytesRead = 0;
+        if (input.Length < 1)
+        {
+            return false;
+        }
+
+        var firstByte = input[0];
+        switch (firstByte >> 6)
+        {
+            case 0:
+                bytesRead = 1;
+                length = firstByte & 0x3F;
+                return true;
+            case 1 when input.Length > 1:
+                bytesRead = 2;
+                length = ((firstByte & 0x3F) << 8) | input[1];
+                return true;
+            case 2:
+                bytesRead = 5;
+                return BinaryPrimitives.TryReadInt32BigEndian(input, out length);
+            default:
+                return false;
+        }
+    }
+
+    /// <summary>
+    /// Try to write RESP-encoded length
+    /// </summary>
+    /// <param name="length"></param>
+    /// <param name="output"></param>
+    /// <param name="bytesWritten"></param>
+    /// <returns></returns>
+    public static bool TryWriteLength(int length, Span<byte> output, out int bytesWritten)
+    {
+        bytesWritten = 0;
+
+        if (length > MaxLength)
+        {
+            return false;
+        }
+
+        // 6-bit encoding (length ≤ 63)
+        if (length < 1 << 6)
+        {
+            if (output.Length < 1)
+            {
+                return false;
+            }
+
+            output[0] = (byte)(length & 0x3F);
+
+            bytesWritten = 1;
+            return true;
+        }
+
+        // 14-bit encoding (64 ≤ length ≤ 16,383)
+        if (length < 1 << 14)
+        {
+            if (output.Length < 2)
+            {
+                return false;
+            }
+
+            output[0] = (byte)(((length >> 8) & 0x3F) | (1 << 6));
+            output[1] = (byte)(length & 0xFF);
+
+            bytesWritten = 2;
+            return true;
+        }
+
+        // 32-bit encoding (length ≤ 4,294,967,295)
+        if (output.Length < 5)
+        {
+            return false;
+        }
+
+        output[0] = 2 << 6;
+        BinaryPrimitives.WriteUInt32BigEndian(output.Slice(1), (uint)length);
+
+        bytesWritten = 5;
+        return true;
+    }
+}

libs/host/Configuration/Options.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation.
+// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
 using System;
@@ -512,6 +512,10 @@ internal sealed class Options
         [Option("fail-on-recovery-error", Default = false, Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")]
         public bool? FailOnRecoveryError { get; set; }
 
+        [OptionValidation]
+        [Option("skip-rdb-restore-checksum-validation", Default = false, Required = false, HelpText = "Skip RDB restore checksum validation")]
+        public bool? SkipRDBRestoreChecksumValidation { get; set; }
+
         [Option("lua-memory-management-mode", Default = LuaMemoryManagementMode.Native, Required = false, HelpText = "Memory management mode for Lua scripts, must be set to LimittedNative or Managed to impose script limits")]
         public LuaMemoryManagementMode LuaMemoryManagementMode { get; set; }
 
@@ -732,6 +736,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 IndexResizeThreshold = IndexResizeThreshold,
                 LoadModuleCS = LoadModuleCS,
                 FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(),
+                SkipRDBRestoreChecksumValidation = SkipRDBRestoreChecksumValidation.GetValueOrDefault(),
                 LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, logger) : null,
             };
         }

libs/host/defaults.conf

@@ -347,6 +347,9 @@
 	/* Fails if encounters error during AOF replay or checkpointing */
 	"FailOnRecoveryError": false,
 
+	/* Skips crc64 validation in restore command */
+	"SkipRDBRestoreChecksumValidation": false,
+
 	/* Lua uses the default, unmanaged and untracked, allocator */
 	"LuaMemoryManagementMode": "Native",
 

libs/resources/RespCommandsDocs.json

@@ -1698,6 +1698,22 @@
     "Group": "Transactions",
     "Complexity": "O(N), when N is the number of queued commands"
   },
+  {
+    "Command": "DUMP",
+    "Name": "DUMP",
+    "Summary": "Returns a serialized representation of the value stored at a key.",
+    "Group": "Generic",
+    "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1).",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      }
+    ]
+  },
   {
     "Command": "ECHO",
     "Name": "ECHO",
@@ -5429,6 +5445,34 @@
       }
     ]
   },
+  {
+    "Command": "RESTORE",
+    "Name": "RESTORE",
+    "Summary": "Creates a key from the serialized representation of a value.",
+    "Group": "Generic",
+    "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "TTL",
+        "DisplayText": "ttl",
+        "Type": "Integer"
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "SERIALIZEDVALUE",
+        "DisplayText": "serialized-value",
+        "Type": "String"
+      }
+    ]
+  },
   {
     "Command": "RPOP",
     "Name": "RPOP",

libs/resources/RespCommandsInfo.json

@@ -1048,6 +1048,31 @@
     "Flags": "Fast, Loading, NoScript, Stale, AllowBusy",
     "AclCategories": "Fast, Transaction"
   },
+  {
+    "Command": "DUMP",
+    "Name": "DUMP",
+    "Arity": 2,
+    "Flags": "ReadOnly",
+    "FirstKey": 1,
+    "LastKey": 1,
+    "Step": 1,
+    "AclCategories": "KeySpace, Read",
+    "KeySpecifications": [
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 1
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 1,
+          "Limit": 0
+        },
+        "Flags": "RO, Access"
+      }
+    ]
+  },
   {
     "Command": "ECHO",
     "Name": "ECHO",
@@ -3403,6 +3428,31 @@
     "Flags": "Admin, NoAsyncLoading, NoScript, Stale",
     "AclCategories": "Admin, Dangerous, Slow"
   },
+  {
+    "Command": "RESTORE",
+    "Name": "RESTORE",
+    "Arity": -4,
+    "Flags": "DenyOom, Write",
+    "FirstKey": 1,
+    "LastKey": 1,
+    "Step": 1,
+    "AclCategories": "Dangerous, KeySpace, Slow, Write",
+    "KeySpecifications": [
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 1
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 0,
+          "Limit": 0
+        },
+        "Flags": "OW, Update"
+      }
+    ]
+  },
   {
     "Command": "RPOP",
     "Name": "RPOP",

libs/server/Resp/CmdStrings.cs

@@ -231,6 +231,7 @@ static partial class CmdStrings
         public static ReadOnlySpan<byte> RESP_ERR_INCR_SUPPORTS_ONLY_SINGLE_PAIR => "ERR INCR option supports a single increment-element pair"u8;
         public static ReadOnlySpan<byte> RESP_ERR_INVALID_BITFIELD_TYPE => "ERR Invalid bitfield type. Use something like i16 u8. Note that u64 is not supported but i64 is"u8;
         public static ReadOnlySpan<byte> RESP_ERR_SCRIPT_FLUSH_OPTIONS => "ERR SCRIPT FLUSH only support SYNC|ASYNC option"u8;
+        public static ReadOnlySpan<byte> RESP_ERR_BUSSYKEY => "BUSYKEY Target key name already exists."u8;
         public static ReadOnlySpan<byte> RESP_ERR_LENGTH_AND_INDEXES => "If you want both the length and indexes, please just use IDX."u8;
         public static ReadOnlySpan<byte> RESP_ERR_INVALID_EXPIRE_TIME => "ERR invalid expire time, must be >= 0"u8;
         public static ReadOnlySpan<byte> RESP_ERR_HCOLLECT_ALREADY_IN_PROGRESS => "ERR HCOLLECT scan already in progress"u8;