Merge branch 'microsoft:main' into main

.github/workflows/ci-bdnbenchmark.yml

@@ -42,7 +42,7 @@ jobs:
         os: [ ubuntu-latest, windows-latest ]
         framework: [ 'net8.0' ]
         configuration: [ 'Release' ]
-        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts', 'Operations.CustomOperations', 'Operations.RawStringOperations', 'Operations.ScriptOperations','Network.BasicOperations', 'Network.RawStringOperations' ]
+        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts', 'Lua.LuaScriptCacheOperations','Lua.LuaRunnerOperations','Operations.CustomOperations', 'Operations.RawStringOperations', 'Operations.ScriptOperations', 'Operations.ModuleOperations', 'Operations.PubSubOperations', 'Network.BasicOperations', 'Network.RawStringOperations' ]
     steps:
       - name: Check out code
         uses: actions/checkout@v4

.github/workflows/ci.yml

@@ -121,6 +121,10 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+      - name: Set workaround for libaio on Ubuntu 24.04 (see https://askubuntu.com/questions/1512196/libaio1-on-noble/1512197#1512197)
+        run: |
+          sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
+        if: ${{ matrix.os == 'ubuntu-latest' }}
       - name: Set environment variable for Linux
         run: echo "RunAzureTests=yes" >> $GITHUB_ENV
         if: ${{ matrix.os == 'ubuntu-latest' }}

Garnet.sln

@@ -109,6 +109,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Garnet.worker", "hosting\Wi
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Garnet.resources", "libs\resources\Garnet.resources.csproj", "{A48412B4-FD60-467E-A5D9-F155CAB4F907}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NoOpModule", "playground\NoOpModule\NoOpModule.csproj", "{D4C9A1A0-7053-F072-21F5-4E0C5827136D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -325,6 +327,14 @@ Global
 		{A48412B4-FD60-467E-A5D9-F155CAB4F907}.Release|Any CPU.Build.0 = Release|Any CPU
 		{A48412B4-FD60-467E-A5D9-F155CAB4F907}.Release|x64.ActiveCfg = Release|Any CPU
 		{A48412B4-FD60-467E-A5D9-F155CAB4F907}.Release|x64.Build.0 = Release|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Debug|x64.Build.0 = Debug|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|x64.ActiveCfg = Release|Any CPU
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -359,6 +369,7 @@ Global
 		{697766CD-2046-46D9-958A-0FD3B46C98D4} = {01823EA4-4446-4D66-B268-DFEE55951964}
 		{DF2DD03E-87EE-482A-9FBA-6C8FBC23BDC5} = {697766CD-2046-46D9-958A-0FD3B46C98D4}
 		{A48412B4-FD60-467E-A5D9-F155CAB4F907} = {147FCE31-EC09-4C90-8E4D-37CA87ED18C3}
+		{D4C9A1A0-7053-F072-21F5-4E0C5827136D} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {2C02C405-4798-41CA-AF98-61EDFEF6772E}

Version.props

@@ -1,6 +1,6 @@
 <Project>
 	<!-- Versioning property for builds and packages -->
 	<PropertyGroup>
-		<VersionPrefix>1.0.49</VersionPrefix>
+		<VersionPrefix>1.0.52</VersionPrefix>
 	</PropertyGroup>
 </Project>

benchmark/BDN.benchmark/BDN.benchmark.csproj

@@ -18,6 +18,8 @@
     <ProjectReference Include="..\..\libs\host\Garnet.host.csproj" />
     <ProjectReference Include="..\..\libs\common\Garnet.common.csproj" />
     <ProjectReference Include="..\..\libs\server\Garnet.server.csproj" />
+    <ProjectReference Include="..\..\playground\GarnetJSON\GarnetJSON.csproj" />
+    <ProjectReference Include="..\..\playground\NoOpModule\NoOpModule.csproj" />
   </ItemGroup>
 
   <ItemGroup>

benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs

@@ -2,8 +2,10 @@
 // Licensed under the MIT license.
 
 using Garnet;
+using Garnet.common;
 using Garnet.server;
 using Microsoft.Extensions.Logging;
+using Tsavorite.core;
 
 namespace Embedded.server
 {
@@ -13,6 +15,7 @@ namespace Embedded.server
     internal sealed class EmbeddedRespServer : GarnetServer
     {
         readonly GarnetServerEmbedded garnetServerEmbedded;
+        readonly SubscribeBroker<SpanByte, SpanByte, IKeySerializer<SpanByte>> subscribeBroker;
 
         /// <summary>
         /// Creates an EmbeddedRespServer instance
@@ -23,6 +26,13 @@ internal sealed class EmbeddedRespServer : GarnetServer
         public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) : base(opts, loggerFactory, server)
         {
             this.garnetServerEmbedded = server;
+            this.subscribeBroker = opts.DisablePubSub ? null :
+                new SubscribeBroker<SpanByte, SpanByte, IKeySerializer<SpanByte>>(
+                    new SpanByteKeySerializer(),
+                    null,
+                    opts.PubSubPageSizeBytes(),
+                    opts.SubscriberRefreshFrequencyMs,
+                    true);
         }
 
         /// <summary>
@@ -38,7 +48,7 @@ public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory
         /// <returns>A new RESP server session</returns>
         internal RespServerSession GetRespSession()
         {
-            return new RespServerSession(0, new EmbeddedNetworkSender(), storeWrapper, null, null, true);
+            return new RespServerSession(0, new EmbeddedNetworkSender(), storeWrapper, subscribeBroker: subscribeBroker, null, true);
         }
 
         internal EmbeddedNetworkHandler GetNetworkHandler()

benchmark/BDN.benchmark/Lua/LuaParams.cs

@@ -1,26 +1,38 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using BenchmarkDotNet.Code;
+using Garnet.server;
+
 namespace BDN.benchmark.Lua
 {
     /// <summary>
-    /// Cluster parameters
+    /// Lua parameters
     /// </summary>
-    public struct LuaParams
+    public readonly struct LuaParams
     {
+        public readonly LuaMemoryManagementMode Mode { get; }
+        public readonly bool MemoryLimit { get; }
+
         /// <summary>
         /// Constructor
         /// </summary>
-        public LuaParams()
+        public LuaParams(LuaMemoryManagementMode mode, bool memoryLimit)
         {
+            Mode = mode;
+            MemoryLimit = memoryLimit;
         }
 
+        /// <summary>
+        /// Get the equivalent <see cref="LuaOptions"/>.
+        /// </summary>
+        public LuaOptions CreateOptions()
+        => new(Mode, MemoryLimit ? "2m" : "");
+
         /// <summary>
         /// String representation
         /// </summary>
         public override string ToString()
-        {
-            return "None";
-        }
+        => $"{Mode},{(MemoryLimit ? "Limit" : "None")}";
     }
 }

benchmark/BDN.benchmark/Lua/LuaRunnerOperations.cs

@@ -139,9 +139,13 @@ public unsafe class LuaRunnerOperations
         /// Lua parameters provider
         /// </summary>
         public IEnumerable<LuaParams> LuaParamsProvider()
-        {
-            yield return new();
-        }
+        => [
+            new(LuaMemoryManagementMode.Native, false),
+            new(LuaMemoryManagementMode.Tracked, false),
+            new(LuaMemoryManagementMode.Tracked, true),
+            new(LuaMemoryManagementMode.Managed, false),
+            new(LuaMemoryManagementMode.Managed, true),
+        ];
 
         private EmbeddedRespServer server;
         private RespServerSession session;
@@ -151,16 +155,21 @@ public IEnumerable<LuaParams> LuaParamsProvider()
         private LuaRunner smallCompileRunner;
         private LuaRunner largeCompileRunner;
 
+        private LuaOptions opts;
+
         [GlobalSetup]
         public void GlobalSetup()
         {
-            server = new EmbeddedRespServer(new GarnetServerOptions() { EnableLua = true, QuietMode = true });
+            opts = Params.CreateOptions();
+
+            server = new EmbeddedRespServer(new GarnetServerOptions() { EnableLua = true, QuietMode = true, LuaOptions = opts });
 
             session = server.GetRespSession();
-            paramsRunner = new LuaRunner("return nil");
 
-            smallCompileRunner = new LuaRunner(SmallScript);
-            largeCompileRunner = new LuaRunner(LargeScript);
+            paramsRunner = new LuaRunner(opts, "return nil");
+
+            smallCompileRunner = new LuaRunner(opts, SmallScript);
+            largeCompileRunner = new LuaRunner(opts, LargeScript);
         }
 
         [GlobalCleanup]
@@ -171,6 +180,18 @@ public void GlobalCleanup()
             paramsRunner.Dispose();
         }
 
+        [IterationSetup]
+        public void IterationSetup()
+        {
+            session.EnterAndGetResponseObject();
+        }
+
+        [IterationCleanup]
+        public void IterationCleanup()
+        {
+            session.ExitAndReturnResponseObject();
+        }
+
         [Benchmark]
         public void ResetParametersSmall()
         {
@@ -194,13 +215,13 @@ public void ResetParametersLarge()
         [Benchmark]
         public void ConstructSmall()
         {
-            using var runner = new LuaRunner(SmallScript);
+            using var runner = new LuaRunner(opts, SmallScript);
         }
 
         [Benchmark]
         public void ConstructLarge()
         {
-            using var runner = new LuaRunner(LargeScript);
+            using var runner = new LuaRunner(opts, LargeScript);
         }
 
         [Benchmark]

benchmark/BDN.benchmark/Lua/LuaScriptCacheOperations.cs

@@ -22,9 +22,13 @@ public class LuaScriptCacheOperations
         /// Lua parameters provider
         /// </summary>
         public IEnumerable<LuaParams> LuaParamsProvider()
-        {
-            yield return new();
-        }
+        => [
+            new(LuaMemoryManagementMode.Native, false),
+            new(LuaMemoryManagementMode.Tracked, false),
+            new(LuaMemoryManagementMode.Tracked, true),
+            new(LuaMemoryManagementMode.Managed, false),
+            new(LuaMemoryManagementMode.Managed, true),
+        ];
 
         private EmbeddedRespServer server;
         private StoreWrapper storeWrapper;
@@ -38,7 +42,9 @@ public IEnumerable<LuaParams> LuaParamsProvider()
         [GlobalSetup]
         public void GlobalSetup()
         {
-            server = new EmbeddedRespServer(new GarnetServerOptions() { EnableLua = true, QuietMode = true });
+            var options = Params.CreateOptions();
+
+            server = new EmbeddedRespServer(new GarnetServerOptions() { EnableLua = true, QuietMode = true, LuaOptions = options });
             storeWrapper = server.StoreWrapper;
             sessionScriptCache = new SessionScriptCache(storeWrapper, new GarnetNoAuthAuthenticator());
             session = server.GetRespSession();

benchmark/BDN.benchmark/Lua/LuaScripts.cs

@@ -22,23 +22,29 @@ public unsafe class LuaScripts
         /// Lua parameters provider
         /// </summary>
         public IEnumerable<LuaParams> LuaParamsProvider()
-        {
-            yield return new();
-        }
+        => [
+            new(LuaMemoryManagementMode.Native, false),
+            new(LuaMemoryManagementMode.Tracked, false),
+            new(LuaMemoryManagementMode.Tracked, true),
+            new(LuaMemoryManagementMode.Managed, false),
+            new(LuaMemoryManagementMode.Managed, true),
+        ];
 
         LuaRunner r1, r2, r3, r4;
         readonly string[] keys = ["key1"];
 
         [GlobalSetup]
         public void GlobalSetup()
         {
-            r1 = new LuaRunner("return");
+            var options = Params.CreateOptions();
+
+            r1 = new LuaRunner(options, "return");
             r1.CompileForRunner();
-            r2 = new LuaRunner("return 1 + 1");
+            r2 = new LuaRunner(options, "return 1 + 1");
             r2.CompileForRunner();
-            r3 = new LuaRunner("return KEYS[1]");
+            r3 = new LuaRunner(options, "return KEYS[1]");
             r3.CompileForRunner();
-            r4 = new LuaRunner("return redis.call(KEYS[1])");
+            r4 = new LuaRunner(options, "return redis.call(KEYS[1])");
             r4.CompileForRunner();
         }