From 3d597c398407d027980344d853930d5df85547cd Mon Sep 17 00:00:00 2001 From: shiranshalom Date: Tue, 9 Apr 2024 17:33:10 +0300 Subject: [PATCH 001/243] RavenDB-22107 - fix failing tests for RavenDB v6.0 ZSTD Linux x64 --- .../Replication/GetReplicationOutgoingsFailureInfoCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Raven.Server/Documents/Commands/Replication/GetReplicationOutgoingsFailureInfoCommand.cs b/src/Raven.Server/Documents/Commands/Replication/GetReplicationOutgoingsFailureInfoCommand.cs index f06c0e65b4b6..356cdb624d3e 100644 --- a/src/Raven.Server/Documents/Commands/Replication/GetReplicationOutgoingsFailureInfoCommand.cs +++ b/src/Raven.Server/Documents/Commands/Replication/GetReplicationOutgoingsFailureInfoCommand.cs @@ -36,7 +36,7 @@ public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, Serve public override async Task ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url) { - await using var stream = await response.Content.ReadAsStreamAsync(); + await using var stream = await response.Content.ReadAsStreamWithZstdSupportAsync().ConfigureAwait(false); using var memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream); From 81393d7d8a159f48ebb7bbdad0dd955ba97d231a Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Wed, 10 Apr 2024 14:50:48 +0200 Subject: [PATCH 002/243] RavenDB-22151 Added support for new types of memory dumps in /admin/debug/dump endpoint --- src/Raven.Server/Web/System/AdminDumpHandler.cs | 4 +++- tools/Raven.Debug/CommandLineApp.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Raven.Server/Web/System/AdminDumpHandler.cs b/src/Raven.Server/Web/System/AdminDumpHandler.cs index fbe305b4ea0a..07a6f2baae8d 100644 --- a/src/Raven.Server/Web/System/AdminDumpHandler.cs +++ b/src/Raven.Server/Web/System/AdminDumpHandler.cs @@ -191,7 +191,9 @@ public void Dispose() private enum DumpType { Mini, - Heap + Heap, + Full, + Triage } } } diff --git a/tools/Raven.Debug/CommandLineApp.cs b/tools/Raven.Debug/CommandLineApp.cs index 1939ef3d889b..eafa19272a4f 100644 --- a/tools/Raven.Debug/CommandLineApp.cs +++ b/tools/Raven.Debug/CommandLineApp.cs @@ -131,7 +131,7 @@ public static int Run(string[] args) var pidOption = cmd.Option("--pid", "Process ID to which the tool will attach to", CommandOptionType.SingleValue); var outputOption = cmd.Option("--output", "Output file path", CommandOptionType.SingleValue); - var typeOption = cmd.Option("--type", "Type of dump (Heap or Mini). ", CommandOptionType.SingleValue); + var typeOption = cmd.Option("--type", "Type of dump (Full, Heap, Mini or Triage). ", CommandOptionType.SingleValue); var outputOwnerOption = cmd.Option("--output-owner", "The owner of the output file (applicable only to Posix based OS)", CommandOptionType.SingleValue); cmd.OnExecute(() => From 843effc225b7fa38f1b6924536f211ec953d4ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pekr=C3=B3l?= Date: Thu, 11 Apr 2024 10:57:42 +0200 Subject: [PATCH 003/243] RavenDB-7070 updated .NET to 7.0.18 --- bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj | 2 +- bench/Indexing.Benchmark/Indexing.Benchmark.csproj | 2 +- bench/Micro.Benchmark/Micro.Benchmark.csproj | 2 +- bench/Regression.Benchmark/Regression.Benchmark.csproj | 2 +- .../SubscriptionFailover.Benchmark.csproj | 2 +- bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj | 2 +- bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj | 2 +- bench/Voron.Benchmark/Voron.Benchmark.csproj | 2 +- src/Corax/Corax.csproj | 2 +- src/Raven.Embedded/ServerOptions.cs | 2 +- src/Raven.Server/Raven.Server.csproj | 2 +- src/Raven.Studio/Raven.Studio.csproj | 2 +- src/Sparrow.Server/Sparrow.Server.csproj | 2 +- src/Voron/Voron.csproj | 2 +- test/BenchmarkTests/BenchmarkTests.csproj | 2 +- test/EmbeddedTests/EmbeddedTests.csproj | 2 +- test/FastTests/FastTests.csproj | 2 +- test/InterversionTests/InterversionTests.csproj | 2 +- test/LicenseTests/LicenseTests.csproj | 2 +- test/RachisTests/RachisTests.csproj | 2 +- test/SlowTests/SlowTests.csproj | 2 +- test/StressTests/StressTests.csproj | 2 +- test/Tests.Infrastructure/Tests.Infrastructure.csproj | 2 +- test/Tryouts/Tryouts.csproj | 2 +- tools/Raven.Debug/Raven.Debug.csproj | 2 +- tools/Raven.Migrator/Raven.Migrator.csproj | 2 +- tools/TypingsGenerator/TypingsGenerator.csproj | 2 +- .../Voron.Dictionary.Generator.csproj | 2 +- tools/Voron.Recovery/Voron.Recovery.csproj | 2 +- tools/rvn/rvn.csproj | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj b/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj index a43dc5d89119..898850237ae6 100644 --- a/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj +++ b/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 BulkInsert.Benchmark Exe BulkInsert.Benchmark diff --git a/bench/Indexing.Benchmark/Indexing.Benchmark.csproj b/bench/Indexing.Benchmark/Indexing.Benchmark.csproj index a727397b3d4f..e13e4e67df95 100644 --- a/bench/Indexing.Benchmark/Indexing.Benchmark.csproj +++ b/bench/Indexing.Benchmark/Indexing.Benchmark.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 Indexing.Benchmark Exe Indexing.Benchmark diff --git a/bench/Micro.Benchmark/Micro.Benchmark.csproj b/bench/Micro.Benchmark/Micro.Benchmark.csproj index 78b5d77a6550..f82f4be0b675 100644 --- a/bench/Micro.Benchmark/Micro.Benchmark.csproj +++ b/bench/Micro.Benchmark/Micro.Benchmark.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true Micro.Benchmark Exe diff --git a/bench/Regression.Benchmark/Regression.Benchmark.csproj b/bench/Regression.Benchmark/Regression.Benchmark.csproj index eaa623df3b76..93a6fd126f03 100644 --- a/bench/Regression.Benchmark/Regression.Benchmark.csproj +++ b/bench/Regression.Benchmark/Regression.Benchmark.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true Regression.Benchmark Regression.Benchmark diff --git a/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj b/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj index 488f94f92057..d5e636086108 100644 --- a/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj +++ b/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj @@ -2,7 +2,7 @@ Exe net7.0 - 7.0.17 + 7.0.18 ..\..\RavenDB.ruleset diff --git a/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj b/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj index 3c1f97c502b7..3984962102b4 100644 --- a/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj +++ b/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 Subscriptions.Benchmark Exe Subscriptions.Benchmark diff --git a/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj b/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj index 473d64ddd3c9..bc08ecac0ab4 100644 --- a/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj +++ b/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj @@ -3,7 +3,7 @@ Exe net7.0 - 7.0.17 + 7.0.18 diff --git a/bench/Voron.Benchmark/Voron.Benchmark.csproj b/bench/Voron.Benchmark/Voron.Benchmark.csproj index a1a596003dbd..a26962cda8d2 100644 --- a/bench/Voron.Benchmark/Voron.Benchmark.csproj +++ b/bench/Voron.Benchmark/Voron.Benchmark.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true Voron.Benchmark Exe diff --git a/src/Corax/Corax.csproj b/src/Corax/Corax.csproj index 16e4ed5a28e0..25cb337b4bbc 100644 --- a/src/Corax/Corax.csproj +++ b/src/Corax/Corax.csproj @@ -4,7 +4,7 @@ Corax - Low level indexing engine Hibernating Rhinos net7.0 - 7.0.17 + 7.0.18 true Corax Corax diff --git a/src/Raven.Embedded/ServerOptions.cs b/src/Raven.Embedded/ServerOptions.cs index 180381273310..e98fcfbdaef6 100644 --- a/src/Raven.Embedded/ServerOptions.cs +++ b/src/Raven.Embedded/ServerOptions.cs @@ -11,7 +11,7 @@ public class ServerOptions internal static string AltServerDirectory = Path.Combine(AppContext.BaseDirectory, "bin", "RavenDBServer"); - public string FrameworkVersion { get; set; } = "7.0.17+"; + public string FrameworkVersion { get; set; } = "7.0.18+"; public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "RavenDB", "Logs"); diff --git a/src/Raven.Server/Raven.Server.csproj b/src/Raven.Server/Raven.Server.csproj index b3ee3904f120..ffea9e4dab55 100644 --- a/src/Raven.Server/Raven.Server.csproj +++ b/src/Raven.Server/Raven.Server.csproj @@ -3,7 +3,7 @@ Raven.Server is the database server for RavenDB Hibernating Rhinos net7.0 - 7.0.17 + 7.0.18 true Raven.Server Exe diff --git a/src/Raven.Studio/Raven.Studio.csproj b/src/Raven.Studio/Raven.Studio.csproj index f404ed55389d..0869efeb8f3b 100644 --- a/src/Raven.Studio/Raven.Studio.csproj +++ b/src/Raven.Studio/Raven.Studio.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true Raven.Studio Exe diff --git a/src/Sparrow.Server/Sparrow.Server.csproj b/src/Sparrow.Server/Sparrow.Server.csproj index e57534c8667f..5509efef0921 100644 --- a/src/Sparrow.Server/Sparrow.Server.csproj +++ b/src/Sparrow.Server/Sparrow.Server.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true Sparrow.Server Sparrow.Server diff --git a/src/Voron/Voron.csproj b/src/Voron/Voron.csproj index 897a61fb5a18..f3b2249a162a 100644 --- a/src/Voron/Voron.csproj +++ b/src/Voron/Voron.csproj @@ -3,7 +3,7 @@ Voron - Low level storage engine Hibernating Rhinos net7.0 - 7.0.17 + 7.0.18 true Voron Voron diff --git a/test/BenchmarkTests/BenchmarkTests.csproj b/test/BenchmarkTests/BenchmarkTests.csproj index b441c5439466..b4271323b45a 100644 --- a/test/BenchmarkTests/BenchmarkTests.csproj +++ b/test/BenchmarkTests/BenchmarkTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true BenchmarkTests BenchmarkTests diff --git a/test/EmbeddedTests/EmbeddedTests.csproj b/test/EmbeddedTests/EmbeddedTests.csproj index fabbe54e6db3..849a432c3ab9 100644 --- a/test/EmbeddedTests/EmbeddedTests.csproj +++ b/test/EmbeddedTests/EmbeddedTests.csproj @@ -1,6 +1,6 @@  - 7.0.17 + 7.0.18 true EmbeddedTests EmbeddedTests diff --git a/test/FastTests/FastTests.csproj b/test/FastTests/FastTests.csproj index 01b962f5064c..0fc65373bb9a 100644 --- a/test/FastTests/FastTests.csproj +++ b/test/FastTests/FastTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true FastTests FastTests diff --git a/test/InterversionTests/InterversionTests.csproj b/test/InterversionTests/InterversionTests.csproj index 690163ce0187..befc2684fc6e 100644 --- a/test/InterversionTests/InterversionTests.csproj +++ b/test/InterversionTests/InterversionTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true InterversionTests InterversionTests diff --git a/test/LicenseTests/LicenseTests.csproj b/test/LicenseTests/LicenseTests.csproj index 725a99b0efde..e39893ff06a6 100644 --- a/test/LicenseTests/LicenseTests.csproj +++ b/test/LicenseTests/LicenseTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 LicenseTests LicenseTests ..\..\RavenDB.ruleset diff --git a/test/RachisTests/RachisTests.csproj b/test/RachisTests/RachisTests.csproj index 40f703fb541d..576ea9bd8232 100644 --- a/test/RachisTests/RachisTests.csproj +++ b/test/RachisTests/RachisTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true RachisTests RachisTests diff --git a/test/SlowTests/SlowTests.csproj b/test/SlowTests/SlowTests.csproj index ee16a3368a7b..0b816052594d 100644 --- a/test/SlowTests/SlowTests.csproj +++ b/test/SlowTests/SlowTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true SlowTests SlowTests diff --git a/test/StressTests/StressTests.csproj b/test/StressTests/StressTests.csproj index 69b66f319b01..632b90015b63 100644 --- a/test/StressTests/StressTests.csproj +++ b/test/StressTests/StressTests.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true StressTests StressTests diff --git a/test/Tests.Infrastructure/Tests.Infrastructure.csproj b/test/Tests.Infrastructure/Tests.Infrastructure.csproj index 01f8615a51d6..f76a80e8b236 100644 --- a/test/Tests.Infrastructure/Tests.Infrastructure.csproj +++ b/test/Tests.Infrastructure/Tests.Infrastructure.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 Tests.Infrastructure Tests.Infrastructure ..\..\RavenDB.ruleset diff --git a/test/Tryouts/Tryouts.csproj b/test/Tryouts/Tryouts.csproj index 3db890ff6f66..4bd09a5517e4 100644 --- a/test/Tryouts/Tryouts.csproj +++ b/test/Tryouts/Tryouts.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 true Tryouts Exe diff --git a/tools/Raven.Debug/Raven.Debug.csproj b/tools/Raven.Debug/Raven.Debug.csproj index de5fc72f8b14..313eb552ba30 100644 --- a/tools/Raven.Debug/Raven.Debug.csproj +++ b/tools/Raven.Debug/Raven.Debug.csproj @@ -3,7 +3,7 @@ Exe net7.0 - 7.0.17 + 7.0.18 win7-x64;win8-x64;win81-x64;win10-x64;win7-x86;win8-x86;win81-x86;win10-x86;ubuntu.14.04-x64;ubuntu.16.04-x64;ubuntu.18.04-x64 ..\..\RavenDB.ruleset true diff --git a/tools/Raven.Migrator/Raven.Migrator.csproj b/tools/Raven.Migrator/Raven.Migrator.csproj index 8f914db52f65..8dba99324aea 100644 --- a/tools/Raven.Migrator/Raven.Migrator.csproj +++ b/tools/Raven.Migrator/Raven.Migrator.csproj @@ -2,7 +2,7 @@ Exe net7.0 - 7.0.17 + 7.0.18 win7-x64;win8-x64;win81-x64;win10-x64;win7-x86;win8-x86;win81-x86;win10-x86;ubuntu.14.04-x64;ubuntu.16.04-x64;ubuntu.18.04-x64 ..\..\RavenDB.ruleset diff --git a/tools/TypingsGenerator/TypingsGenerator.csproj b/tools/TypingsGenerator/TypingsGenerator.csproj index 322ed12002ae..37d99682733a 100644 --- a/tools/TypingsGenerator/TypingsGenerator.csproj +++ b/tools/TypingsGenerator/TypingsGenerator.csproj @@ -1,7 +1,7 @@  net7.0 - 7.0.17 + 7.0.18 TypingsGenerator Exe TypingsGenerator diff --git a/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj b/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj index fc760a809a02..3526a0bdc0f2 100644 --- a/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj +++ b/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj @@ -3,7 +3,7 @@ Exe net7.0 - 7.0.17 + 7.0.18 enable enable True diff --git a/tools/Voron.Recovery/Voron.Recovery.csproj b/tools/Voron.Recovery/Voron.Recovery.csproj index 674ce3beba57..f8b735fc439a 100644 --- a/tools/Voron.Recovery/Voron.Recovery.csproj +++ b/tools/Voron.Recovery/Voron.Recovery.csproj @@ -3,7 +3,7 @@ Voron Recovery Tool Hibernating Rhinos net7.0 - 7.0.17 + 7.0.18 true Voron.Recovery Exe diff --git a/tools/rvn/rvn.csproj b/tools/rvn/rvn.csproj index 1edb7071d6dd..d1eece9740ba 100644 --- a/tools/rvn/rvn.csproj +++ b/tools/rvn/rvn.csproj @@ -4,7 +4,7 @@ Hibernating Rhinos Exe net7.0 - 7.0.17 + 7.0.18 true database;nosql;doc db https://ravendb.net From 3a4381c6e1bd5c6674fd859f0e8eec2bebd8f2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pekr=C3=B3l?= Date: Thu, 11 Apr 2024 10:58:33 +0200 Subject: [PATCH 004/243] RavenDB-7070 updated GH actions to .NET SDK 7.0.408 --- .github/workflows/FastTests.yml | 2 +- .github/workflows/FastTestsARM.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/FastTests.yml b/.github/workflows/FastTests.yml index 1c93e1ca265d..838ebdfd8f29 100644 --- a/.github/workflows/FastTests.yml +++ b/.github/workflows/FastTests.yml @@ -10,7 +10,7 @@ on: - v5.4 env: - DOTNET_VERSION: 7.0.407 + DOTNET_VERSION: 7.0.408 jobs: diff --git a/.github/workflows/FastTestsARM.yml b/.github/workflows/FastTestsARM.yml index 6e813926b0c7..1faffdbfd868 100644 --- a/.github/workflows/FastTestsARM.yml +++ b/.github/workflows/FastTestsARM.yml @@ -7,7 +7,7 @@ on: - cron: '30 6 * * *' env: - DOTNET_VERSION: 7.0.407 + DOTNET_VERSION: 7.0.408 jobs: release: From e562f10b18d570bab91062f0f39fa7e3a28fd420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pekr=C3=B3l?= Date: Thu, 11 Apr 2024 11:01:07 +0200 Subject: [PATCH 005/243] RavenDB-7070 updated NuGet packages --- src/Raven.Client/Raven.Client.csproj | 2 +- src/Raven.Server/Raven.Server.csproj | 16 ++++++++-------- test/FastTests/FastTests.csproj | 2 +- tools/Raven.Migrator/Raven.Migrator.csproj | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Raven.Client/Raven.Client.csproj b/src/Raven.Client/Raven.Client.csproj index a081a0aff186..52692a0f3f48 100644 --- a/src/Raven.Client/Raven.Client.csproj +++ b/src/Raven.Client/Raven.Client.csproj @@ -134,7 +134,7 @@ - + diff --git a/src/Raven.Server/Raven.Server.csproj b/src/Raven.Server/Raven.Server.csproj index ffea9e4dab55..e458ed28b784 100644 --- a/src/Raven.Server/Raven.Server.csproj +++ b/src/Raven.Server/Raven.Server.csproj @@ -151,23 +151,23 @@ - - - + + + - + All - - + + @@ -180,7 +180,7 @@ - + @@ -201,7 +201,7 @@ - + diff --git a/test/FastTests/FastTests.csproj b/test/FastTests/FastTests.csproj index 0fc65373bb9a..432bb89be659 100644 --- a/test/FastTests/FastTests.csproj +++ b/test/FastTests/FastTests.csproj @@ -63,7 +63,7 @@ - + diff --git a/tools/Raven.Migrator/Raven.Migrator.csproj b/tools/Raven.Migrator/Raven.Migrator.csproj index 8dba99324aea..9168c9633afc 100644 --- a/tools/Raven.Migrator/Raven.Migrator.csproj +++ b/tools/Raven.Migrator/Raven.Migrator.csproj @@ -13,9 +13,9 @@ - + - + @@ -25,6 +25,6 @@ - + \ No newline at end of file From 7f47445339868e1ebd5b897995e87a97483dc94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pekr=C3=B3l?= Date: Thu, 11 Apr 2024 11:05:05 +0200 Subject: [PATCH 006/243] RavenDB-7070 updated .NET to 8.0.4 --- bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj | 2 +- bench/Indexing.Benchmark/Indexing.Benchmark.csproj | 2 +- bench/Micro.Benchmark/Micro.Benchmark.csproj | 2 +- bench/Regression.Benchmark/Regression.Benchmark.csproj | 2 +- .../ServerStoreTxMerger.Benchmark.csproj | 2 +- .../SubscriptionFailover.Benchmark.csproj | 2 +- bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj | 2 +- bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj | 2 +- bench/Voron.Benchmark/Voron.Benchmark.csproj | 2 +- src/Corax/Corax.csproj | 2 +- src/Raven.Embedded/ServerOptions.cs | 2 +- src/Raven.Server/Raven.Server.csproj | 2 +- src/Raven.Studio/Raven.Studio.csproj | 2 +- src/Sparrow.Server/Sparrow.Server.csproj | 2 +- src/Voron/Voron.csproj | 2 +- test/BenchmarkTests/BenchmarkTests.csproj | 2 +- test/EmbeddedTests/EmbeddedTests.csproj | 2 +- test/FastTests/FastTests.csproj | 2 +- test/InterversionTests/InterversionTests.csproj | 2 +- test/LicenseTests/LicenseTests.csproj | 2 +- test/RachisTests/RachisTests.csproj | 2 +- test/SlowTests/SlowTests.csproj | 2 +- test/StressTests/StressTests.csproj | 2 +- test/Tests.Infrastructure/Tests.Infrastructure.csproj | 2 +- test/Tryouts/Tryouts.csproj | 2 +- tools/Raven.Debug/Raven.Debug.csproj | 2 +- tools/Raven.Migrator/Raven.Migrator.csproj | 2 +- tools/TypingsGenerator/TypingsGenerator.csproj | 2 +- .../Voron.Dictionary.Generator.csproj | 2 +- tools/Voron.Recovery/Voron.Recovery.csproj | 2 +- tools/rvn/rvn.csproj | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj b/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj index 091020cce196..b5be7433799f 100644 --- a/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj +++ b/bench/BulkInsert.Benchmark/BulkInsert.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 BulkInsert.Benchmark Exe BulkInsert.Benchmark diff --git a/bench/Indexing.Benchmark/Indexing.Benchmark.csproj b/bench/Indexing.Benchmark/Indexing.Benchmark.csproj index c27c033d2a47..a335d0b6babe 100644 --- a/bench/Indexing.Benchmark/Indexing.Benchmark.csproj +++ b/bench/Indexing.Benchmark/Indexing.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 Indexing.Benchmark Exe Indexing.Benchmark diff --git a/bench/Micro.Benchmark/Micro.Benchmark.csproj b/bench/Micro.Benchmark/Micro.Benchmark.csproj index 8c55ec4feed3..93eaf478c201 100644 --- a/bench/Micro.Benchmark/Micro.Benchmark.csproj +++ b/bench/Micro.Benchmark/Micro.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true Micro.Benchmark Exe diff --git a/bench/Regression.Benchmark/Regression.Benchmark.csproj b/bench/Regression.Benchmark/Regression.Benchmark.csproj index 33bc0a77411e..7f0adc5a5ef1 100644 --- a/bench/Regression.Benchmark/Regression.Benchmark.csproj +++ b/bench/Regression.Benchmark/Regression.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true Regression.Benchmark Regression.Benchmark diff --git a/bench/ServerStoreTxMerger.Benchmark/ServerStoreTxMerger.Benchmark.csproj b/bench/ServerStoreTxMerger.Benchmark/ServerStoreTxMerger.Benchmark.csproj index 4fbbc00a9eb3..220ca6e2eefa 100644 --- a/bench/ServerStoreTxMerger.Benchmark/ServerStoreTxMerger.Benchmark.csproj +++ b/bench/ServerStoreTxMerger.Benchmark/ServerStoreTxMerger.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 ServerStoreTxMerger.Benchmark Exe ServerStoreTxMerger.Benchmark diff --git a/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj b/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj index 306f75901278..adcbc40f00bb 100644 --- a/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj +++ b/bench/SubscriptionFailover.Benchmark/SubscriptionFailover.Benchmark.csproj @@ -2,7 +2,7 @@ Exe net8.0 - 8.0.3 + 8.0.4 ..\..\RavenDB.ruleset diff --git a/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj b/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj index 515d82a54621..cd68f797c67e 100644 --- a/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj +++ b/bench/Subscriptions.Benchmark/Subscriptions.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 Subscriptions.Benchmark Exe Subscriptions.Benchmark diff --git a/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj b/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj index 653a2edd9f21..8d874e792102 100644 --- a/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj +++ b/bench/TimeSeries.Benchmark/TimeSeries.Benchmark.csproj @@ -3,7 +3,7 @@ Exe net8.0 - 8.0.3 + 8.0.4 diff --git a/bench/Voron.Benchmark/Voron.Benchmark.csproj b/bench/Voron.Benchmark/Voron.Benchmark.csproj index 5617235523b8..2dc52ee4c224 100644 --- a/bench/Voron.Benchmark/Voron.Benchmark.csproj +++ b/bench/Voron.Benchmark/Voron.Benchmark.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true Voron.Benchmark Exe diff --git a/src/Corax/Corax.csproj b/src/Corax/Corax.csproj index 0e6735599131..4196b5511987 100644 --- a/src/Corax/Corax.csproj +++ b/src/Corax/Corax.csproj @@ -4,7 +4,7 @@ Corax - Low level indexing engine Hibernating Rhinos net8.0 - 8.0.3 + 8.0.4 true Corax Corax diff --git a/src/Raven.Embedded/ServerOptions.cs b/src/Raven.Embedded/ServerOptions.cs index 237602e2a276..023e340e5e4c 100644 --- a/src/Raven.Embedded/ServerOptions.cs +++ b/src/Raven.Embedded/ServerOptions.cs @@ -11,7 +11,7 @@ public class ServerOptions internal static string AltServerDirectory = Path.Combine(AppContext.BaseDirectory, "bin", "RavenDBServer"); - public string FrameworkVersion { get; set; } = "8.0.3+"; + public string FrameworkVersion { get; set; } = "8.0.4+"; public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "RavenDB", "Logs"); diff --git a/src/Raven.Server/Raven.Server.csproj b/src/Raven.Server/Raven.Server.csproj index 8f7e76737aff..2647d39680fd 100644 --- a/src/Raven.Server/Raven.Server.csproj +++ b/src/Raven.Server/Raven.Server.csproj @@ -3,7 +3,7 @@ Raven.Server is the database server for RavenDB Hibernating Rhinos net8.0 - 8.0.3 + 8.0.4 true Raven.Server Exe diff --git a/src/Raven.Studio/Raven.Studio.csproj b/src/Raven.Studio/Raven.Studio.csproj index 8744d2686564..43fa23003940 100644 --- a/src/Raven.Studio/Raven.Studio.csproj +++ b/src/Raven.Studio/Raven.Studio.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true Raven.Studio Exe diff --git a/src/Sparrow.Server/Sparrow.Server.csproj b/src/Sparrow.Server/Sparrow.Server.csproj index 93cc515b10e3..088d39d88f5e 100644 --- a/src/Sparrow.Server/Sparrow.Server.csproj +++ b/src/Sparrow.Server/Sparrow.Server.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true Sparrow.Server Sparrow.Server diff --git a/src/Voron/Voron.csproj b/src/Voron/Voron.csproj index 9062ba850a17..de24eea9247c 100644 --- a/src/Voron/Voron.csproj +++ b/src/Voron/Voron.csproj @@ -3,7 +3,7 @@ Voron - Low level storage engine Hibernating Rhinos net8.0 - 8.0.3 + 8.0.4 true Voron Voron diff --git a/test/BenchmarkTests/BenchmarkTests.csproj b/test/BenchmarkTests/BenchmarkTests.csproj index 4da7ab04250c..3b495d4cad37 100644 --- a/test/BenchmarkTests/BenchmarkTests.csproj +++ b/test/BenchmarkTests/BenchmarkTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true BenchmarkTests BenchmarkTests diff --git a/test/EmbeddedTests/EmbeddedTests.csproj b/test/EmbeddedTests/EmbeddedTests.csproj index cbde061e085c..135125afd8ac 100644 --- a/test/EmbeddedTests/EmbeddedTests.csproj +++ b/test/EmbeddedTests/EmbeddedTests.csproj @@ -1,6 +1,6 @@  - 8.0.3 + 8.0.4 true EmbeddedTests EmbeddedTests diff --git a/test/FastTests/FastTests.csproj b/test/FastTests/FastTests.csproj index 8df04a8d26d0..b8b15fed95f4 100644 --- a/test/FastTests/FastTests.csproj +++ b/test/FastTests/FastTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true FastTests FastTests diff --git a/test/InterversionTests/InterversionTests.csproj b/test/InterversionTests/InterversionTests.csproj index e1cb128d79ad..1488d57beba7 100644 --- a/test/InterversionTests/InterversionTests.csproj +++ b/test/InterversionTests/InterversionTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true InterversionTests InterversionTests diff --git a/test/LicenseTests/LicenseTests.csproj b/test/LicenseTests/LicenseTests.csproj index 01130ef9dcc9..3fd416f0ec03 100644 --- a/test/LicenseTests/LicenseTests.csproj +++ b/test/LicenseTests/LicenseTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 LicenseTests LicenseTests ..\..\RavenDB.ruleset diff --git a/test/RachisTests/RachisTests.csproj b/test/RachisTests/RachisTests.csproj index d2c3808f89e1..69a96873d110 100644 --- a/test/RachisTests/RachisTests.csproj +++ b/test/RachisTests/RachisTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true RachisTests RachisTests diff --git a/test/SlowTests/SlowTests.csproj b/test/SlowTests/SlowTests.csproj index 5630bc9a5418..fd646f81d40b 100644 --- a/test/SlowTests/SlowTests.csproj +++ b/test/SlowTests/SlowTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true SlowTests SlowTests diff --git a/test/StressTests/StressTests.csproj b/test/StressTests/StressTests.csproj index 7b275980682f..3a7b63d44282 100644 --- a/test/StressTests/StressTests.csproj +++ b/test/StressTests/StressTests.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true StressTests StressTests diff --git a/test/Tests.Infrastructure/Tests.Infrastructure.csproj b/test/Tests.Infrastructure/Tests.Infrastructure.csproj index c075f8180121..5e1b7d399572 100644 --- a/test/Tests.Infrastructure/Tests.Infrastructure.csproj +++ b/test/Tests.Infrastructure/Tests.Infrastructure.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 Tests.Infrastructure Tests.Infrastructure ..\..\RavenDB.ruleset diff --git a/test/Tryouts/Tryouts.csproj b/test/Tryouts/Tryouts.csproj index a09808c8333c..7dd70e72f3e7 100644 --- a/test/Tryouts/Tryouts.csproj +++ b/test/Tryouts/Tryouts.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 true Tryouts Exe diff --git a/tools/Raven.Debug/Raven.Debug.csproj b/tools/Raven.Debug/Raven.Debug.csproj index 8fc9eea10fed..976246401c46 100644 --- a/tools/Raven.Debug/Raven.Debug.csproj +++ b/tools/Raven.Debug/Raven.Debug.csproj @@ -3,7 +3,7 @@ Exe net8.0 - 8.0.3 + 8.0.4 win-x64;win-x86;linux-x64;osx-x64 ..\..\RavenDB.ruleset true diff --git a/tools/Raven.Migrator/Raven.Migrator.csproj b/tools/Raven.Migrator/Raven.Migrator.csproj index c969c0a6aa20..efded79ac7fb 100644 --- a/tools/Raven.Migrator/Raven.Migrator.csproj +++ b/tools/Raven.Migrator/Raven.Migrator.csproj @@ -2,7 +2,7 @@ Exe net8.0 - 8.0.3 + 8.0.4 win-x64;win-x86;linux-x64;osx-x64 ..\..\RavenDB.ruleset diff --git a/tools/TypingsGenerator/TypingsGenerator.csproj b/tools/TypingsGenerator/TypingsGenerator.csproj index 8e1a2b7f884b..5e6feeb33104 100644 --- a/tools/TypingsGenerator/TypingsGenerator.csproj +++ b/tools/TypingsGenerator/TypingsGenerator.csproj @@ -1,7 +1,7 @@  net8.0 - 8.0.3 + 8.0.4 TypingsGenerator Exe TypingsGenerator diff --git a/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj b/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj index 5392e8eb4411..633ad8488592 100644 --- a/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj +++ b/tools/Voron.Dictionary.Generator/Voron.Dictionary.Generator.csproj @@ -3,7 +3,7 @@ Exe net8.0 - 8.0.3 + 8.0.4 enable enable True diff --git a/tools/Voron.Recovery/Voron.Recovery.csproj b/tools/Voron.Recovery/Voron.Recovery.csproj index b8bc601a5930..2d2edcafbc19 100644 --- a/tools/Voron.Recovery/Voron.Recovery.csproj +++ b/tools/Voron.Recovery/Voron.Recovery.csproj @@ -3,7 +3,7 @@ Voron Recovery Tool Hibernating Rhinos net8.0 - 8.0.3 + 8.0.4 true Voron.Recovery Exe diff --git a/tools/rvn/rvn.csproj b/tools/rvn/rvn.csproj index 00f346722d49..0fe3b07eaa0a 100644 --- a/tools/rvn/rvn.csproj +++ b/tools/rvn/rvn.csproj @@ -4,7 +4,7 @@ Hibernating Rhinos Exe net8.0 - 8.0.3 + 8.0.4 true database;nosql;doc db https://ravendb.net From 68b6e1ba0741f4271cb2686599d0059ef13468e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pekr=C3=B3l?= Date: Thu, 11 Apr 2024 11:05:51 +0200 Subject: [PATCH 007/243] RavenDB-7070 updated GH actions to .NET SDK 8.0.204 --- .github/workflows/FastTests.yml | 2 +- .github/workflows/FastTestsARM.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/FastTests.yml b/.github/workflows/FastTests.yml index 251ce21f622a..0da7c199a939 100644 --- a/.github/workflows/FastTests.yml +++ b/.github/workflows/FastTests.yml @@ -10,7 +10,7 @@ on: - v6.0 env: - DOTNET_VERSION: 8.0.202 + DOTNET_VERSION: 8.0.204 jobs: diff --git a/.github/workflows/FastTestsARM.yml b/.github/workflows/FastTestsARM.yml index ca19f7b45578..b138eaf7d389 100644 --- a/.github/workflows/FastTestsARM.yml +++ b/.github/workflows/FastTestsARM.yml @@ -7,7 +7,7 @@ on: - cron: '30 6 * * *' env: - DOTNET_VERSION: 8.0.202 + DOTNET_VERSION: 8.0.204 jobs: release: From 082a5480a2e0e02482b774aa5602c9bbf678f0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pekr=C3=B3l?= Date: Thu, 11 Apr 2024 11:09:33 +0200 Subject: [PATCH 008/243] RavenDB-7070 updated NuGet packages --- src/Raven.Client/Raven.Client.csproj | 2 +- src/Raven.Server/Raven.Server.csproj | 20 ++++++++++---------- test/FastTests/FastTests.csproj | 2 +- tools/Raven.Debug/Raven.Debug.csproj | 2 +- tools/Raven.Migrator/Raven.Migrator.csproj | 6 +++--- tools/Voron.Recovery/Voron.Recovery.csproj | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Raven.Client/Raven.Client.csproj b/src/Raven.Client/Raven.Client.csproj index 908e4f03dc1c..349f44069f4f 100644 --- a/src/Raven.Client/Raven.Client.csproj +++ b/src/Raven.Client/Raven.Client.csproj @@ -125,7 +125,7 @@ - + diff --git a/src/Raven.Server/Raven.Server.csproj b/src/Raven.Server/Raven.Server.csproj index 2647d39680fd..9ed2c5a94a6e 100644 --- a/src/Raven.Server/Raven.Server.csproj +++ b/src/Raven.Server/Raven.Server.csproj @@ -152,25 +152,25 @@ - - - + + + - + - + All - - + + @@ -179,7 +179,7 @@ - + @@ -188,7 +188,7 @@ - + @@ -199,7 +199,7 @@ - + diff --git a/test/FastTests/FastTests.csproj b/test/FastTests/FastTests.csproj index b8b15fed95f4..02cb9d49e1ce 100644 --- a/test/FastTests/FastTests.csproj +++ b/test/FastTests/FastTests.csproj @@ -74,7 +74,7 @@ - + diff --git a/tools/Raven.Debug/Raven.Debug.csproj b/tools/Raven.Debug/Raven.Debug.csproj index 976246401c46..badb24915c87 100644 --- a/tools/Raven.Debug/Raven.Debug.csproj +++ b/tools/Raven.Debug/Raven.Debug.csproj @@ -21,7 +21,7 @@ - + diff --git a/tools/Raven.Migrator/Raven.Migrator.csproj b/tools/Raven.Migrator/Raven.Migrator.csproj index efded79ac7fb..9e2bf5237489 100644 --- a/tools/Raven.Migrator/Raven.Migrator.csproj +++ b/tools/Raven.Migrator/Raven.Migrator.csproj @@ -13,14 +13,14 @@ - + - + - + \ No newline at end of file diff --git a/tools/Voron.Recovery/Voron.Recovery.csproj b/tools/Voron.Recovery/Voron.Recovery.csproj index 2d2edcafbc19..7597de4d280a 100644 --- a/tools/Voron.Recovery/Voron.Recovery.csproj +++ b/tools/Voron.Recovery/Voron.Recovery.csproj @@ -19,7 +19,7 @@ - + From de037f56a60c5b0d05230ee4fc604b16592413c5 Mon Sep 17 00:00:00 2001 From: egor Date: Thu, 4 Apr 2024 11:58:09 +0300 Subject: [PATCH 009/243] RavenDB-22226 - Fix updating value in dynamic index --- src/Voron/Data/Tables/Table.cs | 35 +++---- test/FastTests/Voron/Tables/RavenDB_22226.cs | 99 ++++++++++++++++++++ 2 files changed, 118 insertions(+), 16 deletions(-) create mode 100644 test/FastTests/Voron/Tables/RavenDB_22226.cs diff --git a/src/Voron/Data/Tables/Table.cs b/src/Voron/Data/Tables/Table.cs index 6d5fba5c6afc..a9415fb93bee 100644 --- a/src/Voron/Data/Tables/Table.cs +++ b/src/Voron/Data/Tables/Table.cs @@ -956,10 +956,8 @@ private void UpdateValuesFromIndex(long id, ref TableValueReader oldVer, TableVa { var indexTree = GetTree(dynamicKeyIndexDef); indexTree.Delete(oldVal); - using (indexTree.DirectAdd(newVal, sizeof(long), TreeNodeFlags.Data, out var ptr)) - { - *(long*)ptr = id; - } + + AddValueToDynamicIndex(id, dynamicKeyIndexDef, indexTree, newVal, TreeNodeFlags.Data); } } } @@ -980,6 +978,22 @@ private void UpdateValuesFromIndex(long id, ref TableValueReader oldVer, TableVa } } + private void AddValueToDynamicIndex(long id, DynamicKeyIndexDef dynamicKeyIndexDef, Tree indexTree, Slice newVal, TreeNodeFlags flags) + { + if (dynamicKeyIndexDef.SupportDuplicateKeys == false) + { + using (indexTree.DirectAdd(newVal, sizeof(long), flags, out var ptr)) + { + *(long*)ptr = id; + } + } + else + { + var index = GetFixedSizeTree(indexTree, newVal, 0, dynamicKeyIndexDef.IsGlobal); + index.Add(id); + } + } + internal long Insert(ref TableValueReader reader) { AssertWritableTable(); @@ -1086,18 +1100,7 @@ private void InsertIndexValuesFor(long id, ref TableValueReader value) dynamicKeyIndexDef.OnIndexEntryChanged(_tx, dynamicKey, oldValue: ref TableValueReaderUtils.EmptyReader, newValue: ref value); var dynamicIndex = GetTree(dynamicKeyIndexDef); - if (dynamicKeyIndexDef.SupportDuplicateKeys == false) - { - using (dynamicIndex.DirectAdd(dynamicKey, sizeof(long), TreeNodeFlags.Data | TreeNodeFlags.NewOnly, out var ptr)) - { - *(long*)ptr = id; - } - - continue; - } - - var index = GetFixedSizeTree(dynamicIndex, dynamicKey, 0, dynamicKeyIndexDef.IsGlobal); - index.Add(id); + AddValueToDynamicIndex(id, dynamicKeyIndexDef, dynamicIndex, dynamicKey, TreeNodeFlags.Data | TreeNodeFlags.NewOnly); } } diff --git a/test/FastTests/Voron/Tables/RavenDB_22226.cs b/test/FastTests/Voron/Tables/RavenDB_22226.cs new file mode 100644 index 000000000000..57266933bac0 --- /dev/null +++ b/test/FastTests/Voron/Tables/RavenDB_22226.cs @@ -0,0 +1,99 @@ +using System; +using System.Runtime.InteropServices; +using Sparrow.Server; +using Voron; +using Voron.Data.Tables; +using Voron.Impl; +using Xunit; +using Xunit.Abstractions; + +namespace FastTests.Voron.Tables +{ + public unsafe class RavenDB_22226 : TableStorageTest + { + public static readonly Slice IndexName; + + public RavenDB_22226(ITestOutputHelper output) : base(output) + { + } + + static RavenDB_22226() + { + using (StorageEnvironment.GetStaticContext(out var ctx)) + { + Slice.From(ctx, "DynamicKeyIndex", ByteStringType.Immutable, out IndexName); + } + } + + [Fact] + public void CanInsertUpdateThenReadByDynamic() + { + using (var tx = Env.WriteTransaction()) + { + Slice.From(tx.Allocator, "RevisionsChangeVector", ByteStringType.Immutable, out var changeVectorSlice); + Slice.From(tx.Allocator, "Etag", ByteStringType.Immutable, out var etag); + var revisionsSchema = new TableSchema(); + revisionsSchema.DefineKey(new TableSchema.IndexDef + { + StartIndex = 0, + Count = 1, + Name = changeVectorSlice, + IsGlobal = false + }); + var index = new TableSchema.DynamicKeyIndexDef { IsGlobal = true, GenerateKey = IndexCvEtagKeyGenerator, Name = IndexName, SupportDuplicateKeys = true }; + revisionsSchema.DefineIndex(index); + + var cv = Guid.NewGuid().ToString(); + + revisionsSchema.Create(tx, "users", 32); + + var usersTbl = tx.OpenTable(revisionsSchema, "users"); + + using (usersTbl.Allocate(out var builder)) + using (Slice.From(tx.Allocator, cv, out var key)) + { + builder.Add(key); + builder.Add(0L); + + usersTbl.Insert(builder); + } + + using (usersTbl.Allocate(out var builder)) + using (Slice.From(tx.Allocator, cv, out var key)) + { + usersTbl.ReadByKey(key, out var tvr); + builder.Add(key); + builder.Add(322L); + usersTbl.Update(tvr.Id, builder); + } + + foreach (var x in usersTbl.SeekForwardFrom(index, Slices.BeforeAllKeys, 0)) + { + var cv1 = x.Result.Reader.ReadString(0); + var etag1 = x.Result.Reader.ReadLong(1); + + Assert.Equal(cv, cv1); + Assert.Equal(322, etag1); + } + } + } + + [StorageIndexEntryKeyGenerator] + internal static ByteStringContext.Scope IndexCvEtagKeyGenerator(Transaction tx, ref TableValueReader tvr, out Slice slice) + { + var cvPtr = tvr.Read(0, out var cvSize); + var etag = tvr.ReadLong(1); + + var scope = tx.Allocator.Allocate(sizeof(long) + cvSize, out var buffer); + + var span = new Span(buffer.Ptr, buffer.Length); + MemoryMarshal.AsBytes(new Span(ref etag)).CopyTo(span); + new ReadOnlySpan(cvPtr, cvSize).CopyTo(span[sizeof(long)..]); + + slice = new Slice(buffer); + return scope; + } + + + } +} From 185663c034e269f7adc48ff2036487ef8f94436a Mon Sep 17 00:00:00 2001 From: egor Date: Mon, 8 Apr 2024 14:59:52 +0300 Subject: [PATCH 010/243] RavenDB-22226 - Fix removing value in dynamic index --- src/Voron/Data/Tables/Table.cs | 32 +++++---- test/FastTests/Voron/Tables/RavenDB_22226.cs | 76 +++++++++++++++----- 2 files changed, 76 insertions(+), 32 deletions(-) diff --git a/src/Voron/Data/Tables/Table.cs b/src/Voron/Data/Tables/Table.cs index a9415fb93bee..d64570aa61ca 100644 --- a/src/Voron/Data/Tables/Table.cs +++ b/src/Voron/Data/Tables/Table.cs @@ -655,19 +655,9 @@ private void DeleteValueFromIndex(long id, ref TableValueReader value) using (dynamicKeyIndexDef.GetValue(_tx, ref value, out Slice val)) { dynamicKeyIndexDef.OnIndexEntryChanged(_tx, val, oldValue: ref value, newValue: ref TableValueReaderUtils.EmptyReader); - var tree = GetTree(dynamicKeyIndexDef); - if (dynamicKeyIndexDef.SupportDuplicateKeys == false) - { - tree.Delete(val); - continue; - } - - var fst = GetFixedSizeTree(tree, val.Clone(_tx.Allocator), 0, dynamicKeyIndexDef.IsGlobal); - if (fst.Delete(id).NumberOfEntriesDeleted == 0) - { - ThrowInvalidAttemptToRemoveValueFromIndexAndNotFindingIt(id, dynamicKeyIndexDef.Name); - } + var tree = GetTree(dynamicKeyIndexDef); + RemoveValueFromDynamicIndex(id, dynamicKeyIndexDef, tree, val); } } @@ -955,7 +945,7 @@ private void UpdateValuesFromIndex(long id, ref TableValueReader oldVer, TableVa forceUpdate) { var indexTree = GetTree(dynamicKeyIndexDef); - indexTree.Delete(oldVal); + RemoveValueFromDynamicIndex(id, dynamicKeyIndexDef, indexTree, oldVal); AddValueToDynamicIndex(id, dynamicKeyIndexDef, indexTree, newVal, TreeNodeFlags.Data); } @@ -994,6 +984,22 @@ private void AddValueToDynamicIndex(long id, DynamicKeyIndexDef dynamicKeyIndexD } } + private void RemoveValueFromDynamicIndex(long id, DynamicKeyIndexDef dynamicKeyIndexDef, Tree tree, Slice val) + { + if (dynamicKeyIndexDef.SupportDuplicateKeys == false) + { + tree.Delete(val); + } + else + { + var fst = GetFixedSizeTree(tree, val.Clone(_tx.Allocator), 0, dynamicKeyIndexDef.IsGlobal); + if (fst.Delete(id).NumberOfEntriesDeleted == 0) + { + ThrowInvalidAttemptToRemoveValueFromIndexAndNotFindingIt(id, dynamicKeyIndexDef.Name); + } + } + } + internal long Insert(ref TableValueReader reader) { AssertWritableTable(); diff --git a/test/FastTests/Voron/Tables/RavenDB_22226.cs b/test/FastTests/Voron/Tables/RavenDB_22226.cs index 57266933bac0..99d8f07239b5 100644 --- a/test/FastTests/Voron/Tables/RavenDB_22226.cs +++ b/test/FastTests/Voron/Tables/RavenDB_22226.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Runtime.InteropServices; using Sparrow.Server; +using Tests.Infrastructure; using Voron; using Voron.Data.Tables; using Voron.Impl; @@ -25,12 +27,13 @@ static RavenDB_22226() } } - [Fact] + [RavenFact(RavenTestCategory.Voron)] public void CanInsertUpdateThenReadByDynamic() { using (var tx = Env.WriteTransaction()) { Slice.From(tx.Allocator, "RevisionsChangeVector", ByteStringType.Immutable, out var changeVectorSlice); + Slice.From(tx.Allocator, "DocumentsChangeVector", ByteStringType.Immutable, out var documentsSlice); Slice.From(tx.Allocator, "Etag", ByteStringType.Immutable, out var etag); var revisionsSchema = new TableSchema(); revisionsSchema.DefineKey(new TableSchema.IndexDef @@ -43,46 +46,83 @@ public void CanInsertUpdateThenReadByDynamic() var index = new TableSchema.DynamicKeyIndexDef { IsGlobal = true, GenerateKey = IndexCvEtagKeyGenerator, Name = IndexName, SupportDuplicateKeys = true }; revisionsSchema.DefineIndex(index); - var cv = Guid.NewGuid().ToString(); + var cv1 = Guid.NewGuid().ToString(); + var cv2 = Guid.NewGuid().ToString(); + var cv3 = Guid.NewGuid().ToString(); + var shared = Guid.NewGuid().ToString(); revisionsSchema.Create(tx, "users", 32); var usersTbl = tx.OpenTable(revisionsSchema, "users"); - using (usersTbl.Allocate(out var builder)) - using (Slice.From(tx.Allocator, cv, out var key)) - { - builder.Add(key); - builder.Add(0L); - - usersTbl.Insert(builder); - } + PopulateTable(tx, usersTbl, cv1, shared, 0L); + PopulateTable(tx, usersTbl, cv2, shared, 0L); + PopulateTable(tx, usersTbl, cv3, shared, 228L); using (usersTbl.Allocate(out var builder)) - using (Slice.From(tx.Allocator, cv, out var key)) + using (Slice.From(tx.Allocator, cv1, out var key)) { usersTbl.ReadByKey(key, out var tvr); + var ptr = tvr.Read(1, out int size); + using var x = Slice.From(tx.Allocator, ptr, size, ByteStringType.Immutable, out var sharedSlice); + builder.Add(key); + builder.Add(sharedSlice); builder.Add(322L); usersTbl.Update(tvr.Id, builder); } + var results = new List<(string key, string shared, long etag)>(); foreach (var x in usersTbl.SeekForwardFrom(index, Slices.BeforeAllKeys, 0)) { - var cv1 = x.Result.Reader.ReadString(0); - var etag1 = x.Result.Reader.ReadLong(1); + results.Add((x.Result.Reader.ReadString(0), x.Result.Reader.ReadString(1), x.Result.Reader.ReadLong(2))); + } + + //Assert results + Assert.Equal(3, results.Count); + + Assert.Contains(results, x=> x.key == cv1 && x.shared == shared && x.etag == 322L); + Assert.Contains(results, x=> x.key == cv2 && x.shared == shared && x.etag == 0L); + Assert.Contains(results, x=> x.key == cv3 && x.shared == shared && x.etag == 228L); + + using (Slice.From(tx.Allocator, cv1, out var key)) + { + usersTbl.ReadByKey(key, out var tvr); + usersTbl.Delete(tvr.Id); + } - Assert.Equal(cv, cv1); - Assert.Equal(322, etag1); + results = new List<(string key, string shared, long etag)>(); + foreach (var x in usersTbl.SeekForwardFrom(index, Slices.BeforeAllKeys, 0)) + { + results.Add((x.Result.Reader.ReadString(0), x.Result.Reader.ReadString(1), x.Result.Reader.ReadLong(2))); } + + //Assert results + Assert.Equal(2, results.Count); + Assert.Contains(results, x => x.key == cv2 && x.shared == shared && x.etag == 0L); + Assert.Contains(results, x => x.key == cv3 && x.shared == shared && x.etag == 228L); + } + } + + private void PopulateTable(Transaction tx, Table usersTbl, string cv, string shared, long etag) + { + using (usersTbl.Allocate(out var builder)) + using (Slice.From(tx.Allocator, cv, out var key)) + using (Slice.From(tx.Allocator, shared, out var sharedSlice)) + { + builder.Add(key); + builder.Add(sharedSlice); + builder.Add(etag); + + usersTbl.Insert(builder); } } [StorageIndexEntryKeyGenerator] internal static ByteStringContext.Scope IndexCvEtagKeyGenerator(Transaction tx, ref TableValueReader tvr, out Slice slice) { - var cvPtr = tvr.Read(0, out var cvSize); - var etag = tvr.ReadLong(1); + var cvPtr = tvr.Read(1, out var cvSize); + var etag = tvr.ReadLong(2); var scope = tx.Allocator.Allocate(sizeof(long) + cvSize, out var buffer); @@ -93,7 +133,5 @@ internal static ByteStringContext.Scope IndexCvEtagKeyGenerator(Transaction tx, slice = new Slice(buffer); return scope; } - - } } From 63e8c52d36526d3936390b6771f95cdcd0fc537e Mon Sep 17 00:00:00 2001 From: Grisha Kotler Date: Sun, 7 Apr 2024 23:37:01 +0300 Subject: [PATCH 011/243] RavenDB-22237 - add transaction limits to DeleteBucketCommand --- .../Documents/Sharding/ShardedDocumentDatabase.cs | 4 +++- .../Documents/Sharding/ShardedDocumentsStorage.cs | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Raven.Server/Documents/Sharding/ShardedDocumentDatabase.cs b/src/Raven.Server/Documents/Sharding/ShardedDocumentDatabase.cs index db3830e864f0..257bd626720e 100644 --- a/src/Raven.Server/Documents/Sharding/ShardedDocumentDatabase.cs +++ b/src/Raven.Server/Documents/Sharding/ShardedDocumentDatabase.cs @@ -231,6 +231,7 @@ public async Task DeleteBucketAsync(int bucket, long migrationIndex, long confir return; // we have more docs, batch limit reached. case DeleteBucketCommand.DeleteBucketResult.FullBatch: + case DeleteBucketCommand.DeleteBucketResult.ReachedTransactionLimit: continue; default: throw new ArgumentOutOfRangeException(); @@ -304,7 +305,8 @@ public enum DeleteBucketResult { Empty, Skipped, - FullBatch + FullBatch, + ReachedTransactionLimit } } } diff --git a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs index db3cd56025d9..b93bfc3ef810 100644 --- a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs +++ b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs @@ -6,12 +6,14 @@ using Raven.Server.Documents.Replication.ReplicationItems; using Raven.Server.ServerWide.Context; using Raven.Server.Utils; +using Sparrow; using Sparrow.Binary; using Sparrow.Server; using Sparrow.Server.Utils; using Sparrow.Utils; using Voron; using Voron.Data.Tables; +using Voron.Global; using Voron.Impl; using static Raven.Server.Documents.Schemas.Attachments; using static Raven.Server.Documents.Schemas.Conflicts; @@ -332,7 +334,9 @@ public IEnumerable RetrieveTombstonesByBucketFrom(DocumentsOperationC } } - public const long MaxDocumentsToDeleteInBucket = 1024; + private const long MaxDocumentsToDeleteInBucket = 1024; + + private const long MaxTransactionSize = 16 * Constants.Size.Megabyte; public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBucket(DocumentsOperationContext context, int bucket, ChangeVector upTo) { @@ -363,6 +367,10 @@ public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBuck // delete revisions for document RevisionsStorage.ForceDeleteAllRevisionsFor(context, document.Id); deleted++; + + if (context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages + + context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > MaxTransactionSize) + return ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult.ReachedTransactionLimit; } if (deleted >= MaxDocumentsToDeleteInBucket) From 5ee1ad2e5e80cffce2a3a10046e8060d576c5d86 Mon Sep 17 00:00:00 2001 From: Grisha Kotler Date: Mon, 8 Apr 2024 16:08:51 +0300 Subject: [PATCH 012/243] RavenDB-22237 - get the TransactionSizeInPages from the LowLevelTransaction --- src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs | 3 +-- src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs | 3 +-- src/Voron/Impl/LowLevelTransaction.cs | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs b/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs index 2d78c2ec2c68..8e2649682774 100644 --- a/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs +++ b/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs @@ -74,8 +74,7 @@ public override long Execute(DocumentsOperationContext context, AbstractTransact break; if (_maxTransactionSizeInPages != null && - context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages + - context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > _maxTransactionSizeInPages) + context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSizeInPages > _maxTransactionSizeInPages) break; if (context.CachedProperties.NeedClearPropertiesCache()) diff --git a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs index b93bfc3ef810..d9436a476571 100644 --- a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs +++ b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs @@ -368,8 +368,7 @@ public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBuck RevisionsStorage.ForceDeleteAllRevisionsFor(context, document.Id); deleted++; - if (context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages + - context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > MaxTransactionSize) + if (context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSizeInPages > MaxTransactionSize) return ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult.ReachedTransactionLimit; } diff --git a/src/Voron/Impl/LowLevelTransaction.cs b/src/Voron/Impl/LowLevelTransaction.cs index eccdd78f97b0..0c281762d031 100644 --- a/src/Voron/Impl/LowLevelTransaction.cs +++ b/src/Voron/Impl/LowLevelTransaction.cs @@ -96,6 +96,8 @@ public void Reset() public event Action LastChanceToReadFromWriteTransactionBeforeCommit; + public long TransactionSizeInPages => NumberOfModifiedPages + AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize; + public Size AdditionalMemoryUsageSize { get From 8d8d2fd837190edbea8d4d7e34adf37c73ff7a0d Mon Sep 17 00:00:00 2001 From: Grisha Kotler Date: Tue, 9 Apr 2024 13:34:47 +0300 Subject: [PATCH 013/243] RavenDB-22237 - fix MaxTransactionSizeInPages --- .../Documents/Sharding/ShardedDocumentsStorage.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs index d9436a476571..823d1a3d85ce 100644 --- a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs +++ b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs @@ -336,7 +336,7 @@ public IEnumerable RetrieveTombstonesByBucketFrom(DocumentsOperationC private const long MaxDocumentsToDeleteInBucket = 1024; - private const long MaxTransactionSize = 16 * Constants.Size.Megabyte; + private const long MaxTransactionSizeInPages = 16 * Constants.Size.Megabyte / Constants.Storage.PageSize; public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBucket(DocumentsOperationContext context, int bucket, ChangeVector upTo) { @@ -368,7 +368,7 @@ public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBuck RevisionsStorage.ForceDeleteAllRevisionsFor(context, document.Id); deleted++; - if (context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSizeInPages > MaxTransactionSize) + if (context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSizeInPages > MaxTransactionSizeInPages) return ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult.ReachedTransactionLimit; } From b4858ab5688ac0af6204085220f8240acc85924a Mon Sep 17 00:00:00 2001 From: Grisha Kotler Date: Wed, 10 Apr 2024 17:08:27 +0300 Subject: [PATCH 014/243] RavenDB-22237 - use Size --- .../Documents/ExecuteRateLimitedOperations.cs | 8 ++++---- .../Documents/Sharding/ShardedDocumentsStorage.cs | 4 ++-- src/Voron/Impl/LowLevelTransaction.cs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs b/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs index 8e2649682774..69a4f37be465 100644 --- a/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs +++ b/src/Raven.Server/Documents/ExecuteRateLimitedOperations.cs @@ -17,7 +17,7 @@ public sealed class ExecuteRateLimitedOperations : MergedTransactionCommand> _commandToExecute; private readonly RateGate _rateGate; private readonly OperationCancelToken _token; - private readonly int? _maxTransactionSizeInPages; + private readonly Size? _maxTransactionSize; private readonly int? _batchSize; private readonly CancellationToken _cancellationToken; @@ -34,7 +34,7 @@ internal ExecuteRateLimitedOperations( _rateGate = rateGate; _token = token; if (maxTransactionSize != null) - _maxTransactionSizeInPages = Math.Max(1, maxTransactionSize.Value / Constants.Storage.PageSize); + _maxTransactionSize = new Size(maxTransactionSize.Value, SizeUnit.Bytes); _batchSize = batchSize; _cancellationToken = token.Token; } @@ -73,8 +73,8 @@ public override long Execute(DocumentsOperationContext context, AbstractTransact if (_batchSize != null && Processed >= _batchSize) break; - if (_maxTransactionSizeInPages != null && - context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSizeInPages > _maxTransactionSizeInPages) + if (_maxTransactionSize != null && + context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSize > _maxTransactionSize) break; if (context.CachedProperties.NeedClearPropertiesCache()) diff --git a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs index 823d1a3d85ce..4c2185e698d5 100644 --- a/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs +++ b/src/Raven.Server/Documents/Sharding/ShardedDocumentsStorage.cs @@ -336,7 +336,7 @@ public IEnumerable RetrieveTombstonesByBucketFrom(DocumentsOperationC private const long MaxDocumentsToDeleteInBucket = 1024; - private const long MaxTransactionSizeInPages = 16 * Constants.Size.Megabyte / Constants.Storage.PageSize; + private readonly Size _maxTransactionSize = new Size(16 * Constants.Size.Megabyte, SizeUnit.Bytes); public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBucket(DocumentsOperationContext context, int bucket, ChangeVector upTo) { @@ -368,7 +368,7 @@ public ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult DeleteBuck RevisionsStorage.ForceDeleteAllRevisionsFor(context, document.Id); deleted++; - if (context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSizeInPages > MaxTransactionSizeInPages) + if (context.Transaction.InnerTransaction.LowLevelTransaction.TransactionSize > _maxTransactionSize) return ShardedDocumentDatabase.DeleteBucketCommand.DeleteBucketResult.ReachedTransactionLimit; } diff --git a/src/Voron/Impl/LowLevelTransaction.cs b/src/Voron/Impl/LowLevelTransaction.cs index 0c281762d031..ff0e744bbe12 100644 --- a/src/Voron/Impl/LowLevelTransaction.cs +++ b/src/Voron/Impl/LowLevelTransaction.cs @@ -96,7 +96,7 @@ public void Reset() public event Action LastChanceToReadFromWriteTransactionBeforeCommit; - public long TransactionSizeInPages => NumberOfModifiedPages + AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize; + public Size TransactionSize => new Size(NumberOfModifiedPages * Constants.Storage.PageSize, SizeUnit.Bytes) + AdditionalMemoryUsageSize; public Size AdditionalMemoryUsageSize { From a6d4d58868569602e626ed3f5771060fef8cde43 Mon Sep 17 00:00:00 2001 From: Federico Lois Date: Tue, 9 Apr 2024 14:56:31 -0300 Subject: [PATCH 015/243] RavenDB-22245: Supporting Vector256 for quantization --- bench/Voron.Benchmark/Corax/CoraxEntryIdEncodings.cs | 6 +++--- src/Corax/Utils/EntryIdEncodings.cs | 12 ++++++------ test/FastTests/Corax/Ranking/QuantizationTest.cs | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bench/Voron.Benchmark/Corax/CoraxEntryIdEncodings.cs b/bench/Voron.Benchmark/Corax/CoraxEntryIdEncodings.cs index dbe175af59be..30aaae4bc96b 100644 --- a/bench/Voron.Benchmark/Corax/CoraxEntryIdEncodings.cs +++ b/bench/Voron.Benchmark/Corax/CoraxEntryIdEncodings.cs @@ -30,9 +30,9 @@ public void GlobalSetup() [Benchmark] public Span DiscardWithSimd() { - EntryIdEncodings.DecodeAndDiscardFrequencyAvx2(_idsWithEncodingsForSimd, BufferSize); - EntryIdEncodings.DecodeAndDiscardFrequencyAvx2(_idsWithEncodingsForSimd, BufferSize); - EntryIdEncodings.DecodeAndDiscardFrequencyAvx2(_idsWithEncodingsForSimd, BufferSize); + EntryIdEncodings.DecodeAndDiscardFrequencyVector256(_idsWithEncodingsForSimd, BufferSize); + EntryIdEncodings.DecodeAndDiscardFrequencyVector256(_idsWithEncodingsForSimd, BufferSize); + EntryIdEncodings.DecodeAndDiscardFrequencyVector256(_idsWithEncodingsForSimd, BufferSize); return _idsWithEncodingsForSimd; } diff --git a/src/Corax/Utils/EntryIdEncodings.cs b/src/Corax/Utils/EntryIdEncodings.cs index 04241a8c3d59..ced93312e86b 100644 --- a/src/Corax/Utils/EntryIdEncodings.cs +++ b/src/Corax/Utils/EntryIdEncodings.cs @@ -67,7 +67,7 @@ public static void Encode(Span entries, Span frequencies) public static long DecodeAndDiscardFrequency(long entryId) => entryId >> EntryIdOffset; [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe void DecodeAndDiscardFrequencyAvx2(Span entries, int read) + public static void DecodeAndDiscardFrequencyVector256(Span entries, int read) { int idX = read - (read % Vector256.Count); if (read < Vector256.Count) @@ -79,7 +79,7 @@ public static unsafe void DecodeAndDiscardFrequencyAvx2(Span entries, int { ref var currentPtr = ref Unsafe.Add(ref start, currentIdx); var innerBuffer = Vector256.LoadUnsafe(ref currentPtr); - var shiftRightLogical = Avx2.ShiftRightLogical(innerBuffer, EntryIdOffset); + var shiftRightLogical = Vector256.ShiftRightLogical(innerBuffer, EntryIdOffset); Vector256.StoreUnsafe(shiftRightLogical, ref currentPtr); currentIdx += Vector256.Count; @@ -92,7 +92,7 @@ public static unsafe void DecodeAndDiscardFrequencyAvx2(Span entries, int } [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe void DecodeAndDiscardFrequencyNeon(Span entries, int read) + public static void DecodeAndDiscardFrequencyNeon(Span entries, int read) { int idX = read - (read % Vector128.Count); if (read < Vector128.Count) @@ -128,8 +128,8 @@ public static void DecodeAndDiscardFrequencyClassic(Span entries, int read [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] public static void DecodeAndDiscardFrequency(Span entries, int read) { - if (Avx2.IsSupported) - DecodeAndDiscardFrequencyAvx2(entries, read); + if (Vector256.IsHardwareAccelerated) + DecodeAndDiscardFrequencyVector256(entries, read); else if (AdvSimd.IsSupported) DecodeAndDiscardFrequencyNeon(entries, read); else @@ -180,7 +180,7 @@ private static long ArmLzcntFrequencyQuantization(short frequency) if (frequency < 16) return frequency; - var leadingZeros = ArmBase.Arm64.LeadingZeroCount((long)frequency); + var leadingZeros = ArmBase.Arm64.LeadingZeroCount(frequency); var level = (60 - leadingZeros + (leadingZeros & 0b1)) >> 1; var mod = (frequency - Step[level - 1]) / LevelSizeInStep[level]; Debug.Assert((long)mod < 16); diff --git a/test/FastTests/Corax/Ranking/QuantizationTest.cs b/test/FastTests/Corax/Ranking/QuantizationTest.cs index 950f77da23c4..86ada0eb2bc6 100644 --- a/test/FastTests/Corax/Ranking/QuantizationTest.cs +++ b/test/FastTests/Corax/Ranking/QuantizationTest.cs @@ -33,7 +33,7 @@ public void CanEncodeAndDecodeEveryNumberUnderShort() } } - [RavenMultiplatformTheory(RavenTestCategory.Corax | RavenTestCategory.Intrinsics, RavenIntrinsics.Avx2)] + [RavenMultiplatformTheory(RavenTestCategory.Corax | RavenTestCategory.Intrinsics)] [InlineData(7)] [InlineData(8)] [InlineData(16)] @@ -41,7 +41,7 @@ public void CanEncodeAndDecodeEveryNumberUnderShort() [InlineData(32)] [InlineData(33)] [InlineData(1)] - public void Avx2InstructionCorrectlyIgnoresFrequency(int size) + public void Vector256InstructionCorrectlyIgnoresFrequency(int size) { var random = new Random(2337); var ids = Enumerable.Range(0, size).Select(i => (long)random.Next(31_111, 59_999)).ToArray(); @@ -50,7 +50,7 @@ public void Avx2InstructionCorrectlyIgnoresFrequency(int size) var idsWithShiftedCopy = idsWithShifted.ToArray(); EntryIdEncodings.DecodeAndDiscardFrequencyClassic(idsWithShiftedCopy.AsSpan(), size); - EntryIdEncodings.DecodeAndDiscardFrequencyAvx2(idsWithShifted.AsSpan(), size); + EntryIdEncodings.DecodeAndDiscardFrequencyVector256(idsWithShifted.AsSpan(), size); Assert.Equal(ids, idsWithShifted); Assert.Equal(idsWithShifted, idsWithShiftedCopy); From bb336b92304c9383144eef70f8c408ae65b3bb59 Mon Sep 17 00:00:00 2001 From: Federico Lois Date: Tue, 9 Apr 2024 14:57:08 -0300 Subject: [PATCH 016/243] RavenDB-22245: Missing return in FrequencyQuantization cause running method twice --- src/Corax/Utils/EntryIdEncodings.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Corax/Utils/EntryIdEncodings.cs b/src/Corax/Utils/EntryIdEncodings.cs index ced93312e86b..9e1daaf19ba4 100644 --- a/src/Corax/Utils/EntryIdEncodings.cs +++ b/src/Corax/Utils/EntryIdEncodings.cs @@ -163,13 +163,13 @@ internal static long QuantizeAndDequantize(short frequency) { return FrequencyReconstructionFromQuantization(FrequencyQuantization(frequency)); } - + internal static long FrequencyQuantization(short frequency) { if (Lzcnt.IsSupported) return LzcntFrequencyQuantization(frequency); if (ArmBase.Arm64.IsSupported) - ArmLzcntFrequencyQuantization(frequency); + return ArmLzcntFrequencyQuantization(frequency); return FrequencyQuantizationWithoutAcceleration(frequency); } From 6dd440bd4343aef1ef33defd9cfb85b6ffaff39e Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Wed, 3 Apr 2024 15:08:38 +0200 Subject: [PATCH 017/243] RavenDB-21996 Include documentation --- src/Raven.Client/DocumentationUrls.cs | 9 +- .../Conventions/DocumentConventions.cs | 1 + src/Raven.Client/Documents/LinqExtensions.cs | 33 ++----- .../Session/AbstractDocumentQuery.Includes.cs | 9 +- .../Session/AsyncDocumentSession.Includes.cs | 25 +---- .../Session/AsyncDocumentSession.Lazy.cs | 15 +-- .../Session/DocumentSession.Includes.cs | 38 +++----- .../Documents/Session/DocumentSession.Lazy.cs | 10 +- .../Session/IAbstractDocumentQuery.cs | 22 ++--- .../Session/IAsyncDocumentSession.Load.cs | 2 + .../Documents/Session/IDocumentQueryBase.cs | 5 +- .../Session/IDocumentSession.Include.cs | 25 +---- .../Loaders/AsyncMultiLoaderWithInclude.cs | 28 +----- .../Loaders/IAsyncLoaderWithInclude.cs | 36 ++----- .../Session/Loaders/ILazyLoaderWithInclude.cs | 30 ++---- .../Session/Loaders/ILoaderWithInclude.cs | 30 +----- .../Session/Loaders/IncludeBuilder.cs | 94 ++++++++++++++++++- .../Loaders/LazyMultiLoaderWithInclude.cs | 31 ++---- .../Operations/Lazy/ILazySessionOperations.cs | 31 ++---- 19 files changed, 186 insertions(+), 288 deletions(-) diff --git a/src/Raven.Client/DocumentationUrls.cs b/src/Raven.Client/DocumentationUrls.cs index 1bab116a5f40..77887e135be2 100644 --- a/src/Raven.Client/DocumentationUrls.cs +++ b/src/Raven.Client/DocumentationUrls.cs @@ -98,13 +98,17 @@ internal static class StreamQueryResults public const string GroupByQuery = nameof(GroupByQuery); /// - /// + ///
/// /// public const string GroupByArrayQuery = nameof(GroupByArrayQuery); /// public const string GroupByArrayContent = nameof(GroupByArrayContent); + + /// + public const string Includes = nameof(Includes); + } internal static class Counters @@ -126,6 +130,9 @@ internal static class Options /// public const string NoCaching = nameof(NoCaching); + + /// + public const string Conventions = nameof(Conventions); } internal static class Sharding diff --git a/src/Raven.Client/Documents/Conventions/DocumentConventions.cs b/src/Raven.Client/Documents/Conventions/DocumentConventions.cs index 5b6db50b7f07..e1fefc9a914f 100644 --- a/src/Raven.Client/Documents/Conventions/DocumentConventions.cs +++ b/src/Raven.Client/Documents/Conventions/DocumentConventions.cs @@ -32,6 +32,7 @@ namespace Raven.Client.Documents.Conventions /// The set of conventions used by the which allow the users to customize /// the way the Raven client API behaves /// + /// public sealed class DocumentConventions : Client.Conventions { public delegate LinqPathProvider.Result CustomQueryTranslator(LinqPathProvider provider, Expression expression); diff --git a/src/Raven.Client/Documents/LinqExtensions.cs b/src/Raven.Client/Documents/LinqExtensions.cs index 9808a178d38d..63af9ddd1b2e 100644 --- a/src/Raven.Client/Documents/LinqExtensions.cs +++ b/src/Raven.Client/Documents/LinqExtensions.cs @@ -31,26 +31,16 @@ namespace Raven.Client.Documents /// public static class LinqExtensions { - /// - /// Includes the specified path in the query, loading the document specified in that path - /// - /// The type of the object that holds the id that you want to include. + + /// /// The source for querying - /// The path, which is name of the property that holds the id of the object to include. - /// public static IRavenQueryable Include(this IQueryable source, Expression> path) { return source.Include(path.ToPropertyPath(((IRavenQueryProvider)source.Provider).QueryGenerator.Conventions)); } - /// - /// Includes the specified path in the query, loading the document specified in that path - /// - /// The type of the object that holds the id that you want to include. - /// The type of the object that you want to include. + /// /// The source for querying - /// The path, which is name of the property that holds the id of the object to include. - /// public static IRavenQueryable Include(this IQueryable source, Expression> path) { var queryInspector = (IRavenQueryInspector)source; @@ -58,14 +48,9 @@ public static IRavenQueryable Include(this IQueryabl return Include(source, IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(((IRavenQueryProvider)source.Provider).QueryGenerator.Conventions), conventions)); } - - /// - /// Includes the specified path in the query, loading the document specified in that path - /// - /// The type of the object that holds the id that you want to include. + + /// /// The source for querying - /// The path, which is name of the property that holds the id of the object to include. - /// public static IRavenQueryable Include(this IQueryable source, string path) { var currentMethod = (MethodInfo)MethodBase.GetCurrentMethod(); @@ -115,13 +100,9 @@ public static IRavenQueryable Filter( return (IRavenQueryable)queryable; } - /// - /// Includes the specified documents and/or counters in the query - /// - /// The type of the object that holds the id that you want to include. + /// + /// Includes builder. Specifies the documents/counters/revisions/time series to load from the server. See more at: . /// The source for querying - /// Specifies the documents and/or counters to include - /// public static IRavenQueryable Include(this IQueryable source, Action> includes) { var queryInspector = (IRavenQueryInspector)source; diff --git a/src/Raven.Client/Documents/Session/AbstractDocumentQuery.Includes.cs b/src/Raven.Client/Documents/Session/AbstractDocumentQuery.Includes.cs index 6e820b4d3f60..a5a1131683bd 100644 --- a/src/Raven.Client/Documents/Session/AbstractDocumentQuery.Includes.cs +++ b/src/Raven.Client/Documents/Session/AbstractDocumentQuery.Includes.cs @@ -23,11 +23,7 @@ public abstract partial class AbstractDocumentQuery /// protected HashSet DocumentIncludes = new HashSet(); - /// - /// Allows to fetch related documents without having to make multiple remote calls. - /// - /// Path to the property referencing related document. - /// + /// public void Include(string path) { TheSession?.AssertNoIncludesInNonTrackingSession(); @@ -35,12 +31,13 @@ public void Include(string path) DocumentIncludes.Add(path); } - /// + /// public void Include(Expression> path) { Include(path.ToPropertyPath(_conventions)); } + /// public void Include(IncludeBuilder includes) { if (includes == null) diff --git a/src/Raven.Client/Documents/Session/AsyncDocumentSession.Includes.cs b/src/Raven.Client/Documents/Session/AsyncDocumentSession.Includes.cs index 9cb0db3a690f..ed095f2fcc20 100644 --- a/src/Raven.Client/Documents/Session/AsyncDocumentSession.Includes.cs +++ b/src/Raven.Client/Documents/Session/AsyncDocumentSession.Includes.cs @@ -16,46 +16,31 @@ namespace Raven.Client.Documents.Session /// public partial class AsyncDocumentSession { - /// - /// Begin a load while including the specified path - /// - /// The path. + /// public IAsyncLoaderWithInclude Include(string path) { return new AsyncMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// public IAsyncLoaderWithInclude Include(Expression> path) { return new AsyncMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// public IAsyncLoaderWithInclude Include(Expression> path) { return new AsyncMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// public IAsyncLoaderWithInclude Include(Expression>> path) { return new AsyncMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// public IAsyncLoaderWithInclude Include(Expression>> path) { return new AsyncMultiLoaderWithInclude(this).Include(path); diff --git a/src/Raven.Client/Documents/Session/AsyncDocumentSession.Lazy.cs b/src/Raven.Client/Documents/Session/AsyncDocumentSession.Lazy.cs index 3cf859d7c5fe..5386d671e335 100644 --- a/src/Raven.Client/Documents/Session/AsyncDocumentSession.Lazy.cs +++ b/src/Raven.Client/Documents/Session/AsyncDocumentSession.Lazy.cs @@ -166,28 +166,19 @@ private async Task ExecuteLazyOperationsSingleStep(ResponseTimeInformation return false; } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// IAsyncLazyLoaderWithInclude IAsyncLazySessionOperations.Include(Expression> path) { return new AsyncLazyMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// IAsyncLazyLoaderWithInclude IAsyncLazySessionOperations.Include(Expression>> path) { return new AsyncLazyMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// IAsyncLazyLoaderWithInclude IAsyncLazySessionOperations.Include(string path) { return new AsyncLazyMultiLoaderWithInclude(this).Include(path); diff --git a/src/Raven.Client/Documents/Session/DocumentSession.Includes.cs b/src/Raven.Client/Documents/Session/DocumentSession.Includes.cs index c0e20ea60c36..ea9c02cbb6c4 100644 --- a/src/Raven.Client/Documents/Session/DocumentSession.Includes.cs +++ b/src/Raven.Client/Documents/Session/DocumentSession.Includes.cs @@ -16,54 +16,40 @@ namespace Raven.Client.Documents.Session /// public partial class DocumentSession { - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// + /// Type of main loaded document public ILoaderWithInclude Include(Expression> path) { return new MultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// + /// Type of main loaded document + /// Type of included document public ILoaderWithInclude Include(Expression> path) { return new MultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// + /// Type of main loaded document public ILoaderWithInclude Include(Expression>> path) { return new MultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// + /// Type of main loaded document + /// Type of included document public ILoaderWithInclude Include(Expression>> path) { return new MultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// public ILoaderWithInclude Include(string path) { return new MultiLoaderWithInclude(this).Include(path); } } -} \ No newline at end of file +} diff --git a/src/Raven.Client/Documents/Session/DocumentSession.Lazy.cs b/src/Raven.Client/Documents/Session/DocumentSession.Lazy.cs index 67f548e1ec73..09144e1b3cec 100644 --- a/src/Raven.Client/Documents/Session/DocumentSession.Lazy.cs +++ b/src/Raven.Client/Documents/Session/DocumentSession.Lazy.cs @@ -19,19 +19,13 @@ namespace Raven.Client.Documents.Session /// public partial class DocumentSession { - /// - /// Begin a load while including the specified path - /// - /// The path. + /// ILazyLoaderWithInclude ILazySessionOperations.Include(Expression> path) { return new LazyMultiLoaderWithInclude(this).Include(path); } - /// - /// Begin a load while including the specified path - /// - /// The path. + /// ILazyLoaderWithInclude ILazySessionOperations.Include(Expression>> path) { return new LazyMultiLoaderWithInclude(this).Include(path); diff --git a/src/Raven.Client/Documents/Session/IAbstractDocumentQuery.cs b/src/Raven.Client/Documents/Session/IAbstractDocumentQuery.cs index d9caf9ba7088..37a486a1b644 100644 --- a/src/Raven.Client/Documents/Session/IAbstractDocumentQuery.cs +++ b/src/Raven.Client/Documents/Session/IAbstractDocumentQuery.cs @@ -30,6 +30,7 @@ public interface IAbstractDocumentQuery /// /// Get the document conventions used for this query. /// + /// DocumentConventions Conventions { get; } /// @@ -50,15 +51,10 @@ public interface IAbstractDocumentQuery /// IEnumerable GetProjectionFields(); - /// - /// Order the query results randomly. - /// + /// void RandomOrdering(); - /// - /// Orders the query results randomly using the specified seed. - /// Allows to repeat random query results. - /// + /// void RandomOrdering(string seed); #if FEATURE_CUSTOM_SORTING @@ -71,19 +67,13 @@ public interface IAbstractDocumentQuery void CustomSortUsing(string typeName, bool descending = false); #endif - /// - /// Includes document using the specified path, loading it into session. - /// - /// Path to loaded document. + /// void Include(string path); - /// > + /// void Include(Expression> path); - /// - /// Includes the specified documents and/or counters in the query, specified by IncludeBuilder - /// - /// + /// void Include(IncludeBuilder includes); /// diff --git a/src/Raven.Client/Documents/Session/IAsyncDocumentSession.Load.cs b/src/Raven.Client/Documents/Session/IAsyncDocumentSession.Load.cs index b6a54e7cf79f..8a85c624c0a5 100644 --- a/src/Raven.Client/Documents/Session/IAsyncDocumentSession.Load.cs +++ b/src/Raven.Client/Documents/Session/IAsyncDocumentSession.Load.cs @@ -36,6 +36,7 @@ public partial interface IAsyncDocumentSession /// /// An action that specifies which documents and\or counters /// to include, by using the IIncludeBuilder interface. + /// See more at IncludeBuilder. /// /// The cancellation token. @@ -48,6 +49,7 @@ public partial interface IAsyncDocumentSession /// /// An action that specifies which documents and\or counters /// to include, by using the IIncludeBuilder interface. + /// See more at IncludeBuilder. /// /// The cancellation token. Task> LoadAsync(IEnumerable ids, Action> includes, CancellationToken token = default(CancellationToken)); diff --git a/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs b/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs index b67f926e092a..9796ca74c430 100644 --- a/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs +++ b/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs @@ -8,6 +8,7 @@ using Raven.Client.Documents.Queries.Highlighting; using Raven.Client.Documents.Queries.MoreLikeThis; using Raven.Client.Documents.Queries.Spatial; +using Raven.Client.Documents.Session.Loaders; namespace Raven.Client.Documents.Session { @@ -511,10 +512,10 @@ public interface IDocumentQueryBase : IPagingDocumentQueryBase TSelf Highlight(Expression> path, int fragmentLength, int fragmentCount, HighlightingOptions options, out Highlightings highlightings); - /// + /// TSelf Include(string path); - /// + /// TSelf Include(Expression> path); /// diff --git a/src/Raven.Client/Documents/Session/IDocumentSession.Include.cs b/src/Raven.Client/Documents/Session/IDocumentSession.Include.cs index b48d7013d62e..32fa7cf12fb4 100644 --- a/src/Raven.Client/Documents/Session/IDocumentSession.Include.cs +++ b/src/Raven.Client/Documents/Session/IDocumentSession.Include.cs @@ -16,34 +16,19 @@ namespace Raven.Client.Documents.Session /// public partial interface IDocumentSession { - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILoaderWithInclude Include(string path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILoaderWithInclude Include(Expression>> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILoaderWithInclude Include(Expression>> path); } } diff --git a/src/Raven.Client/Documents/Session/Loaders/AsyncMultiLoaderWithInclude.cs b/src/Raven.Client/Documents/Session/Loaders/AsyncMultiLoaderWithInclude.cs index 66969e72015f..a61223647fde 100644 --- a/src/Raven.Client/Documents/Session/Loaders/AsyncMultiLoaderWithInclude.cs +++ b/src/Raven.Client/Documents/Session/Loaders/AsyncMultiLoaderWithInclude.cs @@ -22,50 +22,32 @@ public sealed class AsyncMultiLoaderWithInclude : IAsyncLoaderWithInclude private readonly IAsyncDocumentSessionImpl _session; private readonly List _includes = new List(); - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// public AsyncMultiLoaderWithInclude Include(string path) { _includes.Add(path); return this; } - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// public AsyncMultiLoaderWithInclude Include(Expression> path) { return Include(path.ToPropertyPath(_session.Conventions)); } - /// - /// Includes the specified path. - /// - /// The path. + /// public AsyncMultiLoaderWithInclude Include(Expression> path) { return Include(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_session.Conventions), _session.Conventions)); } - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// public AsyncMultiLoaderWithInclude Include(Expression>> path) { return Include(path.ToPropertyPath(_session.Conventions)); } - /// - /// Includes the specified path. - /// - /// The path. + /// public AsyncMultiLoaderWithInclude Include(Expression>> path) { return Include(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_session.Conventions), _session.Conventions)); diff --git a/src/Raven.Client/Documents/Session/Loaders/IAsyncLoaderWithInclude.cs b/src/Raven.Client/Documents/Session/Loaders/IAsyncLoaderWithInclude.cs index 07071128cabb..b549061eea9b 100644 --- a/src/Raven.Client/Documents/Session/Loaders/IAsyncLoaderWithInclude.cs +++ b/src/Raven.Client/Documents/Session/Loaders/IAsyncLoaderWithInclude.cs @@ -12,39 +12,21 @@ namespace Raven.Client.Documents.Session.Loaders /// public interface IAsyncLoaderWithInclude { - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// AsyncMultiLoaderWithInclude Include(string path); - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// AsyncMultiLoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// - /// The path. - /// + + /// + /// Type of included document. AsyncMultiLoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// The path. - /// + /// AsyncMultiLoaderWithInclude Include(Expression>> path); - /// - /// Begin a load while including the specified path - /// - /// - /// The path. - /// + + /// + /// Type of included document. AsyncMultiLoaderWithInclude Include(Expression>> path); /// diff --git a/src/Raven.Client/Documents/Session/Loaders/ILazyLoaderWithInclude.cs b/src/Raven.Client/Documents/Session/Loaders/ILazyLoaderWithInclude.cs index 7299a7e28c2a..3b36d0524819 100644 --- a/src/Raven.Client/Documents/Session/Loaders/ILazyLoaderWithInclude.cs +++ b/src/Raven.Client/Documents/Session/Loaders/ILazyLoaderWithInclude.cs @@ -11,22 +11,13 @@ namespace Raven.Client.Documents.Session.Loaders /// public interface ILazyLoaderWithInclude { - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILazyLoaderWithInclude Include(string path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILazyLoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILazyLoaderWithInclude Include(Expression>> path); /// @@ -68,22 +59,13 @@ public interface ILazyLoaderWithInclude public interface IAsyncLazyLoaderWithInclude { - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// IAsyncLazyLoaderWithInclude Include(string path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// IAsyncLazyLoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// IAsyncLazyLoaderWithInclude Include(Expression>> path); /// diff --git a/src/Raven.Client/Documents/Session/Loaders/ILoaderWithInclude.cs b/src/Raven.Client/Documents/Session/Loaders/ILoaderWithInclude.cs index 9e15ede743e5..7b6a1e527529 100644 --- a/src/Raven.Client/Documents/Session/Loaders/ILoaderWithInclude.cs +++ b/src/Raven.Client/Documents/Session/Loaders/ILoaderWithInclude.cs @@ -16,39 +16,19 @@ namespace Raven.Client.Documents.Session.Loaders /// public interface ILoaderWithInclude { - /// - /// Includes the specified path. - /// - /// The path. - /// + /// ILoaderWithInclude Include(string path); - /// - /// Includes the specified path. - /// - /// The path. - /// + /// ILoaderWithInclude Include(Expression> path); - /// - /// Includes the specified path. - /// - /// The path. - /// + /// ILoaderWithInclude Include(Expression> path); - /// - /// Includes the specified path. - /// - /// The path. - /// + /// ILoaderWithInclude Include(Expression>> path); - /// - /// Includes the specified path. - /// - /// The path. - /// + /// ILoaderWithInclude Include(Expression>> path); /// diff --git a/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs b/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs index 9ebf44c1bf3a..48cb5232f284 100644 --- a/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs +++ b/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs @@ -9,16 +9,42 @@ namespace Raven.Client.Documents.Session.Loaders { + /// + /// The server is instructed to pre-load referenced documents concurrently with retrieving the documents.
+ /// The documents are added to the session unit of work, and subsequent requests to load them are served directly from the session cache, + /// without requiring any additional queries to the server.
+ /// The server can then be instructed to pre-load the referenced object at the same time that the root object is retrieved, for example using: + /// + /// + /// Order order = session.Include<Order>(x => x.CustomerId).Load("orders/1-A"); + /// // this will not require querying the server: + /// Customer customer = session.Load<Customer>(order.CustomerId); + /// + /// + /// public interface IDocumentIncludeBuilder { + + /// + /// Name of property which contains id(s) of document(s) to include from queried document. + /// TBuilder IncludeDocuments(string path); + /// + /// Path to the property which contains id of document to include. TBuilder IncludeDocuments(Expression> path); + /// + /// Path to the property which contains ids of documents to include. + /// TBuilder IncludeDocuments(Expression>> path); + /// + /// Type of included document TBuilder IncludeDocuments(Expression> path); + /// + /// Type of included document TBuilder IncludeDocuments(Expression>> path); } @@ -96,6 +122,7 @@ public interface IIncludeBuilder : IDocumentIncludeBuilder public interface IIncludeBuilder : IIncludeBuilder> { } @@ -106,12 +133,16 @@ public interface ISubscriptionIncludeBuilder : IDocumentIncludeBuilder : IIncludeBuilder> { + /// IQueryIncludeBuilder IncludeCounter(Expression> path, string name); - + + /// IQueryIncludeBuilder IncludeCounters(Expression> path, string[] names); + /// IQueryIncludeBuilder IncludeAllCounters(Expression> path); - + + /// IQueryIncludeBuilder IncludeTimeSeries(Expression> path, string name, DateTime from, DateTime to); } @@ -178,228 +209,267 @@ internal IncludeBuilder(DocumentConventions conventions) _conventions = conventions; } + /// IQueryIncludeBuilder IQueryIncludeBuilder.IncludeCounter(Expression> path, string name) { IncludeCounterWithAlias(path, name); return this; } + /// IQueryIncludeBuilder IQueryIncludeBuilder.IncludeCounters(Expression> path, string[] names) { IncludeCountersWithAlias(path, names); return this; } + /// IQueryIncludeBuilder IQueryIncludeBuilder.IncludeAllCounters(Expression> path) { IncludeAllCountersWithAlias(path); return this; } + /// IQueryIncludeBuilder ICounterIncludeBuilder>.IncludeCounter(string name) { IncludeCounter(string.Empty, name); return this; } + /// IQueryIncludeBuilder ICounterIncludeBuilder>.IncludeCounters(string[] names) { IncludeCounters(string.Empty, names); return this; } + /// IQueryIncludeBuilder ICounterIncludeBuilder>.IncludeAllCounters() { IncludeAllCounters(string.Empty); return this; } + /// ISubscriptionIncludeBuilder ICounterIncludeBuilder>.IncludeCounter(string name) { IncludeCounter(string.Empty, name); return this; } + /// ISubscriptionIncludeBuilder ICounterIncludeBuilder>.IncludeCounters(string[] names) { IncludeCounters(string.Empty, names); return this; } + /// ISubscriptionIncludeBuilder ICounterIncludeBuilder>.IncludeAllCounters() { IncludeAllCounters(string.Empty); return this; } + /// ISubscriptionIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression> path) { IncludeDocuments(path.ToPropertyPath(_conventions)); return this; } + /// ISubscriptionIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression>> path) { IncludeDocuments(path.ToPropertyPath(_conventions)); return this; } + /// ISubscriptionIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression> path) { IncludeDocuments(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_conventions), _conventions)); return this; } + /// ISubscriptionIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression>> path) { IncludeDocuments(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_conventions), _conventions)); return this; } + /// ISubscriptionIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(string path) { IncludeDocuments(path); return this; } + /// IQueryIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression> path) { IncludeDocuments(path.ToPropertyPath(_conventions)); return this; } + /// IQueryIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression>> path) { IncludeDocuments(path.ToPropertyPath(_conventions)); return this; } + /// IQueryIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression> path) { IncludeDocuments(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_conventions), _conventions)); return this; } + /// IQueryIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression>> path) { IncludeDocuments(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_conventions), _conventions)); return this; } + /// IQueryIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(string path) { IncludeDocuments(path); return this; } + /// IIncludeBuilder ICounterIncludeBuilder>.IncludeCounter(string name) { IncludeCounter(string.Empty, name); return this; } + /// IIncludeBuilder ICounterIncludeBuilder>.IncludeCounters(string[] names) { IncludeCounters(string.Empty, names); return this; } + /// IIncludeBuilder ICounterIncludeBuilder>.IncludeAllCounters() { IncludeAllCounters(string.Empty); return this; } + /// IIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(string path) { IncludeDocuments(path); return this; } + /// IIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression> path) { IncludeDocuments(path.ToPropertyPath(_conventions)); return this; } + /// IIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression>> path) { IncludeDocuments(path.ToPropertyPath(_conventions)); return this; } + /// IIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression> path) { IncludeDocuments(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_conventions), _conventions)); return this; } + /// IIncludeBuilder IDocumentIncludeBuilder>.IncludeDocuments(Expression>> path) { IncludeDocuments(IncludesUtil.GetPrefixedIncludePath(path.ToPropertyPath(_conventions), _conventions)); return this; } + /// IIncludeBuilder ITimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, DateTime? from, DateTime? to) { IncludeTimeSeriesFromTo(string.Empty, name, from, to); return this; } + /// IQueryIncludeBuilder ITimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, DateTime? from, DateTime? to) { IncludeTimeSeriesFromTo(string.Empty, name, from, to); return this; } + /// IQueryIncludeBuilder IQueryIncludeBuilder.IncludeTimeSeries(Expression> path, string name, DateTime from, DateTime to) { WithAlias(path); IncludeTimeSeriesFromTo(path.ToPropertyPath(_conventions), name, from, to); return this; } + + /// IQueryIncludeBuilder ICompareExchangeValueIncludeBuilder>.IncludeCompareExchangeValue(string path) { IncludeCompareExchangeValue(path); return this; } + /// IQueryIncludeBuilder ICompareExchangeValueIncludeBuilder>.IncludeCompareExchangeValue(Expression> path) { IncludeCompareExchangeValue(path.ToPropertyPath(_conventions)); return this; } + /// IQueryIncludeBuilder ICompareExchangeValueIncludeBuilder>.IncludeCompareExchangeValue(Expression>> path) { IncludeCompareExchangeValue(path.ToPropertyPath(_conventions)); return this; } + /// IIncludeBuilder ICompareExchangeValueIncludeBuilder>.IncludeCompareExchangeValue(string path) { IncludeCompareExchangeValue(path); return this; } + /// IIncludeBuilder ICompareExchangeValueIncludeBuilder>.IncludeCompareExchangeValue(Expression> path) { IncludeCompareExchangeValue(path.ToPropertyPath(_conventions)); return this; } + /// IIncludeBuilder ICompareExchangeValueIncludeBuilder>.IncludeCompareExchangeValue(Expression>> path) { IncludeCompareExchangeValue(path.ToPropertyPath(_conventions)); return this; } - + + /// ITimeSeriesIncludeBuilder ITimeSeriesIncludeBuilder.IncludeTags() { IncludeTimeSeriesTags = true; return this; } + /// ITimeSeriesIncludeBuilder ITimeSeriesIncludeBuilder.IncludeDocument() { IncludeTimeSeriesDocument = true; @@ -412,108 +482,126 @@ public ITimeSeriesIncludeBuilder IncludeTimeSeries(string name, DateTime? from = return this; } + /// IIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, TimeSeriesRangeType type, TimeValue time) { IncludeTimeSeriesByRangeTypeAndTime(string.Empty, name, type, time); return this; } + /// IIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, TimeSeriesRangeType type, int count) { IncludeTimeSeriesByRangeTypeAndCount(string.Empty, name, type, count); return this; } + /// IIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string[] names, TimeSeriesRangeType type, TimeValue time) { IncludeArrayOfTimeSeriesByRangeTypeAndTime(names, type, time); return this; } + /// IIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string[] names, TimeSeriesRangeType type, int count) { IncludeArrayOfTimeSeriesByRangeTypeAndCount(names, type, count); return this; } + /// IIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeAllTimeSeries(TimeSeriesRangeType type, TimeValue time) { IncludeTimeSeriesByRangeTypeAndTime(string.Empty, Constants.TimeSeries.All, type, time); return this; } + /// IIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeAllTimeSeries(TimeSeriesRangeType type, int count) { IncludeTimeSeriesByRangeTypeAndCount(string.Empty, Constants.TimeSeries.All, type, count); return this; } + /// IQueryIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, TimeSeriesRangeType type, TimeValue time) { IncludeTimeSeriesByRangeTypeAndTime(string.Empty, name, type, time); return this; } + /// IQueryIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, TimeSeriesRangeType type, int count) { IncludeTimeSeriesByRangeTypeAndCount(string.Empty, name, type, count); return this; } + /// IQueryIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string[] names, TimeSeriesRangeType type, TimeValue time) { IncludeArrayOfTimeSeriesByRangeTypeAndTime(names, type, time); return this; } + /// IQueryIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string[] names, TimeSeriesRangeType type, int count) { IncludeArrayOfTimeSeriesByRangeTypeAndCount(names, type, count); return this; } + /// IQueryIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeAllTimeSeries(TimeSeriesRangeType type, TimeValue time) { IncludeTimeSeriesByRangeTypeAndTime(string.Empty, Constants.TimeSeries.All, type, time); return this; } + /// IQueryIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeAllTimeSeries(TimeSeriesRangeType type, int count) { IncludeTimeSeriesByRangeTypeAndCount(string.Empty, Constants.TimeSeries.All, type, count); return this; } + /// ISubscriptionIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, TimeSeriesRangeType type, TimeValue time) { IncludeTimeSeriesByRangeTypeAndTime(string.Empty, name, type, time); return this; } + /// ISubscriptionIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string name, TimeSeriesRangeType type, int count) { IncludeTimeSeriesByRangeTypeAndCount(string.Empty, name, type, count); return this; } + /// ISubscriptionIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string[] names, TimeSeriesRangeType type, TimeValue time) { IncludeArrayOfTimeSeriesByRangeTypeAndTime(names, type, time); return this; } + /// ISubscriptionIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeTimeSeries(string[] names, TimeSeriesRangeType type, int count) { IncludeArrayOfTimeSeriesByRangeTypeAndCount(names, type, count); return this; } + /// ISubscriptionIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeAllTimeSeries(TimeSeriesRangeType type, TimeValue time) { IncludeTimeSeriesByRangeTypeAndTime(string.Empty, Constants.TimeSeries.All, type, time); return this; } + /// ISubscriptionIncludeBuilder IAbstractTimeSeriesIncludeBuilder>.IncludeAllTimeSeries(TimeSeriesRangeType type, int count) { IncludeTimeSeriesByRangeTypeAndCount(string.Empty, Constants.TimeSeries.All, type, count); diff --git a/src/Raven.Client/Documents/Session/Loaders/LazyMultiLoaderWithInclude.cs b/src/Raven.Client/Documents/Session/Loaders/LazyMultiLoaderWithInclude.cs index a1efea6036fe..65b1a35c9d16 100644 --- a/src/Raven.Client/Documents/Session/Loaders/LazyMultiLoaderWithInclude.cs +++ b/src/Raven.Client/Documents/Session/Loaders/LazyMultiLoaderWithInclude.cs @@ -21,29 +21,20 @@ internal LazyMultiLoaderWithInclude(IDocumentSessionImpl session) _session = session; } - /// - /// Includes the specified path. - /// - /// The path. + /// public ILazyLoaderWithInclude Include(string path) { _includes.Add(path); return this; } - /// - /// Includes the specified path. - /// - /// The path. + /// public ILazyLoaderWithInclude Include(Expression> path) { return Include(path.ToPropertyPath(_session.Conventions)); } - /// - /// Includes the specified path. - /// - /// The path. + /// public ILazyLoaderWithInclude Include(Expression>> path) { return Include(path.ToPropertyPath(_session.Conventions)); @@ -123,30 +114,20 @@ internal AsyncLazyMultiLoaderWithInclude(IAsyncDocumentSessionImpl session) _session = session; } - /// - /// Includes the specified path. - /// - /// The path. + /// public IAsyncLazyLoaderWithInclude Include(string path) { _includes.Add(path); return this; } - /// - /// Includes the specified path. - /// - /// The path. + /// public IAsyncLazyLoaderWithInclude Include(Expression> path) { return Include(path.ToPropertyPath(_session.Conventions)); } - - /// - /// Includes the specified path. - /// - /// The path. + /// public IAsyncLazyLoaderWithInclude Include(Expression>> path) { return Include(path.ToPropertyPath(_session.Conventions)); diff --git a/src/Raven.Client/Documents/Session/Operations/Lazy/ILazySessionOperations.cs b/src/Raven.Client/Documents/Session/Operations/Lazy/ILazySessionOperations.cs index 52a0762f344e..8cea96d25b29 100644 --- a/src/Raven.Client/Documents/Session/Operations/Lazy/ILazySessionOperations.cs +++ b/src/Raven.Client/Documents/Session/Operations/Lazy/ILazySessionOperations.cs @@ -12,22 +12,13 @@ namespace Raven.Client.Documents.Session.Operations.Lazy /// public interface ILazySessionOperations { - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILazyLoaderWithInclude Include(string path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILazyLoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// ILazyLoaderWithInclude Include(Expression>> path); /// @@ -90,22 +81,14 @@ public interface ILazySessionOperations /// public interface IAsyncLazySessionOperations { - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + + /// IAsyncLazyLoaderWithInclude Include(string path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// IAsyncLazyLoaderWithInclude Include(Expression> path); - /// - /// Begin a load while including the specified path - /// - /// Path in documents in which server should look for a 'referenced' documents. + /// IAsyncLazyLoaderWithInclude Include(Expression>> path); /// From 9fd358c8da5a85a24e850d09d69f270eff369962 Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Thu, 4 Apr 2024 16:01:34 +0300 Subject: [PATCH 018/243] RavenDB-21050 - When restoring from incremental backup with atomic guard, a tombstone can be handled before creating the document (v5.4) --- .../Restore/RestoreBackupTaskBase.cs | 9 ++- test/SlowTests/Issues/RavenDB-21050.cs | 70 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 test/SlowTests/Issues/RavenDB-21050.cs diff --git a/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs b/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs index f1ecd575a3f9..d54e8d7d1f61 100644 --- a/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs +++ b/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs @@ -718,6 +718,7 @@ protected async Task SmugglerRestore(DocumentDatabase database, List fil var oldOperateOnTypes = Raven.Client.Documents.Smuggler.DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options); var destination = new DatabaseDestination(database); + long totalExecutedCommands = 0; for (var i = 0; i < filesToRestore.Count - 1; i++) { @@ -736,6 +737,7 @@ await ImportSingleBackupFile(database, onProgress, result, filePath, context, de // need to enable revisions before import database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord); }); + ExecuteClusterTransactions(database, onProgress, result, ref totalExecutedCommands); } options.OperateOnTypes = oldOperateOnTypes; @@ -817,9 +819,12 @@ await ImportSingleBackupFile(database, onProgress, result, lastFilePath, context result.Files.CurrentFileName = null; - long totalExecutedCommands = 0; - //when restoring from a backup, the database doesn't exist yet and we cannot rely on the DocumentDatabase to execute the database cluster transaction commands + ExecuteClusterTransactions(database, onProgress, result, ref totalExecutedCommands); + } + + private void ExecuteClusterTransactions(DocumentDatabase database, Action onProgress, RestoreResult result, ref long totalExecutedCommands) + { while (true) { _operationCancelToken.Token.ThrowIfCancellationRequested(); diff --git a/test/SlowTests/Issues/RavenDB-21050.cs b/test/SlowTests/Issues/RavenDB-21050.cs new file mode 100644 index 000000000000..0bae078dcdc3 --- /dev/null +++ b/test/SlowTests/Issues/RavenDB-21050.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using FastTests; +using Raven.Client.Documents; +using Raven.Client.Documents.Operations.Backups; +using Raven.Client.Documents.Session; +using Tests.Infrastructure; +using Xunit; +using Xunit.Abstractions; + +namespace SlowTests.Issues; + +public class RavenDB_21050 : RavenTestBase +{ + public RavenDB_21050(ITestOutputHelper output) : base(output) + { + } + + [RavenFact(RavenTestCategory.BackupExportImport)] + public async Task ClusterWideTransaction_WhenRestoreFromIncrementalBackupAfterStoreAndDelete_ShouldDeleteInTheDestination() + { + var backupPath = NewDataPath(suffix: "BackupFolder"); + const string id = "TestObjs/0"; + + using (var source = GetDocumentStore()) + using (var destination = new DocumentStore { Urls = new[] { Server.WebUrl }, Database = $"restored_{source.Database}" }.Initialize()) + { + var config = new PeriodicBackupConfiguration { LocalSettings = new LocalSettings { FolderPath = backupPath }, IncrementalBackupFrequency = "0 0 */12 * *" }; + var backupTaskId = (await source.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; + + using (var session = source.OpenAsyncSession(new SessionOptions { TransactionMode = TransactionMode.ClusterWide })) + { + await session.StoreAsync(new TestObj(), id); + await session.SaveChangesAsync(); + } + + var backupStatus = await source.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); + await backupStatus.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); + + using (var session = source.OpenAsyncSession(new SessionOptions { TransactionMode = TransactionMode.ClusterWide })) + { + session.Delete(id); + await session.SaveChangesAsync(); + } + + var backupStatus2 = await source.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); + await backupStatus2.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); + + var restoreConfig = new RestoreBackupConfiguration { BackupLocation = Directory.GetDirectories(backupPath).First(), DatabaseName = destination.Database }; + using (Backup.RestoreDatabase(destination, restoreConfig)) + { + using (var session = destination.OpenAsyncSession()) + { + var shouldBeDeleted = await session.LoadAsync(id); + Assert.Null(shouldBeDeleted); //Fails here + } + } + + + } + } + + private class TestObj + { + } +} From 784a9f5d89f03bdbea8eb1fb53d5270a1f39de4a Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Tue, 9 Apr 2024 16:14:16 +0300 Subject: [PATCH 019/243] RavenDB-21050 - Move ExecuteClusterTransactions to DatabaseCompareExchangeActions dispose in case of full/incremental BackupKind --- .../Restore/RestoreBackupTaskBase.cs | 26 ----------- .../Commands/ClusterTransactionCommand.cs | 2 +- .../Smuggler/Documents/DatabaseDestination.cs | 45 +++++++++++++++++-- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs b/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs index d54e8d7d1f61..b31b79440977 100644 --- a/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs +++ b/src/Raven.Server/Documents/PeriodicBackup/Restore/RestoreBackupTaskBase.cs @@ -718,7 +718,6 @@ protected async Task SmugglerRestore(DocumentDatabase database, List fil var oldOperateOnTypes = Raven.Client.Documents.Smuggler.DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options); var destination = new DatabaseDestination(database); - long totalExecutedCommands = 0; for (var i = 0; i < filesToRestore.Count - 1; i++) { @@ -737,7 +736,6 @@ await ImportSingleBackupFile(database, onProgress, result, filePath, context, de // need to enable revisions before import database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord); }); - ExecuteClusterTransactions(database, onProgress, result, ref totalExecutedCommands); } options.OperateOnTypes = oldOperateOnTypes; @@ -818,30 +816,6 @@ await ImportSingleBackupFile(database, onProgress, result, lastFilePath, context }); result.Files.CurrentFileName = null; - - //when restoring from a backup, the database doesn't exist yet and we cannot rely on the DocumentDatabase to execute the database cluster transaction commands - ExecuteClusterTransactions(database, onProgress, result, ref totalExecutedCommands); - } - - private void ExecuteClusterTransactions(DocumentDatabase database, Action onProgress, RestoreResult result, ref long totalExecutedCommands) - { - while (true) - { - _operationCancelToken.Token.ThrowIfCancellationRequested(); - - using (database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext)) - using (serverContext.OpenReadTransaction()) - { - // the commands are already batched (10k or 16MB), so we are executing only 1 at a time - var executed = database.ExecuteClusterTransaction(serverContext, batchSize: 1); - if (executed.Count == 0) - break; - - totalExecutedCommands += executed.Sum(x => x.Commands.Count); - result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); - onProgress.Invoke(result.Progress); - } - } } private bool IsDefaultDataDirectory(string dataDirectory, string databaseName) diff --git a/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs b/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs index 67771b45b48d..4864bbc15852 100644 --- a/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs @@ -433,7 +433,7 @@ public DynamicJsonArray SaveCommandsBatch(ClusterOperationContext context, RawDa return result; } - private static long GetPrevCount(ClusterOperationContext context, Tree commandsCountPerDatabase, string databaseName) + internal static long GetPrevCount(ClusterOperationContext context, Tree commandsCountPerDatabase, string databaseName) { using (GetPrefix(context, databaseName, out var databaseSlice)) { diff --git a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs index db2f1bb0e758..345dac050984 100644 --- a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs @@ -65,6 +65,7 @@ public class DatabaseDestination : ISmugglerDestination private readonly Logger _log; private BuildVersionType _buildType; private DatabaseSmugglerOptionsServerSide _options; + protected SmugglerResult _result; public DatabaseDestination(DocumentDatabase database, CancellationToken token = default) { @@ -78,6 +79,7 @@ public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide option { _buildType = BuildVersion.Type(buildVersion); _options = options; + _result = result; return new AsyncDisposableAction(() => { @@ -135,13 +137,13 @@ public ICompareExchangeActions CompareExchange(JsonOperationContext context, Bac DatabaseCompareExchangeActions CreateActions() { - return new DatabaseCompareExchangeActions(_database, context, backupKind, _token); + return new DatabaseCompareExchangeActions(_database, context, backupKind, _result, _token); } } public ICompareExchangeActions CompareExchangeTombstones(JsonOperationContext context) { - return new DatabaseCompareExchangeActions(_database, context, backupKind: null, _token); + return new DatabaseCompareExchangeActions(_database, context, backupKind: null, _result, _token); } public ICounterActions Counters(SmugglerResult result) @@ -601,7 +603,9 @@ private class DatabaseCompareExchangeActions : ICompareExchangeActions private long? _lastClusterTransactionIndex; private readonly BackupKind? _backupKind; private readonly CancellationToken _token; - public DatabaseCompareExchangeActions(DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, CancellationToken token) + private readonly SmugglerResult _result; + + public DatabaseCompareExchangeActions(DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, CancellationToken token) { _database = database; _context = context; @@ -614,6 +618,7 @@ public DatabaseCompareExchangeActions(DocumentDatabase database, JsonOperationCo _clusterTransactionCommandsBatchSize = new Size(database.Is32Bits ? 2 : 16, SizeUnit.Megabytes); _clusterTransactionCommandsSize = new Size(0, SizeUnit.Megabytes); + _result = result; } public async ValueTask WriteKeyValueAsync(string key, BlittableJsonReaderObject value, Document existingDocument) @@ -714,6 +719,40 @@ public async ValueTask DisposeAsync() // waiting for the commands to be applied await _database.RachisLogIndexNotifications.WaitForIndexNotification(_lastClusterTransactionIndex.Value, _token); } + else + { + Debug.Assert(_backupKind == BackupKind.Full || _backupKind == BackupKind.Incremental); + ExecuteClusterTransactions(); + } + } + } + } + + private void ExecuteClusterTransactions() + { + long totalExecutedCommands; + using (_database.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext serverContext)) + using (serverContext.OpenReadTransaction()) + { + var commandsCountPerDatabase = serverContext.Transaction.InnerTransaction.ReadTree(ClusterStateMachine.TransactionCommandsCountPerDatabase); + var prevCount = ClusterTransactionCommand.GetPrevCount(serverContext, commandsCountPerDatabase, _database.Name); + totalExecutedCommands = prevCount; + } + + while (true) + { + _token.ThrowIfCancellationRequested(); + + using (_database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext)) + using (serverContext.OpenReadTransaction()) + { + // the commands are already batched (10k or 16MB), so we are executing only 1 at a time + var executed = _database.ExecuteClusterTransaction(serverContext, batchSize: 1); + if (executed.Count == 0) + break; + + totalExecutedCommands += executed.Sum(x => x.Commands.Count); + _result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); } } } From aa8eb81c44079a8dc89a8ac26e3c49a8d183b407 Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Thu, 11 Apr 2024 09:36:04 +0300 Subject: [PATCH 020/243] RavenDB-21050 - pass 'onProgress' action to 'DatabaseCompareExchangeActions' for invoking on ExecuteClusterTransactions --- .../Handlers/LegacyReplicationHandler.cs | 2 +- .../Documents/Data/ISmugglerDestination.cs | 3 ++- .../Smuggler/Documents/DatabaseDestination.cs | 19 ++++++++++++++----- .../Smuggler/Documents/DatabaseSmuggler.cs | 2 +- .../Smuggler/Documents/StreamDestination.cs | 3 ++- .../Smuggler/Migration/Migrator_V2.cs | 2 +- .../Smuggler/Migration/Migrator_V3.cs | 2 +- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs b/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs index c308ee4b2c01..2cb94e907b7e 100644 --- a/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs +++ b/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs @@ -72,7 +72,7 @@ public async Task Attachments() SkipRevisionCreation = true }; - await using (destination.InitializeAsync(options, null, buildVersion: default)) + await using (destination.InitializeAsync(options, result: null, onProgress: null, buildVersion: default)) await using (var documentActions = destination.Documents()) await using (var buffered = new BufferedStream(RequestBodyStream())) #pragma warning disable CS0618 // Type or member is obsolete diff --git a/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs b/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs index 05aee9851622..20b2c3fe1b11 100644 --- a/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs @@ -3,6 +3,7 @@ using System.IO; using System.Threading.Tasks; using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Counters; using Raven.Client.Documents.Operations.Replication; using Raven.Client.Documents.Smuggler; @@ -19,7 +20,7 @@ namespace Raven.Server.Smuggler.Documents.Data { public interface ISmugglerDestination { - IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion); + IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion); IDatabaseRecordActions DatabaseRecord(); diff --git a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs index 345dac050984..62e0274adb27 100644 --- a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs @@ -12,6 +12,7 @@ using Raven.Client.Documents.Attachments; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Attachments; using Raven.Client.Documents.Operations.Counters; using Raven.Client.Documents.Operations.OngoingTasks; @@ -66,6 +67,7 @@ public class DatabaseDestination : ISmugglerDestination private BuildVersionType _buildType; private DatabaseSmugglerOptionsServerSide _options; protected SmugglerResult _result; + protected Action _onProgress; public DatabaseDestination(DocumentDatabase database, CancellationToken token = default) { @@ -75,11 +77,12 @@ public DatabaseDestination(DocumentDatabase database, CancellationToken token = _duplicateDocsHandler = new DuplicateDocsHandler(_database); } - public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) + public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion) { _buildType = BuildVersion.Type(buildVersion); _options = options; _result = result; + _onProgress = onProgress; return new AsyncDisposableAction(() => { @@ -137,13 +140,13 @@ public ICompareExchangeActions CompareExchange(JsonOperationContext context, Bac DatabaseCompareExchangeActions CreateActions() { - return new DatabaseCompareExchangeActions(_database, context, backupKind, _result, _token); + return new DatabaseCompareExchangeActions(_database, context, backupKind, _result, _onProgress, _token); } } public ICompareExchangeActions CompareExchangeTombstones(JsonOperationContext context) { - return new DatabaseCompareExchangeActions(_database, context, backupKind: null, _result, _token); + return new DatabaseCompareExchangeActions(_database, context, backupKind: null, _result, _onProgress, _token); } public ICounterActions Counters(SmugglerResult result) @@ -604,8 +607,9 @@ private class DatabaseCompareExchangeActions : ICompareExchangeActions private readonly BackupKind? _backupKind; private readonly CancellationToken _token; private readonly SmugglerResult _result; + private readonly Action _onProgress; - public DatabaseCompareExchangeActions(DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, CancellationToken token) + public DatabaseCompareExchangeActions(DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, Action onProgress, CancellationToken token) { _database = database; _context = context; @@ -619,6 +623,7 @@ public DatabaseCompareExchangeActions(DocumentDatabase database, JsonOperationCo _clusterTransactionCommandsBatchSize = new Size(database.Is32Bits ? 2 : 16, SizeUnit.Megabytes); _clusterTransactionCommandsSize = new Size(0, SizeUnit.Megabytes); _result = result; + _onProgress = onProgress; } public async ValueTask WriteKeyValueAsync(string key, BlittableJsonReaderObject value, Document existingDocument) @@ -752,7 +757,11 @@ private void ExecuteClusterTransactions() break; totalExecutedCommands += executed.Sum(x => x.Commands.Count); - _result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); + if (_result != null) + { + _result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); + _onProgress?.Invoke(_result.Progress); + } } } } diff --git a/src/Raven.Server/Smuggler/Documents/DatabaseSmuggler.cs b/src/Raven.Server/Smuggler/Documents/DatabaseSmuggler.cs index 2fc151a26b21..dfb014251827 100644 --- a/src/Raven.Server/Smuggler/Documents/DatabaseSmuggler.cs +++ b/src/Raven.Server/Smuggler/Documents/DatabaseSmuggler.cs @@ -88,7 +88,7 @@ public async Task ExecuteAsync(bool ensureStepsProcessed = true, using (_patcher?.Initialize()) using (var initializeResult = await _source.InitializeAsync(_options, result)) - await using (_destination.InitializeAsync(_options, result, initializeResult.BuildNumber)) + await using (_destination.InitializeAsync(_options, result, _onProgress, initializeResult.BuildNumber)) { ModifyV41OperateOnTypes(initializeResult.BuildNumber, isLastFile); diff --git a/src/Raven.Server/Smuggler/Documents/StreamDestination.cs b/src/Raven.Server/Smuggler/Documents/StreamDestination.cs index f1ab89f9983b..0367aa3b84c2 100644 --- a/src/Raven.Server/Smuggler/Documents/StreamDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/StreamDestination.cs @@ -7,6 +7,7 @@ using Raven.Client; using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Indexes.Analysis; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Attachments; using Raven.Client.Documents.Operations.Backups; using Raven.Client.Documents.Operations.Configuration; @@ -66,7 +67,7 @@ public StreamDestination(Stream stream, DocumentsOperationContext context, Datab _compressionLevel = compressionLevel; } - public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) + public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion) { _outputStream = BackupUtils.GetCompressionStream(_stream, _compressionAlgorithm, _compressionLevel); _writer = new AsyncBlittableJsonTextWriter(_context, _outputStream); diff --git a/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs b/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs index e95cade56cca..44970bf26d40 100644 --- a/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs +++ b/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs @@ -100,7 +100,7 @@ private async Task MigrateAttachments(string lastEtag, SmugglerResult parameters SkipRevisionCreation = true }; - destination.InitializeAsync(options, parametersResult, buildVersion: default); + destination.InitializeAsync(options, parametersResult, onProgress: null, buildVersion: default); using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext)) await using (var documentActions = destination.Documents()) diff --git a/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs b/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs index 60265c7f1123..4357f187316d 100644 --- a/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs +++ b/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs @@ -123,7 +123,7 @@ private async Task MigrateRavenFs(string lastEtag, SmugglerResult parame SkipRevisionCreation = true }; - destination.InitializeAsync(options, parametersResult, _buildVersion); + destination.InitializeAsync(options, parametersResult, onProgress: null, _buildVersion); using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var documentActions = destination.Documents()) From 5b8c2f3c77319f224615a415b7c9712e113d8ce9 Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Thu, 4 Apr 2024 09:54:48 +0300 Subject: [PATCH 021/243] RavenDB-21050 - When restoring from incremental backup with atomic guard, a tombstone can be handled before creating the document --- .../Restore/AbstractRestoreBackupTask.cs | 8 +-- test/SlowTests/Issues/RavenDB-21050.cs | 70 +++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 test/SlowTests/Issues/RavenDB-21050.cs diff --git a/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs b/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs index bb36b806ce36..19699db6abe8 100644 --- a/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs +++ b/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs @@ -316,6 +316,7 @@ protected async Task SmugglerRestoreAsync(DocumentDatabase database, JsonOperati var oldOperateOnTypes = Client.Documents.Smuggler.DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options); var destination = database.Smuggler.CreateDestination(); + long totalExecutedCommands = 0; for (var i = 0; i < FilesToRestore.Count - 1; i++) { @@ -329,6 +330,7 @@ protected async Task SmugglerRestoreAsync(DocumentDatabase database, JsonOperati var filePath = RestoreSource.GetBackupPath(fileName); await ImportSingleBackupFileAsync(database, Progress, Result, filePath, context, destination, options, isLastFile: false); + ExecuteClusterTransactions(database, ref totalExecutedCommands); } options.OperateOnTypes = oldOperateOnTypes; @@ -345,7 +347,7 @@ protected async Task SmugglerRestoreAsync(DocumentDatabase database, JsonOperati var lastFilePath = RestoreSource.GetBackupPath(lastFileName); await ImportSingleBackupFileAsync(database, Progress, Result, lastFilePath, context, lastFileDestination, options, isLastFile: true); - ExecuteClusterTransactions(database); + ExecuteClusterTransactions(database, ref totalExecutedCommands); } protected Stream GetInputStream(Stream stream, byte[] databaseEncryptionKey) @@ -555,10 +557,8 @@ protected virtual void ConfigureSettingsForSmugglerRestore(DocumentDatabase data database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord); } - private void ExecuteClusterTransactions(DocumentDatabase database) + private void ExecuteClusterTransactions(DocumentDatabase database, ref long totalExecutedCommands) { - long totalExecutedCommands = 0; - //when restoring from a backup, the database doesn't exist yet and we cannot rely on the DocumentDatabase to execute the database cluster transaction commands while (true) { diff --git a/test/SlowTests/Issues/RavenDB-21050.cs b/test/SlowTests/Issues/RavenDB-21050.cs new file mode 100644 index 000000000000..0bae078dcdc3 --- /dev/null +++ b/test/SlowTests/Issues/RavenDB-21050.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using FastTests; +using Raven.Client.Documents; +using Raven.Client.Documents.Operations.Backups; +using Raven.Client.Documents.Session; +using Tests.Infrastructure; +using Xunit; +using Xunit.Abstractions; + +namespace SlowTests.Issues; + +public class RavenDB_21050 : RavenTestBase +{ + public RavenDB_21050(ITestOutputHelper output) : base(output) + { + } + + [RavenFact(RavenTestCategory.BackupExportImport)] + public async Task ClusterWideTransaction_WhenRestoreFromIncrementalBackupAfterStoreAndDelete_ShouldDeleteInTheDestination() + { + var backupPath = NewDataPath(suffix: "BackupFolder"); + const string id = "TestObjs/0"; + + using (var source = GetDocumentStore()) + using (var destination = new DocumentStore { Urls = new[] { Server.WebUrl }, Database = $"restored_{source.Database}" }.Initialize()) + { + var config = new PeriodicBackupConfiguration { LocalSettings = new LocalSettings { FolderPath = backupPath }, IncrementalBackupFrequency = "0 0 */12 * *" }; + var backupTaskId = (await source.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; + + using (var session = source.OpenAsyncSession(new SessionOptions { TransactionMode = TransactionMode.ClusterWide })) + { + await session.StoreAsync(new TestObj(), id); + await session.SaveChangesAsync(); + } + + var backupStatus = await source.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); + await backupStatus.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); + + using (var session = source.OpenAsyncSession(new SessionOptions { TransactionMode = TransactionMode.ClusterWide })) + { + session.Delete(id); + await session.SaveChangesAsync(); + } + + var backupStatus2 = await source.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); + await backupStatus2.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); + + var restoreConfig = new RestoreBackupConfiguration { BackupLocation = Directory.GetDirectories(backupPath).First(), DatabaseName = destination.Database }; + using (Backup.RestoreDatabase(destination, restoreConfig)) + { + using (var session = destination.OpenAsyncSession()) + { + var shouldBeDeleted = await session.LoadAsync(id); + Assert.Null(shouldBeDeleted); //Fails here + } + } + + + } + } + + private class TestObj + { + } +} From 6c6dcf09eafb21b80f1bab542cf2fe8045e7ebee Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Sun, 7 Apr 2024 10:34:33 +0300 Subject: [PATCH 022/243] RavenDB-21050 - test for sharded database --- test/SlowTests/Issues/RavenDB-21050.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/SlowTests/Issues/RavenDB-21050.cs b/test/SlowTests/Issues/RavenDB-21050.cs index 0bae078dcdc3..83edb584ae94 100644 --- a/test/SlowTests/Issues/RavenDB-21050.cs +++ b/test/SlowTests/Issues/RavenDB-21050.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using FastTests; using Raven.Client.Documents; @@ -20,13 +21,14 @@ public RavenDB_21050(ITestOutputHelper output) : base(output) { } - [RavenFact(RavenTestCategory.BackupExportImport)] - public async Task ClusterWideTransaction_WhenRestoreFromIncrementalBackupAfterStoreAndDelete_ShouldDeleteInTheDestination() + [RavenTheory(RavenTestCategory.BackupExportImport)] + [RavenData(DatabaseMode = RavenDatabaseMode.All)] + public async Task ClusterWideTransaction_WhenRestoreFromIncrementalBackupAfterStoreAndDelete_ShouldDeleteInTheDestination(Options options) { var backupPath = NewDataPath(suffix: "BackupFolder"); const string id = "TestObjs/0"; - using (var source = GetDocumentStore()) + using (var source = GetDocumentStore(options)) using (var destination = new DocumentStore { Urls = new[] { Server.WebUrl }, Database = $"restored_{source.Database}" }.Initialize()) { var config = new PeriodicBackupConfiguration { LocalSettings = new LocalSettings { FolderPath = backupPath }, IncrementalBackupFrequency = "0 0 */12 * *" }; From 208f55d20131a0b3231f6bd66d0a7c8a1fe890d6 Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Tue, 9 Apr 2024 15:57:50 +0300 Subject: [PATCH 023/243] RavenDB-21050 - Move ExecuteClusterTransactions to DatabaseCompareExchangeActions dispose in case of full/incremental BackupKind --- .../Restore/AbstractRestoreBackupTask.cs | 27 +----------- .../Smuggler/ShardedDatabaseDestination.cs | 7 +-- .../Commands/ClusterTransactionCommand.cs | 2 +- .../Actions/DatabaseCompareExchangeActions.cs | 43 ++++++++++++++++++- .../Smuggler/Documents/DatabaseDestination.cs | 6 ++- 5 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs b/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs index 19699db6abe8..a66f78dd58fc 100644 --- a/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs +++ b/src/Raven.Server/Documents/PeriodicBackup/Restore/AbstractRestoreBackupTask.cs @@ -315,8 +315,7 @@ protected async Task SmugglerRestoreAsync(DocumentDatabase database, JsonOperati options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions; var oldOperateOnTypes = Client.Documents.Smuggler.DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options); - var destination = database.Smuggler.CreateDestination(); - long totalExecutedCommands = 0; + var destination = database.Smuggler.CreateDestination(OperationCancelToken.Token); for (var i = 0; i < FilesToRestore.Count - 1; i++) { @@ -330,7 +329,6 @@ protected async Task SmugglerRestoreAsync(DocumentDatabase database, JsonOperati var filePath = RestoreSource.GetBackupPath(fileName); await ImportSingleBackupFileAsync(database, Progress, Result, filePath, context, destination, options, isLastFile: false); - ExecuteClusterTransactions(database, ref totalExecutedCommands); } options.OperateOnTypes = oldOperateOnTypes; @@ -347,7 +345,6 @@ protected async Task SmugglerRestoreAsync(DocumentDatabase database, JsonOperati var lastFilePath = RestoreSource.GetBackupPath(lastFileName); await ImportSingleBackupFileAsync(database, Progress, Result, lastFilePath, context, lastFileDestination, options, isLastFile: true); - ExecuteClusterTransactions(database, ref totalExecutedCommands); } protected Stream GetInputStream(Stream stream, byte[] databaseEncryptionKey) @@ -557,28 +554,6 @@ protected virtual void ConfigureSettingsForSmugglerRestore(DocumentDatabase data database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord); } - private void ExecuteClusterTransactions(DocumentDatabase database, ref long totalExecutedCommands) - { - //when restoring from a backup, the database doesn't exist yet and we cannot rely on the DocumentDatabase to execute the database cluster transaction commands - while (true) - { - OperationCancelToken.Token.ThrowIfCancellationRequested(); - - using (ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext serverContext)) - using (serverContext.OpenReadTransaction()) - { - // the commands are already batched (10k or 16MB), so we are executing only 1 at a time - var executed = database.ExecuteClusterTransaction(serverContext, batchSize: 1); - if (executed.BatchSize == 0) - break; - - totalExecutedCommands += executed.CommandsCount; - Result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); - Progress.Invoke(Result.Progress); - } - } - } - private async Task OnErrorAsync(Action onProgress, Exception e) { if (Logger.IsOperationsEnabled) diff --git a/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs b/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs index cf2a73303a20..972141889e57 100644 --- a/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs +++ b/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using JetBrains.Annotations; +using Raven.Client.Documents.Smuggler; using Raven.Server.Documents.PeriodicBackup; using Raven.Server.ServerWide.Commands; using Raven.Server.Smuggler.Documents; @@ -19,13 +20,13 @@ public ShardedDatabaseDestination(ShardedDocumentDatabase database, Cancellation protected override ICompareExchangeActions CreateCompareExchangeActions(string databaseName, JsonOperationContext context, BackupKind? backupKind) { - return new ShardedDatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _token); + return new ShardedDatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _result, _token); } private sealed class ShardedDatabaseCompareExchangeActions : DatabaseCompareExchangeActions { - public ShardedDatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, CancellationToken token) - : base(databaseName, database, context, backupKind, token) + public ShardedDatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, CancellationToken token) + : base(databaseName, database, context, backupKind, result, token) { } diff --git a/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs b/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs index 167db335dc5d..08073342983c 100644 --- a/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/ClusterTransactionCommand.cs @@ -699,7 +699,7 @@ public DynamicJsonArray SaveCommandsBatch(ClusterOperationContext context, RawDa return result; } - private static long GetPrevCount(ClusterOperationContext context, Tree commandsCountPerDatabase, string databaseName) + internal static long GetPrevCount(ClusterOperationContext context, Tree commandsCountPerDatabase, string databaseName) { using (GetPrefix(context, databaseName, out var databaseSlice)) { diff --git a/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs b/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs index b5e50f132cc4..ae2dc2aff178 100644 --- a/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs +++ b/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs @@ -1,10 +1,13 @@ using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; using Raven.Client.Documents.Commands.Batches; +using Raven.Client.Documents.Smuggler; using Raven.Server.Documents; using Raven.Server.Documents.PeriodicBackup; +using Raven.Server.ServerWide; using Raven.Server.ServerWide.Commands; using Raven.Server.ServerWide.Context; using Sparrow; @@ -18,11 +21,14 @@ internal class DatabaseCompareExchangeActions : AbstractDatabaseCompareExchangeA private readonly DocumentDatabase _database; - public DatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, CancellationToken token) + private readonly SmugglerResult _result; + + public DatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, CancellationToken token) : base(database.ServerStore, databaseName, database.IdentityPartsSeparator, context, backupKind, token) { _database = database ?? throw new ArgumentNullException(nameof(database)); _documentContextHolder = new DocumentContextHolder(database); + _result = result; } protected override bool TryHandleAtomicGuard(string key, string documentId, BlittableJsonReaderObject value, Document existingDocument) @@ -85,6 +91,41 @@ protected override async ValueTask WaitForIndexNotificationAsync(long? lastAddOr // waiting for the commands to be applied await _database.RachisLogIndexNotifications.WaitForIndexNotification(_lastClusterTransactionIndex.Value, _token); } + else + { + Debug.Assert(_backupKind == BackupKind.Full || _backupKind == BackupKind.Incremental); + ExecuteClusterTransactions(); + } + } + } + + private void ExecuteClusterTransactions() + { + long totalExecutedCommands; + using (_database.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext serverContext)) + using (serverContext.OpenReadTransaction()) + { + var commandsCountPerDatabase = serverContext.Transaction.InnerTransaction.ReadTree(ClusterStateMachine.TransactionCommandsCountPerDatabase); + var prevCount = ClusterTransactionCommand.GetPrevCount(serverContext, commandsCountPerDatabase, _database.Name); + totalExecutedCommands = prevCount; + } + + //when restoring from a backup, the database doesn't exist yet and we cannot rely on the DocumentDatabase to execute the database cluster transaction commands + while (true) + { + _token.ThrowIfCancellationRequested(); + + using (_database.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext serverContext)) + using (serverContext.OpenReadTransaction()) + { + // the commands are already batched (10k or 16MB), so we are executing only 1 at a time + var executed = _database.ExecuteClusterTransaction(serverContext, batchSize: 1); + if (executed.BatchSize == 0) + break; + + totalExecutedCommands += executed.CommandsCount; + _result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); + } } } diff --git a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs index 722976600539..95db713a7c98 100644 --- a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs @@ -45,6 +45,7 @@ public class DatabaseDestination : ISmugglerDestination private readonly Logger _log; private BuildVersionType _buildType; private DatabaseSmugglerOptionsServerSide _options; + protected SmugglerResult _result; public DatabaseDestination(DocumentDatabase database, CancellationToken token = default) { @@ -58,6 +59,7 @@ public ValueTask InitializeAsync(DatabaseSmugglerOptionsServer { _buildType = BuildVersion.Type(buildVersion); _options = options; + _result = result; var d = new AsyncDisposableAction(() => { @@ -118,12 +120,12 @@ public ICompareExchangeActions CompareExchange(string databaseName, JsonOperatio protected virtual ICompareExchangeActions CreateCompareExchangeActions(string databaseName, JsonOperationContext context, BackupKind? backupKind) { - return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _token); + return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _result, _token); } public ICompareExchangeActions CompareExchangeTombstones(string databaseName, JsonOperationContext context) { - return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind: null, _token); + return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind: null, _result, _token); } public ICounterActions Counters(SmugglerResult result) From 433e4e465ca9313af067c32e2142bb4d8a1277ad Mon Sep 17 00:00:00 2001 From: shaharhikri Date: Thu, 11 Apr 2024 10:15:09 +0300 Subject: [PATCH 024/243] RavenDB-21050 - pass 'onProgress' action to 'DatabaseCompareExchangeActions' for invoking on ExecuteClusterTransactions --- .../Documents/Handlers/LegacyReplicationHandler.cs | 2 +- .../Sharding/Smuggler/ShardedDatabaseDestination.cs | 7 ++++--- .../Actions/DatabaseCompareExchangeActions.cs | 11 +++++++++-- .../Smuggler/Documents/Data/ISmugglerDestination.cs | 3 ++- .../Smuggler/Documents/DatabaseDestination.cs | 9 ++++++--- .../Smuggler/Documents/MultiShardedDestination.cs | 9 +++++---- src/Raven.Server/Smuggler/Documents/SmugglerBase.cs | 2 +- .../Smuggler/Documents/StreamDestination.cs | 3 ++- src/Raven.Server/Smuggler/Migration/Migrator_V2.cs | 2 +- src/Raven.Server/Smuggler/Migration/Migrator_V3.cs | 2 +- 10 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs b/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs index bc86588b2015..366663306820 100644 --- a/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs +++ b/src/Raven.Server/Documents/Handlers/LegacyReplicationHandler.cs @@ -74,7 +74,7 @@ public async Task Attachments() SkipRevisionCreation = true }; - await using (await destination.InitializeAsync(options, null, buildVersion: default)) + await using (await destination.InitializeAsync(options, result: null, onProgress: null, buildVersion: default)) await using (var documentActions = destination.Documents()) await using (var buffered = new BufferedStream(RequestBodyStream())) #pragma warning disable CS0618 // Type or member is obsolete diff --git a/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs b/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs index 972141889e57..f99ef9d49a45 100644 --- a/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs +++ b/src/Raven.Server/Documents/Sharding/Smuggler/ShardedDatabaseDestination.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using JetBrains.Annotations; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Smuggler; using Raven.Server.Documents.PeriodicBackup; using Raven.Server.ServerWide.Commands; @@ -20,13 +21,13 @@ public ShardedDatabaseDestination(ShardedDocumentDatabase database, Cancellation protected override ICompareExchangeActions CreateCompareExchangeActions(string databaseName, JsonOperationContext context, BackupKind? backupKind) { - return new ShardedDatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _result, _token); + return new ShardedDatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _result, _onProgress, _token); } private sealed class ShardedDatabaseCompareExchangeActions : DatabaseCompareExchangeActions { - public ShardedDatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, CancellationToken token) - : base(databaseName, database, context, backupKind, result, token) + public ShardedDatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, Action onProgress, CancellationToken token) + : base(databaseName, database, context, backupKind, result, onProgress, token) { } diff --git a/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs b/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs index ae2dc2aff178..5b8f932a19a7 100644 --- a/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs +++ b/src/Raven.Server/Smuggler/Documents/Actions/DatabaseCompareExchangeActions.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using JetBrains.Annotations; using Raven.Client.Documents.Commands.Batches; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Smuggler; using Raven.Server.Documents; using Raven.Server.Documents.PeriodicBackup; @@ -22,13 +23,15 @@ internal class DatabaseCompareExchangeActions : AbstractDatabaseCompareExchangeA private readonly DocumentDatabase _database; private readonly SmugglerResult _result; + private readonly Action _onProgress; - public DatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, CancellationToken token) + public DatabaseCompareExchangeActions([NotNull] string databaseName, [NotNull] DocumentDatabase database, JsonOperationContext context, BackupKind? backupKind, SmugglerResult result, Action onProgress, CancellationToken token) : base(database.ServerStore, databaseName, database.IdentityPartsSeparator, context, backupKind, token) { _database = database ?? throw new ArgumentNullException(nameof(database)); _documentContextHolder = new DocumentContextHolder(database); _result = result; + _onProgress = onProgress; } protected override bool TryHandleAtomicGuard(string key, string documentId, BlittableJsonReaderObject value, Document existingDocument) @@ -124,7 +127,11 @@ private void ExecuteClusterTransactions() break; totalExecutedCommands += executed.CommandsCount; - _result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); + if (_result != null) + { + _result.AddInfo($"Executed {totalExecutedCommands:#,#;;0} cluster transaction commands."); + _onProgress?.Invoke(_result.Progress); + } } } } diff --git a/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs b/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs index 39f258903483..5c718aa23660 100644 --- a/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/Data/ISmugglerDestination.cs @@ -3,6 +3,7 @@ using System.IO; using System.Threading.Tasks; using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Counters; using Raven.Client.Documents.Operations.Replication; using Raven.Client.Documents.Smuggler; @@ -18,7 +19,7 @@ namespace Raven.Server.Smuggler.Documents.Data { public interface ISmugglerDestination { - ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion); + ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion); IDatabaseRecordActions DatabaseRecord(); diff --git a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs index 95db713a7c98..a2ccf3dc2aeb 100644 --- a/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/DatabaseDestination.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using Raven.Client.Documents.Attachments; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Attachments; using Raven.Client.Documents.Operations.Counters; using Raven.Client.Documents.Smuggler; @@ -46,6 +47,7 @@ public class DatabaseDestination : ISmugglerDestination private BuildVersionType _buildType; private DatabaseSmugglerOptionsServerSide _options; protected SmugglerResult _result; + protected Action _onProgress; public DatabaseDestination(DocumentDatabase database, CancellationToken token = default) { @@ -55,11 +57,12 @@ public DatabaseDestination(DocumentDatabase database, CancellationToken token = _duplicateDocsHandler = new DuplicateDocsHandler(_database); } - public ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) + public ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion) { _buildType = BuildVersion.Type(buildVersion); _options = options; _result = result; + _onProgress = onProgress; var d = new AsyncDisposableAction(() => { @@ -120,12 +123,12 @@ public ICompareExchangeActions CompareExchange(string databaseName, JsonOperatio protected virtual ICompareExchangeActions CreateCompareExchangeActions(string databaseName, JsonOperationContext context, BackupKind? backupKind) { - return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _result, _token); + return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind, _result, _onProgress, _token); } public ICompareExchangeActions CompareExchangeTombstones(string databaseName, JsonOperationContext context) { - return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind: null, _result, _token); + return new DatabaseCompareExchangeActions(databaseName, _database, context, backupKind: null, _result, _onProgress, _token); } public ICounterActions Counters(SmugglerResult result) diff --git a/src/Raven.Server/Smuggler/Documents/MultiShardedDestination.cs b/src/Raven.Server/Smuggler/Documents/MultiShardedDestination.cs index 9ffc68741cd3..045e464a5298 100644 --- a/src/Raven.Server/Smuggler/Documents/MultiShardedDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/MultiShardedDestination.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using JetBrains.Annotations; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.Counters; using Raven.Client.Documents.Smuggler; using Raven.Client.Util; @@ -44,7 +45,7 @@ public MultiShardedDestination([NotNull] ISmugglerSource source, [NotNull] Shard _allocator = new ByteStringContext(SharedMultipleUseFlag.None); } - public async ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) + public async ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion) { _options = options; @@ -64,7 +65,7 @@ public async ValueTask InitializeAsync(DatabaseSmugglerOptions foreach (int shardNumber in holders.Keys) { - await PrepareShardStreamDestination(holders, shardNumber, result, buildVersion); + await PrepareShardStreamDestination(holders, shardNumber, result, onProgress, buildVersion); } return new AsyncDisposableAction(async () => @@ -78,13 +79,13 @@ public async ValueTask InitializeAsync(DatabaseSmugglerOptions }); } - private async Task PrepareShardStreamDestination(Dictionary holders, int shardNumber, SmugglerResult result, long buildVersion) + private async Task PrepareShardStreamDestination(Dictionary holders, int shardNumber, SmugglerResult result, Action onProgress, long buildVersion) { var stream = ShardedSmugglerHandlerProcessorForImport.GetOutputStream(holders[shardNumber].OutStream.OutputStream.Result, _options); holders[shardNumber].InputStream = stream; holders[shardNumber].ContextReturn = _handler.ContextPool.AllocateOperationContext(out JsonOperationContext context); var destination = new StreamDestination(stream, context, _source, _options.CompressionAlgorithm ?? _databaseContext.Configuration.ExportImport.CompressionAlgorithm, compressionLevel: _databaseContext.Configuration.Sharding.CompressionLevel); - holders[shardNumber].DestinationAsyncDisposable = await destination.InitializeAsync(_options, result, buildVersion); + holders[shardNumber].DestinationAsyncDisposable = await destination.InitializeAsync(_options, result, onProgress, buildVersion); _destinations[shardNumber] = destination; } diff --git a/src/Raven.Server/Smuggler/Documents/SmugglerBase.cs b/src/Raven.Server/Smuggler/Documents/SmugglerBase.cs index 79ee90f7b353..790f7955a81f 100644 --- a/src/Raven.Server/Smuggler/Documents/SmugglerBase.cs +++ b/src/Raven.Server/Smuggler/Documents/SmugglerBase.cs @@ -81,7 +81,7 @@ public virtual async Task ExecuteAsync(bool ensureStepsProcessed var result = _result ?? new SmugglerResult(); using (var initializeResult = await _source.InitializeAsync(_options, result)) using (_patcher?.Initialize()) - await using (await _destination.InitializeAsync(_options, result, initializeResult.BuildNumber)) + await using (await _destination.InitializeAsync(_options, result, _onProgress, initializeResult.BuildNumber)) { ModifyV41OperateOnTypes(initializeResult.BuildNumber, isLastFile); diff --git a/src/Raven.Server/Smuggler/Documents/StreamDestination.cs b/src/Raven.Server/Smuggler/Documents/StreamDestination.cs index 3a3afe9dd4da..3628406bee57 100644 --- a/src/Raven.Server/Smuggler/Documents/StreamDestination.cs +++ b/src/Raven.Server/Smuggler/Documents/StreamDestination.cs @@ -7,6 +7,7 @@ using Raven.Client; using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Indexes.Analysis; +using Raven.Client.Documents.Operations; using Raven.Client.Documents.Operations.DataArchival; using Raven.Client.Documents.Operations.Attachments; using Raven.Client.Documents.Operations.Backups; @@ -66,7 +67,7 @@ public StreamDestination(Stream stream, JsonOperationContext context, ISmugglerS _compressionLevel = compressionLevel; } - public ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) + public ValueTask InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action onProgress, long buildVersion) { _outputStream = BackupUtils.GetCompressionStream(_stream, _compressionAlgorithm, _compressionLevel); _writer = new AsyncBlittableJsonTextWriter(_context, _outputStream); diff --git a/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs b/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs index 1ee4ea7be0e7..e5909c12ac27 100644 --- a/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs +++ b/src/Raven.Server/Smuggler/Migration/Migrator_V2.cs @@ -102,7 +102,7 @@ private async Task MigrateAttachments(string lastEtag, SmugglerResult parameters SkipRevisionCreation = true }; - await using (await destination.InitializeAsync(options, parametersResult, buildVersion: default)) + await using (await destination.InitializeAsync(options, parametersResult, onProgress: null, buildVersion: default)) using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext)) await using (var documentActions = destination.Documents()) { diff --git a/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs b/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs index f9ea2e03c325..2664ec5bb208 100644 --- a/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs +++ b/src/Raven.Server/Smuggler/Migration/Migrator_V3.cs @@ -125,7 +125,7 @@ private async Task MigrateRavenFs(string lastEtag, SmugglerResult parame SkipRevisionCreation = true }; - await using (await destination.InitializeAsync(options, parametersResult, buildVersion: default)) + await using (await destination.InitializeAsync(options, parametersResult, onProgress: null, buildVersion: default)) using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var documentActions = destination.Documents()) From 9d24bee147d1316d561e4f3716346184cf9491c3 Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Thu, 11 Apr 2024 09:39:01 +0200 Subject: [PATCH 025/243] RavenDB-13830 / RavenDB-10621 / RavenDB-21987 Unskipping the test which no longer fails (even on 32 bits) --- test/SlowTests/Issues/RavenDB_10621.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/SlowTests/Issues/RavenDB_10621.cs b/test/SlowTests/Issues/RavenDB_10621.cs index 95afaa47b52b..95c8499b5beb 100644 --- a/test/SlowTests/Issues/RavenDB_10621.cs +++ b/test/SlowTests/Issues/RavenDB_10621.cs @@ -14,10 +14,11 @@ public RavenDB_10621(ITestOutputHelper output) : base(output) { } - [Fact(Skip = "Waiting for RavenDB-13830 to be resolved")] + [Fact] public void ShouldNotErrorIndexOnInvalidProgramException() { - // if this test fails it's very likely the following issue got fixed: https://github.com/dotnet/coreclr/issues/14672 + // this test has been added initially to workaround the following issue: https://github.com/dotnet/coreclr/issues/14672 + // initially it was ShouldErrorIndexOnInvalidProgramException but after some time it got changed to ShouldNotErrorIndexOnInvalidProgramException using (var store = GetDocumentStore()) { From c96e5cdc077b2dc7667c67b6ea3d596bfa4835ff Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Tue, 2 Apr 2024 16:39:24 +0200 Subject: [PATCH 026/243] RavenDB-22176 Exposing notifications via GET endpoint --- .../DatabaseNotificationCenterHandler.cs | 31 ++++++ test/SlowTests/Issues/RavenDB-17068.cs | 42 ++++---- test/SlowTests/Issues/RavenDB_22176.cs | 99 +++++++++++++++++++ 3 files changed, 153 insertions(+), 19 deletions(-) create mode 100644 test/SlowTests/Issues/RavenDB_22176.cs diff --git a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs index ad37564134c1..ffae098b0d08 100644 --- a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs +++ b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs @@ -6,11 +6,42 @@ using Raven.Server.Documents; using Raven.Server.NotificationCenter.Notifications; using Raven.Server.Routing; +using Raven.Server.ServerWide.Context; +using Raven.Server.Utils; +using Sparrow.Json; namespace Raven.Server.NotificationCenter.Handlers { public class DatabaseNotificationCenterHandler : DatabaseRequestHandler { + [RavenAction("/databases/*/notification-center/get", "GET", AuthorizationStatus.ValidUser, EndpointType.Read, SkipUsagesCount = true)] + public async Task GetNotifications() + { + var postponed = GetBoolValueQueryString("postponed", required: false) ?? true; + using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) + await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) + using (Database.NotificationCenter.GetStored(out var storedNotifications, postponed)) + { + writer.WriteStartObject(); + writer.WritePropertyName("Results"); + writer.WriteStartArray(); + var isFirst = true; + foreach (var notification in storedNotifications) + { + if (isFirst == false) + { + writer.WriteComma(); + } + + writer.WriteObject(notification.Json); + isFirst = false; + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + } + [RavenAction("/databases/*/notification-center/watch", "GET", AuthorizationStatus.ValidUser, EndpointType.Read, SkipUsagesCount = true)] public async Task Get() { diff --git a/test/SlowTests/Issues/RavenDB-17068.cs b/test/SlowTests/Issues/RavenDB-17068.cs index 014b1621d7e5..7d05b3a2c72e 100644 --- a/test/SlowTests/Issues/RavenDB-17068.cs +++ b/test/SlowTests/Issues/RavenDB-17068.cs @@ -65,25 +65,29 @@ public async Task CheckIfMismatchesAreRemovedOnMatchingLoad() } while (alertRaised.Item2["Type"].ToString() != NotificationType.AlertRaised.ToString()); var details = alertRaised.Item2[nameof(AlertRaised.Details)] as DynamicJsonValue; + Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(details); + } + } - using (var ctx = JsonOperationContext.ShortTermSingleUse()) - { - var json = ctx.ReadObject(details, "foo"); + protected void Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(DynamicJsonValue details) + { + using (var ctx = JsonOperationContext.ShortTermSingleUse()) + { + var json = ctx.ReadObject(details, "foo"); - var detailsObject = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable(json, "Warnings"); + var detailsObject = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable(json, "Warnings"); - Assert.Equal("DummyIndex",detailsObject.IndexName); - Assert.Equal(1, detailsObject.Warnings.Count); + Assert.Equal("DummyIndex",detailsObject.IndexName); + Assert.Equal(1, detailsObject.Warnings.Count); - detailsObject.Warnings.TryGetValue("orders/2-A", out var warnings); + detailsObject.Warnings.TryGetValue("orders/2-A", out var warnings); - Assert.NotNull(warnings); - Assert.Equal(1, warnings.Count); - Assert.Equal("animals/1-A", warnings.First().ReferenceId); - Assert.Equal("orders/2-A", warnings.First().SourceId); - Assert.Equal("Animals", warnings.First().ActualCollection); - Assert.Equal(2, warnings.First().MismatchedCollections.Count); - } + Assert.NotNull(warnings); + Assert.Equal(1, warnings.Count); + Assert.Equal("animals/1-A", warnings.First().ReferenceId); + Assert.Equal("orders/2-A", warnings.First().SourceId); + Assert.Equal("Animals", warnings.First().ActualCollection); + Assert.Equal(2, warnings.First().MismatchedCollections.Count); } } @@ -204,7 +208,7 @@ public void CheckIfHandlerLimitsAreWorking() } } - private class DummyIndex : AbstractIndexCreationTask + protected class DummyIndex : AbstractIndexCreationTask { public DummyIndex() { @@ -216,7 +220,7 @@ public DummyIndex() } } - private class Order + protected class Order { public string Id { get; set; } public string AnimalId { get; set; } @@ -224,19 +228,19 @@ private class Order public int Price { get; set; } } - private class Animal + protected class Animal { public string Id { get; set; } public string Name { get; set; } } - private class Cat + protected class Cat { public string Id { get; set; } public string Name { get; set; } } - private class Dog + protected class Dog { public string Id { get; set; } public string Name { get; set; } diff --git a/test/SlowTests/Issues/RavenDB_22176.cs b/test/SlowTests/Issues/RavenDB_22176.cs new file mode 100644 index 000000000000..42af84029cf5 --- /dev/null +++ b/test/SlowTests/Issues/RavenDB_22176.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Http; +using Raven.Server.NotificationCenter.Notifications; +using Sparrow.Json; +using Sparrow.Json.Parsing; +using Sparrow.Server.Collections; +using Tests.Infrastructure; +using Xunit; +using Xunit.Abstractions; + +namespace SlowTests.Issues; + +public class RavenDB_22176 : RavenDB_17068 +{ + [RavenFact(RavenTestCategory.Indexes)] + public async Task GetCurrentNotificationsAsJson() + { + using var store = GetDocumentStore(); + var db = await GetDatabase(store.Database); + + var notificationsQueue = new AsyncQueue(); + + using (db.NotificationCenter.TrackActions(notificationsQueue, null)) + using (var session = store.OpenSession()) + { + var c1 = new Cat() { Name = "Bingus" }; + session.Store(c1); + var a1 = new Animal() { Name = "CoolAnimal" }; + session.Store(a1); + var o1 = new Order() { AnimalId = c1.Id, Price = 22 }; + var o2 = new Order() { AnimalId = a1.Id, Price = 33 }; + session.Store(o1); + session.Store(o2); + session.SaveChanges(); + var index = new DummyIndex(); + await index.ExecuteAsync(store); + Indexes.WaitForIndexing(store); + + //Notification should be raised soon, let's get it as json from get + var notifications = store.Maintenance.Send(new GetNotifications()); + Assert.Contains("We have detected usage of LoadDocument(doc, collectionName) where loaded document collection is different than given parameter", + notifications.Results[0].ToString(), StringComparison.InvariantCultureIgnoreCase); + + + Tuple alertRaised; + do + { + alertRaised = await notificationsQueue.TryDequeueAsync(TimeSpan.FromSeconds(5)); + } while (alertRaised.Item2["Type"].ToString() != NotificationType.AlertRaised.ToString()); + + var details = alertRaised.Item2[nameof(AlertRaised.Details)] as DynamicJsonValue; + //Should be dequeued + Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(details); + db.NotificationCenter.Dismiss("AlertRaised/MismatchedReferenceLoad/Indexing"); + notifications = store.Maintenance.Send(new GetNotifications()); + Assert.Equal(0, notifications.Results.Count); + } + } + + public RavenDB_22176(ITestOutputHelper output) : base(output) + { + } + + private record Notification(List Results); + + private class GetNotifications : IMaintenanceOperation + { + public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext context) + { + return new GetNotificationsCommand(); + } + + private class GetNotificationsCommand : RavenCommand + { + public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url) + { + url = $"{node.Url}/databases/{node.Database}/notification-center/get"; + + return new HttpRequestMessage { Method = HttpMethod.Get }; + } + + public override void SetResponse(JsonOperationContext context, BlittableJsonReaderObject response, bool fromCache) + { + if (response == null) + ThrowInvalidResponse(); + + Result = JsonConvert.DeserializeObject(response!.ToString()); + } + + public override bool IsReadRequest => true; + } + } +} From 766282f7bf07870e9115f39fa3d34fe8f3c49fc4 Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Tue, 9 Apr 2024 11:11:36 +0200 Subject: [PATCH 027/243] RavenDB-22176 Allow filtering notifications by type | Create baseclass for `RavenDB_17068` and `RavenDB_22176` tests to avoid repeating tests. --- .../DatabaseNotificationCenterHandler.cs | 16 +++++- test/SlowTests/Issues/RavenDB-17068.cs | 57 +++++++++++-------- test/SlowTests/Issues/RavenDB_22176.cs | 33 +++++++++-- 3 files changed, 75 insertions(+), 31 deletions(-) diff --git a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs index ffae098b0d08..2731b0856c17 100644 --- a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs +++ b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs @@ -7,7 +7,6 @@ using Raven.Server.NotificationCenter.Notifications; using Raven.Server.Routing; using Raven.Server.ServerWide.Context; -using Raven.Server.Utils; using Sparrow.Json; namespace Raven.Server.NotificationCenter.Handlers @@ -18,6 +17,11 @@ public class DatabaseNotificationCenterHandler : DatabaseRequestHandler public async Task GetNotifications() { var postponed = GetBoolValueQueryString("postponed", required: false) ?? true; + var type = GetStringQueryString("type", false); + var shouldFilter = type != null; + if (shouldFilter && Enum.TryParse(typeof(NotificationType), type, out _) == false) + throw new ArgumentException($"The 'type' parameter must be a type of '{{nameof(NotificationType)}}'. Instead, got '{type}'."); + using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) using (Database.NotificationCenter.GetStored(out var storedNotifications, postponed)) @@ -28,6 +32,16 @@ public async Task GetNotifications() var isFirst = true; foreach (var notification in storedNotifications) { + if (shouldFilter && notification.Json != null) + { + if (notification.Json.TryGet(nameof(Notification.Type), out string notificationType) == false) + continue; + + if (notificationType != type) + continue; + } + + if (isFirst == false) { writer.WriteComma(); diff --git a/test/SlowTests/Issues/RavenDB-17068.cs b/test/SlowTests/Issues/RavenDB-17068.cs index 7d05b3a2c72e..a955e04bd319 100644 --- a/test/SlowTests/Issues/RavenDB-17068.cs +++ b/test/SlowTests/Issues/RavenDB-17068.cs @@ -18,9 +18,9 @@ namespace SlowTests.Issues; -public class RavenDB_17068: RavenTestBase +public sealed class RavenDb17068: RavenDB_17068_Base { - public RavenDB_17068(ITestOutputHelper output) : base(output) + public RavenDb17068(ITestOutputHelper output) : base(output) { } @@ -68,29 +68,7 @@ public async Task CheckIfMismatchesAreRemovedOnMatchingLoad() Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(details); } } - - protected void Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(DynamicJsonValue details) - { - using (var ctx = JsonOperationContext.ShortTermSingleUse()) - { - var json = ctx.ReadObject(details, "foo"); - - var detailsObject = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable(json, "Warnings"); - - Assert.Equal("DummyIndex",detailsObject.IndexName); - Assert.Equal(1, detailsObject.Warnings.Count); - - detailsObject.Warnings.TryGetValue("orders/2-A", out var warnings); - - Assert.NotNull(warnings); - Assert.Equal(1, warnings.Count); - Assert.Equal("animals/1-A", warnings.First().ReferenceId); - Assert.Equal("orders/2-A", warnings.First().SourceId); - Assert.Equal("Animals", warnings.First().ActualCollection); - Assert.Equal(2, warnings.First().MismatchedCollections.Count); - } - } - + [RavenFact(RavenTestCategory.Indexes)] public async Task CheckIfNotificationIsNotSendWithAllLoadsEventuallyMatching() { @@ -207,6 +185,35 @@ public void CheckIfHandlerLimitsAreWorking() Assert.Equal(mismatchedDocumentLoadsPerIndex, handler.GetLoadFailures().Count); } } +} + +public abstract class RavenDB_17068_Base : RavenTestBase +{ + protected RavenDB_17068_Base(ITestOutputHelper output) : base(output) + { + } + + protected void Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(DynamicJsonValue details) + { + using (var ctx = JsonOperationContext.ShortTermSingleUse()) + { + var json = ctx.ReadObject(details, "foo"); + + var detailsObject = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable(json, "Warnings"); + + Assert.Equal("DummyIndex",detailsObject.IndexName); + Assert.Equal(1, detailsObject.Warnings.Count); + + detailsObject.Warnings.TryGetValue("orders/2-A", out var warnings); + + Assert.NotNull(warnings); + Assert.Equal(1, warnings.Count); + Assert.Equal("animals/1-A", warnings.First().ReferenceId); + Assert.Equal("orders/2-A", warnings.First().SourceId); + Assert.Equal("Animals", warnings.First().ActualCollection); + Assert.Equal(2, warnings.First().MismatchedCollections.Count); + } + } protected class DummyIndex : AbstractIndexCreationTask { diff --git a/test/SlowTests/Issues/RavenDB_22176.cs b/test/SlowTests/Issues/RavenDB_22176.cs index 42af84029cf5..a730d92ef267 100644 --- a/test/SlowTests/Issues/RavenDB_22176.cs +++ b/test/SlowTests/Issues/RavenDB_22176.cs @@ -16,10 +16,10 @@ namespace SlowTests.Issues; -public class RavenDB_22176 : RavenDB_17068 +public sealed class RavenDB_22176 : RavenDB_17068_Base { [RavenFact(RavenTestCategory.Indexes)] - public async Task GetCurrentNotificationsAsJson() + public async Task TestGetNotificationsEndpoint() { using var store = GetDocumentStore(); var db = await GetDatabase(store.Database); @@ -42,8 +42,12 @@ public async Task GetCurrentNotificationsAsJson() await index.ExecuteAsync(store); Indexes.WaitForIndexing(store); + //filter nonexisting one + var notifications = store.Maintenance.Send(new GetNotifications("OperationChanged")); + Assert.Empty(notifications.Results); + //Notification should be raised soon, let's get it as json from get - var notifications = store.Maintenance.Send(new GetNotifications()); + notifications = store.Maintenance.Send(new GetNotifications("AlertRaised")); Assert.Contains("We have detected usage of LoadDocument(doc, collectionName) where loaded document collection is different than given parameter", notifications.Results[0].ToString(), StringComparison.InvariantCultureIgnoreCase); @@ -58,7 +62,7 @@ public async Task GetCurrentNotificationsAsJson() //Should be dequeued Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(details); db.NotificationCenter.Dismiss("AlertRaised/MismatchedReferenceLoad/Indexing"); - notifications = store.Maintenance.Send(new GetNotifications()); + notifications = store.Maintenance.Send(new GetNotifications("AlertRaised")); Assert.Equal(0, notifications.Results.Count); } } @@ -71,17 +75,36 @@ private record Notification(List Results); private class GetNotifications : IMaintenanceOperation { + private string _alertType; + + public GetNotifications(string alertType = null) + { + _alertType = alertType; + } + public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext context) { - return new GetNotificationsCommand(); + return new GetNotificationsCommand(_alertType); } private class GetNotificationsCommand : RavenCommand { + private string _alertType; + + public GetNotificationsCommand(string alertType) + { + _alertType = alertType; + } + public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url) { url = $"{node.Url}/databases/{node.Database}/notification-center/get"; + if (_alertType != null) + { + url += $"?type={_alertType}"; + } + return new HttpRequestMessage { Method = HttpMethod.Get }; } From 95c5072d76c4ba34478d121859d40f55b80b0f30 Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Wed, 10 Apr 2024 12:25:42 +0200 Subject: [PATCH 028/243] RavenDB-22176 Adjusted endpoint name | Reverted class name change to persist test history in CI | Added the ability to retrieve notifications in a paging fashion. --- .../DatabaseNotificationCenterHandler.cs | 33 ++++++++++-- test/SlowTests/Issues/RavenDB-17068.cs | 4 +- test/SlowTests/Issues/RavenDB_22176.cs | 51 ++++++++++++++----- 3 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs index 2731b0856c17..c827ef7b62a0 100644 --- a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs +++ b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs @@ -13,11 +13,14 @@ namespace Raven.Server.NotificationCenter.Handlers { public class DatabaseNotificationCenterHandler : DatabaseRequestHandler { - [RavenAction("/databases/*/notification-center/get", "GET", AuthorizationStatus.ValidUser, EndpointType.Read, SkipUsagesCount = true)] + [RavenAction("/databases/*/notifications", "GET", AuthorizationStatus.ValidUser, EndpointType.Read, SkipUsagesCount = true)] public async Task GetNotifications() { var postponed = GetBoolValueQueryString("postponed", required: false) ?? true; - var type = GetStringQueryString("type", false); + var type = GetStringQueryString("type", required: false); + var start = GetIntValueQueryString("pageStart", required: false) ?? 0; + var pageSize = GetIntValueQueryString("pageSize", required: false) ?? int.MaxValue; + var shouldFilter = type != null; if (shouldFilter && Enum.TryParse(typeof(NotificationType), type, out _) == false) throw new ArgumentException($"The 'type' parameter must be a type of '{{nameof(NotificationType)}}'. Instead, got '{type}'."); @@ -27,9 +30,13 @@ public async Task GetNotifications() using (Database.NotificationCenter.GetStored(out var storedNotifications, postponed)) { writer.WriteStartObject(); + + var countQuery = pageSize == 0; + var totalResults = 0; + var isFirst = true; + writer.WritePropertyName("Results"); writer.WriteStartArray(); - var isFirst = true; foreach (var notification in storedNotifications) { if (shouldFilter && notification.Json != null) @@ -41,6 +48,20 @@ public async Task GetNotifications() continue; } + if (start > 0) + { + start--; + continue; + } + + totalResults++; + + if (pageSize == 0 && countQuery == false) + break; + pageSize--; + + if (countQuery) + continue; if (isFirst == false) { @@ -50,8 +71,12 @@ public async Task GetNotifications() writer.WriteObject(notification.Json); isFirst = false; } - writer.WriteEndArray(); + + writer.WriteComma(); + writer.WritePropertyName("TotalResults"); + writer.WriteInteger(totalResults); + writer.WriteEndObject(); } } diff --git a/test/SlowTests/Issues/RavenDB-17068.cs b/test/SlowTests/Issues/RavenDB-17068.cs index a955e04bd319..52a25078797c 100644 --- a/test/SlowTests/Issues/RavenDB-17068.cs +++ b/test/SlowTests/Issues/RavenDB-17068.cs @@ -18,9 +18,9 @@ namespace SlowTests.Issues; -public sealed class RavenDb17068: RavenDB_17068_Base +public sealed class RavenDB_17068: RavenDB_17068_Base { - public RavenDb17068(ITestOutputHelper output) : base(output) + public RavenDB_17068(ITestOutputHelper output) : base(output) { } diff --git a/test/SlowTests/Issues/RavenDB_22176.cs b/test/SlowTests/Issues/RavenDB_22176.cs index a730d92ef267..b2e03c6ee77c 100644 --- a/test/SlowTests/Issues/RavenDB_22176.cs +++ b/test/SlowTests/Issues/RavenDB_22176.cs @@ -43,15 +43,28 @@ public async Task TestGetNotificationsEndpoint() Indexes.WaitForIndexing(store); //filter nonexisting one - var notifications = store.Maintenance.Send(new GetNotifications("OperationChanged")); + var notifications = store.Maintenance.Send(new GetNotifications(alertType: "OperationChanged")); Assert.Empty(notifications.Results); //Notification should be raised soon, let's get it as json from get - notifications = store.Maintenance.Send(new GetNotifications("AlertRaised")); - Assert.Contains("We have detected usage of LoadDocument(doc, collectionName) where loaded document collection is different than given parameter", - notifications.Results[0].ToString(), StringComparison.InvariantCultureIgnoreCase); - + notifications = store.Maintenance.Send(new GetNotifications(alertType: "AlertRaised")); + Assert.Contains("We have detected usage of LoadDocument(doc, collectionName) where loaded document collection is different than given parameter", notifications.Results[0].ToString(), StringComparison.InvariantCultureIgnoreCase); + //paging test: + { + var notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 0, alertType: "AlertRaised")); + Assert.Equal(1, notificationsTemp.TotalResults); + Assert.Empty(notificationsTemp.Results); + + notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 1, pageStart: 1, alertType: "AlertRaised")); + Assert.Equal(0, notificationsTemp.TotalResults); + Assert.Empty(notificationsTemp.Results); + + notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 1, pageStart: 0, alertType: "AlertRaised")); + Assert.Equal(1, notificationsTemp.TotalResults); + Assert.NotEmpty(notificationsTemp.Results); + } + Tuple alertRaised; do { @@ -62,7 +75,7 @@ public async Task TestGetNotificationsEndpoint() //Should be dequeued Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(details); db.NotificationCenter.Dismiss("AlertRaised/MismatchedReferenceLoad/Indexing"); - notifications = store.Maintenance.Send(new GetNotifications("AlertRaised")); + notifications = store.Maintenance.Send(new GetNotifications(alertType: "AlertRaised")); Assert.Equal(0, notifications.Results.Count); } } @@ -71,39 +84,51 @@ public RavenDB_22176(ITestOutputHelper output) : base(output) { } - private record Notification(List Results); + private record Notification(int TotalResults, List Results); private class GetNotifications : IMaintenanceOperation { private string _alertType; + private int? _pageStart; + private int? _pageSize; - public GetNotifications(string alertType = null) + public GetNotifications(int? pageStart = null, int? pageSize = null, string alertType = null) { + _pageStart = pageStart; + _pageSize = pageSize; _alertType = alertType; } public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext context) { - return new GetNotificationsCommand(_alertType); + return new GetNotificationsCommand(_alertType, _pageStart, _pageSize); } private class GetNotificationsCommand : RavenCommand { private string _alertType; + private int? _pageStart; + private int? _pageSize; - public GetNotificationsCommand(string alertType) + public GetNotificationsCommand(string alertType, int? pageStart, int? pageSize) { _alertType = alertType; + _pageStart = pageStart; + _pageSize = pageSize; } public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url) { - url = $"{node.Url}/databases/{node.Database}/notification-center/get"; + url = $"{node.Url}/databases/{node.Database}/notifications"; if (_alertType != null) - { url += $"?type={_alertType}"; - } + + if (_pageSize != null) + url += $"&pageSize={_pageSize.Value}"; + + if (_pageStart != null) + url += $"&pageStart={_pageStart.Value}"; return new HttpRequestMessage { Method = HttpMethod.Get }; } From 26ded91a6aba3f87de688839dbcf97a5e873500e Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Thu, 11 Apr 2024 09:44:43 +0200 Subject: [PATCH 029/243] RavenDB-22176 Allow filter notifications with a mask, paging fix --- .../DatabaseNotificationCenterHandler.cs | 51 +++++- test/SlowTests/Issues/RavenDB_22176.cs | 172 +++++++++++++++--- 2 files changed, 186 insertions(+), 37 deletions(-) diff --git a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs index c827ef7b62a0..b586cdbe1d69 100644 --- a/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs +++ b/src/Raven.Server/NotificationCenter/Handlers/DatabaseNotificationCenterHandler.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Raven.Client.Exceptions; using Raven.Client.Util; using Raven.Server.Documents; using Raven.Server.NotificationCenter.Notifications; @@ -13,6 +14,16 @@ namespace Raven.Server.NotificationCenter.Handlers { public class DatabaseNotificationCenterHandler : DatabaseRequestHandler { + [Flags] + private enum NotificationTypeParameter : short + { + None = 0, + Alert = 1, + PerformanceHint = 1 << 1, + } + + private static readonly short SupportedFilterFlags = (short)(NotificationTypeParameter.Alert | NotificationTypeParameter.PerformanceHint); + [RavenAction("/databases/*/notifications", "GET", AuthorizationStatus.ValidUser, EndpointType.Read, SkipUsagesCount = true)] public async Task GetNotifications() { @@ -20,10 +31,19 @@ public async Task GetNotifications() var type = GetStringQueryString("type", required: false); var start = GetIntValueQueryString("pageStart", required: false) ?? 0; var pageSize = GetIntValueQueryString("pageSize", required: false) ?? int.MaxValue; - + + NotificationTypeParameter filter = NotificationTypeParameter.None; var shouldFilter = type != null; - if (shouldFilter && Enum.TryParse(typeof(NotificationType), type, out _) == false) - throw new ArgumentException($"The 'type' parameter must be a type of '{{nameof(NotificationType)}}'. Instead, got '{type}'."); + if (shouldFilter && (Enum.TryParse(type.AsSpan(), ignoreCase: true, out filter) == false || filter == NotificationTypeParameter.None || ((short)filter & ~SupportedFilterFlags) != 0)) + { + var supportedNotificationTypeParameters = Enum.GetValues(typeof(NotificationTypeParameter)) + .OfType() + .Where(x => x != NotificationTypeParameter.None) + .ToArray(); + + throw new BadRequestException($"Accepted values for type parameter are: [{string.Join(", ", supportedNotificationTypeParameters)}]. Instead, got '{type}'. " + + $"Type parameter is a flag, passing a list of types e.g. 'type=alert,performancehint' is also supported."); + } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) @@ -41,23 +61,25 @@ public async Task GetNotifications() { if (shouldFilter && notification.Json != null) { - if (notification.Json.TryGet(nameof(Notification.Type), out string notificationType) == false) + if (notification.Json.TryGet(nameof(Notification.Type), out string notificationType) == false + || Enum.TryParse(notificationType.AsSpan(), out NotificationType alertType) == false) continue; - - if (notificationType != type) + + if (ShouldIncludeNotification(alertType) == false) continue; } + totalResults++; + if (start > 0) { start--; continue; } - - totalResults++; if (pageSize == 0 && countQuery == false) - break; + countQuery = true; + pageSize--; if (countQuery) @@ -71,6 +93,7 @@ public async Task GetNotifications() writer.WriteObject(notification.Json); isFirst = false; } + writer.WriteEndArray(); writer.WriteComma(); @@ -79,6 +102,16 @@ public async Task GetNotifications() writer.WriteEndObject(); } + + bool ShouldIncludeNotification(in NotificationType notificationType) + { + return notificationType switch + { + NotificationType.AlertRaised => filter.HasFlag(NotificationTypeParameter.Alert), + NotificationType.PerformanceHint => filter.HasFlag(NotificationTypeParameter.PerformanceHint), + _ => false + }; + } } [RavenAction("/databases/*/notification-center/watch", "GET", AuthorizationStatus.ValidUser, EndpointType.Read, SkipUsagesCount = true)] diff --git a/test/SlowTests/Issues/RavenDB_22176.cs b/test/SlowTests/Issues/RavenDB_22176.cs index b2e03c6ee77c..3b048be6723c 100644 --- a/test/SlowTests/Issues/RavenDB_22176.cs +++ b/test/SlowTests/Issues/RavenDB_22176.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Newtonsoft.Json; @@ -18,8 +19,10 @@ namespace SlowTests.Issues; public sealed class RavenDB_22176 : RavenDB_17068_Base { - [RavenFact(RavenTestCategory.Indexes)] - public async Task TestGetNotificationsEndpoint() + [RavenTheory(RavenTestCategory.Indexes)] + [InlineData(true)] + [InlineData(false)] + public async Task TestGetNotificationsEndpoint(bool typeAsNumber) { using var store = GetDocumentStore(); var db = await GetDatabase(store.Database); @@ -43,24 +46,24 @@ public async Task TestGetNotificationsEndpoint() Indexes.WaitForIndexing(store); //filter nonexisting one - var notifications = store.Maintenance.Send(new GetNotifications(alertType: "OperationChanged")); + var notifications = store.Maintenance.Send(new GetNotifications(alertType: NotificationTypeParameter.PerformanceHint, testAsNumber: typeAsNumber)); Assert.Empty(notifications.Results); //Notification should be raised soon, let's get it as json from get - notifications = store.Maintenance.Send(new GetNotifications(alertType: "AlertRaised")); - Assert.Contains("We have detected usage of LoadDocument(doc, collectionName) where loaded document collection is different than given parameter", notifications.Results[0].ToString(), StringComparison.InvariantCultureIgnoreCase); + notifications = store.Maintenance.Send(new GetNotifications(alertType: NotificationTypeParameter.Alert, testAsNumber: typeAsNumber)); + Assert.Equal(1, notifications.TotalResults); //paging test: { - var notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 0, alertType: "AlertRaised")); + var notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 0, alertType: NotificationTypeParameter.Alert, testAsNumber: typeAsNumber)); Assert.Equal(1, notificationsTemp.TotalResults); Assert.Empty(notificationsTemp.Results); - notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 1, pageStart: 1, alertType: "AlertRaised")); - Assert.Equal(0, notificationsTemp.TotalResults); + notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 1, pageStart: 1, alertType: NotificationTypeParameter.Alert, testAsNumber: typeAsNumber)); + Assert.Equal(1, notificationsTemp.TotalResults); Assert.Empty(notificationsTemp.Results); - notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 1, pageStart: 0, alertType: "AlertRaised")); + notificationsTemp = store.Maintenance.Send(new GetNotifications(pageSize: 1, pageStart: 0, alertType: NotificationTypeParameter.Alert, testAsNumber: typeAsNumber)); Assert.Equal(1, notificationsTemp.TotalResults); Assert.NotEmpty(notificationsTemp.Results); } @@ -75,54 +78,167 @@ public async Task TestGetNotificationsEndpoint() //Should be dequeued Assert_CheckIfMismatchesAreRemovedOnMatchingLoad(details); db.NotificationCenter.Dismiss("AlertRaised/MismatchedReferenceLoad/Indexing"); - notifications = store.Maintenance.Send(new GetNotifications(alertType: "AlertRaised")); + notifications = store.Maintenance.Send(new GetNotifications(alertType: NotificationTypeParameter.Alert, testAsNumber: typeAsNumber)); Assert.Equal(0, notifications.Results.Count); } } - public RavenDB_22176(ITestOutputHelper output) : base(output) + [RavenTheory(RavenTestCategory.Monitoring)] + [InlineData(NotificationType.AlertRaised, true)] + [InlineData(NotificationType.AlertRaised, false)] + [InlineData(NotificationType.PerformanceHint, true)] + [InlineData(NotificationType.PerformanceHint, false)] + public async Task TestGetNotificationsEndpointPaging(NotificationType notificationType, bool flagAsInt) { + using var store = GetDocumentStore(); + var db = await GetDatabase(store.Database); + NotificationTypeParameter notificationTypeParameter = notificationType switch + { + NotificationType.PerformanceHint => NotificationTypeParameter.PerformanceHint, + NotificationType.AlertRaised => NotificationTypeParameter.Alert, + _ => NotificationTypeParameter.All + }; + + var notificationsQueue = new AsyncQueue(); + using (db.NotificationCenter.TrackActions(notificationsQueue, null)) + { + for (var id = 0; id < 10; ++id) + { + Notification notificationEndpointDto = id % 2 == 0 + ? AlertRaised.Create("test", id.ToString(), id.ToString(), (AlertType)id, NotificationSeverity.Info) + : PerformanceHint.Create("test", id.ToString(), id.ToString(), (PerformanceHintType)(id % 5), NotificationSeverity.Info, "indexing"); + db.NotificationCenter.Add(notificationEndpointDto); + } + + var result = store.Maintenance.Send(new GetNotifications(pageSize: 10, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); + NotificationEndpointDto copyForAssertion = result; + Assert.Equal(5, result.Results.Count(x => x.Type == notificationType)); + + result = store.Maintenance.Send(new GetNotifications(pageSize: 0, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); + Assert.Equal(5, result.TotalResults); + Assert.Empty(result.Results); + + //paging + { + result = store.Maintenance.Send(new GetNotifications(pageStart: 0, pageSize: 2, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); + Assert.Equal(5, result.TotalResults); + Assert.Equal(2, result.Results.Count); + Assert.Equal(copyForAssertion.Results[0].Title, result.Results[0].Title); + Assert.Equal(copyForAssertion.Results[1].Title, result.Results[1].Title); + + result = store.Maintenance.Send(new GetNotifications(pageStart: 2, pageSize: 2, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); + Assert.Equal(5, result.TotalResults); + Assert.Equal(2, result.Results.Count); + Assert.Equal(copyForAssertion.Results[2].Title, result.Results[0].Title); + Assert.Equal(copyForAssertion.Results[3].Title, result.Results[1].Title); + + + result = store.Maintenance.Send(new GetNotifications(pageStart: 4, pageSize: 2, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); + Assert.Equal(5, result.TotalResults); + Assert.Equal(1, result.Results.Count); + Assert.Equal(copyForAssertion.Results[4].Title, result.Results[0].Title); + + + result = store.Maintenance.Send(new GetNotifications(pageStart: 6, pageSize: 2, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); + Assert.Equal(5, result.TotalResults); + Assert.Equal(0, result.Results.Count); + } + + result = store.Maintenance.Send(new GetNotifications(pageStart: 0, pageSize: 12, testAsNumber: flagAsInt)); + Assert.Equal(10, result.TotalResults); + Assert.Equal(10, result.Results.Count); + + result = store.Maintenance.Send(new GetNotifications(pageSize: 0, testAsNumber: flagAsInt)); + Assert.Equal(10, result.TotalResults); + Assert.Equal(0, result.Results.Count); + + //Flags + result = store.Maintenance.Send(new GetNotifications(pageStart: 0, pageSize: 12, alertType: NotificationTypeParameter.Alert | NotificationTypeParameter.PerformanceHint, testAsNumber: flagAsInt)); + Assert.Equal(10, result.TotalResults); + Assert.Equal(10, result.Results.Count); + + //Exception + var ex = Assert.ThrowsAny(() => store.Maintenance.Send(new GetNotifications(alertType: NotificationTypeParameter.NonExistingFlag, testAsNumber: flagAsInt))); + Assert.Contains("Accepted values for type parameter are: [Alert, PerformanceHint]", ex.Message); + Assert.Contains("BadRequestException", ex.Message); + } } - private record Notification(int TotalResults, List Results); + public RavenDB_22176(ITestOutputHelper output) : base(output) + { + } + + // ReSharper disable once ClassNeverInstantiated.Local + private class SerializableNotification : Notification + { + public SerializableNotification(NotificationType type, string database) : base(type, database) + { + } - private class GetNotifications : IMaintenanceOperation + public override string Id { get; } + } + private record NotificationEndpointDto(int TotalResults, List Results); + + [Flags] + private enum NotificationTypeParameter : short + { + All = 0, + Alert = 1, + PerformanceHint = 1 << 1, + NonExistingFlag = 1 << 2, + } + private class GetNotifications : IMaintenanceOperation { - private string _alertType; - private int? _pageStart; - private int? _pageSize; + private readonly NotificationTypeParameter _alertType; + private readonly bool _testAsNumber; + private readonly bool _includeDefaultFilter; + private readonly int? _pageStart; + private readonly int? _pageSize; - public GetNotifications(int? pageStart = null, int? pageSize = null, string alertType = null) + public GetNotifications(int? pageStart = null, int? pageSize = null, NotificationTypeParameter alertType = NotificationTypeParameter.All, bool testAsNumber = false, bool includeDefaultFilter = false) { _pageStart = pageStart; _pageSize = pageSize; _alertType = alertType; + _testAsNumber = testAsNumber; + _includeDefaultFilter = includeDefaultFilter; } - public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext context) + public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext context) { - return new GetNotificationsCommand(_alertType, _pageStart, _pageSize); + return new GetNotificationsCommand(_alertType, _pageStart, _pageSize, _testAsNumber, _includeDefaultFilter); } - private class GetNotificationsCommand : RavenCommand + private class GetNotificationsCommand : RavenCommand { - private string _alertType; - private int? _pageStart; - private int? _pageSize; + private readonly NotificationTypeParameter _alertType; + private readonly int? _pageStart; + private readonly int? _pageSize; + private readonly bool _typeAsNumber; + private readonly bool _includeDefaultFilter; + - public GetNotificationsCommand(string alertType, int? pageStart, int? pageSize) + public GetNotificationsCommand(NotificationTypeParameter alertType, int? pageStart, int? pageSize, bool typeAsNumber, bool includeDefaultFilter) { _alertType = alertType; _pageStart = pageStart; _pageSize = pageSize; + _typeAsNumber = typeAsNumber; + _includeDefaultFilter = includeDefaultFilter; } public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url) { - url = $"{node.Url}/databases/{node.Database}/notifications"; + url = $"{node.Url}/databases/{node.Database}/notifications?"; - if (_alertType != null) - url += $"?type={_alertType}"; + if (_alertType != NotificationTypeParameter.All || _includeDefaultFilter) + { + var type = _typeAsNumber + ? ((short)_alertType).ToString() + : UrlEncode(_alertType.ToString()); + + url += $"&type={type}"; + } if (_pageSize != null) url += $"&pageSize={_pageSize.Value}"; @@ -138,7 +254,7 @@ public override void SetResponse(JsonOperationContext context, BlittableJsonRead if (response == null) ThrowInvalidResponse(); - Result = JsonConvert.DeserializeObject(response!.ToString()); + Result = JsonConvert.DeserializeObject(response!.ToString()); } public override bool IsReadRequest => true; From 26d8c91d38d9baa996e77dae00acd61522f99866 Mon Sep 17 00:00:00 2001 From: "efrat@ravendb.net" Date: Tue, 26 Mar 2024 11:47:51 +0200 Subject: [PATCH 030/243] RavenDB-21427 Prevent downgrade of the license when features are being used --- src/Raven.Server/Commercial/LicenseManager.cs | 35 +- src/Raven.Server/RavenServer.cs | 2 +- .../ServerWide/ClusterStateMachine.License.cs | 639 ++++++++++-------- .../ServerWide/ClusterStateMachine.cs | 1 + .../ServerWide/Commands/PutLicenseCommand.cs | 14 +- src/Raven.Server/ServerWide/ServerStore.cs | 4 +- test/SlowTests/Issues/RavenDB-21427.cs | 476 +++++++++++++ .../MultiLicenseRequiredFactAttribute.cs | 38 ++ 8 files changed, 923 insertions(+), 286 deletions(-) create mode 100644 test/SlowTests/Issues/RavenDB-21427.cs create mode 100644 test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs diff --git a/src/Raven.Server/Commercial/LicenseManager.cs b/src/Raven.Server/Commercial/LicenseManager.cs index e25527a7d9d8..ad24b6bf7f6f 100644 --- a/src/Raven.Server/Commercial/LicenseManager.cs +++ b/src/Raven.Server/Commercial/LicenseManager.cs @@ -380,7 +380,7 @@ public async Task ChangeLicenseLimits(string nodeTag, int? maxUtilizedCores, str } } - public async Task ActivateAsync(License license, string raftRequestId, bool skipGettingUpdatedLicense = false) + public async Task ActivateAsync(License license, string raftRequestId, bool skipGettingUpdatedLicense = false, bool fromApi = false) { var licenseStatus = GetLicenseStatus(license); if (licenseStatus.Expiration.HasValue == false) @@ -400,7 +400,7 @@ public async Task ActivateAsync(License license, string raftRequestId, bool skip { // license expired, we'll try to update it var updatedLicense = await GetUpdatedLicenseForActivation(license); - if (updatedLicense == null) + if (updatedLicense.license == null) { var errorMessage = $"License already expired on: {licenseStatus.FormattedExpiration} and we failed to get an updated one from {ApiHttpClient.ApiRavenDbNet}."; @@ -412,7 +412,7 @@ public async Task ActivateAsync(License license, string raftRequestId, bool skip throw new LicenseExpiredException(errorMessage); } - await ActivateAsync(updatedLicense, raftRequestId, skipGettingUpdatedLicense: true); + await ActivateAsync(updatedLicense.license, raftRequestId, skipGettingUpdatedLicense: true, updatedLicense.FromApi); return; } catch (LicenseExpiredException) @@ -439,14 +439,14 @@ public async Task ActivateAsync(License license, string raftRequestId, bool skip try { var updatedLicense = await GetUpdatedLicenseForActivation(license); - if (updatedLicense == null) + if (updatedLicense.license == null) { throw new LicenseLimitException($"Your license ('{licenseStatus.Id}') version '{licenseStatus.Version}' doesn't allow you to upgrade to server version '{RavenVersionAttribute.Instance.FullVersion}'. " + $"We failed to get an updated one from {ApiHttpClient.ApiRavenDbNet}. " + $"Please proceed to the https://ravendb.net/l/8O2YU1 website to perform the license upgrade first."); } - await ActivateAsync(updatedLicense, raftRequestId, skipGettingUpdatedLicense: true); + await ActivateAsync(updatedLicense.license, raftRequestId, skipGettingUpdatedLicense: true, updatedLicense.FromApi); return; } catch (Exception e) @@ -461,7 +461,7 @@ public async Task ActivateAsync(License license, string raftRequestId, bool skip try { - await _serverStore.PutLicenseAsync(license, raftRequestId).ConfigureAwait(false); + await _serverStore.PutLicenseAsync(license, raftRequestId, fromApi).ConfigureAwait(false); } catch (Exception e) { @@ -639,7 +639,7 @@ private static StudioConfiguration.StudioEnvironment GetStudioEnvironment(Transa } } - private async Task GetUpdatedLicenseForActivation(License currentLicense) + private async Task<(License license, bool FromApi)> GetUpdatedLicenseForActivation(License currentLicense) { try { @@ -651,14 +651,15 @@ private async Task GetUpdatedLicenseForActivation(License currentLicens var configurationKey = RavenConfiguration.GetKey(x => x.Licensing.DisableAutoUpdate); Logger.Info($"Skipping updating of the license from string or path or from api.ravendb.net because '{configurationKey}' was set to true"); } - return null; + + return (null, false); } if (_serverStore.Configuration.Licensing.DisableAutoUpdateFromApi) { var license = TryGetUpdatedLicenseFromStringOrPath(currentLicense); if (license != null) - return license; + return (license, false); if (_skipLeasingErrorsLogging == false && Logger.IsInfoEnabled) { @@ -666,7 +667,7 @@ private async Task GetUpdatedLicenseForActivation(License currentLicens var configurationKey = RavenConfiguration.GetKey(x => x.Licensing.DisableAutoUpdateFromApi); Logger.Info($"Skipping updating of the license from api.ravendb.net because '{configurationKey}' was set to true"); } - return null; + return (null, false); } var response = await GetUpdatedLicenseResponseMessage(currentLicense, _serverStore.ContextPool).ConfigureAwait(false); @@ -677,11 +678,11 @@ private async Task GetUpdatedLicenseForActivation(License currentLicens // we'll try to get it from the json string or path var license = TryGetUpdatedLicenseFromStringOrPath(currentLicense); if (license != null) - return license; + return (license, false); var responseString = await response.Content.ReadAsStringWithZstdSupportAsync().ConfigureAwait(false); AddLeaseLicenseError($"status code: {response.StatusCode}, response: {responseString}"); - return null; + return (null, false); } var leasedLicense = await ConvertResponseToLeasedLicense(response).ConfigureAwait(false); @@ -709,13 +710,13 @@ private async Task GetUpdatedLicenseForActivation(License currentLicens LicenseStatus.ErrorMessage = leasedLicense.ErrorMessage; } - return licenseChanged ? leasedLicense.License : null; + return licenseChanged ? (leasedLicense.License, true) : (null,false); } catch (HttpRequestException) { var license = TryGetUpdatedLicenseFromStringOrPath(currentLicense); if (license != null) - return license; + return (license, false); throw; } @@ -803,15 +804,15 @@ public async Task LeaseLicense(string raftRequestId, bool th return leaseStatus; var updatedLicense = await GetUpdatedLicenseForActivation(loadedLicense); - if (updatedLicense == null) + if (updatedLicense.license == null) return leaseStatus; - var licenseStatus = GetLicenseStatus(updatedLicense); + var licenseStatus = GetLicenseStatus(updatedLicense.license); try { // we'll activate the license from the license server - await _serverStore.PutLicenseAsync(updatedLicense, raftRequestId).ConfigureAwait(false); + await _serverStore.PutLicenseAsync(updatedLicense.license, raftRequestId, updatedLicense.FromApi).ConfigureAwait(false); } catch { diff --git a/src/Raven.Server/RavenServer.cs b/src/Raven.Server/RavenServer.cs index ef44f0d76ceb..f28bee307f85 100644 --- a/src/Raven.Server/RavenServer.cs +++ b/src/Raven.Server/RavenServer.cs @@ -3049,7 +3049,7 @@ private static void VerifyLicenseVersion(LicenseStatus licenseStatus, License li if (licenseStatus.Version.Major >= 6) { serverStore.LicenseManager.OnBeforeInitialize += () => - serverStore.LicenseManager.ActivateAsync(licenseFromApi, RaftIdGenerator.NewId()) + serverStore.LicenseManager.ActivateAsync(licenseFromApi, RaftIdGenerator.NewId(), fromApi: true) .Wait(serverStore.ServerShutdown); return; } diff --git a/src/Raven.Server/ServerWide/ClusterStateMachine.License.cs b/src/Raven.Server/ServerWide/ClusterStateMachine.License.cs index 2fd03e842926..7ab4e561d2a2 100644 --- a/src/Raven.Server/ServerWide/ClusterStateMachine.License.cs +++ b/src/Raven.Server/ServerWide/ClusterStateMachine.License.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; -using Raven.Client.Documents.Subscriptions; +using Raven.Client.Documents.Operations.Revisions; using Raven.Client.Exceptions.Commercial; using Raven.Client.ServerWide; using Raven.Server.Commercial; @@ -17,6 +17,7 @@ using Raven.Server.ServerWide.Commands.Sorters; using Raven.Server.ServerWide.Commands.Subscriptions; using Raven.Server.ServerWide.Context; +using Raven.Server.Utils; using Sparrow.Json; using Sparrow.Server; using Voron; @@ -27,6 +28,7 @@ namespace Raven.Server.ServerWide; public sealed partial class ClusterStateMachine { private const int MinBuildVersion60000 = 60_000; + private const int MinBuildVersion60102 = 60_102; private static readonly List _licenseLimitsCommandsForCreateDatabase = new() { @@ -53,343 +55,450 @@ private void AssertLicenseLimits(string type, ServerStore serverStore, DatabaseR { case nameof(AddDatabaseCommand): case nameof(UpdateTopologyCommand): - if (databaseRecord.IsSharded == false) - return; - - var maxReplicationFactorForSharding = serverStore.LicenseManager.LicenseStatus.MaxReplicationFactorForSharding; - var multiNodeSharding = serverStore.LicenseManager.LicenseStatus.HasMultiNodeSharding; - if (maxReplicationFactorForSharding == null && multiNodeSharding) - return; - - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - var nodes = new HashSet(); - foreach (var shard in databaseRecord.Sharding.Shards) - { - var topology = shard.Value; - if (maxReplicationFactorForSharding != null && topology.ReplicationFactor > maxReplicationFactorForSharding) - { - throw new LicenseLimitException(LimitType.Sharding, $"Your license doesn't allow to use a replication factor of more than {maxReplicationFactorForSharding} for sharding"); - } - - foreach (var nodeTag in topology.AllNodes) - { - nodes.Add(nodeTag); - } - } - - if (multiNodeSharding == false && nodes.Count > 1) - { - throw new LicenseLimitException(LimitType.Sharding, $"Your license allows to create a sharded database only on a single node while you tried to create it on nodes {string.Join(", ", nodes)}"); - } - + AssertMultiNodeSharding(databaseRecord, serverStore.LicenseManager.LicenseStatus, context); break; case nameof(PutIndexCommand): - AssertStaticIndexesCount(); + AssertStaticIndexesCount(databaseRecord, serverStore.LicenseManager.LicenseStatus, context, items, type); break; case nameof(PutAutoIndexCommand): - AssertAutoIndexesCount(); + AssertAutoIndexesCount(databaseRecord, serverStore.LicenseManager.LicenseStatus, context, items, type); break; case nameof(PutIndexesCommand): - AssertStaticIndexesCount(); - AssertAutoIndexesCount(); + AssertStaticIndexesCount(databaseRecord, serverStore.LicenseManager.LicenseStatus, context, items, type); + AssertAutoIndexesCount(databaseRecord, serverStore.LicenseManager.LicenseStatus, context, items, type); break; case nameof(EditRevisionsConfigurationCommand): - if (databaseRecord.Revisions == null) - return; - - if (databaseRecord.Revisions.Default == null && - (databaseRecord.Revisions.Collections == null || databaseRecord.Revisions.Collections.Count == 0)) - return; - - var maxRevisionsToKeep = serverStore.LicenseManager.LicenseStatus.MaxNumberOfRevisionsToKeep; - var maxRevisionAgeToKeepInDays = serverStore.LicenseManager.LicenseStatus.MaxNumberOfRevisionAgeToKeepInDays; - if (serverStore.LicenseManager.LicenseStatus.CanSetupDefaultRevisionsConfiguration && - maxRevisionsToKeep == null && maxRevisionAgeToKeepInDays == null) - return; - - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - if (serverStore.LicenseManager.LicenseStatus.CanSetupDefaultRevisionsConfiguration == false && - databaseRecord.Revisions.Default != null) - { - throw new LicenseLimitException(LimitType.RevisionsConfiguration, "Your license doesn't allow the creation of a default configuration for revisions."); - } - - if (databaseRecord.Revisions.Collections != null) - { - foreach (var revisionPerCollectionConfiguration in databaseRecord.Revisions.Collections) - { - if (revisionPerCollectionConfiguration.Value.MinimumRevisionsToKeep != null && - maxRevisionsToKeep != null && - revisionPerCollectionConfiguration.Value.MinimumRevisionsToKeep > maxRevisionsToKeep) - { - throw new LicenseLimitException(LimitType.RevisionsConfiguration, - $"The defined minimum revisions to keep '{revisionPerCollectionConfiguration.Value.MinimumRevisionsToKeep}' " + - $" exceeds the licensed one '{maxRevisionsToKeep}'"); - } - - if (revisionPerCollectionConfiguration.Value.MinimumRevisionAgeToKeep != null && - maxRevisionAgeToKeepInDays != null && - revisionPerCollectionConfiguration.Value.MinimumRevisionAgeToKeep.Value.TotalDays > maxRevisionAgeToKeepInDays) - { - throw new LicenseLimitException(LimitType.RevisionsConfiguration, - $"The defined minimum revisions age to keep '{revisionPerCollectionConfiguration.Value.MinimumRevisionAgeToKeep}' " + - $" exceeds the licensed one '{maxRevisionAgeToKeepInDays}'"); - } - } - } - + AssertRevisionConfiguration(databaseRecord, serverStore.LicenseManager.LicenseStatus, context); break; case nameof(EditExpirationCommand): - var minPeriodForExpirationInHours = serverStore.LicenseManager.LicenseStatus.MinPeriodForExpirationInHours; - if (minPeriodForExpirationInHours != null && databaseRecord.Expiration is { Disabled: false }) - { - var deleteFrequencyInSec = databaseRecord.Expiration.DeleteFrequencyInSec ?? ExpiredDocumentsCleaner.DefaultDeleteFrequencyInSec; - var deleteFrequency = new TimeSetting(deleteFrequencyInSec, TimeUnit.Seconds); - var minPeriodForExpiration = new TimeSetting(minPeriodForExpirationInHours.Value, TimeUnit.Hours); - if (deleteFrequency.AsTimeSpan < minPeriodForExpiration.AsTimeSpan) - { - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - throw new LicenseLimitException(LimitType.Expiration, $"Your license doesn't allow modifying the expiration frequency below {minPeriodForExpirationInHours} hours."); - } - } - + AssertExpirationConfiguration(databaseRecord, serverStore.LicenseManager.LicenseStatus, context); break; case nameof(EditRefreshCommand): - var minPeriodForRefreshInHours = serverStore.LicenseManager.LicenseStatus.MinPeriodForRefreshInHours; - if (minPeriodForRefreshInHours != null && databaseRecord.Refresh is { Disabled: false }) - { - var refreshFrequencyInSec = databaseRecord.Refresh.RefreshFrequencyInSec ?? ExpiredDocumentsCleaner.DefaultRefreshFrequencyInSec; - var refreshFrequency = new TimeSetting(refreshFrequencyInSec, TimeUnit.Seconds); - var minPeriodForRefresh = new TimeSetting(minPeriodForRefreshInHours.Value, TimeUnit.Hours); - if (refreshFrequency.AsTimeSpan < minPeriodForRefresh.AsTimeSpan) - { - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - throw new LicenseLimitException(LimitType.Refresh, $"Your license doesn't allow modifying the refresh frequency below {minPeriodForRefreshInHours} hours."); - } - } - + AssertRefreshFrequency(databaseRecord, serverStore.LicenseManager.LicenseStatus, context); break; case nameof(PutSortersCommand): - var maxCustomSortersPerDatabase = serverStore.LicenseManager.LicenseStatus.MaxNumberOfCustomSortersPerDatabase; - if (maxCustomSortersPerDatabase != null && maxCustomSortersPerDatabase >= 0 && databaseRecord.Sorters.Count > maxCustomSortersPerDatabase) - { - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - throw new LicenseLimitException(LimitType.CustomSorters, $"The maximum number of custom sorters per database cannot exceed the limit of: {maxCustomSortersPerDatabase}"); - } - - var maxCustomSortersPerCluster = serverStore.LicenseManager.LicenseStatus.MaxNumberOfCustomSortersPerCluster; - if (maxCustomSortersPerCluster != null && maxCustomSortersPerCluster >= 0) - { - var totalSortersCount = GetTotal(DatabaseRecordElementType.CustomSorters, databaseRecord.DatabaseName) + databaseRecord.Sorters.Count; - if (totalSortersCount <= maxCustomSortersPerCluster) - return; - - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - throw new LicenseLimitException(LimitType.CustomSorters, $"The maximum number of custom sorters per cluster cannot exceed the limit of: {maxCustomSortersPerCluster}"); - } + AssertSorters(databaseRecord, serverStore.LicenseManager.LicenseStatus, context, items, type); break; case nameof(PutAnalyzersCommand): - var maxAnalyzersPerDatabase = serverStore.LicenseManager.LicenseStatus.MaxNumberOfCustomAnalyzersPerDatabase; - if (maxAnalyzersPerDatabase != null && maxAnalyzersPerDatabase >= 0 && databaseRecord.Analyzers.Count > maxAnalyzersPerDatabase) - { - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + AssertAnalyzers(databaseRecord, serverStore.LicenseManager.LicenseStatus, context, items, type); + break; - throw new LicenseLimitException(LimitType.CustomAnalyzers, $"The maximum number of analyzers per database cannot exceed the limit of: {maxAnalyzersPerDatabase}"); - } + case nameof(UpdatePeriodicBackupCommand): + if (AssertPeriodicBackup(serverStore.LicenseManager.LicenseStatus, context) == false) + throw new LicenseLimitException(LimitType.PeriodicBackup, "Your license doesn't support adding periodic backups."); + break; - var maxAnalyzersPerCluster = serverStore.LicenseManager.LicenseStatus.MaxNumberOfCustomAnalyzersPerCluster; - if (maxAnalyzersPerCluster != null && maxAnalyzersPerCluster >= 0) - { - var totalAnalyzersCount = GetTotal(DatabaseRecordElementType.Analyzers, databaseRecord.DatabaseName) + databaseRecord.Analyzers.Count; - if (totalAnalyzersCount <= maxAnalyzersPerCluster) - return; + case nameof(PutDatabaseClientConfigurationCommand): + case nameof(EditDatabaseClientConfigurationCommand): + AssertDatabaseClientConfiguration(databaseRecord, serverStore.LicenseManager.LicenseStatus, context); + break; - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + case nameof(PutClientConfigurationCommand): + if (AssertClientConfiguration(serverStore.LicenseManager.LicenseStatus, context) == false) + throw new LicenseLimitException(LimitType.ClientConfiguration, "Your license doesn't support adding the client configuration."); + break; - throw new LicenseLimitException(LimitType.CustomAnalyzers, $"The maximum number of analyzers per cluster cannot exceed the limit of: {maxAnalyzersPerCluster}"); - } + case nameof(PutDatabaseStudioConfigurationCommand): + AssertDatabaseStudioConfiguration(databaseRecord, serverStore.LicenseManager.LicenseStatus, context); break; - case nameof(UpdatePeriodicBackupCommand): - if (serverStore.LicenseManager.LicenseStatus.HasPeriodicBackup) - return; + case nameof(PutServerWideStudioConfigurationCommand): + if (AssertServerWideStudioConfiguration(serverStore.LicenseManager.LicenseStatus, context) == false) + throw new LicenseLimitException(LimitType.StudioConfiguration, "Your license doesn't support adding the studio configuration."); + break; - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + case nameof(AddQueueSinkCommand): + case nameof(UpdateQueueSinkCommand): + if (AssertQueueSink(serverStore.LicenseManager.LicenseStatus, context) == false) + throw new LicenseLimitException(LimitType.QueueSink, "Your license doesn't support using the queue sink feature."); + break; - throw new LicenseLimitException(LimitType.PeriodicBackup, "Your license doesn't support adding periodic backups."); + case nameof(EditDataArchivalCommand): + if (AssertDataArchival(serverStore.LicenseManager.LicenseStatus, context) == false ) + throw new LicenseLimitException(LimitType.DataArchival, "Your license doesn't support using the data archival feature."); + break; + } + } - case nameof(PutDatabaseClientConfigurationCommand): - case nameof(EditDatabaseClientConfigurationCommand): - if (serverStore.LicenseManager.LicenseStatus.HasClientConfiguration) - return; + private void AssertLicense(ClusterOperationContext context, string type, BlittableJsonReaderObject bjro, ServerStore serverStore) + { + LicenseStatus newLicenseLimits; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60102) == false) + return; - if (databaseRecord.Client == null || databaseRecord.Client.Disabled) - return; + var command = (PutLicenseCommand)CommandBase.CreateFrom(bjro); + if (command.SkipLicenseAssertion) + return; + try + { + newLicenseLimits = LicenseManager.GetLicenseStatus(command.Value); + } + catch (Exception e) + { + throw new LicenseLimitException(LimitType.InvalidLicense, e.Message); + } - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + var items = context.Transaction.InnerTransaction.OpenTable(ItemsSchema, Items); + foreach (var database in serverStore.DatabasesLandlord.DatabasesCache.Values.GetEnumerator()) + { + DatabaseRecord databaseRecord = serverStore.Cluster.ReadDatabase(context, ShardHelper.ToDatabaseName(database.Result.Name)); + + AssertMultiNodeSharding(databaseRecord, newLicenseLimits, context); + AssertStaticIndexesCount(databaseRecord, newLicenseLimits, context, items, type); + AssertAutoIndexesCount(databaseRecord, newLicenseLimits, context, items, type); + AssertRevisionConfiguration(databaseRecord, newLicenseLimits, context); + AssertExpirationConfiguration(databaseRecord, newLicenseLimits, context); + AssertRefreshFrequency(databaseRecord, newLicenseLimits, context); + AssertSorters(databaseRecord, newLicenseLimits, context, items, type); + AssertAnalyzers(databaseRecord, newLicenseLimits, context, items, type); + if (AssertPeriodicBackup(newLicenseLimits, context) == false && databaseRecord.PeriodicBackups.Count > 0) + throw new LicenseLimitException(LimitType.PeriodicBackup, $"Your license doesn't support periodic backup."); + AssertDatabaseClientConfiguration(databaseRecord, newLicenseLimits, context); + if (AssertClientConfiguration(newLicenseLimits, context) == false && databaseRecord.Client is { Disabled: false }) throw new LicenseLimitException(LimitType.ClientConfiguration, "Your license doesn't support adding the client configuration."); + AssertDatabaseStudioConfiguration(databaseRecord, newLicenseLimits, context); + if (AssertServerWideStudioConfiguration(newLicenseLimits, context) == false && databaseRecord.Studio is { Disabled: false }) + throw new LicenseLimitException(LimitType.StudioConfiguration, "Your license doesn't support adding the studio configuration."); + if (AssertQueueSink(newLicenseLimits, context) == false && databaseRecord.QueueSinks.Count > 0) + throw new LicenseLimitException(LimitType.QueueSink, "Your license doesn't support using the queue sink feature."); + if (AssertDataArchival(newLicenseLimits, context) == false && databaseRecord.DataArchival is { Disabled: false}) + throw new LicenseLimitException(LimitType.DataArchival, "Your license doesn't support using the data archival feature."); + } + } - case nameof(PutClientConfigurationCommand): - if (serverStore.LicenseManager.LicenseStatus.HasClientConfiguration) - return; + private void AssertMultiNodeSharding(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (databaseRecord.IsSharded == false) + return; - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + if (licenseStatus.MaxReplicationFactorForSharding == null && licenseStatus.HasMultiNodeSharding) + return; - throw new LicenseLimitException(LimitType.ClientConfiguration, "Your license doesn't support adding the client configuration."); + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; - case nameof(PutDatabaseStudioConfigurationCommand): - if (serverStore.LicenseManager.LicenseStatus.HasStudioConfiguration) - return; + var nodes = new HashSet(); + foreach (var shard in databaseRecord.Sharding.Shards) + { + DatabaseTopology topology = shard.Value; + if (licenseStatus.MaxReplicationFactorForSharding != null && topology.ReplicationFactor > licenseStatus.MaxReplicationFactorForSharding) + { + throw new LicenseLimitException(LimitType.Sharding, + $"Your license doesn't allow to use a replication factor of more than {topology.ReplicationFactor} for sharding"); + } - if (databaseRecord.Studio == null || databaseRecord.Studio.Disabled) - return; + foreach (var nodeTag in topology.AllNodes) + { + nodes.Add(nodeTag); + } + } - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + if (licenseStatus.HasMultiNodeSharding == false && nodes.Count > 1) + { + throw new LicenseLimitException(LimitType.Sharding, + $"Your license allows to create a sharded database only on a single node while you tried to create it on nodes {string.Join(", ", nodes)}"); + } + } - throw new LicenseLimitException(LimitType.StudioConfiguration, "Your license doesn't support adding the studio configuration."); + private void AssertStaticIndexesCount(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context, Table items, string type) + { + var maxStaticIndexesPerDatabase = licenseStatus.MaxNumberOfStaticIndexesPerDatabase; + if (maxStaticIndexesPerDatabase is >= 0 && databaseRecord.Indexes.Count > maxStaticIndexesPerDatabase) + { + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; - case nameof(PutServerWideStudioConfigurationCommand): - if (serverStore.LicenseManager.LicenseStatus.HasStudioConfiguration) - return; + throw new LicenseLimitException(LimitType.Indexes, + $"The maximum number of static indexes per database cannot exceed the limit of: {maxStaticIndexesPerDatabase}"); + } - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + var maxStaticIndexesPerCluster = licenseStatus.MaxNumberOfStaticIndexesPerCluster; + if (maxStaticIndexesPerCluster is null or < 0) + return; - throw new LicenseLimitException(LimitType.StudioConfiguration, "Your license doesn't support adding the studio configuration."); + var totalStaticIndexesCount = GetTotal(DatabaseRecordElementType.StaticIndex, databaseRecord.DatabaseName, context, items, type) + databaseRecord.Indexes.Count; + if (totalStaticIndexesCount <= maxStaticIndexesPerCluster) + return; - case nameof(AddQueueSinkCommand): - case nameof(UpdateQueueSinkCommand): - if (serverStore.LicenseManager.LicenseStatus.HasQueueSink) - return; + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + throw new LicenseLimitException(LimitType.Indexes, $"The maximum number of static indexes per cluster cannot exceed the limit of: {maxStaticIndexesPerCluster}"); + } - throw new LicenseLimitException(LimitType.QueueSink, "Your license doesn't support using the queue sink feature."); + private void AssertAutoIndexesCount(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context, Table items, string type) + { + var maxAutoIndexesPerDatabase = licenseStatus.MaxNumberOfAutoIndexesPerDatabase; + if (maxAutoIndexesPerDatabase is >= 0 && databaseRecord.AutoIndexes.Count > maxAutoIndexesPerDatabase) + { + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; - case nameof(EditDataArchivalCommand): - if (serverStore.LicenseManager.LicenseStatus.HasDataArchival) - return; + throw new LicenseLimitException(LimitType.Indexes, $"The maximum number of auto indexes per database cannot exceed the limit of: {maxAutoIndexesPerDatabase}"); + } + + var maxAutoIndexesPerCluster = licenseStatus.MaxNumberOfAutoIndexesPerCluster; + if (maxAutoIndexesPerCluster is >= 0) + { + var totalAutoIndexesCount = GetTotal(DatabaseRecordElementType.AutoIndex, databaseRecord.DatabaseName, context, items, type) + databaseRecord.AutoIndexes.Count; + if (totalAutoIndexesCount <= maxAutoIndexesPerCluster) + return; - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; - throw new LicenseLimitException(LimitType.DataArchival, "Your license doesn't support using the data archival feature."); + throw new LicenseLimitException(LimitType.Indexes, $"The maximum number of auto indexes per cluster cannot exceed the limit of: {maxAutoIndexesPerDatabase}"); } + } - return; + private void AssertRevisionConfiguration(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (databaseRecord.Revisions == null) + return; - long GetTotal(DatabaseRecordElementType resultType, string exceptDb) - { - long total = 0; + if (databaseRecord.Revisions.Default == null && + (databaseRecord.Revisions.Collections == null || databaseRecord.Revisions.Collections.Count == 0)) + return; - using (Slice.From(context.Allocator, "db/", out var loweredPrefix)) - { - foreach (var result in items.SeekByPrimaryKeyPrefix(loweredPrefix, Slices.Empty, 0)) - { - var (_, _, record) = GetCurrentItem(context, result.Value); - var rawRecord = new RawDatabaseRecord(context, record); - if (rawRecord.DatabaseName.Equals(exceptDb, StringComparison.OrdinalIgnoreCase)) - continue; - - switch (resultType) - { - case DatabaseRecordElementType.StaticIndex: - total += rawRecord.CountOfStaticIndexes; - break; - case DatabaseRecordElementType.AutoIndex: - total += rawRecord.CountOfAutoIndexes; - break; - case DatabaseRecordElementType.CustomSorters: - total += rawRecord.CountOfSorters; - break; - case DatabaseRecordElementType.Analyzers: - total += rawRecord.CountOfAnalyzers; - break; - default: - throw new ArgumentOutOfRangeException(nameof(type), type, null); - } - } + var maxRevisionsToKeep = licenseStatus.MaxNumberOfRevisionsToKeep; + var maxRevisionAgeToKeepInDays = licenseStatus.MaxNumberOfRevisionAgeToKeepInDays; + if (licenseStatus.CanSetupDefaultRevisionsConfiguration && + maxRevisionsToKeep == null && maxRevisionAgeToKeepInDays == null) + return; - return total; - } + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + if (licenseStatus.CanSetupDefaultRevisionsConfiguration == false && + databaseRecord.Revisions.Default != null) + { + throw new LicenseLimitException(LimitType.RevisionsConfiguration, "Your license doesn't allow the creation of a default configuration for revisions."); } - void AssertStaticIndexesCount() + if (databaseRecord.Revisions.Collections == null) + return; + + foreach (KeyValuePair revisionPerCollectionConfiguration in databaseRecord.Revisions.Collections) { - var maxStaticIndexesPerDatabase = serverStore.LicenseManager.LicenseStatus.MaxNumberOfStaticIndexesPerDatabase; - if (maxStaticIndexesPerDatabase != null && maxStaticIndexesPerDatabase >= 0 && databaseRecord.Indexes.Count > maxStaticIndexesPerDatabase) + if (revisionPerCollectionConfiguration.Value.MinimumRevisionsToKeep != null && + maxRevisionsToKeep != null && + revisionPerCollectionConfiguration.Value.MinimumRevisionsToKeep > maxRevisionsToKeep) { - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; - - throw new LicenseLimitException(LimitType.Indexes, - $"The maximum number of static indexes per database cannot exceed the limit of: {maxStaticIndexesPerDatabase}"); + throw new LicenseLimitException(LimitType.RevisionsConfiguration, + $"The defined minimum revisions to keep '{revisionPerCollectionConfiguration.Value.MinimumRevisionsToKeep}' " + + $" exceeds the licensed one '{maxRevisionsToKeep}'"); } - var maxStaticIndexesPerCluster = serverStore.LicenseManager.LicenseStatus.MaxNumberOfStaticIndexesPerCluster; - if (maxStaticIndexesPerCluster != null && maxStaticIndexesPerCluster >= 0) + if (revisionPerCollectionConfiguration.Value.MinimumRevisionAgeToKeep != null && + maxRevisionAgeToKeepInDays != null && + revisionPerCollectionConfiguration.Value.MinimumRevisionAgeToKeep.Value.TotalDays > maxRevisionAgeToKeepInDays) { - var totalStaticIndexesCount = GetTotal(DatabaseRecordElementType.StaticIndex, databaseRecord.DatabaseName) + databaseRecord.Indexes.Count; - if (totalStaticIndexesCount <= maxStaticIndexesPerCluster) - return; + throw new LicenseLimitException(LimitType.RevisionsConfiguration, + $"The defined minimum revisions age to keep '{revisionPerCollectionConfiguration.Value.MinimumRevisionAgeToKeep}' " + + $" exceeds the licensed one '{maxRevisionAgeToKeepInDays}'"); + } + } + } - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + private void AssertExpirationConfiguration(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context) + { + var minPeriodForExpirationInHours = licenseStatus.MinPeriodForExpirationInHours; - throw new LicenseLimitException(LimitType.Indexes, $"The maximum number of static indexes per cluster cannot exceed the limit of: {maxStaticIndexesPerCluster}"); - } + if (minPeriodForExpirationInHours == null || databaseRecord.Expiration == null || databaseRecord.Expiration.Disabled) + return; + + var deleteFrequencyInSec = databaseRecord.Expiration?.DeleteFrequencyInSec ?? ExpiredDocumentsCleaner.DefaultDeleteFrequencyInSec; + var deleteFrequency = new TimeSetting(deleteFrequencyInSec, TimeUnit.Seconds); + var minPeriodForExpiration = new TimeSetting(minPeriodForExpirationInHours.Value, TimeUnit.Hours); + + if (deleteFrequency.AsTimeSpan >= minPeriodForExpiration.AsTimeSpan) + return; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.Expiration, $"Your license doesn't allow modifying the expiration frequency below {minPeriodForExpirationInHours} hours."); + } + + private void AssertRefreshFrequency(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context) + { + var minPeriodForRefreshInHours = licenseStatus.MinPeriodForRefreshInHours; + if (minPeriodForRefreshInHours == null || databaseRecord.Refresh is not { Disabled: false }) + return; + + var refreshFrequencyInSec = databaseRecord.Refresh.RefreshFrequencyInSec ?? ExpiredDocumentsCleaner.DefaultRefreshFrequencyInSec; + var refreshFrequency = new TimeSetting(refreshFrequencyInSec, TimeUnit.Seconds); + var minPeriodForRefresh = new TimeSetting(minPeriodForRefreshInHours.Value, TimeUnit.Hours); + if (refreshFrequency.AsTimeSpan >= minPeriodForRefresh.AsTimeSpan) + return; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.Refresh, $"Your license doesn't allow modifying the refresh frequency below {minPeriodForRefreshInHours} hours."); + } + + private void AssertSorters(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context, Table items, string type) + { + var maxCustomSortersPerDatabase = licenseStatus.MaxNumberOfCustomSortersPerDatabase; + if (maxCustomSortersPerDatabase is >= 0 && databaseRecord.Sorters.Count > maxCustomSortersPerDatabase) + { + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.CustomSorters, $"The maximum number of custom sorters per database cannot exceed the limit of: {maxCustomSortersPerDatabase}"); } - void AssertAutoIndexesCount() + var maxCustomSortersPerCluster = licenseStatus.MaxNumberOfCustomSortersPerCluster; + if (maxCustomSortersPerCluster is not >= 0) + return; + + var totalSortersCount = GetTotal(DatabaseRecordElementType.CustomSorters, databaseRecord.DatabaseName, context, items, type) + databaseRecord.Sorters.Count; + if (totalSortersCount <= maxCustomSortersPerCluster) + return; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.CustomSorters, $"The maximum number of custom sorters per cluster cannot exceed the limit of: {maxCustomSortersPerCluster}"); + } + + private void AssertAnalyzers(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context, Table items, string type) + { + var maxAnalyzersPerDatabase = licenseStatus.MaxNumberOfCustomAnalyzersPerDatabase; + if (maxAnalyzersPerDatabase is >= 0 && databaseRecord.Analyzers.Count > maxAnalyzersPerDatabase) { - var maxAutoIndexesPerDatabase = serverStore.LicenseManager.LicenseStatus.MaxNumberOfAutoIndexesPerDatabase; - if (maxAutoIndexesPerDatabase != null && maxAutoIndexesPerDatabase >= 0 && databaseRecord.AutoIndexes.Count > maxAutoIndexesPerDatabase) - { - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; - throw new LicenseLimitException(LimitType.Indexes, $"The maximum number of auto indexes per database cannot exceed the limit of: {maxAutoIndexesPerDatabase}"); - } + throw new LicenseLimitException(LimitType.CustomAnalyzers, $"The maximum number of analyzers per database cannot exceed the limit of: {maxAnalyzersPerDatabase}"); + } - var maxAutoIndexesPerCluster = serverStore.LicenseManager.LicenseStatus.MaxNumberOfAutoIndexesPerCluster; - if (maxAutoIndexesPerCluster != null && maxAutoIndexesPerCluster >= 0) - { - var totalAutoIndexesCount = GetTotal(DatabaseRecordElementType.AutoIndex, databaseRecord.DatabaseName) + databaseRecord.AutoIndexes.Count; - if (totalAutoIndexesCount <= maxAutoIndexesPerCluster) - return; + var maxAnalyzersPerCluster = licenseStatus.MaxNumberOfCustomAnalyzersPerCluster; - if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) - return; + if (maxAnalyzersPerCluster is not >= 0) + return; - throw new LicenseLimitException(LimitType.Indexes, $"The maximum number of auto indexes per cluster cannot exceed the limit of: {maxAutoIndexesPerDatabase}"); + var totalAnalyzersCount = GetTotal(DatabaseRecordElementType.Analyzers, databaseRecord.DatabaseName, context, items, type) + databaseRecord.Analyzers.Count; + if (totalAnalyzersCount <= maxAnalyzersPerCluster) + return; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.CustomAnalyzers, $"The maximum number of analyzers per cluster cannot exceed the limit of: {maxAnalyzersPerCluster}"); + } + + private bool AssertPeriodicBackup(LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasPeriodicBackup) + return true; + + return CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false; + } + + private void AssertDatabaseClientConfiguration(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasClientConfiguration) + return; + + if (databaseRecord.Client == null || databaseRecord.Client.Disabled) + return; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.ClientConfiguration, "Your license doesn't support adding the client configuration."); + } + + private bool AssertClientConfiguration(LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasClientConfiguration) + return true; + + return CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false; + } + + private void AssertDatabaseStudioConfiguration(DatabaseRecord databaseRecord, LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasStudioConfiguration) + return; + + if (databaseRecord.Studio == null || databaseRecord.Studio.Disabled) + return; + + if (CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false) + return; + + throw new LicenseLimitException(LimitType.StudioConfiguration, "Your license doesn't support adding the studio configuration."); + } + + private bool AssertServerWideStudioConfiguration(LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasStudioConfiguration) + return true; + + return CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false; + } + + private bool AssertQueueSink(LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasQueueSink) + return true; + + return CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false; + } + + private bool AssertDataArchival(LicenseStatus licenseStatus, ClusterOperationContext context) + { + if (licenseStatus.HasDataArchival) + return true; + + return CanAssertLicenseLimits(context, minBuildVersion: MinBuildVersion60000) == false; + } + + private static long GetTotal(DatabaseRecordElementType resultType, string exceptDb, ClusterOperationContext context, Table items, string type) + { + long total = 0; + + using (Slice.From(context.Allocator, "db/", out var loweredPrefix)) + { + foreach (var result in items.SeekByPrimaryKeyPrefix(loweredPrefix, Slices.Empty, 0)) + { + var (_, _, record) = GetCurrentItem(context, result.Value); + var rawRecord = new RawDatabaseRecord(context, record); + if (rawRecord.DatabaseName.Equals(exceptDb, StringComparison.OrdinalIgnoreCase)) + continue; + + switch (resultType) + { + case DatabaseRecordElementType.StaticIndex: + total += rawRecord.CountOfStaticIndexes; + break; + case DatabaseRecordElementType.AutoIndex: + total += rawRecord.CountOfAutoIndexes; + break; + case DatabaseRecordElementType.CustomSorters: + total += rawRecord.CountOfSorters; + break; + case DatabaseRecordElementType.Analyzers: + total += rawRecord.CountOfAnalyzers; + break; + default: + throw new ArgumentOutOfRangeException(nameof(type), type, null); + } } + + return total; } } @@ -403,7 +512,7 @@ private void AssertSubscriptionsLicenseLimits(ServerStore serverStore, Table ite }; var includeRevisions = putSubscriptionCommand.IncludesRevisions(); - if(AssertSubscriptionRevisionFeatureLimits(serverStore, includeRevisions, context)) + if (AssertSubscriptionRevisionFeatureLimits(serverStore, includeRevisions, context)) return; if (AssertNumberOfSubscriptionsPerDatabaseLimits(serverStore, items, context, subscriptionsNamesPerDatabase)) diff --git a/src/Raven.Server/ServerWide/ClusterStateMachine.cs b/src/Raven.Server/ServerWide/ClusterStateMachine.cs index 4a5b250a333a..a47357e138a5 100644 --- a/src/Raven.Server/ServerWide/ClusterStateMachine.cs +++ b/src/Raven.Server/ServerWide/ClusterStateMachine.cs @@ -570,6 +570,7 @@ protected override void Apply(ClusterOperationContext context, BlittableJsonRead break; case nameof(PutLicenseCommand): + AssertLicense(context, type, cmd, serverStore); PutValue(context, type, cmd, index); break; diff --git a/src/Raven.Server/ServerWide/Commands/PutLicenseCommand.cs b/src/Raven.Server/ServerWide/Commands/PutLicenseCommand.cs index 2da475edd5f2..ac81f62e6a44 100644 --- a/src/Raven.Server/ServerWide/Commands/PutLicenseCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/PutLicenseCommand.cs @@ -1,20 +1,32 @@ using Raven.Server.Commercial; using Raven.Server.ServerWide.Context; +using Sparrow.Json; using Sparrow.Json.Parsing; namespace Raven.Server.ServerWide.Commands { public sealed class PutLicenseCommand : PutValueCommand { + public bool SkipLicenseAssertion; + public PutLicenseCommand() { // for deserialization } - public PutLicenseCommand(string name, License license, string uniqueRequestId) : base(uniqueRequestId) + public PutLicenseCommand(string name, License license, string uniqueRequestId, bool fromApi = false) : base(uniqueRequestId) { Name = name; Value = license; + SkipLicenseAssertion = fromApi; + } + + public override DynamicJsonValue ToJson(JsonOperationContext context) + { + var djv = base.ToJson(context); + djv[nameof(SkipLicenseAssertion)] = SkipLicenseAssertion; + + return djv; } public override DynamicJsonValue ValueToJson() diff --git a/src/Raven.Server/ServerWide/ServerStore.cs b/src/Raven.Server/ServerWide/ServerStore.cs index c8bfc47c205a..cdb29566127a 100644 --- a/src/Raven.Server/ServerWide/ServerStore.cs +++ b/src/Raven.Server/ServerWide/ServerStore.cs @@ -3166,9 +3166,9 @@ public LicenseLimits LoadLicenseLimits() } } - public async Task PutLicenseAsync(License license, string raftRequestId) + public async Task PutLicenseAsync(License license, string raftRequestId, bool fromApi = false) { - var command = new PutLicenseCommand(LicenseStorageKey, license, raftRequestId); + var command = new PutLicenseCommand(LicenseStorageKey, license, raftRequestId, fromApi); var result = await SendToLeaderAsync(command); diff --git a/test/SlowTests/Issues/RavenDB-21427.cs b/test/SlowTests/Issues/RavenDB-21427.cs new file mode 100644 index 000000000000..a4c9ec0eec5b --- /dev/null +++ b/test/SlowTests/Issues/RavenDB-21427.cs @@ -0,0 +1,476 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using FastTests.Utils; +using Raven.Client.Documents; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations.Backups; +using Raven.Client.Documents.Operations.Configuration; +using Raven.Client.Documents.Operations.ConnectionStrings; +using Raven.Client.Documents.Operations.DataArchival; +using Raven.Client.Documents.Operations.ETL.Queue; +using Raven.Client.Documents.Operations.Expiration; +using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Documents.Operations.QueueSink; +using Raven.Client.Documents.Operations.Refresh; +using Raven.Client.Documents.Operations.Revisions; +using Raven.Client.Documents.Queries.Sorting; +using Raven.Client.Exceptions.Commercial; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations.Configuration; +using Raven.Client.ServerWide.Sharding; +using Raven.Client.Util; +using Raven.Server; +using Raven.Server.Commercial; +using Raven.Server.Documents.Indexes; +using Raven.Server.Documents.Indexes.Auto; +using Raven.Server.ServerWide.Commands; +using Tests.Infrastructure; +using Xunit; +using Xunit.Abstractions; + +namespace SlowTests.Issues +{ + public class RavenDB_21427 : ReplicationTestBase + { + public RavenDB_21427(ITestOutputHelper output) : base(output) + { + } + + private const string RL_COMM = "RAVEN_LICENSE_COMMUNITY"; + private const string RL_DEV = "RAVEN_LICENSE_DEVELOPER"; + private const string RL_PRO = "RAVEN_LICENSE_PROFESSIONAL"; + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Multi_Node_Sharding() + { + DoNotReuseServer(); + + var (_, leader) = await CreateRaftCluster(3, false, watcherCluster: true); + var options = Options.ForMode(RavenDatabaseMode.Sharded); + options.Server = leader; + options.ReplicationFactor = 3; + options.ModifyDatabaseRecord = r => + { + r.Sharding ??= new ShardingConfiguration(); + r.Sharding.Shards = Enumerable.Range(0, 5) + .Select((shardNumber) => new KeyValuePair(shardNumber, new DatabaseTopology())).ToDictionary(x => x.Key, x => x.Value); + }; + using (var store = GetDocumentStore(options)) + { + await TryToChangeLicense(leader, RL_COMM, LimitType.Sharding); + await TryToChangeLicense(leader, RL_DEV, LimitType.Sharding); + await TryToChangeLicense(leader, RL_PRO, LimitType.Sharding); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_QueueSink() + { + DoNotReuseServer(); + using var store = GetDocumentStore(); + var res = store.Maintenance.Send( + new PutConnectionStringOperation( + new QueueConnectionString + { + Name = "KafkaConStr", + BrokerType = QueueBrokerType.Kafka, + KafkaConnectionSettings = new KafkaConnectionSettings() + { BootstrapServers = "localhost:9092" } + })); + + // Define a Sink script + QueueSinkScript queueSinkScript = new QueueSinkScript + { + // Script name + Name = "orders", + // A list of Kafka topics to connect + Queues = new List() { "orders" }, + // Apply this script + Script = @"this['@metadata']['@collection'] = 'Orders'; + put(this.Id.toString(), this)" + }; + + // Define a Kafka configuration + var config = new QueueSinkConfiguration() + { + // Sink name + Name = "KafkaSinkTaskName", + // The connection string to connect the broker with + ConnectionStringName = "KafkaConStr", + // What queue broker is this task using + BrokerType = QueueBrokerType.Kafka, + // The list of scripts to run + Scripts = { queueSinkScript } + }; + + AddQueueSinkOperationResult addQueueSinkOperationResult = + store.Maintenance.Send(new AddQueueSinkOperation(config)); + + await TryToChangeLicense(Server, RL_COMM, LimitType.QueueSink); + await TryToChangeLicense(Server, RL_PRO, LimitType.QueueSink); + + config = new QueueSinkConfiguration() + { + // Sink name + Name = "KafkaSinkTaskName2", + // The connection string to connect the broker with + ConnectionStringName = "KafkaConStr", + // What queue broker is this task using + BrokerType = QueueBrokerType.RabbitMq, + // The list of scripts to run + Scripts = { queueSinkScript } + }; + + addQueueSinkOperationResult = + store.Maintenance.Send(new AddQueueSinkOperation(config)); + + await TryToChangeLicense(Server, RL_COMM, LimitType.QueueSink); + await TryToChangeLicense(Server, RL_PRO, LimitType.QueueSink); + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_DataArchival() + { + DoNotReuseServer(); + + using var store = GetDocumentStore(); + var config = new DataArchivalConfiguration { Disabled = false, ArchiveFrequencyInSec = 100 }; + + await DataArchivalHelper.SetupDataArchival(store, Server.ServerStore, config); + await TryToChangeLicense(Server, RL_COMM, LimitType.DataArchival); + await TryToChangeLicense(Server, RL_PRO, LimitType.DataArchival); + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Static_Index_Count_Per_Database() + { + DoNotReuseServer(); + + using (var store = GetDocumentStore()) + { + for (int i = 0; i < 15; i++) + { + store.Maintenance.Send(new PutIndexesOperation(new[] + { + new IndexDefinition { Maps = { "from doc in docs.Images select new { doc.Tags }" }, Name = "test" + i } + })); + } + + await TryToChangeLicense(Server, RL_COMM, LimitType.Indexes); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Static_Index_Count_Per_Cluster() + { + DoNotReuseServer(); + var (_, leader) = await CreateRaftCluster(3, false, watcherCluster: true); + var storeList = new List(); + for (int i = 0; i < 7; i++) + { + var store = GetDocumentStore(); + storeList.Add(store); + for (int j = 0; j < 10; j++) + { + store.Maintenance.Send(new PutIndexesOperation(new[] + { + new IndexDefinition { Maps = { "from doc in docs.Images select new { doc.Tags }" }, Name = "test" + j } + })); + } + } + + await TryToChangeLicense(Server, RL_COMM, LimitType.Indexes); + foreach (var store in storeList) + { + store.Dispose(); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Auto_Index_Count_Per_Database() + { + DoNotReuseServer(); + + using (var store = GetDocumentStore()) + { + for (int j = 0; j < 28; j++) + { + var database = await Databases.GetDocumentDatabaseInstanceFor(store); + var index = await database.IndexStore.CreateIndex(new AutoMapIndexDefinition("Users", new[] { new AutoIndexField { Name = "Name" + j } }), + Guid.NewGuid().ToString()); + + } + + await TryToChangeLicense(Server, RL_COMM, LimitType.Indexes); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Auto_Index_Count_Per_Cluster() + { + DoNotReuseServer(); + var (_, leader) = await CreateRaftCluster(3, false, watcherCluster: true); + var storeList = new List(); + for (int i = 0; i < 7; i++) + { + var store = GetDocumentStore(); + storeList.Add(store); + for (int j = 0; j < 20; j++) + { + var database = await Databases.GetDocumentDatabaseInstanceFor(store); + var index = await database.IndexStore.CreateIndex(new AutoMapIndexDefinition("Users", new[] { new AutoIndexField { Name = "Name" + j } }), + Guid.NewGuid().ToString()); + + } + } + + await TryToChangeLicense(Server, RL_COMM, LimitType.Indexes); + foreach (var store in storeList) + { + store.Dispose(); + } + + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Revision_Default_Configuration() + { + DoNotReuseServer(); + + using (var store = GetDocumentStore()) + { + var configuration = new RevisionsConfiguration { Default = new RevisionsCollectionConfiguration { Disabled = false, MinimumRevisionsToKeep = 0 } }; + var result = await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration)); + + await TryToChangeLicense(Server, RL_COMM, LimitType.RevisionsConfiguration); + + configuration = new RevisionsConfiguration + { + Collections = new Dictionary + { + { + "Companies", new RevisionsCollectionConfiguration() + { + Disabled = false, + MinimumRevisionsToKeep = 5 + } + } + } + }; + + result = await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration)); + + await TryToChangeLicense(Server, RL_COMM, LimitType.RevisionsConfiguration); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_PeriodicBackup() + { + DoNotReuseServer(); + var backupPath = NewDataPath(suffix: "BackupFolder"); + using (var store = GetDocumentStore()) + { + var config = Backup.CreateBackupConfiguration(backupPath, fullBackupFrequency: "* */1 * * *", incrementalBackupFrequency: "* */2 * * *"); + await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config)); + + await TryToChangeLicense(Server, RL_COMM, LimitType.PeriodicBackup); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Sorters_Per_Database() + { + DoNotReuseServer(); + var sorterName = GetDatabaseName(); + + using (var store = GetDocumentStore(new Options + { + ModifyDatabaseName = _ => sorterName, + ModifyDatabaseRecord = record => record.Sorters = new Dictionary + { + { + "MySorter", + new SorterDefinition { Name = sorterName + "1", Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName + "1") } + }, + { + "MySorter2", + new SorterDefinition { Name = sorterName + "2", Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter2", sorterName + "2") } + } + } + })) + { + await TryToChangeLicense(Server, RL_COMM, LimitType.CustomSorters); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Sorters_Per_Cluster() + { + DoNotReuseServer(); + var (_, leader) = await CreateRaftCluster(3, false, watcherCluster: true); + var sorterName = GetDatabaseName(); + + var storeList = new List(); + for (int i = 0; i < 6; i++) + { + var store = GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => record.Sorters = new Dictionary + { + { + "MySorter", new SorterDefinition { Name = sorterName + "1", Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName + "1") } + } + } + }); + + storeList.Add(store); + } + + await TryToChangeLicense(Server, RL_COMM, LimitType.CustomSorters); + foreach (var store in storeList) + { + store.Dispose(); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Analyzer_Per_Database() + { + DoNotReuseServer(); + var sorterName = GetDatabaseName(); + + using (var store = GetDocumentStore(new Options + { + ModifyDatabaseName = _ => sorterName, + ModifyDatabaseRecord = record => record.Sorters = new Dictionary + { + { + "MySorter", + new SorterDefinition { Name = sorterName + "1", Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName + "1") } + }, + { + "MySorter2", + new SorterDefinition { Name = sorterName + "2", Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter2", sorterName + "2") } + } + } + })) + { + await TryToChangeLicense(Server, RL_COMM, LimitType.CustomSorters); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Analyzer_Per_Cluster() + { + DoNotReuseServer(); + var (_, leader) = await CreateRaftCluster(3, false, watcherCluster: true); + var sorterName = GetDatabaseName(); + + var storeList = new List(); + for (int i = 0; i < 6; i++) + { + var store = GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => record.Sorters = new Dictionary + { + { + "MySorter", new SorterDefinition { Name = sorterName + "1", Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName + "1") } + } + } + }); + + storeList.Add(store); + } + + await TryToChangeLicense(Server, RL_COMM, LimitType.CustomSorters); + foreach (var store in storeList) + { + store.Dispose(); + } + + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_ClientConfiguration() + { + DoNotReuseServer(); + using (var store = GetDocumentStore(new Options { ModifyDatabaseRecord = r => r.Client = new ClientConfiguration { MaxNumberOfRequestsPerSession = 50 } })) + { + await TryToChangeLicense(Server, RL_COMM, LimitType.ClientConfiguration); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_StudioConfiguration() + { + DoNotReuseServer(); + + using var store = GetDocumentStore(); + var command = new PutDatabaseStudioConfigurationCommand(new ServerWideStudioConfiguration() { DisableAutoIndexCreation = true, }, store.Database, + RaftIdGenerator.NewId()); + await Server.ServerStore.SendToLeaderAsync(command); + + await TryToChangeLicense(Server, RL_COMM, LimitType.StudioConfiguration); + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Expiration_Configuration() + { + DoNotReuseServer(); + + using (var store = GetDocumentStore()) + { + var config = new ExpirationConfiguration { Disabled = false, DeleteFrequencyInSec = 100, }; + + await ExpirationHelper.SetupExpiration(store, Server.ServerStore, config); + await TryToChangeLicense(Server, RL_COMM, LimitType.Expiration); + } + } + + [MultiLicenseRequiredFact] + public async Task Prevent_License_Downgrade_Refresh_Configuration() + { + DoNotReuseServer(); + + using (var store = GetDocumentStore()) + { + var refConfig = new RefreshConfiguration { RefreshFrequencyInSec = 33, Disabled = false }; + await store.Maintenance.SendAsync(new ConfigureRefreshOperation(refConfig)); + await TryToChangeLicense(Server, RL_COMM, LimitType.Refresh); + } + } + + private static async Task TryToChangeLicense(RavenServer leader, string licenseType, LimitType limitType) + { + var license = Environment.GetEnvironmentVariable(licenseType); + LicenseHelper.TryDeserializeLicense(license, out License li); + + var exception = await Assert.ThrowsAsync(async () => await leader.ServerStore.PutLicenseAsync(li, RaftIdGenerator.NewId())); + + Assert.Equal(limitType, exception.Type); + } + + + private static string GetSorter(string resourceName, string originalSorterName, string sorterName) + { + using (var stream = GetDump(resourceName)) + using (var reader = new StreamReader(stream)) + { + var analyzerCode = reader.ReadToEnd(); + analyzerCode = analyzerCode.Replace(originalSorterName, sorterName); + + return analyzerCode; + } + } + + private static Stream GetDump(string name) + { + var assembly = typeof(RavenDB_8355).Assembly; + return assembly.GetManifestResourceStream("SlowTests.Data." + name); + } + } +} diff --git a/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs b/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs new file mode 100644 index 000000000000..a24bf6ec963c --- /dev/null +++ b/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs @@ -0,0 +1,38 @@ +using System; +using Xunit; + +namespace Tests.Infrastructure +{ + public class MultiLicenseRequiredFactAttribute : FactAttribute + { + internal static readonly bool HasLicense; + + internal static string SkipMessage = "Requires License to be set via 'RAVEN_LICENSE' environment variable."; + + static MultiLicenseRequiredFactAttribute() + { + HasLicense = string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE")) == false || + string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_DEVELOPER")) == false || + string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_COMMUNITY")) == false; + } + + public override string Skip + { + get + { + if (ShouldSkip(licenseRequired: true)) + return SkipMessage; + + return null; + } + } + + internal static bool ShouldSkip(bool licenseRequired) + { + if (licenseRequired == false) + return false; + + return HasLicense == false; + } + } +} From d99628904b3bee9d035a60fed698e03b95aa9d76 Mon Sep 17 00:00:00 2001 From: "efrat@ravendb.net" Date: Sun, 31 Mar 2024 10:19:35 +0300 Subject: [PATCH 031/243] RavenDB-21427 fix MultiLicenseRequiredFactAttribute --- .../MultiLicenseRequiredFactAttribute.cs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs b/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs index a24bf6ec963c..de5349965374 100644 --- a/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs +++ b/test/Tests.Infrastructure/MultiLicenseRequiredFactAttribute.cs @@ -5,15 +5,31 @@ namespace Tests.Infrastructure { public class MultiLicenseRequiredFactAttribute : FactAttribute { + private static readonly bool RavenLicense; + private static readonly bool RavenLicenseDeveloper; + private static readonly bool RavenLicenseCommunity; + private static readonly bool RavenLicenseProfessional; + internal static readonly bool HasLicense; - internal static string SkipMessage = "Requires License to be set via 'RAVEN_LICENSE' environment variable."; + internal static string SkipMessage = $"Requires Licenses to be set via environment variable. : " + + $"'RAVEN_LICENSE' - {IsSet(RavenLicense)} . " + + $"'RAVEN_LICENSE_DEVELOPER' - {IsSet(RavenLicenseDeveloper)} . " + + $"'RAVEN_LICENSE_COMMUNITY' - {IsSet(RavenLicenseCommunity)} . " + + $"'RAVEN_LICENSE_PROFESSIONAL' - {IsSet(RavenLicenseProfessional)} . "; + internal static string IsSet(bool licenseSet) + { + return licenseSet ? "is set" : "is not set"; + } static MultiLicenseRequiredFactAttribute() { - HasLicense = string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE")) == false || - string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_DEVELOPER")) == false || - string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_COMMUNITY")) == false; + RavenLicense = string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE")) == false; + RavenLicenseDeveloper = string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_DEVELOPER")) == false; + RavenLicenseCommunity = string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_COMMUNITY")) == false; + RavenLicenseProfessional = string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("RAVEN_LICENSE_PROFESSIONAL")) == false; + + HasLicense = RavenLicense && RavenLicenseDeveloper && RavenLicenseCommunity && RavenLicenseProfessional; } public override string Skip From a90f554822197e149a2e70d5922a8237b7968659 Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Thu, 11 Apr 2024 13:16:05 +0200 Subject: [PATCH 032/243] RavenDB-22242 Support `@AllResults` aggregations in Corax. --- .../Corax/CoraxIndexFacetedReadOperation.cs | 9 ++++-- test/SlowTests/Issues/RavenDB_14952.cs | 29 +++++++++++++++---- test/SlowTests/Tests/TestsInheritanceTests.cs | 2 +- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/Raven.Server/Documents/Indexes/Persistence/Corax/CoraxIndexFacetedReadOperation.cs b/src/Raven.Server/Documents/Indexes/Persistence/Corax/CoraxIndexFacetedReadOperation.cs index 227f8a306332..cda84e9cd499 100644 --- a/src/Raven.Server/Documents/Indexes/Persistence/Corax/CoraxIndexFacetedReadOperation.cs +++ b/src/Raven.Server/Documents/Indexes/Persistence/Corax/CoraxIndexFacetedReadOperation.cs @@ -4,7 +4,6 @@ using System.Runtime.InteropServices; using System.Threading; using Corax; -using Corax.Querying; using Corax.Mappings; using Corax.Utils; using Raven.Client.Documents.Indexes; @@ -219,6 +218,12 @@ private void HandleFacetsPerDocument(ref EntryTermsReader reader, if (facetsByName.TryGetValue(result.Key, out var facetValues) == false) facetsByName[result.Key] = facetValues = new Dictionary(); + if (result.Key == Client.Constants.Documents.Querying.Facet.AllResults || result.Value.AggregateBy == Client.Constants.Documents.Querying.Facet.AllResults) + { + InsertTerm(Encodings.Utf8.GetBytes(result.Value.AggregateBy), ref reader); + return; + } + long fieldRootPage = GetFieldRootPage(result.Value.AggregateBy); var cloned = reader; @@ -230,7 +235,7 @@ private void HandleFacetsPerDocument(ref EntryTermsReader reader, : reader.Current.Decoded(); InsertTerm(key, ref cloned); - } + } void InsertTerm(ReadOnlySpan term, ref EntryTermsReader reader) { diff --git a/test/SlowTests/Issues/RavenDB_14952.cs b/test/SlowTests/Issues/RavenDB_14952.cs index ad508c8c5419..87a9b18b877f 100644 --- a/test/SlowTests/Issues/RavenDB_14952.cs +++ b/test/SlowTests/Issues/RavenDB_14952.cs @@ -7,6 +7,7 @@ using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Queries.Facets; using Sparrow.Json; +using Tests.Infrastructure; using Xunit; using Xunit.Abstractions; @@ -18,10 +19,19 @@ public RavenDB_14952(ITestOutputHelper output) : base(output) { } - [Fact] - public void CanDoFacetQueryWithAliasOnFullRange_WhenAliasIsTheSameAsOneOfTheIndexFields() + [RavenTheory(RavenTestCategory.Facets)] + [RavenData(SearchEngineMode = RavenSearchEngineMode.All, DatabaseMode = RavenDatabaseMode.Single)] + public void CanDoFacetQueryWithAliasOnFullRange_WhenAliasIsTheSameAsOneOfTheIndexFields(Options options) => + FacetAggregationOnFullRange(options, true); + + [RavenTheory(RavenTestCategory.Facets)] + [RavenData(SearchEngineMode = RavenSearchEngineMode.All, DatabaseMode = RavenDatabaseMode.Single)] + public void AggregationFacetOnAllResults(Options options) => + FacetAggregationOnFullRange(options, false); + + private void FacetAggregationOnFullRange(Options options, bool useAlias) { - using (var store = GetDocumentStore()) + using (var store = GetDocumentStore(options)) { new Orders_Totals().Execute(store); @@ -97,20 +107,27 @@ public void CanDoFacetQueryWithAliasOnFullRange_WhenAliasIsTheSameAsOneOfTheInde using (var commands = store.Commands()) { + var aliasForQuery = useAlias ? $" as Total" : string.Empty; + var result = commands.Query(new Raven.Client.Documents.Queries.IndexQuery { - Query = @" + Query = @$" from index 'Orders/Totals' where Total < 1000 -select facet(sum(Total)) as Total" +select facet(sum(Total)) {aliasForQuery}" }); var facetResult = (FacetResult)DocumentConventions.Default.Serialization.DefaultConverter.FromBlittable(typeof(FacetResult), (BlittableJsonReaderObject)result.Results[0], "facet/result"); - Assert.Equal("Total", facetResult.Name); + var alias = useAlias ? "Total" : Constants.Documents.Querying.Facet.AllResults; + Assert.Equal(alias, facetResult.Name); + + Assert.Equal(1, facetResult.Values.Count); Assert.Equal(Constants.Documents.Querying.Facet.AllResults, facetResult.Values[0].Range); Assert.Equal(3, facetResult.Values[0].Count); + + Assert.Equal(640, facetResult.Values[0].Sum); } } } diff --git a/test/SlowTests/Tests/TestsInheritanceTests.cs b/test/SlowTests/Tests/TestsInheritanceTests.cs index 31bf6f87fd3f..bc57723f0a1b 100644 --- a/test/SlowTests/Tests/TestsInheritanceTests.cs +++ b/test/SlowTests/Tests/TestsInheritanceTests.cs @@ -91,7 +91,7 @@ where Filter(method) var array = types.ToArray(); - const int numberToTolerate = 4616; + const int numberToTolerate = 4615; if (array.Length == numberToTolerate) return; From c5a389deaf8065306f9728b7ec4dfb4853145d62 Mon Sep 17 00:00:00 2001 From: egor Date: Thu, 11 Apr 2024 16:37:09 +0300 Subject: [PATCH 033/243] RavenDB-22220 - fix Exception during tests discovery --- .../PeriodicBackup/RetentionStress.cs | 6 ++-- .../AmazonS3RetryFactAttribute.cs | 33 +++++++----------- .../AmazonS3RetryTheoryAttribute.cs | 33 +++++++----------- .../AzureRetryFactAttribute.cs | 32 ++++++----------- .../AzureRetryTheoryAttribute.cs | 34 +++++++------------ .../CloudAttributeHelper.cs | 27 +++++++++++++++ .../MultiTheoryAttribute .cs | 19 ----------- .../NightlyBuildTheoryAttribute.cs | 1 + .../RavenTheoryAttribute.cs | 15 ++++++++ 9 files changed, 93 insertions(+), 107 deletions(-) create mode 100644 test/Tests.Infrastructure/CloudAttributeHelper.cs delete mode 100644 test/Tests.Infrastructure/MultiTheoryAttribute .cs diff --git a/test/StressTests/Server/Documents/PeriodicBackup/RetentionStress.cs b/test/StressTests/Server/Documents/PeriodicBackup/RetentionStress.cs index 6fcced673109..a0686d2c45cf 100644 --- a/test/StressTests/Server/Documents/PeriodicBackup/RetentionStress.cs +++ b/test/StressTests/Server/Documents/PeriodicBackup/RetentionStress.cs @@ -11,7 +11,7 @@ public RetentionStress(ITestOutputHelper output) : base(output) { } - [NightlyBuildTheory] + [RavenTheory(RavenTestCategory.BackupExportImport, NightlyBuildRequired = true)] [InlineData(20, 5, false)] [InlineData(20, 20, false)] [InlineData(25, 10, false)] @@ -38,7 +38,7 @@ public async Task can_delete_backups_by_date(int backupAgeInSeconds, int numberO } } - [NightlyBuildTheory(Skip = "Requires Amazon AWS Credentials")] + [RavenTheory(RavenTestCategory.BackupExportImport, NightlyBuildRequired = true, S3Required = true)] [InlineData(20, 5, false)] [InlineData(20, 20, false)] [InlineData(25, 10, false)] @@ -63,7 +63,7 @@ public async Task can_delete_backups_by_date_s3(int backupAgeInSeconds, int numb } } - [MultiTheory(typeof(NightlyBuildTheoryAttribute), typeof(AzureRetryTheoryAttribute))] + [RavenTheory(RavenTestCategory.BackupExportImport, NightlyBuildRequired = true, AzureRequired = true)] [InlineData(20, 5, false)] [InlineData(20, 20, false)] [InlineData(25, 10, false)] diff --git a/test/Tests.Infrastructure/AmazonS3RetryFactAttribute.cs b/test/Tests.Infrastructure/AmazonS3RetryFactAttribute.cs index aeb6d78a219e..84e88fc25ba4 100644 --- a/test/Tests.Infrastructure/AmazonS3RetryFactAttribute.cs +++ b/test/Tests.Infrastructure/AmazonS3RetryFactAttribute.cs @@ -40,32 +40,23 @@ static AmazonS3RetryFactAttribute() public AmazonS3RetryFactAttribute([CallerMemberName] string memberName = "", int maxRetries = 3, int delayBetweenRetriesMs = 0, params Type[] skipOnExceptions) : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) { - if (RavenTestHelper.SkipIntegrationTests) - { - Skip = RavenTestHelper.SkipIntegrationMessage; - return; - } - - if (RavenTestHelper.IsRunningOnCI) - return; + } - if (EnvVariableMissing) + public override string Skip + { + get { - Skip = $"Test is missing '{S3CredentialEnvironmentVariable}' environment variable."; - return; + ShouldSkip(out var skipMessage); + return skipMessage; } - if (string.IsNullOrEmpty(ParsingError) == false) - { - Skip = $"Failed to parse the Amazon S3 settings, error: {ParsingError}"; - return; - } + set => base.Skip = value; + } - if (_s3Settings == null) - { - Skip = $"S3 {memberName} tests missing Amazon S3 settings."; - return; - } + public static bool ShouldSkip(out string skipMessage) + { + skipMessage = CloudAttributeHelper.TestIsMissingCloudCredentialEnvironmentVariable(EnvVariableMissing, S3CredentialEnvironmentVariable, ParsingError, _s3Settings); + return string.IsNullOrEmpty(skipMessage) == false; } } } diff --git a/test/Tests.Infrastructure/AmazonS3RetryTheoryAttribute.cs b/test/Tests.Infrastructure/AmazonS3RetryTheoryAttribute.cs index 4cbfb62d635b..ceeca8f343de 100644 --- a/test/Tests.Infrastructure/AmazonS3RetryTheoryAttribute.cs +++ b/test/Tests.Infrastructure/AmazonS3RetryTheoryAttribute.cs @@ -40,32 +40,23 @@ static AmazonS3RetryTheoryAttribute() public AmazonS3RetryTheoryAttribute([CallerMemberName] string memberName = "", int maxRetries = 3, int delayBetweenRetriesMs = 0, params Type[] skipOnExceptions) : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) { - if (RavenTestHelper.SkipIntegrationTests) - { - Skip = RavenTestHelper.SkipIntegrationMessage; - return; - } - - if (RavenTestHelper.IsRunningOnCI) - return; + } - if (EnvVariableMissing) + public override string Skip + { + get { - Skip = $"Test is missing '{S3CredentialEnvironmentVariable}' environment variable."; - return; + ShouldSkip(out var skipMessage); + return skipMessage; } - if (string.IsNullOrEmpty(ParsingError) == false) - { - Skip = $"Failed to parse the Amazon S3 settings, error: {ParsingError}"; - return; - } + set => base.Skip = value; + } - if (_s3Settings == null) - { - Skip = $"S3 {memberName} tests missing Amazon S3 settings."; - return; - } + public static bool ShouldSkip(out string skipMessage) + { + skipMessage = CloudAttributeHelper.TestIsMissingCloudCredentialEnvironmentVariable(EnvVariableMissing, S3CredentialEnvironmentVariable, ParsingError, _s3Settings); + return string.IsNullOrEmpty(skipMessage) == false; } } } diff --git a/test/Tests.Infrastructure/AzureRetryFactAttribute.cs b/test/Tests.Infrastructure/AzureRetryFactAttribute.cs index 66b62cff8c1b..3136243a4351 100644 --- a/test/Tests.Infrastructure/AzureRetryFactAttribute.cs +++ b/test/Tests.Infrastructure/AzureRetryFactAttribute.cs @@ -40,32 +40,22 @@ static AzureRetryFactAttribute() public AzureRetryFactAttribute([CallerMemberName] string memberName = "", int maxRetries = 3, int delayBetweenRetriesMs = 0, params Type[] skipOnExceptions) : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) { - if (RavenTestHelper.SkipIntegrationTests) - { - Skip = RavenTestHelper.SkipIntegrationMessage; - return; - } - - if (RavenTestHelper.IsRunningOnCI) - return; + } - if (EnvVariableMissing) + public override string Skip + { + get { - Skip = $"Test is missing '{AzureCredentialEnvironmentVariable}' environment variable."; - return; + return CloudAttributeHelper.TestIsMissingCloudCredentialEnvironmentVariable(EnvVariableMissing, AzureCredentialEnvironmentVariable, ParsingError, _azureSettings); } - if (string.IsNullOrEmpty(ParsingError) == false) - { - Skip = $"Failed to parse the Azure settings, error: {ParsingError}"; - return; - } + set => base.Skip = value; + } - if (_azureSettings == null) - { - Skip = $"Azure {memberName} tests missing {nameof(AzureSettings)}."; - return; - } + public static bool ShouldSkip(out string skipMessage) + { + skipMessage = CloudAttributeHelper.TestIsMissingCloudCredentialEnvironmentVariable(EnvVariableMissing, AzureCredentialEnvironmentVariable, ParsingError, _azureSettings); + return string.IsNullOrEmpty(skipMessage) == false; } } } diff --git a/test/Tests.Infrastructure/AzureRetryTheoryAttribute.cs b/test/Tests.Infrastructure/AzureRetryTheoryAttribute.cs index 576e586efd40..d2e275306c8f 100644 --- a/test/Tests.Infrastructure/AzureRetryTheoryAttribute.cs +++ b/test/Tests.Infrastructure/AzureRetryTheoryAttribute.cs @@ -36,36 +36,26 @@ static AzureRetryTheoryAttribute() ParsingError = e.ToString(); } } - + public AzureRetryTheoryAttribute([CallerMemberName] string memberName = "", int maxRetries = 3, int delayBetweenRetriesMs = 0, params Type[] skipOnExceptions) : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) { - if (RavenTestHelper.SkipIntegrationTests) - { - Skip = RavenTestHelper.SkipIntegrationMessage; - return; - } - - if (RavenTestHelper.IsRunningOnCI) - return; + } - if (EnvVariableMissing) + public override string Skip + { + get { - Skip = $"Test is missing '{AzureCredentialEnvironmentVariable}' environment variable."; - return; + return CloudAttributeHelper.TestIsMissingCloudCredentialEnvironmentVariable(EnvVariableMissing, AzureCredentialEnvironmentVariable, ParsingError, _azureSettings); } - if (string.IsNullOrEmpty(ParsingError) == false) - { - Skip = $"Failed to parse the Azure settings, error: {ParsingError}"; - return; - } + set => base.Skip = value; + } - if (_azureSettings == null) - { - Skip = $"Azure {memberName} tests missing {nameof(AzureSettings)}."; - return; - } + public static bool ShouldSkip(out string skipMessage) + { + skipMessage = CloudAttributeHelper.TestIsMissingCloudCredentialEnvironmentVariable(EnvVariableMissing, AzureCredentialEnvironmentVariable, ParsingError, _azureSettings); + return string.IsNullOrEmpty(skipMessage) == false; } } } diff --git a/test/Tests.Infrastructure/CloudAttributeHelper.cs b/test/Tests.Infrastructure/CloudAttributeHelper.cs new file mode 100644 index 000000000000..dd0df8d549b4 --- /dev/null +++ b/test/Tests.Infrastructure/CloudAttributeHelper.cs @@ -0,0 +1,27 @@ +using Raven.Client.Documents.Operations.Backups; + +namespace Tests.Infrastructure +{ + internal class CloudAttributeHelper + { + public static string TestIsMissingCloudCredentialEnvironmentVariable(bool envVariableMissing, string environmentVariable, string parsingError, BackupSettings settings) + { + if (RavenTestHelper.SkipIntegrationTests) + return RavenTestHelper.SkipIntegrationMessage; + + if (RavenTestHelper.IsRunningOnCI) + return null; + + if (envVariableMissing) + return $"Test is missing '{environmentVariable}' environment variable."; + + if (string.IsNullOrEmpty(parsingError) == false) + return $"Failed to parse the {nameof(BackupSettings)} settings, error: {parsingError}"; + + if (settings == null) + return $"Cloud backup tests missing {nameof(BackupSettings)}."; + + return null; + } + } +} diff --git a/test/Tests.Infrastructure/MultiTheoryAttribute .cs b/test/Tests.Infrastructure/MultiTheoryAttribute .cs deleted file mode 100644 index 75eb6fedd8de..000000000000 --- a/test/Tests.Infrastructure/MultiTheoryAttribute .cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Linq; -using Xunit; - -namespace Tests.Infrastructure -{ - public class MultiTheoryAttribute : TheoryAttribute - { - public MultiTheoryAttribute(params Type[] types) - { - var result = types.Select(Activator.CreateInstance).Cast().ToList(); - - if (result.Any(x => string.IsNullOrEmpty(x.Skip) == false)) - { - Skip = string.Join(", ", result.Where(x => string.IsNullOrEmpty(x.Skip) == false).Select(x => x.Skip)); - } - } - } -} diff --git a/test/Tests.Infrastructure/NightlyBuildTheoryAttribute.cs b/test/Tests.Infrastructure/NightlyBuildTheoryAttribute.cs index 5f2f343fd3f6..4a718339e544 100644 --- a/test/Tests.Infrastructure/NightlyBuildTheoryAttribute.cs +++ b/test/Tests.Infrastructure/NightlyBuildTheoryAttribute.cs @@ -49,6 +49,7 @@ public override string Skip return SkipMessage; } + set => base.Skip = value; } } } diff --git a/test/Tests.Infrastructure/RavenTheoryAttribute.cs b/test/Tests.Infrastructure/RavenTheoryAttribute.cs index 2463ab425568..882cc4d3e2fb 100644 --- a/test/Tests.Infrastructure/RavenTheoryAttribute.cs +++ b/test/Tests.Infrastructure/RavenTheoryAttribute.cs @@ -16,6 +16,12 @@ public RavenTheoryAttribute(RavenTestCategory category) public bool LicenseRequired { get; set; } + public bool NightlyBuildRequired { get; set; } + + public bool S3Required { get; set; } + + public bool AzureRequired { get; set; } + public override string Skip { get @@ -33,6 +39,15 @@ public override string Skip if (LicenseRequired && LicenseRequiredFactAttribute.ShouldSkip()) return LicenseRequiredFactAttribute.SkipMessage; + if (NightlyBuildRequired && NightlyBuildFactAttribute.ShouldSkip(out skip)) + return skip; + + if (S3Required && AmazonS3RetryTheoryAttribute.ShouldSkip(out skip)) + return skip; + + if (AzureRequired && AzureRetryTheoryAttribute.ShouldSkip(out skip)) + return skip; + return null; } From 2c82376e7a3c4490e2562344605c8ebf369b5b9e Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Thu, 11 Apr 2024 15:15:30 +0200 Subject: [PATCH 034/243] RavenDB-7070 Updating Jint --- src/Raven.Server/Raven.Server.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Raven.Server/Raven.Server.csproj b/src/Raven.Server/Raven.Server.csproj index 9ed2c5a94a6e..3078970dbb16 100644 --- a/src/Raven.Server/Raven.Server.csproj +++ b/src/Raven.Server/Raven.Server.csproj @@ -168,7 +168,7 @@ All - + From d7a96a4437d1666839fb411eddb27f4f07d0699d Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Fri, 12 Apr 2024 10:27:54 +0200 Subject: [PATCH 035/243] RavenDB-20446 Fixing occassionally failing. Let's wait explicitly for index to be non stale before issuing a query so we will be sure we won't wait for the replacement to complete the swap. --- test/SlowTests/Issues/RavenDB_12269.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/SlowTests/Issues/RavenDB_12269.cs b/test/SlowTests/Issues/RavenDB_12269.cs index 55bb67d39c50..f061244fe51f 100644 --- a/test/SlowTests/Issues/RavenDB_12269.cs +++ b/test/SlowTests/Issues/RavenDB_12269.cs @@ -80,6 +80,8 @@ public async Task Changing_index_def_must_not_error_index() await session.SaveChangesAsync(); + Indexes.WaitForIndexing(store); + // this will ensure that index isn't in error state try { From 8ce4c5ea0d89ab3bee58f11d57acb40ca3921760 Mon Sep 17 00:00:00 2001 From: aviv Date: Sun, 14 Apr 2024 13:13:53 +0300 Subject: [PATCH 036/243] RDBCL-3407 : server wide backups that are defined using a backup-script should use database name (instead of shard names) when generating backup folder name --- ...PutServerWideBackupConfigurationCommand.cs | 3 +- .../Sharding/Backup/ShardedBackupTests .cs | 131 ++++++++++++++++++ .../Backup/ShardedRestoreBackupTests.cs | 122 ++++++++++++++++ 3 files changed, 255 insertions(+), 1 deletion(-) diff --git a/src/Raven.Server/ServerWide/Commands/PutServerWideBackupConfigurationCommand.cs b/src/Raven.Server/ServerWide/Commands/PutServerWideBackupConfigurationCommand.cs index 5b08f5fb3df6..5a7ca9ef45cb 100644 --- a/src/Raven.Server/ServerWide/Commands/PutServerWideBackupConfigurationCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/PutServerWideBackupConfigurationCommand.cs @@ -5,6 +5,7 @@ using Raven.Client.ServerWide.Operations.Configuration; using Raven.Server.Rachis; using Raven.Server.ServerWide.Context; +using Raven.Server.Utils; using Sparrow.Json; using Sparrow.Json.Parsing; @@ -212,7 +213,7 @@ private static string GetUpdatedPath(string str, string databaseName, char separ if (str.EndsWith(separator) == false) str += separator; - return str + databaseName; + return str + ShardHelper.ToDatabaseName(databaseName); } } } diff --git a/test/SlowTests/Sharding/Backup/ShardedBackupTests .cs b/test/SlowTests/Sharding/Backup/ShardedBackupTests .cs index 97f2db9f244f..02b9f336c0ac 100644 --- a/test/SlowTests/Sharding/Backup/ShardedBackupTests .cs +++ b/test/SlowTests/Sharding/Backup/ShardedBackupTests .cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json; using Raven.Client.Documents; using Raven.Client.Documents.Commands; using Raven.Client.Documents.Operations.Backups; @@ -17,13 +19,16 @@ using Raven.Server.Documents.Sharding; using Raven.Server.ServerWide; using Raven.Server.ServerWide.Context; +using Raven.Server.Utils; using Raven.Tests.Core.Utils.Entities; using Sparrow.Json; +using Sparrow.Platform; using Tests.Infrastructure; using Tests.Infrastructure.Entities; using Xunit; using Xunit.Abstractions; using BackupTask = Raven.Server.Documents.PeriodicBackup.BackupTask; +using Directory = System.IO.Directory; namespace SlowTests.Sharding.Backup { @@ -278,6 +283,108 @@ await store1.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOp } } + [RavenTheory(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)] + [RavenData(DatabaseMode = RavenDatabaseMode.All)] + public async Task CanBackupShardedServerWide_UsingScript(Options options) + { + DoNotReuseServer(); + + const string usersPrefix = "Users"; + const string ordersPrefix = "Orders"; + + var backupPath = NewDataPath(suffix: "_BackupFolder"); + + using (var store1 = Sharding.GetDocumentStore()) + using (var store2 = GetDocumentStore()) + { + // generate data on store1 and store2 + using (var session1 = store1.OpenAsyncSession()) + using (var session2 = store2.OpenAsyncSession()) + { + for (int i = 0; i < 100; i++) + { + await session1.StoreAsync(new User(), $"{usersPrefix}/{i}"); + await session2.StoreAsync(new Order(), $"{ordersPrefix}/{i}"); + } + + await session1.SaveChangesAsync(); + await session2.SaveChangesAsync(); + } + + // use backup configuration script for local settings + var scriptPath = GenerateConfigurationScript(backupPath, out var command); + var config = new ServerWideBackupConfiguration + { + FullBackupFrequency = "0 0 1 1 *", + Disabled = false, + LocalSettings = new LocalSettings + { + FolderPath = backupPath, + GetBackupConfigurationScript = new GetBackupConfigurationScript + { + Exec = command, + Arguments = scriptPath + } + } + }; + + // define server wide backup + await store1.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(config)); + + // wait for backups to complete + var backupsDone = await Sharding.Backup.WaitForBackupToComplete(store1); + var backupsDone2 = await Backup.WaitForBackupToComplete(store2); + + foreach (var store in new[] { store1, store2 }) + { + var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); + Assert.Equal(1, databaseRecord.PeriodicBackups.Count); + + var taskId = databaseRecord.PeriodicBackups[0].TaskId; + if (databaseRecord.IsSharded) + await Sharding.Backup.RunBackupAsync(store.Database, taskId, isFullBackup: true); + else + await Backup.RunBackupAsync(Server, taskId, store, isFullBackup: true); + } + + Assert.True(WaitHandle.WaitAll(backupsDone, TimeSpan.FromMinutes(1))); + Assert.True(WaitHandle.WaitAll(backupsDone2, TimeSpan.FromMinutes(1))); + + // one backup folder per database + var dirs = Directory.GetDirectories(backupPath); + Assert.Equal(2, dirs.Length); + + // should have one root folder for all shards + Assert.Contains(store1.Database, dirs[0]); + Assert.DoesNotContain('$', dirs[0]); + + var store1Backups = Directory.GetDirectories(Path.Combine(backupPath, store1.Database)); + var store2Backup = Directory.GetDirectories(Path.Combine(backupPath, store2.Database)); + + Assert.Equal(3, store1Backups.Length); // one per shard + Assert.Single(store2Backup); + + Assert.Contains(ShardHelper.ToShardName(store1.Database, 0), store1Backups[0]); + Assert.Contains(ShardHelper.ToShardName(store1.Database, 1), store1Backups[1]); + Assert.Contains(ShardHelper.ToShardName(store1.Database, 2), store1Backups[2]); + + // import data to new stores and assert + using (var store3 = GetDocumentStore(options)) + using (var store4 = GetDocumentStore(options)) + { + foreach (var dir in store1Backups) + { + await store3.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), dir); + } + + await store4.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), store2Backup[0]); + + await AssertDocs(store3, idPrefix: usersPrefix, dbMode: options.DatabaseMode); + await AssertDocs(store4, idPrefix: ordersPrefix, dbMode: options.DatabaseMode); + } + } + } + [RavenTheory(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)] [RavenData(DatabaseMode = RavenDatabaseMode.All)] public async Task OneTimeBackupSharded(Options options) @@ -720,5 +827,29 @@ private async Task AssertDocsInShardedDb(IDocumentStore store, Dictionary x.FullPath).ToList(); + + Assert.Contains(ShardHelper.ToShardName(store.Database, 0), backupPaths[0]); + Assert.Contains(ShardHelper.ToShardName(store.Database, 1), backupPaths[1]); + Assert.Contains(ShardHelper.ToShardName(store.Database, 2), backupPaths[2]); + + settings = Sharding.Backup.GenerateShardRestoreSettings(backupPaths, sharding); + } + + var restoredDatabaseName = $"restored_database-{Guid.NewGuid()}"; + using (Backup.RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration + { + DatabaseName = restoredDatabaseName, + ShardRestoreSettings = settings, + Settings = s3Settings + }, timeout: TimeSpan.FromSeconds(60))) + { + var dbRec = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(restoredDatabaseName)); + Assert.Equal(3, dbRec.Sharding.Shards.Count); + + await Sharding.Backup.CheckData(store, RavenDatabaseMode.Sharded, expectedRevisionsCount: 16, database: restoredDatabaseName); + + } + } + } + finally + { + await DeleteObjects(s3Settings); + } + + } private static string GetDirectoryName(string path) { @@ -1831,5 +1930,28 @@ private static async Task DeleteObjects(GoogleCloudSettings settings) } } + private static string GenerateConfigurationScriptForS3(S3Settings settings, out string command) + { + var scriptPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); + var s3SettingsString = JsonConvert.SerializeObject(settings); + + string script; + if (PlatformDetails.RunningOnPosix) + { + command = "bash"; + script = $"#!/bin/bash\r\necho '{s3SettingsString}'"; + File.WriteAllText(scriptPath, script); + Process.Start("chmod", $"700 {scriptPath}"); + } + else + { + command = "powershell"; + script = $"echo '{s3SettingsString}'"; + File.WriteAllText(scriptPath, script); + } + + return scriptPath; + } + } } From 307d1d9bf5399101c1de195de1ffaa5261c04af2 Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Mon, 15 Apr 2024 09:19:58 +0200 Subject: [PATCH 037/243] RavenDB-20446 Fixing occassionally failing. Let's wait explicitly for index to be non stale before issuing a query so we will be sure we won't wait for the replacement to complete the swap. --- test/SlowTests/Issues/RavenDB_12269.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/SlowTests/Issues/RavenDB_12269.cs b/test/SlowTests/Issues/RavenDB_12269.cs index 55bb67d39c50..f061244fe51f 100644 --- a/test/SlowTests/Issues/RavenDB_12269.cs +++ b/test/SlowTests/Issues/RavenDB_12269.cs @@ -80,6 +80,8 @@ public async Task Changing_index_def_must_not_error_index() await session.SaveChangesAsync(); + Indexes.WaitForIndexing(store); + // this will ensure that index isn't in error state try { From fcacad907de746227f5f9f4770cf7f5b2b76c583 Mon Sep 17 00:00:00 2001 From: Maciej Aszyk Date: Mon, 15 Apr 2024 10:47:19 +0200 Subject: [PATCH 038/243] RavenDB-22176 Use class instead of struct for test dto deserialization. --- test/SlowTests/Issues/RavenDB_22176.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/SlowTests/Issues/RavenDB_22176.cs b/test/SlowTests/Issues/RavenDB_22176.cs index 3b048be6723c..00c90f0ca3d8 100644 --- a/test/SlowTests/Issues/RavenDB_22176.cs +++ b/test/SlowTests/Issues/RavenDB_22176.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using Newtonsoft.Json; using Raven.Client.Documents.Conventions; using Raven.Client.Documents.Operations; using Raven.Client.Http; @@ -117,7 +116,7 @@ public async Task TestGetNotificationsEndpointPaging(NotificationType notificati result = store.Maintenance.Send(new GetNotifications(pageSize: 0, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); Assert.Equal(5, result.TotalResults); Assert.Empty(result.Results); - + //paging { result = store.Maintenance.Send(new GetNotifications(pageStart: 0, pageSize: 2, alertType: notificationTypeParameter, testAsNumber: flagAsInt)); @@ -177,8 +176,12 @@ public SerializableNotification(NotificationType type, string database) : base(t public override string Id { get; } } - private record NotificationEndpointDto(int TotalResults, List Results); - + private class NotificationEndpointDto + { + public int TotalResults { get; set; } + public List Results { get; set; } + } + [Flags] private enum NotificationTypeParameter : short { @@ -254,7 +257,7 @@ public override void SetResponse(JsonOperationContext context, BlittableJsonRead if (response == null) ThrowInvalidResponse(); - Result = JsonConvert.DeserializeObject(response!.ToString()); + Result = DocumentConventions.DefaultForServer.Serialization.DeserializeEntityFromBlittable(response); } public override bool IsReadRequest => true; From e39e5a90e7c85e3e2a80d4c47279ee34c382e9dc Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Fri, 12 Apr 2024 14:18:48 +0200 Subject: [PATCH 039/243] RavenDB-22253 Hooking up NewTransactionCreated and AfterCommitWhenNewTransactionsPrevented events to transactions created on async commit. This ensure that we'll be setting and updating DocumentsStorage.DocumentTransactionCache on commit of those transactions. This is important for read transactions calling methods like DocumentsStorage.ReadLastXXXEtag because they use that cache. --- src/Voron/Impl/LowLevelTransaction.cs | 2 + src/Voron/StorageEnvironment.cs | 10 +++ test/SlowTests/Issues/RavenDB_22253.cs | 96 ++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 test/SlowTests/Issues/RavenDB_22253.cs diff --git a/src/Voron/Impl/LowLevelTransaction.cs b/src/Voron/Impl/LowLevelTransaction.cs index ea5e2e839772..785ee8b9733e 100644 --- a/src/Voron/Impl/LowLevelTransaction.cs +++ b/src/Voron/Impl/LowLevelTransaction.cs @@ -1123,6 +1123,8 @@ public LowLevelTransaction BeginAsyncCommitAndStartNewTransaction(TransactionPer _env.ActiveTransactions.Add(nextTx); _env.WriteTransactionStarted(); + nextTx.AfterCommitWhenNewTransactionsPrevented += _env.InvokeAfterCommitWhenNewTransactionsPrevented; + _env.InvokeNewTransactionCreated(nextTx); return nextTx; } diff --git a/src/Voron/StorageEnvironment.cs b/src/Voron/StorageEnvironment.cs index b901dd2b81b6..9b0762da230d 100644 --- a/src/Voron/StorageEnvironment.cs +++ b/src/Voron/StorageEnvironment.cs @@ -770,6 +770,16 @@ internal LowLevelTransaction NewLowLevelTransaction(TransactionPersistentContext } } + internal void InvokeNewTransactionCreated(LowLevelTransaction tx) + { + NewTransactionCreated?.Invoke(tx); + } + + internal void InvokeAfterCommitWhenNewTransactionsPrevented(LowLevelTransaction tx) + { + AfterCommitWhenNewTransactionsPrevented?.Invoke(tx); + } + [Conditional("DEBUG")] private void ThrowOnWriteTransactionOpenedByTheSameThread() { diff --git a/test/SlowTests/Issues/RavenDB_22253.cs b/test/SlowTests/Issues/RavenDB_22253.cs new file mode 100644 index 000000000000..aa79c3fb08a8 --- /dev/null +++ b/test/SlowTests/Issues/RavenDB_22253.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FastTests; +using Raven.Client.Documents; +using Raven.Client.Documents.Indexes; +using Sparrow.Collections; +using Tests.Infrastructure; +using Xunit; +using Xunit.Abstractions; + +namespace SlowTests.Issues; + +public class RavenDB_22253 : RavenTestBase +{ + public RavenDB_22253(ITestOutputHelper output) : base(output) + { + } + + [RavenFact(RavenTestCategory.Querying | RavenTestCategory.Voron)] + public async Task QueryMustNotReturnNoResultsAfterWaitForIndexesAfterSaveChanges() + { + using var store = GetDocumentStore(); + + store.ExecuteIndex(new PupilsIndex()); + + var tasks = new List(); + var failures = new ConcurrentSet(); + + WaitForUserToContinueTheTest(store); + + // Run 10 concurrent tasks: + for (int i = 0; i < 10; i++) + { + var taskId = i.ToString() + "-"; + + tasks.Add(Task.Run(async () => + { + // Add 1 pupil to each of 1000 schools: + foreach (var schoolId in Enumerable + .Range(1, 100) + .Select(j => $"Schools/{taskId}{j}")) + { + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(new Pupil + { + SchoolId = schoolId, + FirstName = "John" + }, $"pupils/{schoolId}"); + session.Advanced.WaitForIndexesAfterSaveChanges(); + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession()) + { + // Given the previous step uses WaitForIndexesAfterSaveChanges, + // we'd assume that the index contains the new pupil. + var pupils = await session.Query() + .Where(x => x.SchoolId == schoolId) + .ToListAsync(); + + if (pupils.Count == 0) + { + failures.Add($"No pupils found for {schoolId}"); + } + } + } + })); + } + + await Task.WhenAll(tasks); + Assert.True(failures.Count == 0, string.Join('\n', failures)); + } + + class Pupil + { + public string Id { get; set; } + public string SchoolId { get; set; } + public string FirstName { get; set; } + } + + class PupilsIndex : AbstractIndexCreationTask + { + + public PupilsIndex() + { + Map = pupils => from pupil in pupils + select new + { + pupil.SchoolId + }; + } + } +} From 706144c2b2e56b42cfb6733d30d048d72c199ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Le=C5=9Bniak?= Date: Fri, 15 Mar 2024 11:44:33 +0100 Subject: [PATCH 040/243] RavenDB-21996 IFilterDocumentQueryBase documentation --- src/Raven.Client/DocumentationUrls.cs | 6 + .../Documents/Session/IDocumentQueryBase.cs | 351 ++++++++++-------- 2 files changed, 200 insertions(+), 157 deletions(-) diff --git a/src/Raven.Client/DocumentationUrls.cs b/src/Raven.Client/DocumentationUrls.cs index 77887e135be2..6d562f3a4322 100644 --- a/src/Raven.Client/DocumentationUrls.cs +++ b/src/Raven.Client/DocumentationUrls.cs @@ -106,6 +106,12 @@ internal static class StreamQueryResults /// public const string GroupByArrayContent = nameof(GroupByArrayContent); + /// + public const string HowToUseNotOperator = nameof(HowToUseNotOperator); + + /// + public const string HowToUseLucene = nameof(HowToUseLucene); + /// public const string Includes = nameof(Includes); diff --git a/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs b/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs index 9796ca74c430..489c46d2ac19 100644 --- a/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs +++ b/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs @@ -15,92 +15,88 @@ namespace Raven.Client.Documents.Session public interface IFilterDocumentQueryBase where TSelf : IDocumentQueryBase { /// - /// Negate the next operation + /// Negates the next subclause. /// + /// TSelf Not { get; } - /// - /// Add an AND to the query - /// + /// TSelf AndAlso(); /// - /// Wraps previous query with clauses and add an AND operation to the given query + /// Adds an 'AND' statement to the query. /// + /// Wraps preceding clauses using parentheses. TSelf AndAlso(bool wrapPreviousQueryClauses); /// - /// Simplified method for closing a clause within the query + /// Closes previously opened subclause. /// - /// TSelf CloseSubclause(); /// - /// Performs a query matching ALL of the provided values against the given field (AND) + /// Matches documents with chosen field containing all provided values. /// + /// Name of the field to match values against. + /// Values that the chosen field has to contain. TSelf ContainsAll(string fieldName, IEnumerable values); - /// - /// Performs a query matching ALL of the provided values against the given field (AND) - /// + /// TSelf ContainsAll(Expression> propertySelector, IEnumerable values); /// - /// Performs a query matching ALL of the provided values against the given field (AND) + /// Matches documents with chosen field containing all provided values. /// + /// Property selector for the field to match values against. + /// Values that the chosen field has to contain. TSelf ContainsAll(Expression>> propertySelector, IEnumerable values); /// - /// Performs a query matching ANY of the provided values against the given field (OR) + /// Matches documents with chosen field containing any of provided values. /// + /// Name of the field to match values against. + /// Values, where at least one must be contained in value in order to match the document. TSelf ContainsAny(string fieldName, IEnumerable values); - /// - /// Performs a query matching ANY of the provided values against the given field (OR) - /// + /// TSelf ContainsAny(Expression> propertySelector, IEnumerable values); /// - /// Performs a query matching ANY of the provided values against the given field (OR) + /// Matches documents with chosen field containing any of provided values. /// + /// Property selector for the field to match values against. + /// Values, where at least one must be contained in value in order to match the document. TSelf ContainsAny(Expression>> propertySelector, IEnumerable values); /// - /// Negate the next operation + /// Negates the next subclause. /// TSelf NegateNext(); /// - /// Simplified method for opening a new clause within the query + /// Opens a new subclause. /// - /// TSelf OpenSubclause(); /// - /// Add an OR to the query + /// Adds an 'OR' statement to the query. /// TSelf OrElse(); /// - /// Perform a search for documents which fields that match the searchTerms. - /// If there is more than a single term, each of them will be checked independently. + /// Matches documents with value of chosen field matching searched terms. /// - /// Marks a field in which terms should be looked for - /// - /// Space separated terms e.g. 'John Adam' means that we will look in selected field for 'John' - /// or 'Adam'. - /// + /// Name of the field that searched terms will be checked against. + /// Space separated terms to search. If there is more than a single term, each of them will be checked independently. + /// Operator to be used for relationship between terms. Default: Or. TSelf Search(string fieldName, string searchTerms, SearchOperator @operator = SearchOperator.Or); /// - /// Perform a search for documents which fields that match the searchTerms. - /// If there is more than a single term, each of them will be checked independently. + /// Matches documents with value of chosen field matching searched terms. /// - /// Expression marking a field in which terms should be looked for - /// - /// Space separated terms e.g. 'John Adam' means that we will look in selected field for 'John' - /// or 'Adam'. - /// + /// Property selector for the field that searched terms will be checked against. + /// Space separated terms to search. If there is more than a single term, each of them will be checked independently. + /// Operator to be used for relationship between terms. Default: Or. TSelf Search(Expression> propertySelector, string searchTerms, SearchOperator @operator = SearchOperator.Or); /// @@ -109,298 +105,339 @@ public interface IFilterDocumentQueryBase where TSelf : IDocumentQuery /// IEnumerable Where(Func predicate); - /// - /// Filter the results from the index using the specified where clause. - /// - /// Name of the field. - /// Lucene-syntax based query predicate. + /// TSelf WhereLucene(string fieldName, string whereClause); /// - /// Filter the results from the index using the specified where clause. + /// Matches documents with chosen field value meeting criteria of specified predicate in Lucene syntax. /// - /// Name of the field. - /// Lucene-syntax based query predicate. + /// Name of the field to get value from. + /// Predicate in Lucene syntax. + /// Specifies if comparison is case sensitive. + /// TSelf WhereLucene(string fieldName, string whereClause, bool exact); /// - /// Matches fields where the value is between the specified start and end, inclusive + /// Matches documents with value of the chosen field between the specified start and end value, inclusive. /// - /// Name of the field. - /// The start. - /// The end. + /// Name of the field to get value from. + /// Start value. + /// End value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereBetween(string fieldName, object start, object end, bool exact = false); /// - /// Matches fields where the value is between the specified start and end, inclusive + /// Matches documents with value of the chosen field between specified the start and end, inclusive. /// - /// Property selector for the field. - /// The start. - /// The end. + /// Property selector for the field to get value from. + /// Start value. + /// End value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereBetween(Expression> propertySelector, TValue start, TValue end, bool exact = false); - /// - /// Matches fields which ends with the specified value. - /// + /// TSelf WhereEndsWith(string fieldName, object value); /// - /// Matches fields which ends with the specified value. + /// Matches documents with value of the chosen field ending with the specified value. /// + /// Name of the field to get value from. + /// Value that the value has to end with in order to match the document. + /// Specifies if comparison is case sensitive. TSelf WhereEndsWith(string fieldName, object value, bool exact); - /// - /// Matches fields which ends with the specified value. - /// + /// TSelf WhereEndsWith(Expression> propertySelector, TValue value); /// - /// Matches fields which ends with the specified value. + /// Matches documents with value of the chosen field ending with the specified value. /// + /// Property selector for the field to get value from. + /// Value that the value has to end with in order to match the document. + /// Specifies if comparison is case sensitive. TSelf WhereEndsWith(Expression> propertySelector, TValue value, bool exact); /// - /// Matches value + /// Matches documents with value of the chosen field equal to the specified value. /// + /// Name of the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereEquals(string fieldName, object value, bool exact = false); /// - /// Matches the evaluated expression + /// Matches documents with value of the chosen field equal to the evaluated provided expression. /// + /// Name of the field to get value from. + /// Expression to evaluate. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereEquals(string fieldName, MethodCall value, bool exact = false); /// - /// Matches value + /// Matches documents with value of the chosen field equal to the specified value. /// + /// Name of the field to get value from. + /// + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereEquals(Expression> propertySelector, TValue value, bool exact = false); /// - /// Matches value + /// Matches documents with value of the chosen field equal to the evaluated provided expression. /// + /// Property selector for the field to get value from. + /// Expression to evaluate. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereEquals(Expression> propertySelector, MethodCall value, bool exact = false); /// - /// Matches value + /// Matches documents that match specified . /// + /// WhereParams containing query parameters. TSelf WhereEquals(WhereParams whereParams); /// - /// Not matches value + /// Matches documents with value of the chosen field different than the specified value. /// + /// Name of the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereNotEquals(string fieldName, object value, bool exact = false); /// - /// Not matches the evaluated expression + /// Matches documents with value of the chosen field different than the evaluated provided expression. /// + /// Name of the field to get value from. + /// Expression to evaluate. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereNotEquals(string fieldName, MethodCall value, bool exact = false); /// - /// Not matches value + /// Matches documents with value of the chosen field different than the specified value. /// + /// Property selector for the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereNotEquals(Expression> propertySelector, TValue value, bool exact = false); /// - /// Matches value + /// Matches documents with value of the chosen field different than the evaluated provided expression. /// + /// Property selector for the field to get value from. + /// Expression to evaluate. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereNotEquals(Expression> propertySelector, MethodCall value, bool exact = false); /// - /// Not matches value + /// Matches documents that do not match specified . /// + /// WhereParams containing query parameters. TSelf WhereNotEquals(WhereParams whereParams); /// - /// Matches fields where the value is greater than the specified value + /// Matches documents with value of the chosen field greater than the specified value. /// - /// Name of the field. - /// The value. + /// Name of the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereGreaterThan(string fieldName, object value, bool exact = false); /// - /// Matches fields where the value is greater than the specified value + /// Matches documents with value of the chosen field greater than the specified value. /// - /// Property selector for the field. - /// The value. + /// Property selector for the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereGreaterThan(Expression> propertySelector, TValue value, bool exact = false); /// - /// Matches fields where the value is greater than or equal to the specified value + /// Matches documents with value of the chosen field greater than or equal to the specified value. /// - /// Name of the field. - /// The value. + /// Name of the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereGreaterThanOrEqual(string fieldName, object value, bool exact = false); /// - /// Matches fields where the value is greater than or equal to the specified value + /// Matches documents with value of the chosen field greater than or equal to the specified value. /// - /// Property selector for the field. - /// The value. + /// Property selector for the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereGreaterThanOrEqual(Expression> propertySelector, TValue value, bool exact = false); /// - /// Check that the field has one of the specified values + /// Matches documents with value of the chosen field contained in provided values. /// + /// Name of the field to get value from. + /// Values that have to contain value in order for the document to be matched. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereIn(string fieldName, IEnumerable values, bool exact = false); /// - /// Check that the field has one of the specified values + /// Matches documents with value of the chosen field contained in provided values. /// + /// Property selector for the field to get value from. + /// Values that have to contain value in order for the document to be matched. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereIn(Expression> propertySelector, IEnumerable values, bool exact = false); /// - /// Matches fields where the value is less than the specified value + /// Matches documents with value of the chosen field less than the specified value. /// - /// Name of the field. - /// The value. + /// Name of the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereLessThan(string fieldName, object value, bool exact = false); /// - /// Matches fields where the value is less than the specified value + /// Matches documents with value of the chosen field less than the specified value. /// - /// Property selector for the field. - /// The value. + /// Property selector for the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereLessThan(Expression> propertySelector, TValue value, bool exact = false); /// - /// Matches fields where the value is less than or equal to the specified value + /// Matches documents with value of the chosen field less than or equal to the specified value. /// - /// Name of the field. - /// The value. + /// Name of the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereLessThanOrEqual(string fieldName, object value, bool exact = false); /// - /// Matches fields where the value is less than or equal to the specified value + /// Matches documents with value of the chosen field less than or equal to the specified value. /// - /// Property selector for the field. - /// The value. + /// Property selector for the field to get value from. + /// Value to compare with value. + /// Specifies if comparison is case sensitive. Default: false. TSelf WhereLessThanOrEqual(Expression> propertySelector, TValue value, bool exact = false); - /// - /// Matches fields which starts with the specified value. - /// + /// TSelf WhereStartsWith(string fieldName, object value); /// - /// Matches fields which starts with the specified value. + /// Matches documents with value of the chosen field starting with the specified value. /// + /// Name of the field to get value from. + /// Value that the value has to start with in order to match the document. + /// Specifies if comparison is case sensitive. TSelf WhereStartsWith(string fieldName, object value, bool exact); - /// - /// Matches fields which starts with the specified value. - /// + /// TSelf WhereStartsWith(Expression> propertySelector, TValue value); /// - /// Matches fields which starts with the specified value. + /// Matches documents with value of the chosen field starting with the specified value. /// + /// Property selector for the field to get value from. + /// Value that the value has to start with in order to match the document. + /// Specifies if comparison is case sensitive. TSelf WhereStartsWith(Expression> propertySelector, TValue value, bool exact); /// - /// Check if the given field exists + /// Matches documents with existing given field. /// - /// Property selector for the field. + /// Property selector for the field to check the existence of. TSelf WhereExists(Expression> propertySelector); /// - /// Check if the given field exists + /// Matches documents with existing given field. /// - /// Name of the field. + /// Name of the field to check the existence of. TSelf WhereExists(string fieldName); /// - /// Checks value of a given field against supplied regular expression pattern + /// Matches documents with the value of a given field matched by provided regular expression. /// + /// Property selector for the field to get value from. + /// Regular expression pattern to check field value against. TSelf WhereRegex(Expression> propertySelector, string pattern); /// - /// Checks value of a given field against supplied regular expression pattern + /// Matches documents with the value of a given field matched by provided regular expression. /// + /// Name of the field to get value from. + /// Regular expression pattern to check field value against. TSelf WhereRegex(string fieldName, string pattern); /// - /// Filter matches to be inside the specified radius + /// Matches documents with the value of specified field in radius of given spatial circle. /// - /// Property selector for the field. + /// Property selector for the spatial field to get the value from. /// Radius (measured in units passed to radiusUnits parameter) in which matches should be found. - /// Latitude pointing to a circle center. - /// Longitude pointing to a circle center. - /// Units that will be used to measure distances (Kilometers, Miles). + /// Latitude of a circle center. + /// Longitude of a circle center. + /// Units that the radius was measured in (kilometers or miles). TSelf WithinRadiusOf(Expression> propertySelector, double radius, double latitude, double longitude, SpatialUnits? radiusUnits = null, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// - /// Filter matches to be inside the specified radius + /// Matches documents with the value of specified field in radius of given spatial circle. /// - /// Spatial field name. + /// Name of the spatial field to get the value from. /// Radius (measured in units passed to radiusUnits parameter) in which matches should be found. - /// Latitude pointing to a circle center. - /// Longitude pointing to a circle center. - /// Units that will be used to measure distances (Kilometers, Miles). + /// Latitude of a circle center. + /// Longitude of a circle center. + /// Units that the radius was measured in (kilometers or miles). TSelf WithinRadiusOf(string fieldName, double radius, double latitude, double longitude, SpatialUnits? radiusUnits = null, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); - /// - /// Filter matches based on a given shape - only documents with the shape defined in fieldName that - /// have a relation rel with the given shapeWkt will be returned - /// - /// Property selector for the field. - /// WKT formatted shape - /// Spatial relation to check (Within, Contains, Disjoint, Intersects, Nearby) - /// The allowed error percentage. By default: 0.025 + /// TSelf RelatesToShape(Expression> propertySelector, string shapeWkt, SpatialRelation relation, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// - /// Filter matches based on a given shape - only documents with the shape defined in fieldName that - /// have a relation rel with the given shapeWkt will be returned + /// Matches documents with the value of specified field in relation with the provided WKT shape. /// - /// Property selector for the field. - /// WKT formatted shape - /// Spatial relation to check (Within, Contains, Disjoint, Intersects, Nearby) - /// Units to be used - /// The allowed error percentage. By default: 0.025 + /// Property selector for the spatial field to get the value from. + /// String representing the WKT shape. + /// Spatial relation to check (Within, Contains, Disjoint, Intersects). + /// Units to be used (kilometers or miles). + /// Allowed error percentage. Default: 0.025. TSelf RelatesToShape(Expression> propertySelector, string shapeWkt, SpatialRelation relation, SpatialUnits units, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); - /// - /// Filter matches based on a given shape - only documents with the shape defined in fieldName that - /// have a relation rel with the given shapeWkt will be returned - /// - /// Spatial field name. - /// WKT formatted shape - /// Spatial relation to check (Within, Contains, Disjoint, Intersects, Nearby) - /// The allowed error percentage. By default: 0.025 + /// TSelf RelatesToShape(string fieldName, string shapeWkt, SpatialRelation relation, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// - /// Filter matches based on a given shape - only documents with the shape defined in fieldName that - /// have a relation rel with the given shapeWkt will be returned + /// Matches documents with the value of specified field in relation with the provided WKT shape. /// - /// Spatial field name. - /// WKT formatted shape - /// Spatial relation to check (Within, Contains, Disjoint, Intersects, Nearby) - /// Units to be used - /// The allowed error percentage. By default: 0.025 + /// Spatial field name to get the value from. + /// String representing the WKT shape. + /// Spatial relation (Within, Contains, Disjoint, Intersects). + /// Units to be used (kilometers or miles). + /// Allowed error percentage. Default: 0.025. TSelf RelatesToShape(string fieldName, string shapeWkt, SpatialRelation relation, SpatialUnits units, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// - /// Ability to use one factory to determine spatial shape that will be used in query. + /// Matches documents based on provided spatial criteria created by factory. /// - /// Spatial field name. - /// function with spatial criteria factory + /// Path to the spatial field to get value from. + /// Function creating spatial criteria. TSelf Spatial(Expression> path, Func clause); /// - /// Ability to use one factory to determine spatial shape that will be used in query. + /// Matches documents based on provided spatial criteria created by factory. /// - /// Spatial field name. - /// function with spatial criteria factory + /// Name of spatial field to get value from. + /// Function creating spatial criteria. TSelf Spatial(string fieldName, Func clause); + /// + /// Matches documents based on provided spatial criteria created by factory. + /// + /// Dynamic spatial field to get value from. + /// Function creating spatial criteria. TSelf Spatial(DynamicSpatialField field, Func clause); + /// + /// Matches documents based on provided factories. + /// + /// Function creating dynamic spatial field using values from chosen fields. + /// Function creating spatial criteria. TSelf Spatial(Func, DynamicSpatialField> field, Func clause); /// - /// Specify MoreLikeThisQuery. + /// Specified MoreLikeThisQuery. TSelf MoreLikeThis(MoreLikeThisBase moreLikeThis); - } public interface IGroupByDocumentQueryBase where TSelf : IDocumentQueryBase From d596e3b507dfa2d8bf21662aaafef38a13600a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Le=C5=9Bniak?= Date: Fri, 12 Apr 2024 15:27:42 +0200 Subject: [PATCH 041/243] RavenDB-21996 Fixed warnings --- .../Documents/Session/IDocumentQueryBase.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs b/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs index 489c46d2ac19..4078a24fef6d 100644 --- a/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs +++ b/src/Raven.Client/Documents/Session/IDocumentQueryBase.cs @@ -94,7 +94,7 @@ public interface IFilterDocumentQueryBase where TSelf : IDocumentQuery /// /// Matches documents with value of chosen field matching searched terms. /// - /// Property selector for the field that searched terms will be checked against. + /// Property selector for the field that searched terms will be checked against. /// Space separated terms to search. If there is more than a single term, each of them will be checked independently. /// Operator to be used for relationship between terms. Default: Or. TSelf Search(Expression> propertySelector, string searchTerms, SearchOperator @operator = SearchOperator.Or); @@ -176,7 +176,7 @@ public interface IFilterDocumentQueryBase where TSelf : IDocumentQuery /// /// Matches documents with value of the chosen field equal to the specified value. /// - /// Name of the field to get value from. + /// Name of the field to get value from. /// /// Specifies if comparison is case sensitive. Default: false. TSelf WhereEquals(Expression> propertySelector, TValue value, bool exact = false); @@ -369,6 +369,7 @@ public interface IFilterDocumentQueryBase where TSelf : IDocumentQuery /// Latitude of a circle center. /// Longitude of a circle center. /// Units that the radius was measured in (kilometers or miles). + /// Allowed error percentage. Default: 0.025. TSelf WithinRadiusOf(Expression> propertySelector, double radius, double latitude, double longitude, SpatialUnits? radiusUnits = null, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// @@ -379,9 +380,10 @@ public interface IFilterDocumentQueryBase where TSelf : IDocumentQuery /// Latitude of a circle center. /// Longitude of a circle center. /// Units that the radius was measured in (kilometers or miles). + /// /// Allowed error percentage. Default: 0.025. TSelf WithinRadiusOf(string fieldName, double radius, double latitude, double longitude, SpatialUnits? radiusUnits = null, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); - /// + /// TSelf RelatesToShape(Expression> propertySelector, string shapeWkt, SpatialRelation relation, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// @@ -394,7 +396,7 @@ public interface IFilterDocumentQueryBase where TSelf : IDocumentQuery /// Allowed error percentage. Default: 0.025. TSelf RelatesToShape(Expression> propertySelector, string shapeWkt, SpatialRelation relation, SpatialUnits units, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); - /// + /// TSelf RelatesToShape(string fieldName, string shapeWkt, SpatialRelation relation, double distanceErrorPct = Constants.Documents.Indexing.Spatial.DefaultDistanceErrorPct); /// From b21172c3750220580d7b3fb7dfb06b8c9c354ec3 Mon Sep 17 00:00:00 2001 From: shiranshalom Date: Sun, 14 Apr 2024 13:01:00 +0300 Subject: [PATCH 042/243] RavenDB-21983 - Update our Client API XML documentation [IAbstractTimeSeriesIncludeBuilder, ITimeSeriesIncludeBuilder] --- src/Raven.Client/DocumentationUrls.cs | 3 ++ .../Session/Loaders/IncludeBuilder.cs | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/Raven.Client/DocumentationUrls.cs b/src/Raven.Client/DocumentationUrls.cs index 6d562f3a4322..b848a2c1f325 100644 --- a/src/Raven.Client/DocumentationUrls.cs +++ b/src/Raven.Client/DocumentationUrls.cs @@ -164,6 +164,9 @@ internal static class TimeSeries /// public const string Include = nameof(Include); + /// + public const string IncludeWithQuery = nameof(IncludeWithQuery); + /// public const string ClientApi = nameof(ClientApi); diff --git a/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs b/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs index 48cb5232f284..8947306982fb 100644 --- a/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs +++ b/src/Raven.Client/Documents/Session/Loaders/IncludeBuilder.cs @@ -76,18 +76,42 @@ public interface ITimeSeriesIncludeBuilder ITimeSeriesIncludeBuilder IncludeDocument(); } + /// + /// The server is instructed to pre-load referenced documents concurrently with retrieving the time series data.
+ /// The time series results are added to the session unit of work, and subsequent requests to load them are served directly from the session cache, + /// without requiring any additional requests to the server. + /// + /// public interface IAbstractTimeSeriesIncludeBuilder { + /// + /// The name of the time series to include. + /// Indicates how to retrieve the time series entries. + /// When set to 'Last', retrieves entries from the end of the time series within the specified time range.
+ /// Note that cannot be 'None' when time is specified. + /// The time range to consider when retrieving time series entries. TBuilder IncludeTimeSeries(string name, TimeSeriesRangeType type, TimeValue time); + /// + /// The name of the time series to include. + /// Indicates how to retrieve the time series entries. + /// When set to 'Last', retrieves the last X entries, where X is determined by the 'count' parameter.
+ /// Note that cannot be 'None' when count is specified. + /// The maximum number of entries to take when retrieving time series entries. TBuilder IncludeTimeSeries(string name, TimeSeriesRangeType type, int count); + /// + /// The names of the time series to include. TBuilder IncludeTimeSeries(string[] names, TimeSeriesRangeType type, TimeValue time); + /// + /// The names of the time series to include. TBuilder IncludeTimeSeries(string[] names, TimeSeriesRangeType type, int count); + /// TBuilder IncludeAllTimeSeries(TimeSeriesRangeType type, TimeValue time); + /// TBuilder IncludeAllTimeSeries(TimeSeriesRangeType type, int count); } @@ -101,9 +125,14 @@ public interface IRevisionIncludeBuilder public interface ITimeSeriesIncludeBuilder : IAbstractTimeSeriesIncludeBuilder { + /// + /// The name of the time series to include. + /// The date and time from which to start including time series entries (inclusive). If not specified, the collection will start from the earliest possible date and time (DateTime.MinValue). + /// The date and time at which to stop including time series entries (inclusive). If not specified, the collection will continue until the latest possible date and time (DateTime.MaxValue). TBuilder IncludeTimeSeries(string name, DateTime? from = null, DateTime? to = null); } + /// public interface ISubscriptionTimeSeriesIncludeBuilder : IAbstractTimeSeriesIncludeBuilder { } From 9ab2852acc961be52f0171bb69e9d54179a76d37 Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Fri, 12 Jan 2024 14:46:39 +0100 Subject: [PATCH 043/243] RavenDB-16865 Prepare CreateDatabase main component --- .../common/CreateDatabase.stories.tsx | 63 -- .../resources/databases/DatabasesPage.tsx | 29 +- .../partials/create}/CreateDatabase.scss | 0 .../create/CreateDatabase.stories.tsx | 15 + .../partials/create/CreateDatabase.tsx | 44 + .../formBackup/CreateDatabaseFromBackup.tsx} | 746 +-------------- .../create/regular/CreateDatabaseRegular.tsx | 884 ++++++++++++++++++ .../createDatabaseRegularValidation.ts | 56 ++ 8 files changed, 1049 insertions(+), 788 deletions(-) delete mode 100644 src/Raven.Studio/typescript/components/common/CreateDatabase.stories.tsx rename src/Raven.Studio/typescript/components/{common => pages/resources/databases/partials/create}/CreateDatabase.scss (100%) create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/CreateDatabase.stories.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/CreateDatabase.tsx rename src/Raven.Studio/typescript/components/{common/CreateDatabase.tsx => pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx} (53%) create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts diff --git a/src/Raven.Studio/typescript/components/common/CreateDatabase.stories.tsx b/src/Raven.Studio/typescript/components/common/CreateDatabase.stories.tsx deleted file mode 100644 index 36a36b5cc8e4..000000000000 --- a/src/Raven.Studio/typescript/components/common/CreateDatabase.stories.tsx +++ /dev/null @@ -1,63 +0,0 @@ -import { Meta } from "@storybook/react"; - -import React, { useState } from "react"; -import { withBootstrap5, withStorybookContexts } from "test/storybookTestUtils"; -import { Button, Card } from "reactstrap"; -import { CreateDatabase } from "./CreateDatabase"; -import { Icon } from "./Icon"; -import { boundCopy } from "components/utils/common"; - -export default { - title: "Bits/CreateDatabase", - component: CreateDatabase, - decorators: [withStorybookContexts, withBootstrap5], - argTypes: { - serverAuthentication: { - control: { - type: "boolean", - }, - }, - createDatabaseModal: { - control: "none", - }, - }, -} satisfies Meta; - -const TemplatePanel = (args: { - serverAuthentication: boolean; - licenseProps: { encryption: boolean; sharding: boolean; dynamicDatabaseDistribution: boolean }; -}) => { - const [createDatabaseModal, setCreateDatabaseModal] = useState(true); - const toggleCreateDatabase = () => setCreateDatabaseModal(!createDatabaseModal); - - return ( - <> - -
- -
- - - - -
-
- - ); -}; - -export const Panel = boundCopy(TemplatePanel, { - serverAuthentication: true, - licenseProps: { - encryption: true, - sharding: true, - dynamicDatabaseDistribution: true, - }, -}); diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx index 63f3be883f4b..691912d66659 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx @@ -3,7 +3,7 @@ import { DatabasePanel } from "./partials/DatabasePanel"; import { DatabasesSelectActions } from "./partials/DatabasesSelectActions"; import { DatabasesFilter } from "./partials/DatabasesFilter"; import { NoDatabases } from "./partials/NoDatabases"; -import { Button, ButtonGroup, DropdownItem, DropdownMenu, DropdownToggle, UncontrolledDropdown } from "reactstrap"; +import { Button } from "reactstrap"; import { useAppDispatch, useAppSelector } from "components/store"; import router from "plugins/router"; import appUrl from "common/appUrl"; @@ -13,7 +13,6 @@ import { DatabaseFilterCriteria } from "components/models/databases"; import { compactDatabase, loadDatabasesDetails, - openCreateDatabaseDialog, openCreateDatabaseFromRestoreDialog, syncDatabaseDetails, } from "components/pages/resources/databases/store/databasesViewActions"; @@ -22,6 +21,8 @@ import { databasesViewSelectors } from "components/pages/resources/databases/sto import { StickyHeader } from "components/common/StickyHeader"; import { Icon } from "components/common/Icon"; import { accessManagerSelectors } from "components/common/shell/accessManagerSlice"; +import useBoolean from "components/hooks/useBoolean"; +import CreateDatabase from "./partials/create/CreateDatabase"; interface DatabasesPageProps { activeDatabase?: string; @@ -109,26 +110,20 @@ export function DatabasesPage(props: DatabasesPageProps) { const selectedDatabases = databases.filter((x) => selectedDatabaseNames.includes(x.name)); + const { value: isCreateDatabaseOpen, toggle: toggleIsCreateDatabaseOpen } = useBoolean(false); + return ( <>
{isOperatorOrAbove && ( - - - - - - - - dispatch(openCreateDatabaseFromRestoreDialog())}> - New database from backup (Restore) - - - + <> + + {isCreateDatabaseOpen && } + )} {showToggleButton && ( ) : ( )} - {showQuickCreate && ( - <> - - - - - - - Encryption - - {encryptionEnabled ? ( - ON - ) : ( - OFF - )} - - - - - Replication - - {replicationFactor > 1 ? ( - ON - ) : ( - OFF - )} - - - - - Sharding - - {shardingEnabled ? ( - ON - ) : ( - OFF - )} - - - {manualNodeSelection && ( - - - Manual node selection - - ON - - )} - - - - {useDefaultPaths ? ( - <> - Default paths - - ) : ( - <> - Custom{" "} - paths - - )} - - - - - - )} - {isLastStep ? ( )} - - ); -} - -interface StepCreateNewProps { - encryptionEnabled: boolean; - toggleEncryption: () => void; - serverAuthentication: boolean; - licenseProps: licenseProps; -} - -export function StepCreateNew(props: StepCreateNewProps) { - const { encryptionEnabled, toggleEncryption, serverAuthentication, licenseProps } = props; - const newDatabaseImg = require("Content/img/createDatabase/new-database.svg"); - return ( -
-
- -
-

Create new database

- - - - - - -
- {licenseProps.encryption ? ( - -

- Authentication is off -

-

- Encryption at Rest is only possible when authentication is - enabled and a server certificate has been defined. -

-

- For more information go to the certificates page -

- - } - > - - - - Encrypt at Rest - - - - ) : ( - - Storage encryption - - } - > - - - - Encrypt at Rest - - - - )} - - -
- - - Data will be encrypted at the storage engine layer, using XChaCha20-Poly1305{" "} - authenticated encryption algorithm. - - - - -
+ ); } -export function StepEncryption() { +function StepEncryption() { const encryptionImg = require("Content/img/createDatabase/encryption.svg"); const qrImg = require("Content/img/createDatabase/qr.jpg"); return ( @@ -515,408 +240,13 @@ export function StepEncryption() { ); } -interface StepReplicationAndShardingProps { - availableNodes: number; - manualNodeSelection: boolean; - toggleManualNodeSelection: () => void; - shardingEnabled: boolean; - setShardingEnabled: (value: boolean) => void; - toggleSharding: () => void; - licenseProps: licenseProps; - replicationFactor: number; - setReplicationFactor: (value: number) => void; - shardCount: number; - setShardCount: (value: number) => void; -} - -export function StepReplicationAndSharding(props: StepReplicationAndShardingProps) { - const { - availableNodes, - manualNodeSelection, - toggleManualNodeSelection, - shardingEnabled, - toggleSharding, - licenseProps, - replicationFactor, - setReplicationFactor, - shardCount, - setShardCount, - } = props; - - const shardingImg = require("Content/img/createDatabase/sharding.svg"); - - const handleReplicationFactorChange = (event: any) => { - setReplicationFactor(event.target.value); - }; - const handleShardCountChange = (event: any) => { - setShardCount(event.target.value); - }; - - return ( -
-
- -
- -

Replication & Sharding

- - - -

- Database replication provides benefits such as improved data availability, increased - scalability, and enhanced disaster recovery capabilities. -

- - - - - - - Available nodes:{" "} - {availableNodes} - - - -
- - What is sharding? - -
-
- - Sharding - - } - className="d-inline-block" - > - toggleSharding()} - color="shard" - disabled={!licenseProps.sharding} - className="mt-1" - > - Enable{" "} - - Sharding - - - -
- - - - - -
- Add more{" "} - - Instance nodes - {" "} - in Manage cluster view -
- - - - - -

- - Sharding - {" "} - is a database partitioning technique that breaks up large databases into smaller, more - manageable pieces called{" "} - - {" "} - - shards - - . -

-

- Each shard contains a subset of the data and can be stored on a separate server, allowing for{" "} - horizontal scalability and improved performance. -

- - Learn more TODO - - - - - - - - - Replication Factor - - - - - - - - - - - - Number of shards - - - - - - - - - - - <> - Data will be divided into{" "} - - {shardCount} - Shards - - .
- - - {replicationFactor > 1 ? ( - <> - {shardingEnabled ? <>Each shard : <>Data} will be replicated to{" "} - - {replicationFactor} Nodes - - . - - ) : ( - <>Data won't be replicated. - )} - - - - - - - {licenseProps.dynamicDatabaseDistribution ? ( - 1} - message="Replication factor is set to 1" - className="d-inline-block" - > - - Allow dynamic database distribution -
- Maintain replication factor upon node failure - - - ) : ( - - - Allow dynamic database distribution -
- Maintain replication factor upon node failure - - - )} - - - - Set replication nodes manually -
- Select nodes from the list in the next step - - - -
- ); -} - -interface StepNodeSelectionProps { - nodeList: databaseLocationSpecifier[]; - shardCount: number; - replicationFactor: number; -} - -type destinationNode = { - id: string; - node: string; -}; - -type shardReplicas = destinationNode[]; - -export function StepNodeSelection(props: StepNodeSelectionProps) { - const { nodeList, shardCount, replicationFactor } = props; - - const initialNodes: shardReplicas[] = []; - - for (let i = 0; i < shardCount; i++) { - initialNodes.push([]); - for (let j = 0; j < replicationFactor; j++) { - initialNodes[i].push({ id: "s" + i + "r" + j, node: null }); - } - } - - const [shardNodes] = useState(initialNodes); - - function updateShardNodes(): () => void { - console.log("TODO Update selected node"); - return; - } - - // TODO: @kalczur - naming when we will implement the new CreateDatabase view - - const allActionContexts = ActionContextUtils.getContexts(nodeList); - const [selectedActionContexts, setSelectedActionContexts] = useState(allActionContexts); - - return ( -
-

Manual Node Selection

- -
- -
- - - - - {shardCount > 1 && - ); - })} - - - - - {shardNodes.map((shard, index) => { - return ( - - {shardCount > 1 && ( - - )} - - {shard.map((replica) => { - return ( - - ); - })} - - ); - })} - -
} - {shardNodes[0].map((replica, index) => { - return ( - - Replica {index + 1} -
- {index} - - {replica.node} - dbLocation.nodeTag)} - id={replica.id} - destinationNode={replica.node} - handleUpdate={updateShardNodes()} - > -
- - -

Orchestrators

-
- minimum 1 -
- -
- ); -} -interface NodeSelectionDropdownProps { - nodeList: string[]; - id: string; - destinationNode: string; - handleUpdate: () => void; -} - -export function NodeSelectionDropdown(props: NodeSelectionDropdownProps) { - const { nodeList, destinationNode, handleUpdate } = props; - return ( - <> - - - {destinationNode == null ? ( - <>select - ) : ( - <> - {destinationNode} - - )} - - - {nodeList.map((nodeTag) => ( - handleUpdate()}> - {nodeTag} - - ))} - - None - - - - - ); -} - interface StepPathsProps { nodeList: string[]; useDefaultPaths: boolean; toggleUseDefaultPaths: () => void; } -export function StepPaths(props: StepPathsProps) { +function StepPaths(props: StepPathsProps) { const { nodeList, useDefaultPaths, toggleUseDefaultPaths } = props; return ( @@ -964,7 +294,7 @@ interface StepCreateFromBackupProps { setShardingEnabled: (value: boolean) => void; } -export function StepCreateFromBackup(props: StepCreateFromBackupProps) { +function StepCreateFromBackup(props: StepCreateFromBackupProps) { const { shardingEnabled, setShardingEnabled, licenseIncludesSharding } = props; const fromBackupImg = require("Content/img/createDatabase/from-backup.svg"); @@ -1039,7 +369,7 @@ interface StepBackupSourceProps { serverAuthentication: boolean; } -export function StepBackupSource(props: StepBackupSourceProps) { +function StepBackupSource(props: StepBackupSourceProps) { const { backupSource, setBackupSource, @@ -1214,7 +544,7 @@ export function StepBackupSource(props: StepBackupSourceProps) {
{/* TODO: Lock encryption when the source file is encrypted */} - {licenseProps.encryption ? ( + {licenseProps?.encryption ? ( ) : ( Storage @@ -1258,7 +588,7 @@ export function StepBackupSource(props: StepBackupSourceProps) { color="primary" selected={encryptionEnabled} toggleSelection={toggleEncryption} - disabled={!licenseProps.encryption} + disabled={!licenseProps?.encryption} > Encrypt at Rest @@ -1278,7 +608,7 @@ interface RestorPointSelectorProps { className?: string; restorePoint?: string; } -export function RestorePointSelector(props: RestorPointSelectorProps) { +function RestorePointSelector(props: RestorPointSelectorProps) { const { className, restorePoint } = props; return ( @@ -1298,7 +628,7 @@ interface BackupSourceFragmentLocalProps { nodeList: string[]; } -export function BackupSourceFragmentLocal(props: BackupSourceFragmentLocalProps) { +function BackupSourceFragmentLocal(props: BackupSourceFragmentLocalProps) { const { shardingEnabled, nodeList } = props; type localBackupSource = { @@ -1385,7 +715,7 @@ export function BackupSourceFragmentLocal(props: BackupSourceFragmentLocalProps) ); } -export function BackupSourceFragmentCloud() { +function BackupSourceFragmentCloud() { return (
@@ -1418,7 +748,7 @@ export function BackupSourceFragmentCloud() { ); } -export function BackupSourceFragmentAws() { +function BackupSourceFragmentAws() { const [useCustomHost, setUseCustomHost] = useState(false); const toggleUseCustomHost = () => { setUseCustomHost(!useCustomHost); @@ -1508,7 +838,7 @@ export function BackupSourceFragmentAws() { ); } -export function BackupSourceFragmentAzure() { +function BackupSourceFragmentAzure() { return (
@@ -1552,7 +882,7 @@ export function BackupSourceFragmentAzure() { ); } -export function BackupSourceFragmentGcp() { +function BackupSourceFragmentGcp() { return (
diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx new file mode 100644 index 000000000000..c6f8e3e11fe3 --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx @@ -0,0 +1,884 @@ +import React, { useState } from "react"; +import { + Alert, + Button, + CloseButton, + Col, + Collapse, + DropdownItem, + DropdownMenu, + DropdownToggle, + FormGroup, + Input, + InputGroup, + InputGroupText, + Label, + ModalBody, + ModalFooter, + PopoverBody, + Row, + Table, + UncontrolledDropdown, + UncontrolledPopover, +} from "reactstrap"; +import { Checkbox, Switch } from "../../../../../../common/Checkbox"; +import { FlexGrow } from "../../../../../../common/FlexGrow"; +import { Icon } from "../../../../../../common/Icon"; +import { PropSummary, PropSummaryItem, PropSummaryName, PropSummaryValue } from "../../../../../../common/PropSummary"; +import { Steps } from "../../../../../../common/Steps"; +import { LicenseRestrictions } from "../../../../../../common/LicenseRestrictions"; +import { + DatabaseActionContexts, + MultipleDatabaseLocationSelector, +} from "../../../../../../common/MultipleDatabaseLocationSelector"; +import ActionContextUtils from "components/utils/actionContextUtils"; + +interface CreateDatabaseRegularProps { + closeModal: () => void; + changeCreateModeToBackup: () => void; +} + +type licenseProps = { + encryption: boolean; + sharding: boolean; + dynamicDatabaseDistribution: boolean; +}; + +type StepId = "createNew" | "encryption" | "replicationAndSharding" | "nodeSelection" | "paths"; + +interface StepItem { + id: StepId; + label: string; + active: boolean; +} + +export default function CreateDatabaseRegular(props: CreateDatabaseRegularProps) { + const { closeModal, changeCreateModeToBackup } = props; + + const isServerAuthenticationEnabled = true; // TODO get from store + const [encryptionEnabled, setEncryptionEnabled] = useState(false); + + const toggleEncryption = () => { + setEncryptionEnabled(!encryptionEnabled); + }; + + const [replicationFactor, setReplicationFactor] = useState(1); + + const [shardingEnabled, setShardingEnabled] = useState(false); + const [shardCount, setShardCount] = useState(1); + const toggleSharding = () => { + setShardingEnabled(!shardingEnabled); + }; + + const [manualNodeSelection, setManualNodeSelection] = useState(false); + + const toggleManualNodeSelection = () => { + setManualNodeSelection(!manualNodeSelection); + }; + + const nodeList: databaseLocationSpecifier[] = [ + { nodeTag: "A" }, + { nodeTag: "B" }, + { nodeTag: "C" }, + { nodeTag: "D" }, + { nodeTag: "DEV" }, + ]; + + const [useDefaultPaths, setUseDefaultPaths] = useState(true); + const toggleUseDefaultPaths = () => { + setUseDefaultPaths(!useDefaultPaths); + }; + + const [currentStep, setCurrentStep] = useState(0); + + const stepsList: StepItem[] = [ + { id: "createNew", label: "Name", active: true }, + { + id: "encryption", + label: "Encryption", + active: encryptionEnabled, + }, + { + id: "replicationAndSharding", + label: "Replication & Sharding", + active: true, + }, + { + id: "nodeSelection", + label: "Manual Node Selection", + active: manualNodeSelection, + }, + { id: "paths", label: "Paths Configuration", active: true }, + ]; + + const activeSteps = stepsList.filter((step) => step.active); + + const isLastStep = activeSteps.length - 2 < currentStep; + const isFirstStep = currentStep < 1; + const showQuickCreate = !isLastStep; + + const goToStep = (stepNum: number) => { + setCurrentStep(stepNum); + }; + + const nextStep = () => { + if (!isLastStep) setCurrentStep(currentStep + 1); + }; + + const prevStep = () => { + if (!isFirstStep) setCurrentStep(currentStep - 1); + }; + + const stepViews: Record = { + createNew: ( + + ), + encryption: , + replicationAndSharding: ( + + ), + nodeSelection: ( + + ), + paths: ( + node.nodeTag)} + useDefaultPaths={useDefaultPaths} + toggleUseDefaultPaths={toggleUseDefaultPaths} + /> + ), + }; + + return ( + <> + +
+ step.label)} + onClick={goToStep} + className="flex-grow me-4" + > + +
+ {stepViews[activeSteps[currentStep].id]} + + + + {isFirstStep ? ( + + ) : ( + + )} + + {showQuickCreate && ( + <> + + + + + + + Encryption + + {encryptionEnabled ? ( + ON + ) : ( + OFF + )} + + + + + Replication + + {replicationFactor > 1 ? ( + ON + ) : ( + OFF + )} + + + + + Sharding + + {shardingEnabled ? ( + ON + ) : ( + OFF + )} + + + {manualNodeSelection && ( + + + Manual node selection + + ON + + )} + + + + {useDefaultPaths ? ( + <> + Default paths + + ) : ( + <> + Custom{" "} + paths + + )} + + + + + + )} + + {isLastStep ? ( + + ) : ( + + )} + + + ); +} + +interface StepCreateNewProps { + encryptionEnabled: boolean; + toggleEncryption: () => void; + serverAuthentication: boolean; + licenseProps: licenseProps; +} + +function StepCreateNew(props: StepCreateNewProps) { + const { encryptionEnabled, toggleEncryption, serverAuthentication, licenseProps } = props; + const newDatabaseImg = require("Content/img/createDatabase/new-database.svg"); + return ( +
+
+ +
+

Create new database

+ + + + + + +
+ {licenseProps?.encryption ? ( + +

+ Authentication is off +

+

+ Encryption at Rest is only possible when authentication is + enabled and a server certificate has been defined. +

+

+ For more information go to the certificates page +

+ + } + > + + + + Encrypt at Rest + + + + ) : ( + + Storage encryption + + } + > + + + + Encrypt at Rest + + + + )} + + +
+ + + Data will be encrypted at the storage engine layer, using XChaCha20-Poly1305{" "} + authenticated encryption algorithm. + + + + +
+ ); +} + +function StepEncryption() { + const encryptionImg = require("Content/img/createDatabase/encryption.svg"); + const qrImg = require("Content/img/createDatabase/qr.jpg"); + return ( +
+
+ +
+ +

Encryption at Rest

+ + + +
Key (Base64 Encoding)
+ + + + + + + + + + + + + + +
+ Save the key in a safe place. It will not be available again. If you lose this key you could + lose access to your data +
+ + + + +
+ + what's this? + +
+ + TODO: write info about qr code + + + + {/* TODO validate encryption key saved */} +
+ + I have saved the encryption key + +
+
+ ); +} + +interface StepReplicationAndShardingProps { + availableNodes: number; + manualNodeSelection: boolean; + toggleManualNodeSelection: () => void; + shardingEnabled: boolean; + setShardingEnabled: (value: boolean) => void; + toggleSharding: () => void; + licenseProps: licenseProps; + replicationFactor: number; + setReplicationFactor: (value: number) => void; + shardCount: number; + setShardCount: (value: number) => void; +} + +function StepReplicationAndSharding(props: StepReplicationAndShardingProps) { + const { + availableNodes, + manualNodeSelection, + toggleManualNodeSelection, + shardingEnabled, + toggleSharding, + licenseProps, + replicationFactor, + setReplicationFactor, + shardCount, + setShardCount, + } = props; + + const shardingImg = require("Content/img/createDatabase/sharding.svg"); + + const handleReplicationFactorChange = (event: any) => { + setReplicationFactor(event.target.value); + }; + const handleShardCountChange = (event: any) => { + setShardCount(event.target.value); + }; + + return ( +
+
+ +
+ +

Replication & Sharding

+ + + +

+ Database replication provides benefits such as improved data availability, increased + scalability, and enhanced disaster recovery capabilities. +

+ + + + + + + Available nodes:{" "} + {availableNodes} + + + +
+ + What is sharding? + +
+
+ + Sharding + + } + className="d-inline-block" + > + toggleSharding()} + color="shard" + disabled={!licenseProps?.sharding} + className="mt-1" + > + Enable{" "} + + Sharding + + + +
+ + + + + +
+ Add more{" "} + + Instance nodes + {" "} + in Manage cluster view +
+ + + + + +

+ + Sharding + {" "} + is a database partitioning technique that breaks up large databases into smaller, more + manageable pieces called{" "} + + {" "} + + shards + + . +

+

+ Each shard contains a subset of the data and can be stored on a separate server, allowing for{" "} + horizontal scalability and improved performance. +

+ + Learn more TODO + + + + + + + + + Replication Factor + + + + + + + + + + + + Number of shards + + + + + + + + + + + <> + Data will be divided into{" "} + + {shardCount} + Shards + + .
+ + + {replicationFactor > 1 ? ( + <> + {shardingEnabled ? <>Each shard : <>Data} will be replicated to{" "} + + {replicationFactor} Nodes + + . + + ) : ( + <>Data won't be replicated. + )} + + + + + + + {licenseProps?.dynamicDatabaseDistribution ? ( + 1} + message="Replication factor is set to 1" + className="d-inline-block" + > + + Allow dynamic database distribution +
+ Maintain replication factor upon node failure + + + ) : ( + + + Allow dynamic database distribution +
+ Maintain replication factor upon node failure + + + )} + + + + Set replication nodes manually +
+ Select nodes from the list in the next step + + + +
+ ); +} + +interface StepNodeSelectionProps { + nodeList: databaseLocationSpecifier[]; + shardCount: number; + replicationFactor: number; +} + +type destinationNode = { + id: string; + node: string; +}; + +type shardReplicas = destinationNode[]; + +function StepNodeSelection(props: StepNodeSelectionProps) { + const { nodeList, shardCount, replicationFactor } = props; + + const initialNodes: shardReplicas[] = []; + + for (let i = 0; i < shardCount; i++) { + initialNodes.push([]); + for (let j = 0; j < replicationFactor; j++) { + initialNodes[i].push({ id: "s" + i + "r" + j, node: null }); + } + } + + const [shardNodes] = useState(initialNodes); + + function updateShardNodes(): () => void { + console.log("TODO Update selected node"); + return; + } + + // TODO: @kalczur - naming when we will implement the new CreateDatabase view + + const allActionContexts = ActionContextUtils.getContexts(nodeList); + const [selectedActionContexts, setSelectedActionContexts] = useState(allActionContexts); + + return ( +
+

Manual Node Selection

+ +
+ +
+ + + + + {shardCount > 1 && + ); + })} + + + + + {shardNodes.map((shard, index) => { + return ( + + {shardCount > 1 && ( + + )} + + {shard.map((replica) => { + return ( + + ); + })} + + ); + })} + +
} + {shardNodes[0].map((replica, index) => { + return ( + + Replica {index + 1} +
+ {index} + + {replica.node} + dbLocation.nodeTag)} + id={replica.id} + destinationNode={replica.node} + handleUpdate={updateShardNodes()} + > +
+ + +

Orchestrators

+
+ minimum 1 +
+ +
+ ); +} +interface NodeSelectionDropdownProps { + nodeList: string[]; + id: string; + destinationNode: string; + handleUpdate: () => void; +} + +function NodeSelectionDropdown(props: NodeSelectionDropdownProps) { + const { nodeList, destinationNode, handleUpdate } = props; + return ( + <> + + + {destinationNode == null ? ( + <>select + ) : ( + <> + {destinationNode} + + )} + + + {nodeList.map((nodeTag) => ( + handleUpdate()}> + {nodeTag} + + ))} + + None + + + + + ); +} + +interface StepPathsProps { + nodeList: string[]; + useDefaultPaths: boolean; + toggleUseDefaultPaths: () => void; +} + +function StepPaths(props: StepPathsProps) { + const { nodeList, useDefaultPaths, toggleUseDefaultPaths } = props; + + return ( +
+

Paths Configuration

+ + + + Use default paths + + + + + + + + + + + + + + {nodeList.map((nodeTag) => ( + + + + + + + ))} + +
+ PathFree spaceTotal
+ + {nodeTag} + + /data/test24.45 GBytes29.40 GBytes
+
+ ); +} diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts new file mode 100644 index 000000000000..168c64e5655f --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts @@ -0,0 +1,56 @@ +import * as yup from "yup"; + +const basicInfoSchema = yup.object({ + databaseName: yup.string().nullable().required(), + isEncrypted: yup.boolean(), +}); + +const encryptionSchema = yup.object({ + isEncryptionKeySaved: yup.boolean().oneOf([true]), +}); + +const replicationAndShardingSchema = yup.object({ + replicationFactor: yup.number().integer().positive().required(), + isSharded: yup.boolean(), + shardsCount: yup + .number() + .nullable() + .when("isSharded", { + is: true, + then: (schema) => schema.integer().positive().required(), + }), + isDynamicDistribution: yup.boolean(), + isManualReplication: yup.boolean(), +}); + +const manualNodeSelectionSchema = yup.object({ + manualNodes: yup + .array() + .of(yup.string()) + .when("isManualReplication", { + is: true, + then: (schema) => schema.min(1), + }), + manualShard: yup.array().nullable().of(yup.array().of(yup.string())), +}); + +const pathsConfigurationsSchema = yup.object({ + isPathsConfigDefault: yup.boolean(), + pathsConfig: yup + .string() + .nullable() + .when("isPathsConfigDefault", { + is: true, + then: (schema) => schema.required(), + }), +}); + +export const createNewDatabaseSchema = yup + .object() + .concat(basicInfoSchema) + .concat(encryptionSchema) + .concat(replicationAndShardingSchema) + .concat(manualNodeSelectionSchema) + .concat(pathsConfigurationsSchema); + +export type CreateNewDatabaseFormData = yup.InferType; From 5c735eadc4b0c991777caa8a79f9bd7c992ff3c9 Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Mon, 15 Jan 2024 11:11:21 +0100 Subject: [PATCH 044/243] RavenDB-16865 Add isSecureServer to store --- .../create/formBackup/CreateDatabaseFromBackup.tsx | 13 ++++++++----- .../create/regular/CreateDatabaseRegular.tsx | 10 +++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx index 3590e03ea12d..f87e50844595 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx @@ -28,6 +28,8 @@ import { Icon } from "../../../../../../common/Icon"; import { Steps } from "../../../../../../common/Steps"; import { LicenseRestrictions } from "../../../../../../common/LicenseRestrictions"; import classNames from "classnames"; +import { accessManagerSelectors } from "components/common/shell/accessManagerSlice"; +import { useAppSelector } from "components/store"; interface CreateDatabaseFromBackupProps { closeModal: () => void; @@ -48,10 +50,11 @@ interface StepItem { active: boolean; } -export default function CreateDatabaseFromBackup(props: CreateDatabaseFromBackupProps) { - const { closeModal, changeCreateModeToRegular } = props; - - const isServerAuthenticationEnabled = true; // TODO get from store +export default function CreateDatabaseFromBackup({ + closeModal, + changeCreateModeToRegular, +}: CreateDatabaseFromBackupProps) { + const isSecureServer = useAppSelector(accessManagerSelectors.isSecureServer); const [encryptionEnabled, setEncryptionEnabled] = useState(false); const toggleEncryption = () => { @@ -125,7 +128,7 @@ export default function CreateDatabaseFromBackup(props: CreateDatabaseFromBackup encryptionEnabled={encryptionEnabled} toggleEncryption={toggleEncryption} licenseProps={null} - serverAuthentication={isServerAuthenticationEnabled} + serverAuthentication={isSecureServer} /> ), encryption: , diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx index c6f8e3e11fe3..54dbd43e5136 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx @@ -32,6 +32,8 @@ import { MultipleDatabaseLocationSelector, } from "../../../../../../common/MultipleDatabaseLocationSelector"; import ActionContextUtils from "components/utils/actionContextUtils"; +import { useAppSelector } from "components/store"; +import { accessManagerSelectors } from "components/common/shell/accessManagerSlice"; interface CreateDatabaseRegularProps { closeModal: () => void; @@ -52,10 +54,8 @@ interface StepItem { active: boolean; } -export default function CreateDatabaseRegular(props: CreateDatabaseRegularProps) { - const { closeModal, changeCreateModeToBackup } = props; - - const isServerAuthenticationEnabled = true; // TODO get from store +export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBackup }: CreateDatabaseRegularProps) { + const isSecureServer = useAppSelector(accessManagerSelectors.isSecureServer); const [encryptionEnabled, setEncryptionEnabled] = useState(false); const toggleEncryption = () => { @@ -134,7 +134,7 @@ export default function CreateDatabaseRegular(props: CreateDatabaseRegularProps) ), From 88a528551d93447ff0d8c21e57c27e87537d7394 Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Tue, 16 Jan 2024 11:15:34 +0100 Subject: [PATCH 045/243] RavenDB-16865 Add resources service and validateName method --- .../typescript/components/hooks/useServices.tsx | 3 +++ .../components/services/ResourcesService.ts | 7 +++++++ .../test/mocks/services/MockResourcesService.ts | 13 +++++++++++++ .../test/mocks/services/MockServices.ts | 3 +++ .../typescript/test/stubs/ResourcesStubs.ts | 15 +++++++++++++++ 5 files changed, 41 insertions(+) create mode 100644 src/Raven.Studio/typescript/components/services/ResourcesService.ts create mode 100644 src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts create mode 100644 src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts diff --git a/src/Raven.Studio/typescript/components/hooks/useServices.tsx b/src/Raven.Studio/typescript/components/hooks/useServices.tsx index e9cb9260d84c..ccd3ecc661ba 100644 --- a/src/Raven.Studio/typescript/components/hooks/useServices.tsx +++ b/src/Raven.Studio/typescript/components/hooks/useServices.tsx @@ -5,6 +5,7 @@ import * as React from "react"; import TasksService from "../services/TasksService"; import ManageServerService from "components/services/ManageServerService"; import LicenseService from "components/services/LicenseService"; +import ResourcesService from "components/services/ResourcesService"; export interface ServicesContextDto { indexesService: IndexesService; @@ -12,6 +13,7 @@ export interface ServicesContextDto { tasksService: TasksService; manageServerService: ManageServerService; licenseService: LicenseService; + resourcesService: ResourcesService; } export let services = { @@ -20,6 +22,7 @@ export let services = { tasksService: new TasksService(), manageServerService: new ManageServerService(), licenseService: new LicenseService(), + resourcesService: new ResourcesService(), }; export function configureMockServices(overloads: typeof services) { diff --git a/src/Raven.Studio/typescript/components/services/ResourcesService.ts b/src/Raven.Studio/typescript/components/services/ResourcesService.ts new file mode 100644 index 000000000000..3465aedd8ca1 --- /dev/null +++ b/src/Raven.Studio/typescript/components/services/ResourcesService.ts @@ -0,0 +1,7 @@ +import validateNameCommand from "commands/resources/validateNameCommand"; + +export default class ResourcesService { + async validateName(type: Raven.Server.Web.Studio.StudioTasksHandler.ItemType, name: string, dataPath?: string) { + return new validateNameCommand(type, name, dataPath).execute(); + } +} diff --git a/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts b/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts new file mode 100644 index 000000000000..54f1cad39e75 --- /dev/null +++ b/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts @@ -0,0 +1,13 @@ +import ResourcesService from "components/services/ResourcesService"; +import { AutoMockService, MockedValue } from "test/mocks/services/AutoMockService"; +import { ResourcesStubs } from "test/stubs/ResourcesStubs"; + +export default class MockResourcesService extends AutoMockService { + constructor() { + super(new ResourcesService()); + } + + withValidateNameCommand(dto?: MockedValue) { + return this.mockResolvedValue(this.mocks.validateName, dto, ResourcesStubs.validValidateName()); + } +} diff --git a/src/Raven.Studio/typescript/test/mocks/services/MockServices.ts b/src/Raven.Studio/typescript/test/mocks/services/MockServices.ts index 3c06f09cb7a8..aaa26b36218f 100644 --- a/src/Raven.Studio/typescript/test/mocks/services/MockServices.ts +++ b/src/Raven.Studio/typescript/test/mocks/services/MockServices.ts @@ -4,6 +4,7 @@ import MockDatabasesService from "./MockDatabasesService"; import MockTasksService from "./MockTasksService"; import MockManageServerService from "./MockManageServerService"; import MockLicenseService from "./MockLicenseService"; +import MockResourcesService from "test/mocks/services/MockResourcesService"; class MockServicesContainer { indexesService = new MockIndexesService(); @@ -11,6 +12,7 @@ class MockServicesContainer { tasksService = new MockTasksService(); manageServerService = new MockManageServerService(); licenseService = new MockLicenseService(); + resourcesService = new MockResourcesService(); get context(): ServicesContextDto { return { @@ -19,6 +21,7 @@ class MockServicesContainer { tasksService: this.tasksService.mock, manageServerService: this.manageServerService.mock, licenseService: this.licenseService.mock, + resourcesService: this.resourcesService.mock, }; } } diff --git a/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts b/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts new file mode 100644 index 000000000000..bc6f582a8fc3 --- /dev/null +++ b/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts @@ -0,0 +1,15 @@ +export class ResourcesStubs { + static validValidateName(): Raven.Client.Util.NameValidation { + return { + IsValid: true, + ErrorMessage: null, + }; + } + + static invalidValidateName(): Raven.Client.Util.NameValidation { + return { + IsValid: false, + ErrorMessage: "Invalid name", + }; + } +} From 3318b11f47eb3da4c45f78bed5b3ab6ae993675c Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Wed, 17 Jan 2024 15:25:47 +0100 Subject: [PATCH 046/243] RavenDB-16865 Prepare CreateDatabaseRegular --- .../resources/createDatabaseCommand.ts | 21 +- .../components/common/Steps.stories.tsx | 2 +- .../typescript/components/common/Steps.tsx | 56 +- .../common/shell/databaseSliceSelectors.ts | 4 + .../resources/databases/DatabasesPage.tsx | 3 +- .../create/CreateDatabase.stories.tsx | 24 +- .../formBackup/CreateDatabaseFromBackup.tsx | 4 +- .../create/regular/CreateDatabaseRegular.tsx | 1001 ++++------------- .../create/regular/QuickCreateButton.tsx | 92 ++ .../createDatabaseRegularValidation.ts | 73 +- .../CreateDatabaseRegularStepBasicInfo.tsx | 111 ++ .../CreateDatabaseRegularStepEncryption.tsx | 72 ++ ...CreateDatabaseRegularStepNodeSelection.tsx | 209 ++++ ...abaseRegularStepReplicationAndSharding.tsx | 234 ++++ .../create/shared/CreateDatabaseStepPaths.tsx | 157 +++ .../shared/createDatabaseSharedValidation.tsx | 11 + .../components/services/DatabasesService.ts | 31 +- .../components/services/ResourcesService.ts | 11 + .../mocks/services/MockResourcesService.ts | 12 + .../typescript/test/stubs/ResourcesStubs.ts | 31 + .../viewmodels/resources/createDatabase.ts | 2 +- 21 files changed, 1266 insertions(+), 895 deletions(-) create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/QuickCreateButton.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepEncryption.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepNodeSelection.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/CreateDatabaseStepPaths.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/createDatabaseSharedValidation.tsx diff --git a/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts b/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts index 68fa1f5bc328..918946711568 100644 --- a/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts +++ b/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts @@ -1,13 +1,24 @@ import commandBase = require("commands/commandBase"); import endpoints = require("endpoints"); -class createDatabaseCommand extends commandBase { - - private databaseDocument: Raven.Client.ServerWide.DatabaseRecord; +export type CreateDatabaseDto = Pick< + Raven.Client.ServerWide.DatabaseRecord, + "DatabaseName" | "Settings" | "Disabled" | "Encrypted" +> & { + Topology?: Pick; + Sharding?: { + Shards: Record; + Orchestrator: { + Topology: Pick + } + }; +}; +export class createDatabaseCommand extends commandBase { + private databaseDocument: CreateDatabaseDto; private replicationFactor: number; - constructor(databaseDocument: Raven.Client.ServerWide.DatabaseRecord, replicationFactor: number) { + constructor(databaseDocument: CreateDatabaseDto, replicationFactor: number) { super(); this.replicationFactor = replicationFactor; this.databaseDocument = databaseDocument; @@ -24,5 +35,3 @@ class createDatabaseCommand extends commandBase { .fail((response: JQueryXHR) => this.reportError("Failed to create database", response.responseText, response.statusText)); } } - -export = createDatabaseCommand; diff --git a/src/Raven.Studio/typescript/components/common/Steps.stories.tsx b/src/Raven.Studio/typescript/components/common/Steps.stories.tsx index d8298efdda1f..bcf348b90bda 100644 --- a/src/Raven.Studio/typescript/components/common/Steps.stories.tsx +++ b/src/Raven.Studio/typescript/components/common/Steps.stories.tsx @@ -1,5 +1,5 @@ import { Meta } from "@storybook/react"; -import { Steps } from "./Steps"; +import Steps from "./Steps"; import React, { useState } from "react"; import { withBootstrap5, withStorybookContexts } from "test/storybookTestUtils"; import { Button, Card } from "reactstrap"; diff --git a/src/Raven.Studio/typescript/components/common/Steps.tsx b/src/Raven.Studio/typescript/components/common/Steps.tsx index 24112cf3dbf5..996032b0b097 100644 --- a/src/Raven.Studio/typescript/components/common/Steps.tsx +++ b/src/Raven.Studio/typescript/components/common/Steps.tsx @@ -1,8 +1,8 @@ import * as React from "react"; import classNames from "classnames"; - import "./Steps.scss"; import { Icon } from "./Icon"; + interface StepsProps { current: number; steps: string[]; @@ -10,36 +10,36 @@ interface StepsProps { className?: string; } -export class Steps extends React.Component { - render() { - const { current, steps, onClick, className } = this.props; +// TODO show invalid state +export default function Steps(props: StepsProps) { + const { current, steps, onClick, className } = props; - const stepNodes = steps.map((stepName, i) => { - const classes = classNames({ - "steps-item": true, - done: i < current, - active: i === current, - }); - const stepItem = ( -
onClick(i)}> -
- - -
- {stepName} + const stepNodes = steps.map((stepName, i) => { + const classes = classNames({ + "steps-item": true, + done: i < current, + active: i === current, + }); + + const stepItem = ( +
onClick(i)}> +
+ +
- ); + {stepName} +
+ ); - const spacer =
; + const spacer =
; - return ( - - {stepItem} - {i !== steps.length - 1 && spacer} - - ); - }); + return ( + + {stepItem} + {i !== steps.length - 1 && spacer} + + ); + }); - return
{stepNodes}
; - } + return
{stepNodes}
; } diff --git a/src/Raven.Studio/typescript/components/common/shell/databaseSliceSelectors.ts b/src/Raven.Studio/typescript/components/common/shell/databaseSliceSelectors.ts index 3aaec7be2d45..8e30fb3acb21 100644 --- a/src/Raven.Studio/typescript/components/common/shell/databaseSliceSelectors.ts +++ b/src/Raven.Studio/typescript/components/common/shell/databaseSliceSelectors.ts @@ -1,6 +1,7 @@ import { RootState } from "components/store"; import DatabaseUtils from "components/utils/DatabaseUtils"; import { databasesSliceInternal } from "components/common/shell/databasesSlice"; +import { createSelector } from "@reduxjs/toolkit"; const selectActiveDatabaseName = (store: RootState) => store.databases.activeDatabaseName; @@ -8,6 +9,8 @@ const { databasesSelectors } = databasesSliceInternal; const selectAllDatabases = (store: RootState) => databasesSelectors.selectAll(store.databases.databases); +const selectAllDatabaseNames = createSelector(selectAllDatabases, (databases) => databases.map((x) => x.name)); + const selectAllDatabasesCount = (store: RootState) => databasesSelectors.selectTotal(store.databases.databases); function selectDatabaseByName(name: string) { @@ -35,6 +38,7 @@ export const databaseSelectors = { activeDatabaseName: selectActiveDatabaseName, activeDatabase: selectActiveDatabase, allDatabases: selectAllDatabases, + allDatabaseNames: selectAllDatabaseNames, allDatabasesCount: selectAllDatabasesCount, databaseByName: selectDatabaseByName, }; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx index 691912d66659..74713bb4a50f 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/DatabasesPage.tsx @@ -3,7 +3,7 @@ import { DatabasePanel } from "./partials/DatabasePanel"; import { DatabasesSelectActions } from "./partials/DatabasesSelectActions"; import { DatabasesFilter } from "./partials/DatabasesFilter"; import { NoDatabases } from "./partials/NoDatabases"; -import { Button } from "reactstrap"; +import { Button, ButtonGroup, DropdownItem, DropdownMenu, DropdownToggle, UncontrolledDropdown } from "reactstrap"; import { useAppDispatch, useAppSelector } from "components/store"; import router from "plugins/router"; import appUrl from "common/appUrl"; @@ -13,6 +13,7 @@ import { DatabaseFilterCriteria } from "components/models/databases"; import { compactDatabase, loadDatabasesDetails, + openCreateDatabaseDialog, openCreateDatabaseFromRestoreDialog, syncDatabaseDetails, } from "components/pages/resources/databases/store/databasesViewActions"; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/CreateDatabase.stories.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/CreateDatabase.stories.tsx index 15b54f64cd41..112ee17a5088 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/CreateDatabase.stories.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/CreateDatabase.stories.tsx @@ -1,15 +1,33 @@ -import { Meta } from "@storybook/react"; +import { Meta, StoryObj } from "@storybook/react"; import CreateDatabase from "./CreateDatabase"; import React from "react"; import { withBootstrap5, withStorybookContexts } from "test/storybookTestUtils"; +import { mockStore } from "test/mocks/store/MockStore"; +import { mockServices } from "test/mocks/services/MockServices"; +import { ResourcesStubs } from "test/stubs/ResourcesStubs"; export default { - title: "Pages/Databases/Create", + title: "Pages/Databases/Create Database/Create Database", decorators: [withStorybookContexts, withBootstrap5], } satisfies Meta; -export const Create = { +export const DefaultCreateDatabase: StoryObj = { + name: "Create Database", render: () => { + const { license, accessManager, cluster } = mockStore; + const { resourcesService } = mockServices; + + resourcesService.withValidateNameCommand(ResourcesStubs.invalidValidateName()); + resourcesService.withDatabaseLocation(); + resourcesService.withValidateNameCommand(); + + license.with_License({ + HasEncryption: true, + }); + + accessManager.with_isServerSecure(true); + cluster.with_Cluster(); + return null} />; }, }; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx index f87e50844595..6e1b08a4bf7b 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/CreateDatabaseFromBackup.tsx @@ -25,7 +25,7 @@ import { import { Checkbox, Switch } from "../../../../../../common/Checkbox"; import { FlexGrow } from "../../../../../../common/FlexGrow"; import { Icon } from "../../../../../../common/Icon"; -import { Steps } from "../../../../../../common/Steps"; +import Steps from "../../../../../../common/Steps"; import { LicenseRestrictions } from "../../../../../../common/LicenseRestrictions"; import classNames from "classnames"; import { accessManagerSelectors } from "components/common/shell/accessManagerSlice"; @@ -50,6 +50,8 @@ interface StepItem { active: boolean; } +// TODO google events + export default function CreateDatabaseFromBackup({ closeModal, changeCreateModeToRegular, diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx index 54dbd43e5136..651bb2e031d0 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx @@ -1,51 +1,36 @@ import React, { useState } from "react"; +import { Button, CloseButton, Form, ModalBody, ModalFooter } from "reactstrap"; +import { FlexGrow } from "components/common/FlexGrow"; +import { Icon } from "components/common/Icon"; +import Steps from "components/common/Steps"; +import { FormProvider, SubmitHandler, useForm, useWatch } from "react-hook-form"; +import { yupResolver } from "@hookform/resolvers/yup"; import { - Alert, - Button, - CloseButton, - Col, - Collapse, - DropdownItem, - DropdownMenu, - DropdownToggle, - FormGroup, - Input, - InputGroup, - InputGroupText, - Label, - ModalBody, - ModalFooter, - PopoverBody, - Row, - Table, - UncontrolledDropdown, - UncontrolledPopover, -} from "reactstrap"; -import { Checkbox, Switch } from "../../../../../../common/Checkbox"; -import { FlexGrow } from "../../../../../../common/FlexGrow"; -import { Icon } from "../../../../../../common/Icon"; -import { PropSummary, PropSummaryItem, PropSummaryName, PropSummaryValue } from "../../../../../../common/PropSummary"; -import { Steps } from "../../../../../../common/Steps"; -import { LicenseRestrictions } from "../../../../../../common/LicenseRestrictions"; -import { - DatabaseActionContexts, - MultipleDatabaseLocationSelector, -} from "../../../../../../common/MultipleDatabaseLocationSelector"; -import ActionContextUtils from "components/utils/actionContextUtils"; + CreateDatabaseRegularFormData as FormData, + useCreateDatabaseRegularSchema, +} from "./createDatabaseRegularValidation"; +import StepBasicInfo from "./steps/CreateDatabaseRegularStepBasicInfo"; +import StepEncryption from "./steps/CreateDatabaseRegularStepEncryption"; +import StepReplicationAndSharding from "./steps/CreateDatabaseRegularStepReplicationAndSharding"; +import StepNodeSelection from "./steps/CreateDatabaseRegularStepNodeSelection"; +import StepPaths from "../shared/CreateDatabaseStepPaths"; +import { DevTool } from "@hookform/devtools"; +import { databaseSelectors } from "components/common/shell/databaseSliceSelectors"; import { useAppSelector } from "components/store"; -import { accessManagerSelectors } from "components/common/shell/accessManagerSlice"; +import { CreateDatabaseDto } from "commands/resources/createDatabaseCommand"; +import { useServices } from "components/hooks/useServices"; +import { useAsyncCallback } from "react-async-hook"; +import databasesManager from "common/shell/databasesManager"; +import ButtonWithSpinner from "components/common/ButtonWithSpinner"; +import { clusterSelectors } from "components/common/shell/clusterSlice"; +import { tryHandleSubmit } from "components/utils/common"; +import QuickCreateButton from "components/pages/resources/databases/partials/create/regular/QuickCreateButton"; interface CreateDatabaseRegularProps { closeModal: () => void; changeCreateModeToBackup: () => void; } -type licenseProps = { - encryption: boolean; - sharding: boolean; - dynamicDatabaseDistribution: boolean; -}; - type StepId = "createNew" | "encryption" | "replicationAndSharding" | "nodeSelection" | "paths"; interface StepItem { @@ -54,39 +39,53 @@ interface StepItem { active: boolean; } -export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBackup }: CreateDatabaseRegularProps) { - const isSecureServer = useAppSelector(accessManagerSelectors.isSecureServer); - const [encryptionEnabled, setEncryptionEnabled] = useState(false); +// TODO google events - const toggleEncryption = () => { - setEncryptionEnabled(!encryptionEnabled); - }; +export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBackup }: CreateDatabaseRegularProps) { + const usedDatabaseNames = useAppSelector(databaseSelectors.allDatabases).map((db) => db.name); + const createNewDatabaseSchema = useCreateDatabaseRegularSchema(); + const allNodeTags = useAppSelector(clusterSelectors.allNodeTags); + + const form = useForm({ + mode: "all", + defaultValues: getDefaultValues(allNodeTags.length), + resolver: yupResolver(createNewDatabaseSchema), + context: { + usedDatabaseNames, + }, + }); + const { control, handleSubmit, formState } = form; - const [replicationFactor, setReplicationFactor] = useState(1); + if (formState.errors) { + console.log("kalczur errors", formState.errors); + } + const formValues = useWatch({ + control, + }); - const [shardingEnabled, setShardingEnabled] = useState(false); - const [shardCount, setShardCount] = useState(1); - const toggleSharding = () => { - setShardingEnabled(!shardingEnabled); - }; + const { databasesService } = useServices(); - const [manualNodeSelection, setManualNodeSelection] = useState(false); + const asyncCreateDatabase = useAsyncCallback( + async (dto: CreateDatabaseDto, replicationFactor: number) => { + return databasesService.createDatabase(dto, replicationFactor); + }, + { + onSuccess: () => { + closeModal(); + }, + } + ); - const toggleManualNodeSelection = () => { - setManualNodeSelection(!manualNodeSelection); - }; + const selectedOrchestrators = + formValues.isSharded && formValues.isManualReplication ? formValues.manualNodes : allNodeTags; - const nodeList: databaseLocationSpecifier[] = [ - { nodeTag: "A" }, - { nodeTag: "B" }, - { nodeTag: "C" }, - { nodeTag: "D" }, - { nodeTag: "DEV" }, - ]; + const onFinish: SubmitHandler = async (formValues) => { + tryHandleSubmit(async () => { + databasesManager.default.activateAfterCreation(formValues.databaseName); - const [useDefaultPaths, setUseDefaultPaths] = useState(true); - const toggleUseDefaultPaths = () => { - setUseDefaultPaths(!useDefaultPaths); + const dto = mapToDto(formValues, selectedOrchestrators); + await asyncCreateDatabase.execute(dto, formValues.replicationFactor); + }); }; const [currentStep, setCurrentStep] = useState(0); @@ -96,7 +95,7 @@ export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBa { id: "encryption", label: "Encryption", - active: encryptionEnabled, + active: formValues.isEncrypted, }, { id: "replicationAndSharding", @@ -106,779 +105,175 @@ export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBa { id: "nodeSelection", label: "Manual Node Selection", - active: manualNodeSelection, + active: formValues.isManualReplication, }, { id: "paths", label: "Paths Configuration", active: true }, ]; const activeSteps = stepsList.filter((step) => step.active); - - const isLastStep = activeSteps.length - 2 < currentStep; - const isFirstStep = currentStep < 1; - const showQuickCreate = !isLastStep; + const isFirstStep = currentStep === 0; + const isLastStep = currentStep === activeSteps.length - 1; const goToStep = (stepNum: number) => { setCurrentStep(stepNum); }; const nextStep = () => { - if (!isLastStep) setCurrentStep(currentStep + 1); + if (!isLastStep) { + setCurrentStep((step) => step + 1); + } }; const prevStep = () => { - if (!isFirstStep) setCurrentStep(currentStep - 1); + if (!isFirstStep) { + setCurrentStep((step) => step - 1); + } }; - const stepViews: Record = { - createNew: ( - - ), + const stepViews: Record = { + createNew: , encryption: , - replicationAndSharding: ( - - ), - nodeSelection: ( - - ), + replicationAndSharding: , + nodeSelection: , paths: ( node.nodeTag)} - useDefaultPaths={useDefaultPaths} - toggleUseDefaultPaths={toggleUseDefaultPaths} + databaseName={formValues.databaseName} + manualSelectedNodes={formValues.isManualReplication ? formValues.manualNodes : null} + isBackupFolder={false} /> ), }; return ( - <> - -
- step.label)} - onClick={goToStep} - className="flex-grow me-4" - > - -
- {stepViews[activeSteps[currentStep].id]} - - - - {isFirstStep ? ( - - ) : ( - - )} - - {showQuickCreate && ( - <> - - - - - - - Encryption - - {encryptionEnabled ? ( - ON - ) : ( - OFF - )} - - - - - Replication - - {replicationFactor > 1 ? ( - ON - ) : ( - OFF - )} - - - - - Sharding - - {shardingEnabled ? ( - ON - ) : ( - OFF - )} - - - {manualNodeSelection && ( - - - Manual node selection - - ON - - )} - - - - {useDefaultPaths ? ( - <> - Default paths - - ) : ( - <> - Custom{" "} - paths - - )} - - - - - - )} - - {isLastStep ? ( - - ) : ( - - )} - - - ); -} - -interface StepCreateNewProps { - encryptionEnabled: boolean; - toggleEncryption: () => void; - serverAuthentication: boolean; - licenseProps: licenseProps; -} - -function StepCreateNew(props: StepCreateNewProps) { - const { encryptionEnabled, toggleEncryption, serverAuthentication, licenseProps } = props; - const newDatabaseImg = require("Content/img/createDatabase/new-database.svg"); - return ( -
-
- -
-

Create new database

- - - - - - -
- {licenseProps?.encryption ? ( - -

- Authentication is off -

-

- Encryption at Rest is only possible when authentication is - enabled and a server certificate has been defined. -

-

- For more information go to the certificates page -

- - } - > - - - - Encrypt at Rest - - - - ) : ( - - Storage encryption - - } - > - - - - Encrypt at Rest - - - - )} - - + +
+ + +
+ step.label)} + onClick={goToStep} + className="flex-grow me-4" + > +
- - - Data will be encrypted at the storage engine layer, using XChaCha20-Poly1305{" "} - authenticated encryption algorithm. - - - - -
- ); -} - -function StepEncryption() { - const encryptionImg = require("Content/img/createDatabase/encryption.svg"); - const qrImg = require("Content/img/createDatabase/qr.jpg"); - return ( -
-
- -
- -

Encryption at Rest

- - - -
Key (Base64 Encoding)
- - - - - - - - - - - - - - -
- Save the key in a safe place. It will not be available again. If you lose this key you could - lose access to your data -
- - - - -
- - what's this? - -
- - TODO: write info about qr code - - - - {/* TODO validate encryption key saved */} -
- - I have saved the encryption key - -
-
- ); -} - -interface StepReplicationAndShardingProps { - availableNodes: number; - manualNodeSelection: boolean; - toggleManualNodeSelection: () => void; - shardingEnabled: boolean; - setShardingEnabled: (value: boolean) => void; - toggleSharding: () => void; - licenseProps: licenseProps; - replicationFactor: number; - setReplicationFactor: (value: number) => void; - shardCount: number; - setShardCount: (value: number) => void; -} - -function StepReplicationAndSharding(props: StepReplicationAndShardingProps) { - const { - availableNodes, - manualNodeSelection, - toggleManualNodeSelection, - shardingEnabled, - toggleSharding, - licenseProps, - replicationFactor, - setReplicationFactor, - shardCount, - setShardCount, - } = props; - - const shardingImg = require("Content/img/createDatabase/sharding.svg"); - - const handleReplicationFactorChange = (event: any) => { - setReplicationFactor(event.target.value); - }; - const handleShardCountChange = (event: any) => { - setShardCount(event.target.value); - }; - - return ( -
-
- -
- -

Replication & Sharding

- - - -

- Database replication provides benefits such as improved data availability, increased - scalability, and enhanced disaster recovery capabilities. -

- - - - - - - Available nodes:{" "} - {availableNodes} - - - -
- - What is sharding? - -
-
- - Sharding - - } - className="d-inline-block" - > - toggleSharding()} - color="shard" - disabled={!licenseProps?.sharding} - className="mt-1" - > - Enable{" "} - - Sharding - - - -
- - - - - -
- Add more{" "} - - Instance nodes - {" "} - in Manage cluster view -
- - - - - -

- - Sharding - {" "} - is a database partitioning technique that breaks up large databases into smaller, more - manageable pieces called{" "} - - {" "} - - shards - - . -

-

- Each shard contains a subset of the data and can be stored on a separate server, allowing for{" "} - horizontal scalability and improved performance. -

- - Learn more TODO - - - - - - - - - Replication Factor - - - - - - - - - - - - Number of shards - - - - - - - - - - - <> - Data will be divided into{" "} - - {shardCount} - Shards - - .
- - - {replicationFactor > 1 ? ( - <> - {shardingEnabled ? <>Each shard : <>Data} will be replicated to{" "} - - {replicationFactor} Nodes - - . - - ) : ( - <>Data won't be replicated. - )} - - - - - - - {licenseProps?.dynamicDatabaseDistribution ? ( - 1} - message="Replication factor is set to 1" - className="d-inline-block" + ) : ( + + )} + + {!isLastStep && } + {isLastStep ? ( + - - Allow dynamic database distribution -
- Maintain replication factor upon node failure - - + Finish + ) : ( - - - Allow dynamic database distribution -
- Maintain replication factor upon node failure - - + Next + )} - - - - Set replication nodes manually -
- Select nodes from the list in the next step - - - -
+ + + ); } -interface StepNodeSelectionProps { - nodeList: databaseLocationSpecifier[]; - shardCount: number; - replicationFactor: number; -} - -type destinationNode = { - id: string; - node: string; +const getDefaultValues = (replicationFactor: number): FormData => { + return { + replicationFactor, + databaseName: "", + isEncrypted: false, + isEncryptionKeySaved: false, + isSharded: false, + shardsCount: 1, + isDynamicDistribution: false, + isManualReplication: false, + manualNodes: [], + manualShard: [], + isPathDefault: true, + path: "", + }; }; -type shardReplicas = destinationNode[]; - -function StepNodeSelection(props: StepNodeSelectionProps) { - const { nodeList, shardCount, replicationFactor } = props; - - const initialNodes: shardReplicas[] = []; - - for (let i = 0; i < shardCount; i++) { - initialNodes.push([]); - for (let j = 0; j < replicationFactor; j++) { - initialNodes[i].push({ id: "s" + i + "r" + j, node: null }); +function mapToDto(formValues: FormData, selectedOrchestrators: string[]): CreateDatabaseDto { + const { + databaseName, + isSharded, + isManualReplication, + manualNodes, + manualShard, + shardsCount, + isPathDefault, + path, + isEncrypted, + isDynamicDistribution, + } = formValues; + + const Settings: CreateDatabaseDto["Settings"] = isPathDefault + ? {} + : { + DataDir: _.trim(path), + }; + + const Topology: CreateDatabaseDto["Topology"] = isSharded + ? null + : { + Members: isManualReplication ? manualNodes : null, + DynamicNodesDistribution: isDynamicDistribution, + }; + + const Shards: CreateDatabaseDto["Sharding"]["Shards"] = {}; + + if (isSharded) { + for (let i = 0; i < shardsCount; i++) { + Shards[i] = isManualReplication + ? { + Members: manualShard[i], + } + : {}; } } - const [shardNodes] = useState(initialNodes); - - function updateShardNodes(): () => void { - console.log("TODO Update selected node"); - return; - } - - // TODO: @kalczur - naming when we will implement the new CreateDatabase view - - const allActionContexts = ActionContextUtils.getContexts(nodeList); - const [selectedActionContexts, setSelectedActionContexts] = useState(allActionContexts); - - return ( -
-

Manual Node Selection

- -
- -
- - - - - {shardCount > 1 && - ); - })} - - - - - {shardNodes.map((shard, index) => { - return ( - - {shardCount > 1 && ( - - )} - - {shard.map((replica) => { - return ( - - ); - })} - - ); - })} - -
} - {shardNodes[0].map((replica, index) => { - return ( - - Replica {index + 1} -
- {index} - - {replica.node} - dbLocation.nodeTag)} - id={replica.id} - destinationNode={replica.node} - handleUpdate={updateShardNodes()} - > -
- - -

Orchestrators

-
- minimum 1 -
- -
- ); -} -interface NodeSelectionDropdownProps { - nodeList: string[]; - id: string; - destinationNode: string; - handleUpdate: () => void; -} - -function NodeSelectionDropdown(props: NodeSelectionDropdownProps) { - const { nodeList, destinationNode, handleUpdate } = props; - return ( - <> - - - {destinationNode == null ? ( - <>select - ) : ( - <> - {destinationNode} - - )} - - - {nodeList.map((nodeTag) => ( - handleUpdate()}> - {nodeTag} - - ))} - - None - - - - - ); -} - -interface StepPathsProps { - nodeList: string[]; - useDefaultPaths: boolean; - toggleUseDefaultPaths: () => void; -} - -function StepPaths(props: StepPathsProps) { - const { nodeList, useDefaultPaths, toggleUseDefaultPaths } = props; - - return ( -
-

Paths Configuration

- - - - Use default paths - - - - - - - - - - - - - - {nodeList.map((nodeTag) => ( - - - - - - - ))} - -
- PathFree spaceTotal
- - {nodeTag} - - /data/test24.45 GBytes29.40 GBytes
-
- ); + const Sharding: CreateDatabaseDto["Sharding"] = isSharded + ? { + Shards, + Orchestrator: { + Topology: { + Members: selectedOrchestrators, + }, + }, + } + : null; + + return { + DatabaseName: databaseName, + Settings, + Disabled: false, + Encrypted: isEncrypted, + Topology, + Sharding, + }; } diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/QuickCreateButton.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/QuickCreateButton.tsx new file mode 100644 index 000000000000..ee27d9d6aa31 --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/QuickCreateButton.tsx @@ -0,0 +1,92 @@ +import ButtonWithSpinner from "components/common/ButtonWithSpinner"; +import { Icon } from "components/common/Icon"; +import { PropSummary, PropSummaryItem, PropSummaryName, PropSummaryValue } from "components/common/PropSummary"; +import React from "react"; +import { UncontrolledPopover } from "reactstrap"; +import { CreateDatabaseRegularFormData } from "./createDatabaseRegularValidation"; + +interface QuickCreateButtonProps { + formValues: CreateDatabaseRegularFormData; + isSubmitting: boolean; +} + +export default function QuickCreateButton({ formValues, isSubmitting }: QuickCreateButtonProps) { + return ( + <> + + Quick Create + + + + + + + Encryption + + {formValues.isEncrypted ? ( + ON + ) : ( + OFF + )} + + + + + Replication + + {formValues.replicationFactor > 1 ? ( + ON + ) : ( + OFF + )} + + + + + Sharding + + {formValues.isSharded ? ( + ON + ) : ( + OFF + )} + + + {formValues.isManualReplication && ( + + + Manual node selection + + ON + + )} + + + + {formValues.isPathDefault ? ( + <> + Default path + + ) : ( + <> + Custom path + + )} + + + + + + ); +} diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts index 168c64e5655f..d53b59c844b3 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts @@ -1,12 +1,46 @@ +import { useServices } from "components/hooks/useServices"; +import { pathsConfigurationsSchema } from "../shared/createDatabaseSharedValidation"; import * as yup from "yup"; -const basicInfoSchema = yup.object({ - databaseName: yup.string().nullable().required(), - isEncrypted: yup.boolean(), -}); +const useBasicInfoSchema = () => { + const { resourcesService } = useServices(); + + return yup.object({ + databaseName: yup + .string() + .required() + .when("$usedDatabaseNames", ([usedDatabaseNames], schema) => + schema.notOneOf(usedDatabaseNames, "Database already exists") + ) + .test({ + test: async function (value, ctx) { + if (value == null || value === "") { + return true; + } + + try { + const result = await resourcesService.validateName("Database", value); + return ( + result.IsValid || + ctx.createError({ + message: result.ErrorMessage, + }) + ); + } catch (_) { + return true; + } + }, + }), + + isEncrypted: yup.boolean(), + }); +}; const encryptionSchema = yup.object({ - isEncryptionKeySaved: yup.boolean().oneOf([true]), + isEncryptionKeySaved: yup.boolean().when("isEncrypted", { + is: true, + then: (schema) => schema.oneOf([true], "Encryption key must be saved"), + }), }); const replicationAndShardingSchema = yup.object({ @@ -34,23 +68,16 @@ const manualNodeSelectionSchema = yup.object({ manualShard: yup.array().nullable().of(yup.array().of(yup.string())), }); -const pathsConfigurationsSchema = yup.object({ - isPathsConfigDefault: yup.boolean(), - pathsConfig: yup - .string() - .nullable() - .when("isPathsConfigDefault", { - is: true, - then: (schema) => schema.required(), - }), -}); +export const useCreateDatabaseRegularSchema = () => { + const basicInfoSchema = useBasicInfoSchema(); -export const createNewDatabaseSchema = yup - .object() - .concat(basicInfoSchema) - .concat(encryptionSchema) - .concat(replicationAndShardingSchema) - .concat(manualNodeSelectionSchema) - .concat(pathsConfigurationsSchema); + return yup + .object() + .concat(basicInfoSchema) + .concat(encryptionSchema) + .concat(replicationAndShardingSchema) + .concat(manualNodeSelectionSchema) + .concat(pathsConfigurationsSchema); +}; -export type CreateNewDatabaseFormData = yup.InferType; +export type CreateDatabaseRegularFormData = yup.InferType>; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx new file mode 100644 index 000000000000..b314c423bead --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx @@ -0,0 +1,111 @@ +import { FormInput, FormSwitch } from "components/common/Form"; +import { Icon } from "components/common/Icon"; +import { LicenseRestrictions } from "components/common/LicenseRestrictions"; +import { accessManagerSelectors } from "components/common/shell/accessManagerSlice"; +import { licenseSelectors } from "components/common/shell/licenseSlice"; +import { CreateDatabaseRegularFormData } from "../createDatabaseRegularValidation"; +import { useAppSelector } from "components/store"; +import React from "react"; +import { useFormContext } from "react-hook-form"; +import { Col, FormGroup, PopoverBody, Row, UncontrolledPopover } from "reactstrap"; + +const newDatabaseImg = require("Content/img/createDatabase/new-database.svg"); + +export default function CreateDatabaseRegularStepBasicInfo() { + const hasEncryption = useAppSelector(licenseSelectors.statusValue("HasEncryption")); + const isSecureServer = useAppSelector(accessManagerSelectors.isSecureServer); + + const { control } = useFormContext(); + + return ( +
+
+ +
+

Create new database

+ + + + {/* */} + + +
+ {hasEncryption ? ( + +

+ Authentication is off +

+

+ Encryption at Rest is only possible when authentication is + enabled and a server certificate has been defined. +

+

+ For more information go to the certificates page +

+ + } + > + + + + Encrypt at Rest + + + + ) : ( + + Storage encryption + + } + > + + + + Encrypt at Rest + + + + )} + + +
+ + + Data will be encrypted at the storage engine layer, using XChaCha20-Poly1305{" "} + authenticated encryption algorithm. + + + + +
+ ); +} diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepEncryption.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepEncryption.tsx new file mode 100644 index 000000000000..a8f02af470fd --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepEncryption.tsx @@ -0,0 +1,72 @@ +import { FormCheckbox } from "components/common/Form"; +import { Icon } from "components/common/Icon"; +import { CreateDatabaseRegularFormData } from "../createDatabaseRegularValidation"; +import React from "react"; +import { useFormContext } from "react-hook-form"; +import { Row, Col, InputGroup, Input, Button, Alert, UncontrolledPopover, PopoverBody } from "reactstrap"; + +const encryptionImg = require("Content/img/createDatabase/encryption.svg"); +const qrImg = require("Content/img/createDatabase/qr.jpg"); + +export default function CreateDatabaseRegularStepEncryption() { + const { control } = useFormContext(); + + // TODO copy/download/print buttons + + return ( +
+
+ +
+ +

Encryption at Rest

+ + + +
Key (Base64 Encoding)
+ + + + + + + + + + + + + + +
+ Save the key in a safe place. It will not be available again. If you lose this key you could + lose access to your data +
+ + + + +
+ + what's this? + +
+ + TODO: write info about qr code + + + +
+ + I have saved the encryption key + +
+
+ ); +} diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepNodeSelection.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepNodeSelection.tsx new file mode 100644 index 000000000000..102842b3f16a --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepNodeSelection.tsx @@ -0,0 +1,209 @@ +import { Icon } from "components/common/Icon"; +import { CreateDatabaseRegularFormData } from "../createDatabaseRegularValidation"; +import React, { useEffect } from "react"; +import { useFormContext, useWatch } from "react-hook-form"; +import { Table, Label } from "reactstrap"; +import { FormSelect } from "components/common/Form"; +import { OptionWithIcon, SelectOptionWithIcon, SingleValueWithIcon } from "components/common/select/Select"; +import { Checkbox } from "components/common/Checkbox"; +import { NodeSet, NodeSetLabel, NodeSetList, NodeSetItem } from "components/common/NodeSet"; +import { useAppSelector } from "components/store"; +import { clusterSelectors } from "components/common/shell/clusterSlice"; +import { todo } from "common/developmentHelper"; + +todo("Feature", "Damian", "Add Auto fill button"); + +export default function CreateDatabaseRegularStepNodeSelection() { + const { control, setValue, formState } = useFormContext(); + const formValues = useWatch({ + control, + }); + + // const availableNodes = useAppSelector(clusterSelectors.allNodes); + // TODO show node url? + const availableNodeTags = useAppSelector(clusterSelectors.allNodeTags); + + // const allActionContexts = ActionContextUtils.getContexts(nodeList); + // const [selectedActionContexts, setSelectedActionContexts] = useState(allActionContexts); + + const nodeOptions: SelectOptionWithIcon[] = [ + { + label: "None", + value: "null", + }, + ...availableNodeTags.map( + (x) => + ({ + label: x, + value: x, + icon: "node", + iconColor: "node", + }) satisfies SelectOptionWithIcon + ), + ]; + + const toggleNodeTag = (nodeTag: string) => { + if (formValues.manualNodes.includes(nodeTag)) { + setValue( + "manualNodes", + formValues.manualNodes.filter((x) => x !== nodeTag) + ); + } else { + setValue("manualNodes", [...formValues.manualNodes, nodeTag]); + } + }; + + const isSelectAllNodesIndeterminate = + formValues.manualNodes.length > 0 && formValues.manualNodes.length < availableNodeTags.length; + + const isSelectedAllNodes = formValues.manualNodes.length === availableNodeTags.length; + + const toggleAllNodeTags = () => { + if (formValues.manualNodes.length === 0) { + setValue("manualNodes", availableNodeTags); + } else { + setValue("manualNodes", []); + } + }; + + useEffect(() => { + if (!formValues.isSharded) { + setValue("replicationFactor", formValues.manualNodes.length); + } + }, [formValues.isSharded, formValues.manualNodes, setValue]); + + const shardNumbers = new Array(formValues.shardsCount).fill(0).map((_, i) => i); + const replicationNumbers = new Array(formValues.replicationFactor).fill(0).map((_, i) => i); + + return ( +
+

Manual Node Selection

+ + {formValues.isSharded && ( + <> + {/* TODO @damian +
+ +
*/} + + + + + {formValues.shardsCount > 1 && + ))} + + + + + {shardNumbers.map((shardNumber) => ( + + {formValues.shardsCount > 1 && ( + + )} + + {replicationNumbers.map((replicationNumber) => ( + + ))} + + ))} + +
} + {replicationNumbers.map((replicationNumber) => ( + + Replica {replicationNumber + 1} +
+ {shardNumber} + + +
+ + )} + + + +

{formValues.isSharded ? "Orchestrator nodes" : "Available nodes"}

+
+ minimum 1 +
+ + + + + + {availableNodeTags.map((nodeTag) => ( + + + + ))} + + {formState.errors.manualNodes && ( +
+ {formState.errors.manualNodes.message} +
+ )} + +
+ ); +} + +// interface NodeSelectionDropdownProps { +// nodeList: string[]; +// id: string; +// destinationNode: string; +// handleUpdate: () => void; +// } + +// function NodeSelectionDropdown(props: NodeSelectionDropdownProps) { +// const { nodeList, destinationNode, handleUpdate } = props; +// return ( +// <> +// +// +// {destinationNode == null ? ( +// <>select +// ) : ( +// <> +// {destinationNode} +// +// )} +// +// +// {nodeList.map((nodeTag) => ( +// handleUpdate()}> +// {nodeTag} +// +// ))} +// +// None +// +// +// +// +// ); +// } diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx new file mode 100644 index 000000000000..225331f44a30 --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx @@ -0,0 +1,234 @@ +import { Switch } from "components/common/Checkbox"; +import { FormInput, FormSwitch } from "components/common/Form"; +import { Icon } from "components/common/Icon"; +import { LicenseRestrictions } from "components/common/LicenseRestrictions"; +import { licenseSelectors } from "components/common/shell/licenseSlice"; +import { CreateDatabaseRegularFormData } from "../createDatabaseRegularValidation"; +import { useAppSelector } from "components/store"; +import React from "react"; +import { useFormContext, useWatch } from "react-hook-form"; +import { Alert, Col, Collapse, InputGroup, InputGroupText, PopoverBody, Row, UncontrolledPopover } from "reactstrap"; +import { clusterSelectors } from "components/common/shell/clusterSlice"; + +const shardingImg = require("Content/img/createDatabase/sharding.svg"); + +export default function CreateDatabaseRegularStepReplicationAndSharding() { + const availableNodesCount = useAppSelector(clusterSelectors.allNodes).length; + + // TODO + // const maxReplicationFactorForSharding = useAppSelector( + // licenseSelectors.statusValue("MaxReplicationFactorForSharding") + // ); + // const hasMultiNodeSharding = useAppSelector(licenseSelectors.statusValue("HasMultiNodeSharding")); + + // TODO remove shard number range input + + const hasDynamicNodesDistribution = useAppSelector(licenseSelectors.statusValue("HasDynamicNodesDistribution")); + + const { control } = useFormContext(); + + const formValues = useWatch({ + control, + }); + + const isReplicationFactorDisabled = formValues.isManualReplication && !formValues.isSharded; + + return ( +
+
+ +
+ +

Replication & Sharding

+ + + +

+ Database replication provides benefits such as improved data availability, increased + scalability, and enhanced disaster recovery capabilities. +

+ + + + + +
+ Add more{" "} + + Instance nodes + {" "} + in Manage cluster view +
+ + + + + +

+ + Sharding + {" "} + is a database partitioning technique that breaks up large databases into smaller, more + manageable pieces called{" "} + + {" "} + + shards + + . +

+

+ Each shard contains a subset of the data and can be stored on a separate server, allowing for{" "} + horizontal scalability and improved performance. +

+ + Learn more TODO + + + + + + + + Available nodes:{" "} + {availableNodesCount} + + + + + + Replication Factor + + + + + + + + + + What is sharding? + + + + + + Sharding + + } + className="d-inline-block" + > + + Enable{" "} + + Sharding + + + + + + + + Number of shards + + + + + + + + <> + Data will be divided into{" "} + + {formValues.shardsCount} + Shards + + .
+ + + {formValues.replicationFactor > 1 ? ( + <> + {formValues.isSharded ? <>Each shard : <>Data} will be replicated to{" "} + + {formValues.replicationFactor} Nodes + + . + + ) : ( + <>Data won't be replicated. + )} + + + + + + + {hasDynamicNodesDistribution ? ( + 1} + message="Replication factor is set to 1" + className="d-inline-block" + > + + Allow dynamic database distribution +
+ Maintain replication factor upon node failure + + + ) : ( + + + Allow dynamic database distribution +
+ Maintain replication factor upon node failure + + + )} + + + + Set replication nodes manually +
+ Select nodes from the list in the next step + + + +
+ ); +} diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/CreateDatabaseStepPaths.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/CreateDatabaseStepPaths.tsx new file mode 100644 index 000000000000..04ac8ec65d0f --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/CreateDatabaseStepPaths.tsx @@ -0,0 +1,157 @@ +import { PathsConfigurations } from "./createDatabaseSharedValidation"; +import React from "react"; +import { useFormContext, useWatch } from "react-hook-form"; +import { Alert, InputGroup, InputGroupText, Spinner } from "reactstrap"; +import { FormCheckbox, FormSelectCreatable } from "components/common/Form"; +import { useAppSelector } from "components/store"; +import { clusterSelectors } from "components/common/shell/clusterSlice"; +import { useServices } from "components/hooks/useServices"; +import activeDatabaseTracker from "common/shell/activeDatabaseTracker"; +import { UseAsyncReturn, useAsync } from "react-async-hook"; +import { InputActionMeta } from "react-select"; +import { InputNotHidden, SelectOption } from "components/common/select/Select"; + +interface CreateDatabaseStepPathsProps { + manualSelectedNodes: string[]; + isBackupFolder: boolean; + databaseName: string; +} + +export default function CreateDatabaseStepPaths({ + databaseName, + manualSelectedNodes, + isBackupFolder, +}: CreateDatabaseStepPathsProps) { + const { control, setValue } = useFormContext(); + const formValues = useWatch({ control }); + const { resourcesService } = useServices(); + + const allNodeTags = useAppSelector(clusterSelectors.allNodeTags); + const selectedNodeTags = manualSelectedNodes ?? allNodeTags; + + // TODO debounce + const asyncGetLocalFolderPathOptions = useAsync( + () => + resourcesService.getLocalFolderPathOptions( + formValues.path, + isBackupFolder, + activeDatabaseTracker.default.database() + ), + [formValues.path] + ); + + // TODO debounce + const asyncGetDatabaseLocation = useAsync(() => { + const path = formValues.isPathDefault || !formValues.path ? "" : formValues.path; + + return resourcesService.getDatabaseLocation(databaseName, path); + }, [formValues.isPathDefault, formValues.path]); + + // TODO make autocomplete component? + const onPathChange = (value: string, action: InputActionMeta) => { + if (action?.action !== "input-blur" && action?.action !== "menu-close") { + setValue("path", value); + } + }; + + const pathOptions = getAvailableFolderOptions(asyncGetLocalFolderPathOptions.result?.List); + + return ( +
+

Path Configuration

+ + + + Use server directory + + + + + +
+ ); +} + +function getAvailableFolderOptions(backupLocation: string[]): SelectOption[] { + if (!backupLocation) { + return []; + } + + return backupLocation.map((x) => ({ value: x, label: x })); +} + +function PathInfo({ + asyncGetDatabaseLocation, + nodeTagsToDisplay, +}: { + asyncGetDatabaseLocation: UseAsyncReturn; + nodeTagsToDisplay?: string[]; +}) { + if (asyncGetDatabaseLocation.status === "not-requested" || asyncGetDatabaseLocation.status === "error") { + return null; + } + + if (asyncGetDatabaseLocation.status === "loading") { + return ( + + + + ); + } + + if (asyncGetDatabaseLocation.status === "success" && asyncGetDatabaseLocation.result?.List?.length > 0) { + const filteredLocations = nodeTagsToDisplay + ? asyncGetDatabaseLocation.result.List.filter((x) => nodeTagsToDisplay.includes(x.NodeTag)) + : asyncGetDatabaseLocation.result.List; + + return ( + + {filteredLocations.map((location) => ( +
+ + + Node tag: {location.NodeTag} + +
+ + Path: {location.FullPath} + +
+ {location.Error ? ( + {location.Error} + ) : ( + <> + {location.FreeSpaceHumane ? ( + + Free space: {location.FreeSpaceHumane}{" "} + {location.TotalSpaceHumane && ( + + {"(Total: "} + {location.TotalSpaceHumane} + {")"} + + )} + + ) : ( + (Path is unreachable) + )} + + )} + +
+ ))} + + ); + } +} diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/createDatabaseSharedValidation.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/createDatabaseSharedValidation.tsx new file mode 100644 index 000000000000..0279664133ab --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/createDatabaseSharedValidation.tsx @@ -0,0 +1,11 @@ +import * as yup from "yup"; + +export const pathsConfigurationsSchema = yup.object({ + isPathDefault: yup.boolean(), + path: yup.string().when("isPathDefault", { + is: false, + then: (schema) => schema.required(), + }), +}); + +export type PathsConfigurations = yup.InferType; diff --git a/src/Raven.Studio/typescript/components/services/DatabasesService.ts b/src/Raven.Studio/typescript/components/services/DatabasesService.ts index 9a4dc8593875..950400a64034 100644 --- a/src/Raven.Studio/typescript/components/services/DatabasesService.ts +++ b/src/Raven.Studio/typescript/components/services/DatabasesService.ts @@ -58,14 +58,9 @@ import getConnectionStringsCommand = require("commands/database/settings/getConn import saveConnectionStringCommand = require("commands/database/settings/saveConnectionStringCommand"); import { ConnectionStringDto } from "components/pages/database/settings/connectionStrings/connectionStringsTypes"; import saveCustomSorterCommand = require("commands/database/settings/saveCustomSorterCommand"); -import queryCommand = require("commands/database/query/queryCommand"); -import getIntegrationsPostgreSqlSupportCommand = require("commands/database/settings/getIntegrationsPostgreSqlSupportCommand"); -import getIntegrationsPostgreSqlCredentialsCommand = require("commands/database/settings/getIntegrationsPostgreSqlCredentialsCommand"); -import saveIntegrationsPostgreSqlCredentialsCommand = require("commands/database/settings/saveIntegrationsPostgreSqlCredentialsCommand"); -import deleteIntegrationsPostgreSqlCredentialsCommand = require("commands/database/settings/deleteIntegrationsPostgreSqlCredentialsCommand"); -import generateSecretCommand = require("commands/database/secrets/generateSecretCommand"); import getDatabaseStatsCommand = require("commands/resources/getDatabaseStatsCommand"); import saveUnusedDatabaseIDsCommand = require("commands/database/settings/saveUnusedDatabaseIDsCommand"); +import { CreateDatabaseDto, createDatabaseCommand } from "commands/resources/createDatabaseCommand"; export default class DatabasesService { async setLockMode(databaseNames: string[], newLockMode: DatabaseLockMode) { @@ -295,28 +290,8 @@ export default class DatabasesService { return new saveDatabaseRecordCommand(databaseName, databaseRecord, etag).execute(); } - async query(...args: ConstructorParameters) { - return new queryCommand(...args).execute(); - } - - async getIntegrationsPostgreSqlSupport(databaseName: string) { - return new getIntegrationsPostgreSqlSupportCommand(databaseName).execute(); - } - - async getIntegrationsPostgreSqlCredentials(databaseName: string) { - return new getIntegrationsPostgreSqlCredentialsCommand(databaseName).execute(); - } - - async saveIntegrationsPostgreSqlCredentials(databaseName: string, username: string, password: string) { - return new saveIntegrationsPostgreSqlCredentialsCommand(databaseName, username, password).execute(); - } - - async deleteIntegrationsPostgreSqlCredentials(databaseName: string, username: string) { - return new deleteIntegrationsPostgreSqlCredentialsCommand(databaseName, username).execute(); - } - - async generateSecret() { - return new generateSecretCommand().execute(); + async createDatabase(dto: CreateDatabaseDto, replicationFactor: number) { + return new createDatabaseCommand(dto, replicationFactor).execute(); } async getDatabaseStats(...args: ConstructorParameters) { diff --git a/src/Raven.Studio/typescript/components/services/ResourcesService.ts b/src/Raven.Studio/typescript/components/services/ResourcesService.ts index 3465aedd8ca1..5a682d19ccff 100644 --- a/src/Raven.Studio/typescript/components/services/ResourcesService.ts +++ b/src/Raven.Studio/typescript/components/services/ResourcesService.ts @@ -1,7 +1,18 @@ +import getDatabaseLocationCommand from "commands/resources/getDatabaseLocationCommand"; +import getFolderPathOptionsCommand from "commands/resources/getFolderPathOptionsCommand"; import validateNameCommand from "commands/resources/validateNameCommand"; +import database from "models/resources/database"; export default class ResourcesService { async validateName(type: Raven.Server.Web.Studio.StudioTasksHandler.ItemType, name: string, dataPath?: string) { return new validateNameCommand(type, name, dataPath).execute(); } + + async getLocalFolderPathOptions(path: string, isBackupFolder: boolean, db: database) { + return getFolderPathOptionsCommand.forServerLocal(path, isBackupFolder, null, db).execute(); + } + + async getDatabaseLocation(dbName: string, path: string) { + return new getDatabaseLocationCommand(dbName, path).execute(); + } } diff --git a/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts b/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts index 54f1cad39e75..fa558d931259 100644 --- a/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts +++ b/src/Raven.Studio/typescript/test/mocks/services/MockResourcesService.ts @@ -10,4 +10,16 @@ export default class MockResourcesService extends AutoMockService) { return this.mockResolvedValue(this.mocks.validateName, dto, ResourcesStubs.validValidateName()); } + + withDatabaseLocation(dto?: Raven.Server.Web.Studio.DataDirectoryResult) { + return this.mockResolvedValue(this.mocks.getDatabaseLocation, dto, ResourcesStubs.databaseLocation()); + } + + withLocalFolderPathOptions(dto?: Raven.Server.Web.Studio.FolderPathOptions) { + return this.mockResolvedValue( + this.mocks.getLocalFolderPathOptions, + dto, + ResourcesStubs.localFolderPathOptions() + ); + } } diff --git a/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts b/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts index bc6f582a8fc3..ecaaad2554ef 100644 --- a/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts +++ b/src/Raven.Studio/typescript/test/stubs/ResourcesStubs.ts @@ -12,4 +12,35 @@ export class ResourcesStubs { ErrorMessage: "Invalid name", }; } + + static databaseLocation(): Raven.Server.Web.Studio.DataDirectoryResult { + return { + List: [ + { + NodeTag: "A", + FullPath: `/`, + FreeSpaceInBytes: null, + FreeSpaceHumane: null, + TotalSpaceInBytes: null, + TotalSpaceHumane: null, + Error: "Cannot write to directory path: /", + }, + { + NodeTag: "B", + FullPath: `C:/Workspace/ravendb/test/RachisTests/bin/Debug/net8.0/Databases/GetNewServer-B.0-3`, + FreeSpaceInBytes: 6126075904, + FreeSpaceHumane: "5.705 GBytes", + TotalSpaceInBytes: 20738408448, + TotalSpaceHumane: "19.314 GBytes", + Error: null, + }, + ], + }; + } + + static localFolderPathOptions(): Raven.Server.Web.Studio.FolderPathOptions { + return { + List: ["/bin", "/boot", "/data", "/dev", "/etc"], + }; + } } diff --git a/src/Raven.Studio/typescript/viewmodels/resources/createDatabase.ts b/src/Raven.Studio/typescript/viewmodels/resources/createDatabase.ts index bc0c35ee3383..2606eb8ac67e 100644 --- a/src/Raven.Studio/typescript/viewmodels/resources/createDatabase.ts +++ b/src/Raven.Studio/typescript/viewmodels/resources/createDatabase.ts @@ -2,7 +2,7 @@ import dialog = require("plugins/dialog"); import database = require("models/resources/database"); import dialogViewModelBase = require("viewmodels/dialogViewModelBase"); import databasesManager = require("common/shell/databasesManager"); -import createDatabaseCommand = require("commands/resources/createDatabaseCommand"); +import {createDatabaseCommand} from "commands/resources/createDatabaseCommand"; import restoreDatabaseFromBackupCommand = require("commands/resources/restoreDatabaseFromBackupCommand"); import getClusterTopologyCommand = require("commands/database/cluster/getClusterTopologyCommand"); import getDatabaseLocationCommand = require("commands/resources/getDatabaseLocationCommand"); From d4f2948ece3db9c40c5478437a1163db470a7da3 Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Mon, 22 Jan 2024 12:46:51 +0100 Subject: [PATCH 047/243] RavenDB-16865 Database name async validation --- .../create/regular/CreateDatabaseRegular.tsx | 12 ++-- .../createDatabaseRegularValidation.ts | 63 ++++++------------- .../shared/useDatabaseNameValidation.tsx | 43 +++++++++++++ 3 files changed, 68 insertions(+), 50 deletions(-) create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/useDatabaseNameValidation.tsx diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx index 651bb2e031d0..4f128b414f9d 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx @@ -4,10 +4,9 @@ import { FlexGrow } from "components/common/FlexGrow"; import { Icon } from "components/common/Icon"; import Steps from "components/common/Steps"; import { FormProvider, SubmitHandler, useForm, useWatch } from "react-hook-form"; -import { yupResolver } from "@hookform/resolvers/yup"; import { CreateDatabaseRegularFormData as FormData, - useCreateDatabaseRegularSchema, + createDatabaseRegularSchema, } from "./createDatabaseRegularValidation"; import StepBasicInfo from "./steps/CreateDatabaseRegularStepBasicInfo"; import StepEncryption from "./steps/CreateDatabaseRegularStepEncryption"; @@ -25,6 +24,8 @@ import ButtonWithSpinner from "components/common/ButtonWithSpinner"; import { clusterSelectors } from "components/common/shell/clusterSlice"; import { tryHandleSubmit } from "components/utils/common"; import QuickCreateButton from "components/pages/resources/databases/partials/create/regular/QuickCreateButton"; +import { yupResolver } from "@hookform/resolvers/yup"; +import { useDatabaseNameValidation } from "components/pages/resources/databases/partials/create/shared/useDatabaseNameValidation"; interface CreateDatabaseRegularProps { closeModal: () => void; @@ -43,18 +44,17 @@ interface StepItem { export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBackup }: CreateDatabaseRegularProps) { const usedDatabaseNames = useAppSelector(databaseSelectors.allDatabases).map((db) => db.name); - const createNewDatabaseSchema = useCreateDatabaseRegularSchema(); const allNodeTags = useAppSelector(clusterSelectors.allNodeTags); const form = useForm({ mode: "all", defaultValues: getDefaultValues(allNodeTags.length), - resolver: yupResolver(createNewDatabaseSchema), + resolver: yupResolver(createDatabaseRegularSchema), context: { usedDatabaseNames, }, }); - const { control, handleSubmit, formState } = form; + const { control, handleSubmit, formState, setError, clearErrors } = form; if (formState.errors) { console.log("kalczur errors", formState.errors); @@ -63,6 +63,8 @@ export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBa control, }); + useDatabaseNameValidation(formValues.databaseName, setError, clearErrors); + const { databasesService } = useServices(); const asyncCreateDatabase = useAsyncCallback( diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts index d53b59c844b3..e0bcb09a9666 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts @@ -1,40 +1,17 @@ -import { useServices } from "components/hooks/useServices"; +// import { useServices } from "components/hooks/useServices"; import { pathsConfigurationsSchema } from "../shared/createDatabaseSharedValidation"; import * as yup from "yup"; -const useBasicInfoSchema = () => { - const { resourcesService } = useServices(); +const basicInfoSchema = yup.object({ + databaseName: yup + .string() + .required() + .when("$usedDatabaseNames", ([usedDatabaseNames], schema) => + schema.notOneOf(usedDatabaseNames, "Database already exists") + ), - return yup.object({ - databaseName: yup - .string() - .required() - .when("$usedDatabaseNames", ([usedDatabaseNames], schema) => - schema.notOneOf(usedDatabaseNames, "Database already exists") - ) - .test({ - test: async function (value, ctx) { - if (value == null || value === "") { - return true; - } - - try { - const result = await resourcesService.validateName("Database", value); - return ( - result.IsValid || - ctx.createError({ - message: result.ErrorMessage, - }) - ); - } catch (_) { - return true; - } - }, - }), - - isEncrypted: yup.boolean(), - }); -}; + isEncrypted: yup.boolean(), +}); const encryptionSchema = yup.object({ isEncryptionKeySaved: yup.boolean().when("isEncrypted", { @@ -68,16 +45,12 @@ const manualNodeSelectionSchema = yup.object({ manualShard: yup.array().nullable().of(yup.array().of(yup.string())), }); -export const useCreateDatabaseRegularSchema = () => { - const basicInfoSchema = useBasicInfoSchema(); - - return yup - .object() - .concat(basicInfoSchema) - .concat(encryptionSchema) - .concat(replicationAndShardingSchema) - .concat(manualNodeSelectionSchema) - .concat(pathsConfigurationsSchema); -}; +export const createDatabaseRegularSchema = yup + .object() + .concat(basicInfoSchema) + .concat(encryptionSchema) + .concat(replicationAndShardingSchema) + .concat(manualNodeSelectionSchema) + .concat(pathsConfigurationsSchema); -export type CreateDatabaseRegularFormData = yup.InferType>; +export type CreateDatabaseRegularFormData = yup.InferType; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/useDatabaseNameValidation.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/useDatabaseNameValidation.tsx new file mode 100644 index 000000000000..5774326c877e --- /dev/null +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/useDatabaseNameValidation.tsx @@ -0,0 +1,43 @@ +import { useServices } from "components/hooks/useServices"; +import { useEffect } from "react"; +import { UseFormSetError, UseFormClearErrors } from "react-hook-form"; + +interface FormData { + databaseName: string; +} + +export const useDatabaseNameValidation = ( + databaseName: string, + setError: UseFormSetError, + clearErrors: UseFormClearErrors +) => { + const { resourcesService } = useServices(); + + useEffect(() => { + if (!databaseName) { + return; + } + + debouncedValidateName(() => resourcesService.validateName("Database", databaseName), setError, clearErrors); + }, [databaseName, resourcesService, setError, clearErrors]); +}; + +const debouncedValidateName = _.debounce( + async ( + validateName: () => Promise, + setError: UseFormSetError, + clearErrors: UseFormClearErrors + ) => { + const result = await validateName(); + + if (result.IsValid) { + clearErrors("databaseName"); + } else { + setError("databaseName", { + type: "manual", + message: result.ErrorMessage, + }); + } + }, + 500 +); From 301c92885fddcc69470280650b9a588884ceb9b4 Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Mon, 29 Jan 2024 12:15:06 +0100 Subject: [PATCH 048/243] RavenDB-16865 Add useAsyncDebounce --- .../components/utils/hooks/useAsyncDebounce.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/Raven.Studio/typescript/components/utils/hooks/useAsyncDebounce.ts diff --git a/src/Raven.Studio/typescript/components/utils/hooks/useAsyncDebounce.ts b/src/Raven.Studio/typescript/components/utils/hooks/useAsyncDebounce.ts new file mode 100644 index 000000000000..cf65b0c81b84 --- /dev/null +++ b/src/Raven.Studio/typescript/components/utils/hooks/useAsyncDebounce.ts @@ -0,0 +1,14 @@ +import { useMemo } from "react"; +import { UseAsyncReturn, useAsync } from "react-async-hook"; + +export function useAsyncDebounce( + callback: (...args: unknown[]) => Promise, + params: ParamsType, + waitTimeMs = 500 +): UseAsyncReturn { + // debounce should be created only once + // eslint-disable-next-line react-hooks/exhaustive-deps + const debounced = useMemo(() => _.debounce(callback, waitTimeMs), []); + + return useAsync(debounced, params); +} From 82c78beabaa1316c7f6ff2776a328316dde0d62f Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Fri, 9 Feb 2024 13:43:22 +0100 Subject: [PATCH 049/243] RavenDB-16865 Refactor form data model and prepare part of restore database --- .../typescript/components/common/Form.tsx | 4 +- .../components/common/Steps.stories.tsx | 23 +- .../typescript/components/common/Steps.tsx | 81 +- .../components/common/select/Select.tsx | 2 +- .../create/CreateDatabase.stories.tsx | 2 + .../partials/create/CreateDatabase.tsx | 13 +- .../formBackup/CreateDatabaseFromBackup.tsx | 1110 ++++------------- .../createDatabaseFromBackupValidation.ts | 147 +++ .../CreateDatabaseFromBackupStepBasicInfo.tsx | 80 ++ .../CreateDatabaseFromBackupStepSource.tsx | 716 +++++++++++ .../create/regular/CreateDatabaseRegular.tsx | 177 +-- .../create/regular/QuickCreateButton.tsx | 10 +- .../createDatabaseRegularValidation.ts | 32 +- .../CreateDatabaseRegularStepBasicInfo.tsx | 15 +- .../CreateDatabaseRegularStepEncryption.tsx | 72 -- ...CreateDatabaseRegularStepNodeSelection.tsx | 189 ++- ...abaseRegularStepReplicationAndSharding.tsx | 38 +- .../shared/CreateDatabaseStepEncryption.tsx | 148 +++ ...epPaths.tsx => CreateDatabaseStepPath.tsx} | 56 +- .../shared/createDatabaseSharedValidation.tsx | 18 +- .../shared/useDatabaseNameValidation.tsx | 8 +- .../components/services/DatabasesService.ts | 15 +- .../components/services/ResourcesService.ts | 23 +- .../mocks/services/MockResourcesService.ts | 10 +- .../typescript/test/stubs/ResourcesStubs.ts | 45 +- src/Raven.Studio/typings/_studio/utils.d.ts | 4 + .../wwwroot/Content/css/_bs5.scss | 27 +- .../wwwroot/Content/img/createDatabase/qr.jpg | Bin 12696 -> 0 bytes .../wwwroot/Content/scss/_layout-tools.scss | 26 + .../wwwroot/Content/scss/_qrcode.scss | 26 + 30 files changed, 1836 insertions(+), 1281 deletions(-) create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/createDatabaseFromBackupValidation.ts create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/steps/CreateDatabaseFromBackupStepBasicInfo.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/steps/CreateDatabaseFromBackupStepSource.tsx delete mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepEncryption.tsx create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/CreateDatabaseStepEncryption.tsx rename src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/shared/{CreateDatabaseStepPaths.tsx => CreateDatabaseStepPath.tsx} (74%) create mode 100644 src/Raven.Studio/typings/_studio/utils.d.ts delete mode 100644 src/Raven.Studio/wwwroot/Content/img/createDatabase/qr.jpg create mode 100644 src/Raven.Studio/wwwroot/Content/scss/_qrcode.scss diff --git a/src/Raven.Studio/typescript/components/common/Form.tsx b/src/Raven.Studio/typescript/components/common/Form.tsx index 0556c415822f..8bfdd761a90b 100644 --- a/src/Raven.Studio/typescript/components/common/Form.tsx +++ b/src/Raven.Studio/typescript/components/common/Form.tsx @@ -140,8 +140,8 @@ export function getFormSelectedOptions