From a44cc3fa6fa25f5c024a4d5d482ec9e86cf8bfc1 Mon Sep 17 00:00:00 2001 From: Federico Lois Date: Wed, 8 May 2024 14:22:57 -0300 Subject: [PATCH 1/3] RavenDB-22348: ReadBigEndianInt64 is missing advance pointer --- src/Voron/ValueReader.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Voron/ValueReader.cs b/src/Voron/ValueReader.cs index 3d242405cc94..f1b2b80a2b2b 100644 --- a/src/Voron/ValueReader.cs +++ b/src/Voron/ValueReader.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Runtime.CompilerServices; using Sparrow; @@ -124,6 +124,7 @@ public long ReadBigEndianInt64() throw new EndOfStreamException(); var val = *(long*) (_val + _pos); + _pos += sizeof(long); return Bits.SwapBytes(val); } From 3894f2909305381ff070a640a1240a6dfb709703 Mon Sep 17 00:00:00 2001 From: Damian Olszewski Date: Wed, 8 May 2024 08:54:50 +0200 Subject: [PATCH 2/3] RavenDB-19844 Add Lucene Unmanaged Allocations to memory usage widget --- .../Cluster/Notifications/MemoryUsageNotificationSender.cs | 3 ++- .../Dashboard/Cluster/Notifications/MemoryUsagePayload.cs | 2 ++ .../typescript/models/resources/widgets/memoryUsage.ts | 3 ++- .../App/views/resources/widgets/memoryUsageWidget.html | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsageNotificationSender.cs b/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsageNotificationSender.cs index 1d211744e6a2..12ee140cbc52 100644 --- a/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsageNotificationSender.cs +++ b/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsageNotificationSender.cs @@ -60,7 +60,8 @@ protected override AbstractClusterDashboardNotification CreateNotification() DirtyMemory = dirtyMemoryState.TotalDirty.GetValue(SizeUnit.Bytes), AvailableMemory = memoryInfo.AvailableMemory.GetValue(SizeUnit.Bytes), AvailableMemoryForProcessing = memoryInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes), - TotalSwapUsage = memoryInfo.TotalSwapUsage.GetValue(SizeUnit.Bytes) + TotalSwapUsage = memoryInfo.TotalSwapUsage.GetValue(SizeUnit.Bytes), + LuceneUnmanagedAllocations = NativeMemory.TotalLuceneUnmanagedAllocationsForSorting }; } } diff --git a/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsagePayload.cs b/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsagePayload.cs index bd85a551fdb1..b543f438372b 100644 --- a/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsagePayload.cs +++ b/src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsagePayload.cs @@ -26,6 +26,7 @@ public class MemoryUsagePayload : AbstractClusterDashboardNotification public long AvailableMemory { get; set; } public long AvailableMemoryForProcessing { get; set; } public long TotalSwapUsage { get; set; } + public long LuceneUnmanagedAllocations { get; set; } public override ClusterDashboardNotificationType Type => ClusterDashboardNotificationType.MemoryUsage; @@ -47,6 +48,7 @@ public override DynamicJsonValue ToJson() json[nameof(AvailableMemory)] = AvailableMemory; json[nameof(AvailableMemoryForProcessing)] = AvailableMemoryForProcessing; json[nameof(TotalSwapUsage)] = TotalSwapUsage; + json[nameof(LuceneUnmanagedAllocations)] = LuceneUnmanagedAllocations; return json; } diff --git a/src/Raven.Studio/typescript/models/resources/widgets/memoryUsage.ts b/src/Raven.Studio/typescript/models/resources/widgets/memoryUsage.ts index eb9744d128a6..0734809ba6c6 100644 --- a/src/Raven.Studio/typescript/models/resources/widgets/memoryUsage.ts +++ b/src/Raven.Studio/typescript/models/resources/widgets/memoryUsage.ts @@ -19,7 +19,8 @@ class memoryUsage extends historyAwareNodeStats x.SystemCommitLimit); totalSwapUsage = this.dataExtractor(x => x.TotalSwapUsage); totalSwap = this.dataExtractor(x => x.PhysicalMemory != null && x.SystemCommitLimit != null ? x.SystemCommitLimit - x.PhysicalMemory : undefined); - + luceneUnmanagedAllocations = this.dataExtractor(x => x.LuceneUnmanagedAllocations); + workingSetFormatted: KnockoutComputed<[string, string]>; machineMemoryUsage: KnockoutComputed; machineMemoryUsagePercentage: KnockoutComputed; diff --git a/src/Raven.Studio/wwwroot/App/views/resources/widgets/memoryUsageWidget.html b/src/Raven.Studio/wwwroot/App/views/resources/widgets/memoryUsageWidget.html index 9028efef3a14..5aad66434380 100644 --- a/src/Raven.Studio/wwwroot/App/views/resources/widgets/memoryUsageWidget.html +++ b/src/Raven.Studio/wwwroot/App/views/resources/widgets/memoryUsageWidget.html @@ -36,6 +36,10 @@

RavenDB Memory usage

Unmanaged Allocations
+
+
Lucene Unmanaged Allocations
+
+
Encryption Buffers in Use
From 8ce3fa9502af4a2c141662d5171376d3eb21b61f Mon Sep 17 00:00:00 2001 From: Arkadiusz Palinski Date: Thu, 9 May 2024 07:26:14 +0200 Subject: [PATCH 3/3] RavenDB-22257 Making the transaction owner check on tx dispose to be Debug only. Removing the usage of await in the scope of a write transaction in the test and making it sync. Added ConfigureAwait(true) to the transaction replay code to in order to try to get to the original thread (that opened the write tx) so the tests for the tx recorder won't fail in debug. --- .../Documents/ReplayTxCommandHelper.cs | 2 +- src/Voron/Impl/LowLevelTransaction.cs | 16 ++++++++++------ test/RachisTests/BasicTests.cs | 11 ++++++----- test/SlowTests/Voron/Issues/RavenDB_22257.cs | 9 ++++++--- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/Raven.Server/Documents/ReplayTxCommandHelper.cs b/src/Raven.Server/Documents/ReplayTxCommandHelper.cs index bb1061b42264..427c5715d67c 100644 --- a/src/Raven.Server/Documents/ReplayTxCommandHelper.cs +++ b/src/Raven.Server/Documents/ReplayTxCommandHelper.cs @@ -44,7 +44,7 @@ internal static async IAsyncEnumerable ReplayAsync(DocumentDatab await using (var readersItr = readers.GetAsyncEnumerator()) { await ReadStartRecordingDetailsAsync(readersItr, context, peepingTomStream); - while (await readersItr.MoveNextAsync()) + while (await readersItr.MoveNextAsync().ConfigureAwait(true)) { using (readersItr.Current) { diff --git a/src/Voron/Impl/LowLevelTransaction.cs b/src/Voron/Impl/LowLevelTransaction.cs index 66fd74730904..dcb66f3c24ca 100644 --- a/src/Voron/Impl/LowLevelTransaction.cs +++ b/src/Voron/Impl/LowLevelTransaction.cs @@ -935,8 +935,7 @@ public void Dispose() if (_txState.HasFlag(TxState.Disposed)) return; - if (Flags == TransactionFlags.ReadWrite && NativeMemory.CurrentThreadStats != CurrentTransactionHolder) - ThrowDisposeOfTxMustBeCalledOnTheSameThreadThatCreatedIt(); + EnsureDisposeOfWriteTxIsOnTheSameThreadThatCreatedIt(); try { @@ -1326,11 +1325,16 @@ private static void ThrowAlreadyCommitted() throw new InvalidOperationException("Cannot commit already committed transaction."); } - private void ThrowDisposeOfTxMustBeCalledOnTheSameThreadThatCreatedIt() + [Conditional("DEBUG")] + private void EnsureDisposeOfWriteTxIsOnTheSameThreadThatCreatedIt() { - throw new InvalidOperationException($"Dispose of the transaction must be called from the same thread that created it. " + - $"Transaction {Id} (Flags: {Flags}) was created by {CurrentTransactionHolder.Name}, thread Id: {CurrentTransactionHolder.ManagedThreadId}. " + - $"The dispose was called from {NativeMemory.CurrentThreadStats.Name}, thread Id: {NativeMemory.CurrentThreadStats.ManagedThreadId}"); + if (Flags == TransactionFlags.ReadWrite && NativeMemory.CurrentThreadStats != CurrentTransactionHolder) + { + throw new InvalidOperationException($"Dispose of the write transaction must be called from the same thread that created it. " + + $"Transaction {Id} (Flags: {Flags}) was created by {CurrentTransactionHolder.Name}, thread Id: {CurrentTransactionHolder.ManagedThreadId}. " + + $"The dispose was called from {NativeMemory.CurrentThreadStats.Name}, thread Id: {NativeMemory.CurrentThreadStats.ManagedThreadId}. " + + $"Do you have any await call in the scope of the write transaction?"); + } } public void Rollback() diff --git a/test/RachisTests/BasicTests.cs b/test/RachisTests/BasicTests.cs index f4aaee35d0d2..b0a469a7ce72 100644 --- a/test/RachisTests/BasicTests.cs +++ b/test/RachisTests/BasicTests.cs @@ -2,6 +2,7 @@ using System.Threading; using System.Threading.Tasks; using Raven.Client.ServerWide; +using Raven.Client.Util; using Raven.Server.Rachis; using Raven.Server.ServerWide.Context; using Sparrow.Json; @@ -59,10 +60,10 @@ await ActionWithLeader(async l => } [Fact] - public async Task RavenDB_13659() + public void RavenDB_13659() { - var leader = await CreateNetworkAndGetLeader(1); - var mre = new AsyncManualResetEvent(); + var leader = AsyncHelpers.RunSync(() => CreateNetworkAndGetLeader(1)); + var mre = new ManualResetEventSlim(); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var currentThread = NativeMemory.CurrentThreadStats.ManagedThreadId; @@ -90,12 +91,12 @@ public async Task RavenDB_13659() using (leader.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (context.OpenWriteTransaction()) { - await mre.WaitAsync(); + mre.Wait(); leader.SetNewStateInTx(context, RachisState.Follower, null, leader.CurrentTerm, "deadlock"); context.Transaction.Commit(); } - await tcs.Task; + AsyncHelpers.RunSync(() => tcs.Task); } } } diff --git a/test/SlowTests/Voron/Issues/RavenDB_22257.cs b/test/SlowTests/Voron/Issues/RavenDB_22257.cs index 90a62256564e..0a6f4269017c 100644 --- a/test/SlowTests/Voron/Issues/RavenDB_22257.cs +++ b/test/SlowTests/Voron/Issues/RavenDB_22257.cs @@ -21,8 +21,11 @@ protected override void Configure(StorageEnvironmentOptions options) options.ManualFlushing = true; } - +#if DEBUG [RavenFact(RavenTestCategory.Voron)] +#else + [RavenFact(RavenTestCategory.Voron, Skip = "This test relies on Debug only check - LowLevelTransaction.EnsureDisposeOfWriteTxIsOnTheSameThreadThatCreatedIt()")] +#endif public void MustNotAllowToHaveTwoWriteTransactionsConcurrently() { RequireFileBasedPager(); @@ -48,7 +51,7 @@ public void MustNotAllowToHaveTwoWriteTransactionsConcurrently() { ex = Assert.Throws(() => { - txw1.Dispose(); // this is supposed to throw because we're attempting to dispose write tx from a different thread + txw1.Dispose(); // this is supposed to throw in Debug because we're attempting to dispose write tx from a different thread txw2 = Env.NewLowLevelTransaction(new TransactionPersistentContext(), TransactionFlags.ReadWrite); @@ -60,7 +63,7 @@ public void MustNotAllowToHaveTwoWriteTransactionsConcurrently() newTransactionThread.Join(); - Assert.StartsWith("Dispose of the transaction must be called from the same thread that created it. Transaction 2 (Flags: ReadWrite) was created by", ex.Message); + Assert.StartsWith("Dispose of the write transaction must be called from the same thread that created it. Transaction 2 (Flags: ReadWrite) was created by", ex.Message); }); /*