Merge pull request ravendb#18486 from arekpalinski/v6.0

5.4 to 6.0 merge

src/Raven.Client/Documents/Operations/Operation.cs

@@ -338,29 +338,43 @@ public async Task<TResult> WaitForCompletionAsync<TResult>(CancellationToken tok
             {
                 var result = await InitializeResult().ConfigureAwait(false);
 
-                _ = Task.Factory.StartNew(Initialize);
+                var initTask = Task.Factory.StartNew(Initialize);
 
                 try
                 {
+                    try
+                    {
 #if NET6_0_OR_GREATER
-                    await result.WaitAsync(token).ConfigureAwait(false);
-                    await _afterOperationCompleted.WaitAsync(token).ConfigureAwait(false);
+                        await result.WaitAsync(token).ConfigureAwait(false);
+                        await _afterOperationCompleted.WaitAsync(token).ConfigureAwait(false);
 #else
-                    await result.WithCancellation(token).ConfigureAwait(false);
-                    await _afterOperationCompleted.WithCancellation(token).ConfigureAwait(false);
+                        await result.WithCancellation(token).ConfigureAwait(false);
+                        await _afterOperationCompleted.WithCancellation(token).ConfigureAwait(false);
 #endif
+                    }
+                    catch (TaskCanceledException e) when (token.IsCancellationRequested)
+                    {
+                        await StopProcessingUnderLock().ConfigureAwait(false);
+                        throw new TimeoutException($"Did not get a reply for operation '{_id}'.", e);
+                    }
+                    catch (Exception ex)
+                    {
+                        await StopProcessingUnderLock(ex).ConfigureAwait(false);
+                    }
+
+                    return (TResult)await result.ConfigureAwait(false); // already done waiting but in failure we want the exception itself and not AggregateException 
                 }
-                catch (TaskCanceledException e) when (token.IsCancellationRequested)
-                {
-                    await StopProcessingUnderLock().ConfigureAwait(false);
-                    throw new TimeoutException($"Did not get a reply for operation '{_id}'.", e);
-                }
-                catch (Exception ex)
+                finally
                 {
-                    await StopProcessingUnderLock(ex).ConfigureAwait(false);
+                    try
+                    {
+                        await initTask.ConfigureAwait(false);
+                    }
+                    catch
+                    {
+                        // ignored
+                    }
                 }
-
-                return (TResult)await result.ConfigureAwait(false); // already done waiting but in failure we want the exception itself and not AggregateException 
             }
         }
 

src/Raven.Server/Commercial/SetupWizard/LetsEncryptSetupUtils.cs

@@ -54,7 +54,7 @@ public static async Task<byte[]> Setup(SetupInfo setupInfo,  SetupProgressAndRes
                 try
                 {
                     var content = new StringContent(serializeObject, Encoding.UTF8, "application/json");
-                    var response = await ApiHttpClient.Instance.PostAsync($"/api/v1/dns-n-cert/claim", content, CancellationToken.None).ConfigureAwait(false);
+                    var response = await ApiHttpClient.Instance.PostAsync($"/api/v1/dns-n-cert/claim", content, token).ConfigureAwait(false);
                     response.EnsureSuccessStatusCode();
                     progress?.AddInfo($"Successfully claimed this domain: {setupInfo.Domain}.");
                 }
@@ -67,7 +67,7 @@ await RavenDnsRecordHelper.UpdateDnsRecordsTask(new UpdateDnsRecordParameters
                     Challenge = challengeResult.Challenge,
                     SetupInfo = setupInfo,
                     Progress = progress,
-                    Token = CancellationToken.None,
+                    Token = token,
                     RegisterTcpDnsRecords = registerTcpDnsRecords
                 });
                 progress?.AddInfo($"Updating DNS record(s) and challenge(s) in {setupInfo.Domain.ToLower()}.{setupInfo.RootDomain.ToLower()}.");
@@ -92,8 +92,7 @@ await CertificateUtils.CompleteAuthorizationAndGetCertificate(new CompleteAuthor
                 SetupInfo = setupInfo,
                 Client = acmeClient,
                 ChallengeResult = challengeResult,
-                Token = CancellationToken.None
-
+                Token = token
             });
 
             progress.Processed++;

src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsageNotificationSender.cs

@@ -57,10 +57,11 @@ protected override AbstractClusterDashboardNotification CreateNotification()
                 EncryptionBuffersInUse = encryptionBuffers.CurrentlyInUseSize,
                 EncryptionBuffersPool = encryptionBuffers.TotalPoolSize,
                 MemoryMapped = totalMapping,
-                DirtyMemory = dirtyMemoryState.TotalDirtyInBytes,
+                DirtyMemory = dirtyMemoryState.TotalDirty.GetValue(SizeUnit.Bytes),
                 AvailableMemory = memoryInfo.AvailableMemory.GetValue(SizeUnit.Bytes),
                 AvailableMemoryForProcessing = memoryInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes),
-                TotalSwapUsage = memoryInfo.TotalSwapUsage.GetValue(SizeUnit.Bytes)
+                TotalSwapUsage = memoryInfo.TotalSwapUsage.GetValue(SizeUnit.Bytes),
+                LuceneUnmanagedAllocations = NativeMemory.TotalLuceneUnmanagedAllocationsForSorting
             };
         }
     }

src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsagePayload.cs

@@ -26,6 +26,7 @@ public sealed class MemoryUsagePayload : AbstractClusterDashboardNotification
         public long AvailableMemory { get; set; }
         public long AvailableMemoryForProcessing { get; set; }
         public long TotalSwapUsage { get; set; }
+        public long LuceneUnmanagedAllocations { get; set; }
 
         public override ClusterDashboardNotificationType Type => ClusterDashboardNotificationType.MemoryUsage;
 
@@ -47,6 +48,7 @@ public override DynamicJsonValue ToJson()
             json[nameof(AvailableMemory)] = AvailableMemory;
             json[nameof(AvailableMemoryForProcessing)] = AvailableMemoryForProcessing;
             json[nameof(TotalSwapUsage)] = TotalSwapUsage;
+            json[nameof(LuceneUnmanagedAllocations)] = LuceneUnmanagedAllocations;
 
             return json;
         }

src/Raven.Server/Documents/ETL/Providers/Elasticsearch/ElasticSearchEtl.cs

@@ -114,7 +114,9 @@ protected override int LoadInternal(IEnumerable<ElasticSearchIndexWithRecords> r
                     string indexName = index.IndexName.ToLower();
 
                     EnsureIndexExistsAndValidateIfNeeded(indexName, index);
-
+
+                    CancellationToken.ThrowIfCancellationRequested();
+
                     if (index.InsertOnlyMode == false)
                         count += DeleteByQueryOnIndexIdProperty(index);
 

src/Raven.Server/Documents/ETL/Providers/Queue/Kafka/KafkaEtl.cs

@@ -159,6 +159,8 @@ void ReportHandler(DeliveryReport<string, byte[]> report)
             {
                 foreach (var queueItem in topic.Items)
                 {
+                    CancellationToken.ThrowIfCancellationRequested();
+
                     var cloudEvent = CreateCloudEvent(queueItem);
 
                     var kafkaMessage = cloudEvent.ToKafkaMessage(ContentMode.Binary, formatter);

src/Raven.Server/Documents/ETL/Providers/Queue/RabbitMq/RabbitMqEtl.cs

@@ -67,6 +67,8 @@ protected override int PublishMessages(List<QueueWithItems<RabbitMqItem>> itemsP
 
             foreach (var queueItem in exchange.Items)
             {
+                CancellationToken.ThrowIfCancellationRequested();
+
                 var properties = producer.CreateBasicProperties();
 
                 properties.Headers = new Dictionary<string, object>();

src/Raven.Server/Documents/Expiration/ExpiredDocumentsCleaner.cs

@@ -241,7 +241,7 @@ internal sealed class DeleteExpiredDocumentsCommandDto : IReplayableCommandDto<D
     {
         public ExpiredDocumentsCleaner.DeleteExpiredDocumentsCommand ToCommand(DocumentsOperationContext context, DocumentDatabase database)
         {
-            var expired = new Dictionary<Slice, List<(Slice LowerId, string Id)>>();
+            var expired = new Dictionary<Slice, List<(Slice LowerId, string Id)>>(SliceComparer.Instance);
             foreach (var item in Expired)
             {
                 expired[item.Key] = item.Value;

src/Raven.Server/Documents/Handlers/Admin/AdminLogsHandler.cs

@@ -116,52 +116,79 @@ public async Task Download()
             var adminLogsFileName = $"admin.logs.download.{Guid.NewGuid():N}";
             var adminLogsFilePath = ServerStore._env.Options.DataPager.Options.TempPath.Combine(adminLogsFileName);
 
-            var from = GetDateTimeQueryString("from", required: false);
-            var to = GetDateTimeQueryString("to", required: false);
+            var startUtc = GetDateTimeQueryString("from", required: false);
+            var endUtc = GetDateTimeQueryString("to", required: false);
 
-            using (var stream = SafeFileStream.Create(adminLogsFilePath.FullPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096,
+            if (startUtc >= endUtc)
+                throw new ArgumentException($"End Date '{endUtc:yyyy-MM-ddTHH:mm:ss.fffffff} UTC' must be greater than Start Date '{startUtc:yyyy-MM-ddTHH:mm:ss.fffffff} UTC'");
+
+            await using (var stream = SafeFileStream.Create(adminLogsFilePath.FullPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096,
                        FileOptions.DeleteOnClose | FileOptions.SequentialScan))
             {
                 using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, true))
                 {
+                    bool isEmptyArchive = true;
+
                     foreach (var filePath in Directory.GetFiles(ServerStore.Configuration.Logs.Path.FullPath))
                     {
                         var fileName = Path.GetFileName(filePath);
-                        if (fileName.EndsWith(LoggingSource.LogInfo.LogExtension, StringComparison.OrdinalIgnoreCase) == false &&
-                            fileName.EndsWith(LoggingSource.LogInfo.FullCompressExtension, StringComparison.OrdinalIgnoreCase) == false)
+                        if (fileName.EndsWith(LoggingSource.LogExtension, StringComparison.OrdinalIgnoreCase) == false &&
+                            fileName.EndsWith(LoggingSource.FullCompressExtension, StringComparison.OrdinalIgnoreCase) == false)
                             continue;
 
-                        var hasLogDateTime = LoggingSource.LogInfo.TryGetDate(filePath, out var logDateTime);
-                        if (hasLogDateTime)
-                        {
-                            if (from != null && logDateTime < from)
-                                continue;
+                        // Skip this file if either the last write time or the creation time could not be determined
+                        if (LoggingSource.TryGetLastWriteTimeUtc(filePath, out var logLastWriteTimeUtc) == false ||
+                            LoggingSource.TryGetCreationTimeUtc(filePath, out var logCreationTimeUtc) == false)
+                            continue;
 
-                            if (to != null && logDateTime > to)
-                                continue;
-                        }
+                        bool isWithinDateRange =
+                            // Check if the file was created before the end date.
+                            (endUtc.HasValue == false || logCreationTimeUtc < endUtc.Value) &&
+                            // Check if the file was last modified after the start date.
+                            (startUtc.HasValue == false || logLastWriteTimeUtc > startUtc.Value);
+
+                        // Skip this file if it does not fall within the specified date range
+                        if (isWithinDateRange == false)
+                            continue;
 
                         try
                         {
                             var entry = archive.CreateEntry(fileName);
-                            if (hasLogDateTime)
-                                entry.LastWriteTime = logDateTime;
-
-                            using (var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
+                            await using (var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                             {
                                 entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;
 
                                 await using (var entryStream = entry.Open())
                                 {
                                     await fs.CopyToAsync(entryStream);
                                 }
+
+                                isEmptyArchive = false;
                             }
                         }
                         catch (Exception e)
                         {
                             await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, fileName);
                         }
                     }
+
+                    // Add an informational file to the archive if no log files match the specified date range,
+                    // ensuring the user receives a non-empty archive with an explanation.
+                    if (isEmptyArchive)
+                    {
+                        const string infoFileName = "No logs matched the date range.txt";
+
+                        // Create a dummy entry in the zip file
+                        var infoEntry = archive.CreateEntry(infoFileName);
+                        await using var entryStream = infoEntry.Open();
+                        await using var streamWriter = new StreamWriter(entryStream);
+
+                        var formattedStartUtc = startUtc.HasValue ? startUtc.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff 'UTC'") : "not specified";
+                        var formattedEndUtc = endUtc.HasValue ? endUtc.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff 'UTC'") : "not specified";
+
+                        await streamWriter.WriteAsync(
+                            $"No log files were found that matched the specified date range from '{formattedStartUtc}' to '{formattedEndUtc}'.");
+                    }
                 }
 
                 stream.Position = 0;

src/Raven.Server/Documents/Handlers/Debugging/MemoryDebugHandler.cs

@@ -9,7 +9,6 @@
 using Raven.Server.Routing;
 using Raven.Server.Utils;
 using Raven.Server.Web;
-using Sparrow;
 using Sparrow.Json;
 using Sparrow.Json.Parsing;
 using Sparrow.LowMemory;
@@ -332,10 +331,9 @@ private static void WriteMemoryStats<TWriter>(TWriter writer, JsonOperationConte
                 [nameof(MemoryInfo.EncryptionBuffersPool)] = Size.Humane(encryptionBuffers.TotalPoolSize),
                 [nameof(MemoryInfo.EncryptionLockedMemory)] = Size.Humane(Sodium.LockedBytes),
                 [nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping),
-                [nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(),
                 [nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
-                [nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
-                [nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
+                [nameof(MemoryInfo.DirtyMemory)] = dirtyMemoryState.TotalDirty.ToString(),
+                [nameof(MemoryInfo.AvailableMemory)] = memInfo.AvailableMemory.ToString(),
                 [nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
             };
             if (memInfo.Remarks != null)

src/Raven.Server/Documents/ReplayTxCommandHelper.cs

@@ -50,7 +50,7 @@ internal static async IAsyncEnumerable<ReplayProgress> ReplayAsync(DocumentDatab
                 await using (var readersItr = readers.GetAsyncEnumerator())
                 {
                     await ReadStartRecordingDetailsAsync(readersItr, context, peepingTomStream);
-                    while (await readersItr.MoveNextAsync())
+                    while (await readersItr.MoveNextAsync().ConfigureAwait(true))
                     {
                         using (readersItr.Current)
                         {

src/Raven.Server/Documents/TransactionMerger/AbstractTransactionOperationsMerger.cs

@@ -709,7 +709,7 @@ private PendingOperations ExecutePendingOperationsInTransaction(
                         $"Operation was cancelled by the transaction merger for transaction #{llt.Id} due to high dirty memory in scratch files." +
                         $" This might be caused by a slow IO storage. Current memory usage: " +
                         $"Total Physical Memory: {MemoryInformation.TotalPhysicalMemory}, " +
-                        $"Total Scratch Allocated Memory: {new Size(dirtyMemoryState.TotalDirtyInBytes, SizeUnit.Bytes)} " +
+                        $"Total Scratch Allocated Memory: {dirtyMemoryState.TotalDirty} " +
                         $"(which is above {_configuration.Memory.TemporaryDirtyMemoryAllowedPercentage * 100}%)");
                 }
 

src/Raven.Server/Monitoring/MetricsProvider.cs

@@ -153,8 +153,7 @@ private MemoryMetrics GetMemoryMetrics()
         result.TotalSwapUsageInMb = memoryInfoResult.TotalSwapUsage.GetValue(SizeUnit.Megabytes);
         result.WorkingSetSwapUsageInMb = memoryInfoResult.WorkingSetSwapUsage.GetValue(SizeUnit.Megabytes);
 
-        var totalDirtyInBytes = MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes;
-        result.TotalDirtyInMb = new Size(totalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes);
+        result.TotalDirtyInMb = MemoryInformation.GetDirtyMemoryState().TotalDirty.GetValue(SizeUnit.Megabytes);
 
         return result;
     }

src/Raven.Server/Monitoring/Snmp/Objects/Server/1.6/ServerDirtyMemory.cs

@@ -12,8 +12,7 @@ public ServerDirtyMemory() : base(SnmpOids.Server.DirtyMemory)
 
         protected override Gauge32 GetData()
         {
-            var totalDirtyInBytes = MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes;
-            return new Gauge32(new Size(totalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes));
+            return new Gauge32(MemoryInformation.GetDirtyMemoryState().TotalDirty.GetValue(SizeUnit.Megabytes));
         }
     }
 }

src/Raven.Server/NotificationCenter/OutOfMemoryNotifications.cs

@@ -75,7 +75,7 @@ private static MessageDetails OutOfMemoryDetails(Exception exception)
 
             return new MessageDetails
             {
-                Message = $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo)} {Environment.NewLine}" +
+                Message = $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo, MemoryInformation.GetDirtyMemoryState())} {Environment.NewLine}" +
                           $"Error: {exception}"
             };
         }

src/Raven.Server/ServerWide/CompareExchangeExpirationStorage.cs

@@ -98,7 +98,7 @@ public static bool TryGetExpires(BlittableJsonReaderObject value, out long ticks
         public static unsafe bool DeleteExpiredCompareExchange(ClusterOperationContext context, Table items, long ticks, long take = long.MaxValue)
         {
             // we have to use a dictionary to remove from expired multi tree, because there is a chance that not all keys for certain ticks will be returned in single delete iteration
-            var expired = new Dictionary<Slice, List<Slice>>();
+            var expired = new Dictionary<Slice, List<Slice>>(SliceComparer.Instance);
 
             foreach ((Slice keySlice, long expiredTicks, Slice ticksSlice) in GetExpiredValues(context, ticks))
             {