From b97eb92ec7668e4b90c59c4148166917a6f61af4 Mon Sep 17 00:00:00 2001 From: sebastianburckhardt Date: Tue, 29 Oct 2024 11:58:48 -0700 Subject: [PATCH] add comments as per PR feedback --- .../StorageLayer/Faster/AzureBlobs/BlobManager.cs | 3 +++ .../StorageLayer/Faster/PartitionStorage.cs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobManager.cs b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobManager.cs index d01589a1..e37a0ee0 100644 --- a/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobManager.cs +++ b/src/DurableTask.Netherite/StorageLayer/Faster/AzureBlobs/BlobManager.cs @@ -1140,6 +1140,9 @@ await this.PerformWithRetriesAsync( await this.WriteCheckpointMetadataAsync(); + // we boost the tracing after three failed attempts. This boosting applies to the recovery part only. + // After thirty attempts, we stop boosting since it seems unlikely + // that there is any more information after that that cannot be found in the logs for the first 30 attempts. if (this.CheckpointInfo.RecoveryAttempts > 3 && this.CheckpointInfo.RecoveryAttempts < 30) { this.TraceHelper.BoostTracing = true; diff --git a/src/DurableTask.Netherite/StorageLayer/Faster/PartitionStorage.cs b/src/DurableTask.Netherite/StorageLayer/Faster/PartitionStorage.cs index 5eb3ae57..c17b663f 100644 --- a/src/DurableTask.Netherite/StorageLayer/Faster/PartitionStorage.cs +++ b/src/DurableTask.Netherite/StorageLayer/Faster/PartitionStorage.cs @@ -191,6 +191,8 @@ async Task TerminationWrapper(Task what) { // replay log as the store checkpoint lags behind the log + // disable prefetch if we have had many unsuccessful recovery attempts, or if the settings say so + // We choose 6 as the threshold since the tracing gets boosted after 3 attempts, and we want to see 3 attempts with boosted tracing before we disable prefetch bool disablePrefetch = this.blobManager.CheckpointInfo.RecoveryAttempts > 6 || this.settings.DisablePrefetchDuringReplay; await this.TerminationWrapper(this.storeWorker.ReplayCommitLog(this.logWorker, prefetch: !disablePrefetch));