From 840095cbe9cc44b25e68cdf4d8fe3582dca789ad Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 16:28:51 +0300 Subject: [PATCH 01/19] OldPartitionsRemover: add partial removablepartitionsfinder --- .../Entities/RemovablePartition.cs | 6 + .../Infrastructure/ResultsCombiner.cs | 10 +- .../RemovablePartitionsFinder.cs | 130 ++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 src/OldPartitionsRemover/Entities/RemovablePartition.cs create mode 100644 src/OldPartitionsRemover/RemovablePartitionsFinder.cs diff --git a/src/OldPartitionsRemover/Entities/RemovablePartition.cs b/src/OldPartitionsRemover/Entities/RemovablePartition.cs new file mode 100644 index 0000000..c9f70be --- /dev/null +++ b/src/OldPartitionsRemover/Entities/RemovablePartition.cs @@ -0,0 +1,6 @@ +using System; +using System.Threading.Tasks; + +namespace OldPartitionsRemover.Entities; + +public record class RemovablePartition(string Id, DateTimeOffset Timestamp, Func Remove); diff --git a/src/OldPartitionsRemover/Infrastructure/ResultsCombiner.cs b/src/OldPartitionsRemover/Infrastructure/ResultsCombiner.cs index fb353e7..4adadcf 100644 --- a/src/OldPartitionsRemover/Infrastructure/ResultsCombiner.cs +++ b/src/OldPartitionsRemover/Infrastructure/ResultsCombiner.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -32,6 +32,12 @@ public async Task>> CollectResults(IEnumerable elems, Fu async (l, p) => (await f(p)).Map(part => { l.AddRange(part); return l; })); } + public async Task>> CollectResults(params Task>>[] fs) + { + return await CombineResults(fs, new List(), + async (l, t) => (await t).Map(part => { l.AddRange(part); return l; })); + } + public async Task> CombineResults(IEnumerable elems, Y seed, Func>> f) { var result = Result.Ok(seed); @@ -59,4 +65,4 @@ private static async Task> Combine(Func>> f return combined.Bind(_ => _); } } -} \ No newline at end of file +} diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs new file mode 100644 index 0000000..b1ec923 --- /dev/null +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BobApi; +using BobApi.BobEntities; +using BobToolsCli; +using BobToolsCli.Helpers; +using OldPartitionsRemover.Entities; +using OldPartitionsRemover.Infrastructure; + +namespace OldPartitionsRemover; + +public class RemovablePartitionsFinder +{ + private readonly ResultsCombiner _resultsCombiner; + private readonly BobApiClientProvider _bobApiProvider; + + public RemovablePartitionsFinder(ResultsCombiner resultsCombiner, CommonArguments args) + { + _resultsCombiner = resultsCombiner; + _bobApiProvider = args.GetBobApiClientProvider(); + } + + public async Task>> Find( + ClusterConfiguration config, + bool allowAliens, + CancellationToken cancellationToken + ) + { + var vDisksByNode = GetVDisksByNode(config); + + return await _resultsCombiner.CollectResults( + config.Nodes, + async node => + { + if (allowAliens) + return await _resultsCombiner.CollectResults( + FindNormalOnNode(config, node, cancellationToken), + FindAlienOnNode(vDisksByNode, node, cancellationToken) + ); + else + return await FindNormalOnNode(config, node, cancellationToken); + } + ); + } + + private Dictionary> GetVDisksByNode(ClusterConfiguration config) + { + var result = config.Nodes.ToDictionary(n => n.Name, _ => new List()); + foreach (var vDisk in config.VDisks) + foreach (var replica in vDisk.Replicas) + { + if (result.TryGetValue(replica.Node, out var vDisks)) + vDisks.Add(vDisk.Id); + } + return result; + } + + private async Task>> FindNormalOnNode( + ClusterConfiguration config, + ClusterConfiguration.Node node, + CancellationToken cancellationToken + ) + { + using var api = _bobApiProvider.GetClient(node); + var vDisksByDisk = GetVDisksByDisk(config, node); + return await _resultsCombiner.CollectResults( + vDisksByDisk, + async kv => + await _resultsCombiner.CollectResults( + kv.Value, + async vDisk => await FindNormal(api, kv.Key, vDisk, cancellationToken) + ) + ); + } + + private async Task>> FindAlienOnNode( + Dictionary> vDisksByNode, + ClusterConfiguration.Node node, + CancellationToken cancellationToken + ) + { + using var api = _bobApiProvider.GetClient(node); + return await _resultsCombiner.CollectResults( + vDisksByNode.Where(kv => kv.Key != node.Name), + async kv => + await _resultsCombiner.CollectResults( + kv.Value, + async vDisk => await FindAlien(api, kv.Key, vDisk, cancellationToken) + ) + ); + } + + private Dictionary> GetVDisksByDisk( + ClusterConfiguration config, + ClusterConfiguration.Node node + ) + { + var result = node.Disks.ToDictionary(d => d.Name, _ => new List()); + foreach (var vDisk in config.VDisks) + foreach (var replica in vDisk.Replicas) + { + if (replica.Node == node.Name && result.TryGetValue(replica.Disk, out var vDisks)) + vDisks.Add(vDisk.Id); + } + return result; + } + + private async Task>> FindNormal( + BobApiClient api, + string disk, + long vdisk, + CancellationToken cancellationToken + ) + { + throw new NotImplementedException(); + } + + private async Task>> FindAlien( + BobApiClient api, + string key, + long vDisk, + CancellationToken cancellationToken + ) + { + throw new NotImplementedException(); + } +} From 0780692c43fb23c070e12ee09baafd3ccf063e06 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 16:52:58 +0300 Subject: [PATCH 02/19] OldPartitionsRemover: add support for new partitions api --- src/BobApi/BobApiClient.cs | 44 +++++++++++++++++++++++++++- src/BobApi/Entities/PartitionSlim.cs | 13 ++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 src/BobApi/Entities/PartitionSlim.cs diff --git a/src/BobApi/BobApiClient.cs b/src/BobApi/BobApiClient.cs index f4d58b4..a51cbe9 100644 --- a/src/BobApi/BobApiClient.cs +++ b/src/BobApi/BobApiClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Net.Http; using System.Net.Http.Headers; @@ -136,6 +136,48 @@ await GetJson( cancellationToken: cancellationToken ); + public async Task>> GetPartitionSlims( + string diskName, + long vDiskId, + CancellationToken cancellationToken = default + ) => + await GetJson>( + $"disks/{diskName}/vdisks/{vDiskId}/partitions", + cancellationToken + ); + + public async Task>> GetAlienPartitionSlims( + string nodeName, + long vDiskId, + CancellationToken cancellationToken = default + ) => + await GetJson>( + $"alien/nodes/{nodeName}/vdisks/{vDiskId}/partitions", + cancellationToken + ); + + public async Task> DeletePartitionById( + string diskName, + long vDiskId, + string partitionId, + CancellationToken cancellationToken + ) => + await DeleteIsOk( + $"disks/{diskName}/vdisks/{vDiskId}/partitions/{partitionId}", + cancellationToken + ); + + public async Task> DeleteAlienPartitionById( + string nodeName, + long vDiskId, + string partitionId, + CancellationToken cancellationToken + ) => + await DeleteIsOk( + $"alien/nodes/{nodeName}/vdisks/{vDiskId}/partitions/{partitionId}", + cancellationToken + ); + public async Task> CountRecordsOnVDisk( long id, CancellationToken cancellationToken = default diff --git a/src/BobApi/Entities/PartitionSlim.cs b/src/BobApi/Entities/PartitionSlim.cs new file mode 100644 index 0000000..e2d777b --- /dev/null +++ b/src/BobApi/Entities/PartitionSlim.cs @@ -0,0 +1,13 @@ +using Newtonsoft.Json; + +namespace BobApi.Entities +{ + public class PartitionSlim + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("timestamp")] + public long Timestamp { get; set; } + } +} From d962c3b5c2228a9539403c3f7879eaf7e7febed3 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 17:01:33 +0300 Subject: [PATCH 03/19] OldPartitionsRemover: finish removable partitions finder --- .../Entities/RemovablePartition.cs | 7 ++- src/OldPartitionsRemover/Program.cs | 13 +---- .../RemovablePartitionsFinder.cs | 58 +++++++++++++++++-- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/OldPartitionsRemover/Entities/RemovablePartition.cs b/src/OldPartitionsRemover/Entities/RemovablePartition.cs index c9f70be..97455ea 100644 --- a/src/OldPartitionsRemover/Entities/RemovablePartition.cs +++ b/src/OldPartitionsRemover/Entities/RemovablePartition.cs @@ -1,6 +1,11 @@ using System; +using System.Threading; using System.Threading.Tasks; namespace OldPartitionsRemover.Entities; -public record class RemovablePartition(string Id, DateTimeOffset Timestamp, Func Remove); +public record class RemovablePartition( + string Id, + DateTimeOffset Timestamp, + Func Remove +); diff --git a/src/OldPartitionsRemover/Program.cs b/src/OldPartitionsRemover/Program.cs index f207eca..d3966bb 100644 --- a/src/OldPartitionsRemover/Program.cs +++ b/src/OldPartitionsRemover/Program.cs @@ -1,20 +1,8 @@ using System; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using BobApi; -using BobApi.Entities; using BobToolsCli; -using BobToolsCli.Helpers; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; using OldPartitionsRemover.Infrastructure; namespace OldPartitionsRemover @@ -38,6 +26,7 @@ private static async Task RemovePartitions(IServiceCollection services, Fu { services.AddTransient(); services.AddTransient(); + services.AddTransient(); using var provider = services.BuildServiceProvider(); var remover = provider.GetRequiredService(); diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs index b1ec923..4ef26bc 100644 --- a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using BobApi; using BobApi.BobEntities; +using BobApi.Entities; using BobToolsCli; using BobToolsCli.Helpers; using OldPartitionsRemover.Entities; @@ -64,7 +65,8 @@ private async Task>> FindNormalOnNode( CancellationToken cancellationToken ) { - using var api = _bobApiProvider.GetClient(node); + // API is not disposed because it will be captured in removable partitions + var api = _bobApiProvider.GetClient(node); var vDisksByDisk = GetVDisksByDisk(config, node); return await _resultsCombiner.CollectResults( vDisksByDisk, @@ -82,7 +84,8 @@ private async Task>> FindAlienOnNode( CancellationToken cancellationToken ) { - using var api = _bobApiProvider.GetClient(node); + // API is not disposed because it will be captured in removable partitions + var api = _bobApiProvider.GetClient(node); return await _resultsCombiner.CollectResults( vDisksByNode.Where(kv => kv.Key != node.Name), async kv => @@ -111,20 +114,63 @@ ClusterConfiguration.Node node private async Task>> FindNormal( BobApiClient api, string disk, - long vdisk, + long vDisk, CancellationToken cancellationToken ) { - throw new NotImplementedException(); + Result> apiResult = await api.GetPartitionSlims( + disk, + vDisk, + cancellationToken + ); + return apiResult.Map( + ps => + ps.Select( + p => + CreateRemovablePartition( + p, + async ct => await api.DeletePartitionById(disk, vDisk, p.Id, ct) + ) + ) + .ToList() + ); } private async Task>> FindAlien( BobApiClient api, - string key, + string node, long vDisk, CancellationToken cancellationToken ) { - throw new NotImplementedException(); + Result> apiResult = await api.GetAlienPartitionSlims( + node, + vDisk, + cancellationToken + ); + return apiResult.Map( + ps => + ps.Select( + p => + CreateRemovablePartition( + p, + async ct => + await api.DeleteAlienPartitionById(node, vDisk, p.Id, ct) + ) + ) + .ToList() + ); + } + + private RemovablePartition CreateRemovablePartition( + PartitionSlim p, + Func remove + ) + { + return new RemovablePartition( + p.Id, + DateTimeOffset.FromUnixTimeSeconds(p.Timestamp), + remove + ); } } From d1ba36481fc1cd31ca01c36b060d3dc75cf327ca Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 17:19:32 +0300 Subject: [PATCH 04/19] OldPartitionsRemover: update space remover --- .../BySpaceRemoving/Arguments.cs | 6 ++- .../BySpaceRemoving/Remover.cs | 39 ++++-------------- .../Entities/RemovablePartition.cs | 4 +- .../RemovablePartitionsFinder.cs | 41 ++++++++++++++----- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs b/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs index 4634a08..7e1be81 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs @@ -1,5 +1,4 @@ -using System; -using System.Text.RegularExpressions; +using System; using BobToolsCli; using ByteSizeLib; using CommandLine; @@ -19,6 +18,9 @@ public class Arguments : CommonArguments [Option("threshold-type", Default = "free", HelpText = "Type of threshold: `free` space on node or bob's `occupied` space")] public string ThresholdTypeString { get; set; } // Enums are case sensitive in CommandLineParser by default, and changing this requires recreating whole help + [Option('a', "allow-alien", Default = false, HelpText = "Allow removal of alien partitions", Required = false)] + public bool AllowAlien { get; set; } + public Result GetThresholdType() { if (Enum.TryParse(ThresholdTypeString, true, out var tt)) diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs index 06a0ccb..cde4d45 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -20,16 +20,18 @@ public class Remover private readonly IConfigurationFinder _configurationFinder; private readonly IBobApiClientFactory _bobApiClientFactory; private readonly ResultsCombiner _resultsCombiner; + private readonly RemovablePartitionsFinder _removablePartitionsFinder; private readonly ILogger _logger; public Remover(Arguments arguments, IConfigurationFinder configurationFinder, IBobApiClientFactory bobApiClientFactory, ResultsCombiner resultsCombiner, - ILogger logger) + RemovablePartitionsFinder removablePartitionsFinder, ILogger logger) { _arguments = arguments; _configurationFinder = configurationFinder; _bobApiClientFactory = bobApiClientFactory; _resultsCombiner = resultsCombiner; + _removablePartitionsFinder = removablePartitionsFinder; _logger = logger; } @@ -89,40 +91,17 @@ private async Task> RemoveOnNode(ClusterConfiguration clusterConfigu { if (isDone) return Result.Ok(n); - var removeResult = await rem(); - return removeResult.Map(removed => n + removed); + var removeResult = await rem.Remove(cancellationToken); + return removeResult.Map(removed => n + (removed ? 1 : 0)); }); })); } - private async Task>>>>> GetRemovalFunctions( + private async Task>> GetRemovalFunctions( ClusterConfiguration clusterConfiguration, ClusterConfiguration.Node node, CancellationToken cancellationToken) { - var partitionsApi = _bobApiClientFactory.GetPartitionsBobApiClient(node); - var vdisksToCheck = clusterConfiguration.VDisks.Where(r => r.Replicas.Any(r => r.Node == node.Name)); - var partitionsResult = await _resultsCombiner.CollectResults(vdisksToCheck, async vd => - { - Result> partitionIdsResult = await partitionsApi.GetPartitions(vd, cancellationToken); - return await partitionIdsResult.Bind(async partitionIds => - { - var partitionsResult = await _resultsCombiner.CollectResults(partitionIds, - async p => await partitionsApi.GetPartition(vd.Id, p, cancellationToken)); - return partitionsResult; - }); - }); - return partitionsResult - .Map(partitions => partitions - .GroupBy(p => (p.Timestamp, p.VDiskId)) - .OrderBy(g => g.Key.Timestamp).ThenBy(g => g.Key.VDiskId) - .Select(g => CreateRemovalFunc(g.Key.VDiskId, g.Key.Timestamp, g.Count()))); - - Func>> CreateRemovalFunc(long vdiskId, long timestamp, int partitionsCount) - => async () => - { - _logger.LogTrace("Removing partitions by timestamp {Timestamp} on {Node}/{VDisk}", timestamp, node.Name, vdiskId); - var result = await partitionsApi.DeletePartitionsByTimestamp(vdiskId, timestamp, cancellationToken); - return result.Map(r => r ? partitionsCount : 0); - }; + var removableResult = await _removablePartitionsFinder.FindOnNode(clusterConfiguration, node, _arguments.AllowAlien, cancellationToken); + return removableResult; } } } diff --git a/src/OldPartitionsRemover/Entities/RemovablePartition.cs b/src/OldPartitionsRemover/Entities/RemovablePartition.cs index 97455ea..708101e 100644 --- a/src/OldPartitionsRemover/Entities/RemovablePartition.cs +++ b/src/OldPartitionsRemover/Entities/RemovablePartition.cs @@ -7,5 +7,7 @@ namespace OldPartitionsRemover.Entities; public record class RemovablePartition( string Id, DateTimeOffset Timestamp, - Func Remove + RemoveRemovablePartition Remove ); + +public delegate Task> RemoveRemovablePartition(CancellationToken ct); diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs index 4ef26bc..cbf770f 100644 --- a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -31,22 +31,41 @@ CancellationToken cancellationToken ) { var vDisksByNode = GetVDisksByNode(config); - return await _resultsCombiner.CollectResults( config.Nodes, async node => - { - if (allowAliens) - return await _resultsCombiner.CollectResults( - FindNormalOnNode(config, node, cancellationToken), - FindAlienOnNode(vDisksByNode, node, cancellationToken) - ); - else - return await FindNormalOnNode(config, node, cancellationToken); - } + await FindOnNode(config, node, allowAliens, vDisksByNode, cancellationToken) ); } + public async Task>> FindOnNode( + ClusterConfiguration config, + ClusterConfiguration.Node node, + bool allowAliens, + CancellationToken cancellationToken + ) + { + var vDisksByNode = GetVDisksByNode(config); + return await FindOnNode(config, node, allowAliens, vDisksByNode, cancellationToken); + } + + private async Task>> FindOnNode( + ClusterConfiguration config, + ClusterConfiguration.Node node, + bool allowAliens, + Dictionary> vDisksByNode, + CancellationToken cancellationToken + ) + { + if (allowAliens) + return await _resultsCombiner.CollectResults( + FindNormalOnNode(config, node, cancellationToken), + FindAlienOnNode(vDisksByNode, node, cancellationToken) + ); + else + return await FindNormalOnNode(config, node, cancellationToken); + } + private Dictionary> GetVDisksByNode(ClusterConfiguration config) { var result = config.Nodes.ToDictionary(n => n.Name, _ => new List()); @@ -164,7 +183,7 @@ await api.DeleteAlienPartitionById(node, vDisk, p.Id, ct) private RemovablePartition CreateRemovablePartition( PartitionSlim p, - Func remove + RemoveRemovablePartition remove ) { return new RemovablePartition( From 5e9015c434e76574e7a6c79311afd9b8c2d5deb5 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 18:06:30 +0300 Subject: [PATCH 05/19] OldPartitionsRemover: update dates remover --- .../ByDateRemoving/Arguments.cs | 7 +- .../ByDateRemoving/Remover.cs | 81 +++---------------- 2 files changed, 18 insertions(+), 70 deletions(-) diff --git a/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs b/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs index 0cefe1c..48156cf 100644 --- a/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs +++ b/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Globalization; using System.Text.RegularExpressions; using BobToolsCli; @@ -15,6 +15,9 @@ public class Arguments : CommonArguments [Option('t', "threshold", HelpText = "Removal threshold. Can be either date, timestamp or in relative days count format, e.g. \"-3d\"", Required = true)] public string ThresholdString { get; set; } + [Option('a', "allow-alien", Default = false, HelpText = "Allow removal of alien partitions", Required = false)] + public bool AllowAlien { get; set; } + public Result GetThreshold() { if (string.IsNullOrWhiteSpace(ThresholdString)) @@ -50,4 +53,4 @@ private static bool TryParseThreshold(string s, DateTime now, out DateTime thres return false; } } -} \ No newline at end of file +} diff --git a/src/OldPartitionsRemover/ByDateRemoving/Remover.cs b/src/OldPartitionsRemover/ByDateRemoving/Remover.cs index 6cd2981..07e0069 100644 --- a/src/OldPartitionsRemover/ByDateRemoving/Remover.cs +++ b/src/OldPartitionsRemover/ByDateRemoving/Remover.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -24,16 +24,18 @@ public partial class Remover private readonly IBobApiClientFactory _bobApiClientFactory; private readonly IConfigurationFinder _configurationFinder; private readonly ResultsCombiner _resultsCombiner; + private readonly RemovablePartitionsFinder _removablePartitionsFinder; public Remover(Arguments arguments, ILogger logger, IBobApiClientFactory bobApiClientFactory, IConfigurationFinder configurationFinder, - ResultsCombiner resultsCombiner) + ResultsCombiner resultsCombiner, RemovablePartitionsFinder removablePartitionsFinder) { _arguments = arguments; _logger = logger; _bobApiClientFactory = bobApiClientFactory; _configurationFinder = configurationFinder; _resultsCombiner = resultsCombiner; + _removablePartitionsFinder = removablePartitionsFinder; } public async Task> RemoveOldPartitions(CancellationToken cancellationToken) @@ -46,80 +48,23 @@ public async Task> RemoveOldPartitions(CancellationToken cancellatio _logger.LogInformation("Removing blobs older than {Threshold}", t); return configResult.Bind(c => FindInCluster(c, t, cancellationToken)); }); - return await removeOperations.Bind(InvokeOperations); + return await removeOperations.Bind(async ops => await InvokeOperations(ops, cancellationToken)); } - private async Task>> FindInCluster(ClusterConfiguration clusterConfig, + private async Task>> FindInCluster(ClusterConfiguration clusterConfig, DateTime threshold, CancellationToken cancellationToken) - => await _resultsCombiner.CollectResults(clusterConfig.Nodes, - async node => await FindOnNode(clusterConfig, node, threshold, cancellationToken)); - - private async Task>> FindOnNode(ClusterConfiguration clusterConfig, - ClusterConfiguration.Node node, DateTime threshold, CancellationToken cancellationToken) - { - _logger.LogInformation("Preparing partitions to remove from node {Node}", node.Name); - - var vdisksOnNode = clusterConfig.VDisks.Where(vd => vd.Replicas.Any(r => r.Node == node.Name)); - var nodeApi = new NodeApi(_bobApiClientFactory.GetPartitionsBobApiClient(node), cancellationToken); - - return await _resultsCombiner.CollectResults(vdisksOnNode, async vdisk => await FindOnVDisk(vdisk, nodeApi, threshold)); - } - - private async Task>> FindOnVDisk(ClusterConfiguration.VDisk vdisk, NodeApi nodeApi, - DateTime threshold) - { - _logger.LogDebug("Preparing partitions to remove from vdisk {VDisk}", vdisk.Id); - - var partitionFunctions = new PartitionFunctions(vdisk, nodeApi); - - var partitionIdsResult = await partitionFunctions.FindPartitionIds(); - return await partitionIdsResult.Bind(partitionIds => FindWithinPatitionIds(partitionIds, partitionFunctions, threshold)); - } - - private async Task>> FindWithinPatitionIds(List partitionIds, - PartitionFunctions partitionFunctions, DateTime threshold) { - var partitionsResult = await GetPartitions(partitionIds, partitionFunctions); - return partitionsResult.Bind(partitions => FindWithinPartitions(partitions, partitionFunctions, threshold)); - } - - private async Task>> GetPartitions(List partitionIds, - PartitionFunctions partitionFunctions) - { - return await _resultsCombiner.CollectResults(partitionIds, async p => await partitionFunctions.FindPartitionById(p)); - } - - private Result> FindWithinPartitions(List partitionInfos, - PartitionFunctions partitionFunctions, DateTime threshold) - { - return FindByTimestamp(partitionInfos, partitionFunctions, threshold); - } - - private Result> FindByTimestamp(List partitionInfos, - PartitionFunctions partitionFunctions, DateTimeOffset threshold) - { - var oldTimestamps = partitionInfos.Select(p => p.Timestamp) - .Where(p => DateTimeOffset.FromUnixTimeSeconds(p) < threshold) - .Distinct() - .ToArray(); - var countByOldTimestamp = partitionInfos.Select(p => p.Timestamp) - .Where(p => DateTimeOffset.FromUnixTimeSeconds(p) < threshold) - .GroupBy(p => p) - .ToDictionary(g => g.Key, g => g.Count()); - if (oldTimestamps.Length > 0) - _logger.LogInformation("Preparing partitions from {TimestampsCount} timestamps to remove", oldTimestamps.Length); - else - _logger.LogInformation("No partitions to be removed"); - - var removeOperations = oldTimestamps.Select(ts => - async () => (await partitionFunctions.RemovePartitionsByTimestamp(ts)).Map(t => t ? countByOldTimestamp[ts] : 0)).ToList(); - return Result>.Ok(removeOperations); + return (await _removablePartitionsFinder.Find(clusterConfig, _arguments.AllowAlien, cancellationToken)) + .Map(rms => { + rms.RemoveAll(rm => rm.Timestamp >= threshold); + return rms; + }); } - private async Task> InvokeOperations(List ops) + private async Task> InvokeOperations(List ops, CancellationToken cancellationToken) { _logger.LogInformation("Invoking {RemoveOperationsCount} remove operations", ops.Count); - return await _resultsCombiner.CombineResults(ops, 0, async (c, n) => (await n()).Map(r => c + r)); + return await _resultsCombiner.CombineResults(ops, 0, async (c, n) => (await n.Remove(cancellationToken)).Map(r => c + (r ? 1 : 0))); } } } From c17365a7660ade22936e24ea0f8ae817e0ed15fc Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 18:09:04 +0300 Subject: [PATCH 06/19] OldPartitionsRemover: refactor api --- src/BobApi/IPartitionsBobApiClient.cs | 41 +++++++++++++++++-- .../ByDateRemoving/Remover.cs | 6 --- .../BySpaceRemoving/Remover.cs | 6 +-- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/BobApi/IPartitionsBobApiClient.cs b/src/BobApi/IPartitionsBobApiClient.cs index 4cf9950..9691233 100644 --- a/src/BobApi/IPartitionsBobApiClient.cs +++ b/src/BobApi/IPartitionsBobApiClient.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BobApi.BobEntities; @@ -8,8 +8,41 @@ namespace BobApi { public interface IPartitionsBobApiClient { - Task> DeletePartitionsByTimestamp(long vDiskId, long timestamp, CancellationToken cancellationToken = default); - Task> GetPartition(long vdiskId, string partition, CancellationToken cancellationToken = default); - Task>> GetPartitions(ClusterConfiguration.VDisk vDisk, CancellationToken cancellationToken = default); + Task> DeletePartitionsByTimestamp( + long vDiskId, + long timestamp, + CancellationToken cancellationToken = default + ); + Task> GetPartition( + long vdiskId, + string partition, + CancellationToken cancellationToken = default + ); + Task>> GetPartitions( + ClusterConfiguration.VDisk vDisk, + CancellationToken cancellationToken = default + ); + Task>> GetPartitionSlims( + string diskName, + long vDiskId, + CancellationToken cancellationToken = default + ); + Task>> GetAlienPartitionSlims( + string nodeName, + long vDiskId, + CancellationToken cancellationToken = default + ); + Task> DeletePartitionById( + string diskName, + long vDiskId, + string partitionId, + CancellationToken cancellationToken + ); + Task> DeleteAlienPartitionById( + string nodeName, + long vDiskId, + string partitionId, + CancellationToken cancellationToken + ); } } diff --git a/src/OldPartitionsRemover/ByDateRemoving/Remover.cs b/src/OldPartitionsRemover/ByDateRemoving/Remover.cs index 07e0069..8f1a784 100644 --- a/src/OldPartitionsRemover/ByDateRemoving/Remover.cs +++ b/src/OldPartitionsRemover/ByDateRemoving/Remover.cs @@ -1,17 +1,11 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using BobApi; using BobApi.BobEntities; -using BobApi.Entities; -using BobToolsCli; using BobToolsCli.BobApliClientFactories; using BobToolsCli.ConfigurationFinding; -using BobToolsCli.Helpers; using Microsoft.Extensions.Logging; -using OldPartitionsRemover.ByDateRemoving.Entities; using OldPartitionsRemover.Entities; using OldPartitionsRemover.Infrastructure; diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs index cde4d45..38a7bc9 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs @@ -1,13 +1,9 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using BobApi; using BobApi.BobEntities; using BobToolsCli.BobApliClientFactories; using BobToolsCli.ConfigurationFinding; -using ByteSizeLib; using Microsoft.Extensions.Logging; using OldPartitionsRemover.Entities; using OldPartitionsRemover.Infrastructure; From 08bad5971a704531f246e1d1f84cc531f9d0d654 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 18:10:35 +0300 Subject: [PATCH 07/19] OldPartitionsRemover: use proper interface for api access --- .../RemovablePartitionsFinder.cs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs index cbf770f..9bc00de 100644 --- a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -6,8 +6,7 @@ using BobApi; using BobApi.BobEntities; using BobApi.Entities; -using BobToolsCli; -using BobToolsCli.Helpers; +using BobToolsCli.BobApliClientFactories; using OldPartitionsRemover.Entities; using OldPartitionsRemover.Infrastructure; @@ -16,12 +15,15 @@ namespace OldPartitionsRemover; public class RemovablePartitionsFinder { private readonly ResultsCombiner _resultsCombiner; - private readonly BobApiClientProvider _bobApiProvider; + private readonly IBobApiClientFactory _bobApiClientFactory; - public RemovablePartitionsFinder(ResultsCombiner resultsCombiner, CommonArguments args) + public RemovablePartitionsFinder( + ResultsCombiner resultsCombiner, + IBobApiClientFactory bobApiClientFactory + ) { _resultsCombiner = resultsCombiner; - _bobApiProvider = args.GetBobApiClientProvider(); + _bobApiClientFactory = bobApiClientFactory; } public async Task>> Find( @@ -85,7 +87,7 @@ CancellationToken cancellationToken ) { // API is not disposed because it will be captured in removable partitions - var api = _bobApiProvider.GetClient(node); + var api = _bobApiClientFactory.GetPartitionsBobApiClient(node); var vDisksByDisk = GetVDisksByDisk(config, node); return await _resultsCombiner.CollectResults( vDisksByDisk, @@ -104,7 +106,7 @@ CancellationToken cancellationToken ) { // API is not disposed because it will be captured in removable partitions - var api = _bobApiProvider.GetClient(node); + var api = _bobApiClientFactory.GetPartitionsBobApiClient(node); return await _resultsCombiner.CollectResults( vDisksByNode.Where(kv => kv.Key != node.Name), async kv => @@ -131,7 +133,7 @@ ClusterConfiguration.Node node } private async Task>> FindNormal( - BobApiClient api, + IPartitionsBobApiClient api, string disk, long vDisk, CancellationToken cancellationToken @@ -156,7 +158,7 @@ CancellationToken cancellationToken } private async Task>> FindAlien( - BobApiClient api, + IPartitionsBobApiClient api, string node, long vDisk, CancellationToken cancellationToken From 2b3e1c6c730e35fbebaa48b062df049acc7bb149 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 18:22:44 +0300 Subject: [PATCH 08/19] OldPartitionsRemover: use single config for all data --- .../RemovablePartitionsFinder.cs | 53 ++++++++++++------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs index 9bc00de..7dacb67 100644 --- a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -7,6 +7,7 @@ using BobApi.BobEntities; using BobApi.Entities; using BobToolsCli.BobApliClientFactories; +using BobToolsCli.Exceptions; using OldPartitionsRemover.Entities; using OldPartitionsRemover.Infrastructure; @@ -32,11 +33,11 @@ public async Task>> Find( CancellationToken cancellationToken ) { - var vDisksByNode = GetVDisksByNode(config); + var vDisksConfiguration = GetVDisksConfiguration(config); return await _resultsCombiner.CollectResults( config.Nodes, async node => - await FindOnNode(config, node, allowAliens, vDisksByNode, cancellationToken) + await FindOnNode(vDisksConfiguration, node, allowAliens, cancellationToken) ); } @@ -47,48 +48,59 @@ public async Task>> FindOnNode( CancellationToken cancellationToken ) { - var vDisksByNode = GetVDisksByNode(config); - return await FindOnNode(config, node, allowAliens, vDisksByNode, cancellationToken); + var vDisksConfiguration = GetVDisksConfiguration(config); + return await FindOnNode(vDisksConfiguration, node, allowAliens, cancellationToken); } private async Task>> FindOnNode( - ClusterConfiguration config, + VDisksConfiguration vDisksConfiguration, ClusterConfiguration.Node node, bool allowAliens, - Dictionary> vDisksByNode, CancellationToken cancellationToken ) { if (allowAliens) return await _resultsCombiner.CollectResults( - FindNormalOnNode(config, node, cancellationToken), - FindAlienOnNode(vDisksByNode, node, cancellationToken) + FindNormalOnNode(vDisksConfiguration, node, cancellationToken), + FindAlienOnNode(vDisksConfiguration, node, cancellationToken) ); else - return await FindNormalOnNode(config, node, cancellationToken); + return await FindNormalOnNode(vDisksConfiguration, node, cancellationToken); } - private Dictionary> GetVDisksByNode(ClusterConfiguration config) + private VDisksConfiguration GetVDisksConfiguration(ClusterConfiguration config) { - var result = config.Nodes.ToDictionary(n => n.Name, _ => new List()); + var vDisksByNode = config.Nodes.ToDictionary(n => n.Name, _ => new HashSet()); + var vDisksByDiskByNode = config.Nodes.ToDictionary( + n => n.Name, + _ => new Dictionary>() + ); foreach (var vDisk in config.VDisks) foreach (var replica in vDisk.Replicas) { - if (result.TryGetValue(replica.Node, out var vDisks)) - vDisks.Add(vDisk.Id); + if (vDisksByNode.TryGetValue(replica.Node, out var vDisksHs)) + vDisksHs.Add(vDisk.Id); + if (vDisksByDiskByNode.TryGetValue(replica.Node, out var vDisksByDisk)) + { + if (vDisksByDisk.TryGetValue(replica.Disk, out var vDisks)) + vDisks.Add(vDisk.Id); + else + vDisksByDisk.Add(replica.Disk, new List { vDisk.Id }); + } } - return result; + return new VDisksConfiguration(vDisksByDiskByNode, vDisksByNode); } private async Task>> FindNormalOnNode( - ClusterConfiguration config, + VDisksConfiguration vDisksConfiguration, ClusterConfiguration.Node node, CancellationToken cancellationToken ) { // API is not disposed because it will be captured in removable partitions var api = _bobApiClientFactory.GetPartitionsBobApiClient(node); - var vDisksByDisk = GetVDisksByDisk(config, node); + if (!vDisksConfiguration.VDisksByDiskByNode.TryGetValue(node.Name, out var vDisksByDisk)) + throw new ConfigurationException($"Node {node} is not presented in replicas"); return await _resultsCombiner.CollectResults( vDisksByDisk, async kv => @@ -100,7 +112,7 @@ await _resultsCombiner.CollectResults( } private async Task>> FindAlienOnNode( - Dictionary> vDisksByNode, + VDisksConfiguration vDisksConfiguration, ClusterConfiguration.Node node, CancellationToken cancellationToken ) @@ -108,7 +120,7 @@ CancellationToken cancellationToken // API is not disposed because it will be captured in removable partitions var api = _bobApiClientFactory.GetPartitionsBobApiClient(node); return await _resultsCombiner.CollectResults( - vDisksByNode.Where(kv => kv.Key != node.Name), + vDisksConfiguration.VDisksByNode.Where(kv => kv.Key != node.Name), async kv => await _resultsCombiner.CollectResults( kv.Value, @@ -194,4 +206,9 @@ RemoveRemovablePartition remove remove ); } + + private record struct VDisksConfiguration( + Dictionary>> VDisksByDiskByNode, + Dictionary> VDisksByNode + ); } From 1cb8675beb05583ee170b6ba4fb26b5e8e7fd450 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 18:26:30 +0300 Subject: [PATCH 09/19] OldPartitionsRemover: add common remover args --- .../ByDateRemoving/Arguments.cs | 6 +----- .../ByDateRemoving/Remover.cs | 10 +++++----- .../BySpaceRemoving/Arguments.cs | 6 +----- .../BySpaceRemoving/Remover.cs | 2 +- src/OldPartitionsRemover/Program.cs | 7 ++++--- .../RemovablePartitionsFinder.cs | 15 +++++++-------- src/OldPartitionsRemover/RemoverArguments.cs | 16 ++++++++++++++++ 7 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 src/OldPartitionsRemover/RemoverArguments.cs diff --git a/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs b/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs index 48156cf..ffca881 100644 --- a/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs +++ b/src/OldPartitionsRemover/ByDateRemoving/Arguments.cs @@ -1,22 +1,18 @@ using System; -using System.Globalization; using System.Text.RegularExpressions; -using BobToolsCli; using CommandLine; using OldPartitionsRemover.Entities; namespace OldPartitionsRemover.ByDateRemoving { [Verb("by-date")] - public class Arguments : CommonArguments + public class Arguments : RemoverArguments { private static readonly Regex s_timeSpanRegex = new(@"^\-(?\d+)(?[dhmy])"); [Option('t', "threshold", HelpText = "Removal threshold. Can be either date, timestamp or in relative days count format, e.g. \"-3d\"", Required = true)] public string ThresholdString { get; set; } - [Option('a', "allow-alien", Default = false, HelpText = "Allow removal of alien partitions", Required = false)] - public bool AllowAlien { get; set; } public Result GetThreshold() { diff --git a/src/OldPartitionsRemover/ByDateRemoving/Remover.cs b/src/OldPartitionsRemover/ByDateRemoving/Remover.cs index 8f1a784..ca980f8 100644 --- a/src/OldPartitionsRemover/ByDateRemoving/Remover.cs +++ b/src/OldPartitionsRemover/ByDateRemoving/Remover.cs @@ -48,11 +48,11 @@ public async Task> RemoveOldPartitions(CancellationToken cancellatio private async Task>> FindInCluster(ClusterConfiguration clusterConfig, DateTime threshold, CancellationToken cancellationToken) { - return (await _removablePartitionsFinder.Find(clusterConfig, _arguments.AllowAlien, cancellationToken)) - .Map(rms => { - rms.RemoveAll(rm => rm.Timestamp >= threshold); - return rms; - }); + return (await _removablePartitionsFinder.Find(clusterConfig, cancellationToken)) + .Map(rms => { + rms.RemoveAll(rm => rm.Timestamp >= threshold); + return rms; + }); } private async Task> InvokeOperations(List ops, CancellationToken cancellationToken) diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs b/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs index 7e1be81..0476bdf 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Arguments.cs @@ -1,5 +1,4 @@ using System; -using BobToolsCli; using ByteSizeLib; using CommandLine; using OldPartitionsRemover.Entities; @@ -7,7 +6,7 @@ namespace OldPartitionsRemover.BySpaceRemoving { [Verb("by-space")] - public class Arguments : CommonArguments + public class Arguments : RemoverArguments { [Option('t', "threshold", HelpText = "Removal threshold", Required = true)] public string ThresholdString { get; set; } @@ -18,9 +17,6 @@ public class Arguments : CommonArguments [Option("threshold-type", Default = "free", HelpText = "Type of threshold: `free` space on node or bob's `occupied` space")] public string ThresholdTypeString { get; set; } // Enums are case sensitive in CommandLineParser by default, and changing this requires recreating whole help - [Option('a', "allow-alien", Default = false, HelpText = "Allow removal of alien partitions", Required = false)] - public bool AllowAlien { get; set; } - public Result GetThresholdType() { if (Enum.TryParse(ThresholdTypeString, true, out var tt)) diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs index 38a7bc9..5983ef0 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs @@ -96,7 +96,7 @@ private async Task> RemoveOnNode(ClusterConfiguration clusterConfigu private async Task>> GetRemovalFunctions( ClusterConfiguration clusterConfiguration, ClusterConfiguration.Node node, CancellationToken cancellationToken) { - var removableResult = await _removablePartitionsFinder.FindOnNode(clusterConfiguration, node, _arguments.AllowAlien, cancellationToken); + var removableResult = await _removablePartitionsFinder.FindOnNode(clusterConfiguration, node, cancellationToken); return removableResult; } } diff --git a/src/OldPartitionsRemover/Program.cs b/src/OldPartitionsRemover/Program.cs index d3966bb..d30b0b7 100644 --- a/src/OldPartitionsRemover/Program.cs +++ b/src/OldPartitionsRemover/Program.cs @@ -16,17 +16,18 @@ private static async Task Main(string[] args) } private static async Task RemovePartitionsByDate(ByDateRemoving.Arguments args, IServiceCollection services, CancellationToken cancellationToken) - => await RemovePartitions(services, r => r.RemoveOldPartitions(cancellationToken)); + => await RemovePartitions(services, args, r => r.RemoveOldPartitions(cancellationToken)); private static async Task RemovePartitionsBySpace(BySpaceRemoving.Arguments args, IServiceCollection services, CancellationToken cancellationToken) - => await RemovePartitions(services, r => r.RemovePartitionsBySpace(cancellationToken)); + => await RemovePartitions(services, args, r => r.RemovePartitionsBySpace(cancellationToken)); - private static async Task RemovePartitions(IServiceCollection services, Func>> remove) + private static async Task RemovePartitions(IServiceCollection services, RemoverArguments args, Func>> remove) where TRem : class { services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddSingleton(args); using var provider = services.BuildServiceProvider(); var remover = provider.GetRequiredService(); diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs index 7dacb67..71874fb 100644 --- a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -17,49 +17,48 @@ public class RemovablePartitionsFinder { private readonly ResultsCombiner _resultsCombiner; private readonly IBobApiClientFactory _bobApiClientFactory; + private readonly RemoverArguments _removerArguments; public RemovablePartitionsFinder( ResultsCombiner resultsCombiner, - IBobApiClientFactory bobApiClientFactory + IBobApiClientFactory bobApiClientFactory, + RemoverArguments removerArguments ) { _resultsCombiner = resultsCombiner; _bobApiClientFactory = bobApiClientFactory; + _removerArguments = removerArguments; } public async Task>> Find( ClusterConfiguration config, - bool allowAliens, CancellationToken cancellationToken ) { var vDisksConfiguration = GetVDisksConfiguration(config); return await _resultsCombiner.CollectResults( config.Nodes, - async node => - await FindOnNode(vDisksConfiguration, node, allowAliens, cancellationToken) + async node => await FindOnNode(vDisksConfiguration, node, cancellationToken) ); } public async Task>> FindOnNode( ClusterConfiguration config, ClusterConfiguration.Node node, - bool allowAliens, CancellationToken cancellationToken ) { var vDisksConfiguration = GetVDisksConfiguration(config); - return await FindOnNode(vDisksConfiguration, node, allowAliens, cancellationToken); + return await FindOnNode(vDisksConfiguration, node, cancellationToken); } private async Task>> FindOnNode( VDisksConfiguration vDisksConfiguration, ClusterConfiguration.Node node, - bool allowAliens, CancellationToken cancellationToken ) { - if (allowAliens) + if (_removerArguments.AllowAlien) return await _resultsCombiner.CollectResults( FindNormalOnNode(vDisksConfiguration, node, cancellationToken), FindAlienOnNode(vDisksConfiguration, node, cancellationToken) diff --git a/src/OldPartitionsRemover/RemoverArguments.cs b/src/OldPartitionsRemover/RemoverArguments.cs new file mode 100644 index 0000000..1b2d82d --- /dev/null +++ b/src/OldPartitionsRemover/RemoverArguments.cs @@ -0,0 +1,16 @@ +using BobToolsCli; +using CommandLine; + +namespace OldPartitionsRemover; + +public class RemoverArguments : CommonArguments +{ + [Option( + 'a', + "allow-alien", + Default = false, + HelpText = "Allow removal of alien partitions", + Required = false + )] + public bool AllowAlien { get; set; } +} From 9ffeecf314ce82bc2b79943f2d719a0de16c5db5 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Fri, 19 Jan 2024 18:27:33 +0300 Subject: [PATCH 10/19] OldPartitionsRemover: remove dead code --- .../ByDateRemoving/Entities/NodeApi.cs | 21 ----------- .../Entities/PartitionFunctions.cs | 35 ------------------- .../Entities/RemoveOperation.cs | 8 ----- 3 files changed, 64 deletions(-) delete mode 100644 src/OldPartitionsRemover/ByDateRemoving/Entities/NodeApi.cs delete mode 100644 src/OldPartitionsRemover/ByDateRemoving/Entities/PartitionFunctions.cs delete mode 100644 src/OldPartitionsRemover/ByDateRemoving/Entities/RemoveOperation.cs diff --git a/src/OldPartitionsRemover/ByDateRemoving/Entities/NodeApi.cs b/src/OldPartitionsRemover/ByDateRemoving/Entities/NodeApi.cs deleted file mode 100644 index 6aa7f75..0000000 --- a/src/OldPartitionsRemover/ByDateRemoving/Entities/NodeApi.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using BobApi; - -namespace OldPartitionsRemover.ByDateRemoving.Entities -{ - internal readonly struct NodeApi - { - private readonly IPartitionsBobApiClient _bobApiClient; - private readonly CancellationToken _cancellationToken; - - public NodeApi(IPartitionsBobApiClient bobApiClient, CancellationToken cancellationToken) - { - _bobApiClient = bobApiClient; - _cancellationToken = cancellationToken; - } - - public async Task Invoke(Func> f) => await f(_bobApiClient, _cancellationToken); - } -} diff --git a/src/OldPartitionsRemover/ByDateRemoving/Entities/PartitionFunctions.cs b/src/OldPartitionsRemover/ByDateRemoving/Entities/PartitionFunctions.cs deleted file mode 100644 index 1fa1980..0000000 --- a/src/OldPartitionsRemover/ByDateRemoving/Entities/PartitionFunctions.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using BobApi; -using BobApi.BobEntities; -using OldPartitionsRemover.Entities; - -namespace OldPartitionsRemover.ByDateRemoving.Entities -{ - internal class PartitionFunctions - { - private readonly ClusterConfiguration.VDisk _vdisk; - private readonly NodeApi _nodeApi; - - public PartitionFunctions(ClusterConfiguration.VDisk vdisk, NodeApi nodeApi) - { - _vdisk = vdisk; - _nodeApi = nodeApi; - } - - public async Task> FindPartitionById(string id) - { - return await _nodeApi.Invoke((c, t) => c.GetPartition(_vdisk.Id, id, t)); - } - public async Task> RemovePartitionsByTimestamp(long timestamp) - { - Result result = await _nodeApi.Invoke((c, t) => c.DeletePartitionsByTimestamp(_vdisk.Id, timestamp, t)); - return result.Bind(r => r ? Result.Ok(true) : Result.Error("Failed to remove partitions though the bob API")); - } - - public async Task>> FindPartitionIds() - { - return await _nodeApi.Invoke((c, t) => c.GetPartitions(_vdisk, t)); - } - } -} diff --git a/src/OldPartitionsRemover/ByDateRemoving/Entities/RemoveOperation.cs b/src/OldPartitionsRemover/ByDateRemoving/Entities/RemoveOperation.cs deleted file mode 100644 index 6c89158..0000000 --- a/src/OldPartitionsRemover/ByDateRemoving/Entities/RemoveOperation.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; -using System.Threading.Tasks; -using OldPartitionsRemover.Entities; - -namespace OldPartitionsRemover.ByDateRemoving.Entities -{ - internal delegate Task> RemoveOperation(); -} \ No newline at end of file From dc9c96b55946b7058f7f0d196b2fcebd77e10964 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Tue, 23 Jan 2024 17:41:17 +0300 Subject: [PATCH 11/19] OldPartitionsRemover: refactor test to more readable form --- src/BobApi/Entities/BobApiResult.cs | 4 +- .../ConfigurationReadingResult.cs | 7 +- .../ByDateRemoving/ArgumentsTests.cs | 26 +- .../ByDateRemoving/RemoverTests.cs | 212 +++++------ .../BySpaceRemoving/ArgumentsTests.cs | 17 +- .../BySpaceRemoving/RemoverTests.cs | 360 +++++++----------- .../Common/GenericRemoverTests.cs | 146 +++++++ .../Common/ResultAssertionsChecker.cs | 77 ++++ .../Common/TestConstants.cs | 35 ++ .../FrozenApiClientsCustomization.cs | 32 -- .../FrozenArgumentsCustomization.cs | 23 -- .../SingleNodeConfigCustomization.cs | 54 --- .../OldPartitionsRemover.UnitTests.csproj | 15 +- 13 files changed, 510 insertions(+), 498 deletions(-) create mode 100644 test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs create mode 100644 test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs create mode 100644 test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs delete mode 100644 test/OldPartitionsRemover.UnitTests/Customizations/FrozenApiClientsCustomization.cs delete mode 100644 test/OldPartitionsRemover.UnitTests/Customizations/FrozenArgumentsCustomization.cs delete mode 100644 test/OldPartitionsRemover.UnitTests/Customizations/SingleNodeConfigCustomization.cs diff --git a/src/BobApi/Entities/BobApiResult.cs b/src/BobApi/Entities/BobApiResult.cs index 591dd4c..2a0db61 100644 --- a/src/BobApi/Entities/BobApiResult.cs +++ b/src/BobApi/Entities/BobApiResult.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -76,5 +76,7 @@ string content content ) ); + + public static implicit operator BobApiResult(T data) => Ok(data); } } diff --git a/src/BobToolsCli/ConfigurationReading/ConfigurationReadingResult.cs b/src/BobToolsCli/ConfigurationReading/ConfigurationReadingResult.cs index 615b6d8..ccd82a8 100644 --- a/src/BobToolsCli/ConfigurationReading/ConfigurationReadingResult.cs +++ b/src/BobToolsCli/ConfigurationReading/ConfigurationReadingResult.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace BobToolsCli.ConfigurationReading { @@ -28,6 +28,9 @@ public ConfigurationReadingResult Map(Func f) } public static ConfigurationReadingResult Ok(T data) => new(data, null); + public static ConfigurationReadingResult Error(string error) => new(default, error); + + public static implicit operator ConfigurationReadingResult(T data) => Ok(data); } -} \ No newline at end of file +} diff --git a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/ArgumentsTests.cs b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/ArgumentsTests.cs index 43f4ab6..f92fcdb 100644 --- a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/ArgumentsTests.cs +++ b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/ArgumentsTests.cs @@ -1,14 +1,13 @@ -using System; +using System; using FluentAssertions; -using NUnit.Framework; using OldPartitionsRemover.ByDateRemoving; -using OldPartitionsRemover.Entities; +using Xunit; namespace OldPartitionsRemover.UnitTests.ByDateRemoving; public class ArgumentsTests { - [Test] + [Fact] public void GetThreshold_WithEmptyThresholdString_ReturnsError() { var arguments = new Arguments(); @@ -18,7 +17,7 @@ public void GetThreshold_WithEmptyThresholdString_ReturnsError() threshold.IsOk(out var _, out var _).Should().BeFalse(); } - [Test] + [Fact] public void GetThreshold_WithGarbageString_ReturnsError() { var arguments = new Arguments() @@ -31,10 +30,11 @@ public void GetThreshold_WithGarbageString_ReturnsError() threshold.IsOk(out var _, out var _).Should().BeFalse(); } - [TestCase("-2y", "800.00:00:00", "700.00:00:00")] - [TestCase("-1m", "40.00:00:00", "20.00:00:00")] - [TestCase("-1d", "1.01:00:00", "0.23:00:00")] - [TestCase("-1h", "01:10:00", "00:50:00")] + [Theory] + [InlineData("-2y", "800.00:00:00", "700.00:00:00")] + [InlineData("-1m", "40.00:00:00", "20.00:00:00")] + [InlineData("-1d", "1.01:00:00", "0.23:00:00")] + [InlineData("-1h", "01:10:00", "00:50:00")] public void GetThreshold_WithThresholdAndEps_LiesWithinInterval(string thresholdString, string minBefore, string maxBefore) { var arguments = new Arguments { ThresholdString = thresholdString }; @@ -47,7 +47,7 @@ public void GetThreshold_WithThresholdAndEps_LiesWithinInterval(string threshold d.Should().BeAfter(dt - TimeSpan.Parse(minBefore)); } - [Test] + [Fact] public void GetThreshold_WithDateString_ReturnsExactThreshold() { var arguments = new Arguments @@ -61,7 +61,7 @@ public void GetThreshold_WithDateString_ReturnsExactThreshold() d.Should().Be(new DateTime(2020, 12, 01)); } - [Test] + [Fact] public void GetThreshold_UnitMillisecondsTimestamp_ReturnsExactThreshold() { var arguments = new Arguments @@ -75,7 +75,7 @@ public void GetThreshold_UnitMillisecondsTimestamp_ReturnsExactThreshold() d.Should().Be(new DateTime(2022, 4, 06, 14, 24, 35)); } - [Test] + [Fact] public void GetThreshold_TooLargeNumberOfYears_ReturnsError() { var arguments = new Arguments @@ -87,4 +87,4 @@ public void GetThreshold_TooLargeNumberOfYears_ReturnsError() threshold.IsOk(out var _, out var _).Should().BeFalse(); } -} \ No newline at end of file +} diff --git a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs index 94f90bc..b449582 100644 --- a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs @@ -1,185 +1,143 @@ -using System; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using AutoFixture; -using AutoFixture.NUnit3; using BobApi; using BobApi.BobEntities; using BobApi.Entities; -using BobToolsCli; -using BobToolsCli.BobApliClientFactories; -using BobToolsCli.ConfigurationFinding; -using BobToolsCli.ConfigurationReading; using FakeItEasy; using FluentAssertions; -using NUnit.Framework; +using Microsoft.Extensions.Logging; using OldPartitionsRemover.ByDateRemoving; -using OldPartitionsRemover.UnitTests.Customizations; +using Xunit; namespace OldPartitionsRemover.UnitTests.ByDateRemoving; -public class RemoverTests +public class RemoverTests : GenericRemoverTests { - [Test, AD] - public async Task RemoveOldPartitions_WithoutConfig_ReturnsError( - IConfigurationFinder configurationFinder, - Remover sut) - { - A.CallTo(() => configurationFinder.FindClusterConfiguration(A.Ignored, A.Ignored)) - .Returns(ConfigurationReadingResult.Error("")); - - var result = await sut.RemoveOldPartitions(CancellationToken.None); + private readonly Arguments _arguments = new(); - result.IsOk(out var _, out var _).Should().BeFalse(); + public RemoverTests() + { + _arguments.ThresholdString = "01.01.2000"; } - [Test, AD] - public async Task RemoveOldPartitions_WithoutConnection_ReturnsError( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithoutConfig_ReturnsError() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Unavailable()); + ConfigurationReadingReturnsError(""); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var err).Should().BeFalse(); - err.Should().ContainEquivalentOf("Unavailable"); + AssertRunFailed(); } - [Test, AD] - public async Task RemoveOldPartitions_WithoutConnectionWithContinueOnErrorFlag_ReturnsOk( - IPartitionsBobApiClient partitionsBobApiClient, - Arguments arguments, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithoutConnection_ReturnsError() { - arguments.ContinueOnError = true; - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Unavailable()); + PartitionSlimsReturns(BobApiResult>.Unavailable()); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var err).Should().BeTrue(); + AssertRunFailed("Unavailable"); } - [Test, AD] - public async Task RemoveOldPartitions_WithFailOnPartitionFetch_ReturnsError( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithoutConnectionWithContinueOnErrorFlag_ReturnsOk() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(A.Ignored, A.Ignored, A.Ignored)) - .Returns(BobApiResult.Unavailable()); + ContinueOnErrorIs(true); + PartitionSlimsReturns(BobApiResult>.Unavailable()); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var err).Should().BeFalse(); - err.Should().ContainEquivalentOf("unavailable"); + AssertRunSucceeded(); } - [Test, AD] - public async Task RemoveOldPartitions_WithFailOnSecondPartitionFetch_ReturnsError( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithPartitionsWithTimestampOverThreshold_DoesNotRemoveAnything() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(A.Ignored, A.Ignored, A.Ignored)) - .ReturnsNextFromSequence(BobApiResult.Ok(new Partition()), BobApiResult.Unavailable()); + PartitionSlimsReturns( + new PartitionSlim { Timestamp = DateTimeOffset.MaxValue.ToUnixTimeSeconds() } + ); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var err).Should().BeFalse(); - err.Should().ContainEquivalentOf("Unavailable"); + AssertDeleteNeverHappened(); } - [Test, AD] - public async Task RemoveOldPartitions_WithPartitionsWithTimestampOverThreshold_DoesNotRemoveAnything( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithPartitionsWithOldTimestampAndNewTimestamp_RemovesOldTimestampPartition() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(A.Ignored, A.Ignored, A.Ignored)) - .Returns(BobApiResult.Ok(new Partition() { Timestamp = DateTimeOffset.MaxValue.ToUnixTimeSeconds() })); + PartitionSlimsReturns( + new PartitionSlim { Timestamp = DateTimeOffset.MaxValue.ToUnixTimeSeconds() }, + new PartitionSlim + { + Id = "deleted", + Timestamp = DateTimeOffset.MinValue.ToUnixTimeSeconds() + } + ); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(A.Ignored, A.Ignored, A.Ignored)) - .MustNotHaveHappened(); + AssertDeleteHappened("deleted"); } - [Test, AD] - public async Task RemoveOldPartitions_WithPartitionsWithOldTimestampAndNewTimestamp_RemovesOldTimestampPartition( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithPartitionsWithOldTimestampAndNewTimestamp_DoesNotRemoveNewTimestampPartition() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - var oldTimestamp = DateTimeOffset.MinValue.ToUnixTimeSeconds(); - A.CallTo(() => partitionsBobApiClient.GetPartition(A.Ignored, A.Ignored, A.Ignored)) - .ReturnsNextFromSequence( - BobApiResult.Ok(new Partition() { Timestamp = DateTimeOffset.MaxValue.ToUnixTimeSeconds() }), - BobApiResult.Ok(new Partition() { Timestamp = oldTimestamp })); - - var result = await sut.RemoveOldPartitions(CancellationToken.None); - - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(A.Ignored, A.That.IsEqualTo(oldTimestamp), A.Ignored)) - .MustHaveHappened(); + PartitionSlimsReturns( + new PartitionSlim + { + Id = "not-deleted", + Timestamp = DateTimeOffset.MaxValue.ToUnixTimeSeconds() + }, + new PartitionSlim { Timestamp = DateTimeOffset.MinValue.ToUnixTimeSeconds() } + ); + + await Run(); + + AssertDeleteNeverHappened("not-deleted"); } - [Test, AD] - public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsOk( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsOk() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(A.Ignored, A.Ignored, A.Ignored)) - .Returns(BobApiResult.Ok(new Partition() { Timestamp = DateTimeOffset.MinValue.ToUnixTimeSeconds() })); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(A.Ignored, A.Ignored, A.Ignored)) - .Returns(BobApiResult.Ok(true)); + NumberOfReturnedPartitionsIs(2); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var _).Should().BeTrue(); + AssertRunSucceeded(); } - [Test, AD] - public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsTrue( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsTrue() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(A.Ignored, A.Ignored, A.Ignored)) - .Returns(BobApiResult.Ok(new Partition() { Timestamp = DateTimeOffset.MinValue.ToUnixTimeSeconds() })); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(A.Ignored, A.Ignored, A.Ignored)) - .Returns(BobApiResult.Ok(true)); + NumberOfReturnedPartitionsIs(2); + EveryPartitionIsOutdated(); - var result = await sut.RemoveOldPartitions(CancellationToken.None); + await Run(); - result.IsOk(out var f, out var _); - f.Should().Be(2); + AssertRemovedCount(2); } - private class ADAttribute : AutoDataAttribute + private async Task Run() { - public ADAttribute() : base(() => - { - var fixture = new Fixture(); - fixture.Customize(new FrozenApiClientsCustomization()); - fixture.Customize(new SingleNodeConfigCustomization()); - fixture.Customize(new FrozenArgumentsCustomization(args => - { - args.ThresholdString = "-1d"; - args.ContinueOnError = false; - })); + var remover = new Remover( + _arguments, + _loggerFactory.CreateLogger(), + _bobApiClientFactory, + _configurationFinder, + _resultsCombiner, + _removablePartitionsFinder + ); + _result = await remover.RemoveOldPartitions(default); + } - return fixture; - }) - { } + private void EveryPartitionIsOutdated() => + _arguments.ThresholdString = DateTime.MaxValue.ToString(); + + protected override RemoverArguments GetArguments() + { + return _arguments; } } diff --git a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/ArgumentsTests.cs b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/ArgumentsTests.cs index f4a0262..99bc9a2 100644 --- a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/ArgumentsTests.cs +++ b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/ArgumentsTests.cs @@ -1,12 +1,12 @@ -using FluentAssertions; -using NUnit.Framework; +using FluentAssertions; using OldPartitionsRemover.BySpaceRemoving; +using Xunit; namespace OldPartitionsRemover.UnitTests.BySpaceRemoving; public class ArgumentsTests { - [Test] + [Fact] public void GetThreshold_WithEmptyThresholdString_ReturnsError() { var arguments = new Arguments(); @@ -16,7 +16,7 @@ public void GetThreshold_WithEmptyThresholdString_ReturnsError() result.IsOk(out var _, out var _).Should().BeFalse(); } - [Test] + [Fact] public void GetThreshold_WithGarbageString_ReturnsError() { var arguments = new Arguments @@ -29,9 +29,10 @@ public void GetThreshold_WithGarbageString_ReturnsError() result.IsOk(out var _, out var _).Should().BeFalse(); } - [TestCase("1000B", 1000)] - [TestCase("2kB", 2000)] - [TestCase("2kb", 2000)] + [Theory] + [InlineData("1000B", 1000)] + [InlineData("2kB", 2000)] + [InlineData("2kb", 2000)] public void GetThreshold_WithNumberWithB_ReturnsBytesSize(string s, double bytes) { var arguments = new Arguments @@ -44,4 +45,4 @@ public void GetThreshold_WithNumberWithB_ReturnsBytesSize(string s, double bytes result.IsOk(out var d, out var _).Should().BeTrue(); d.Bytes.Should().Be(bytes); } -} \ No newline at end of file +} diff --git a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs index 8293c1d..e4aaa2a 100644 --- a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs @@ -1,290 +1,200 @@ -using System.Collections.Generic; -using System.Threading; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; -using AutoFixture; -using AutoFixture.AutoFakeItEasy; -using AutoFixture.NUnit3; -using BobApi; -using BobApi.BobEntities; using BobApi.Entities; -using BobToolsCli; -using BobToolsCli.BobApliClientFactories; -using BobToolsCli.ConfigurationFinding; -using BobToolsCli.ConfigurationReading; using FakeItEasy; -using FluentAssertions; -using NUnit.Framework; +using Microsoft.Extensions.Logging; using OldPartitionsRemover.BySpaceRemoving; -using OldPartitionsRemover.UnitTests.Customizations; +using Xunit; namespace OldPartitionsRemover.UnitTests.BySpaceRemoving; -public class RemoverTests +public class RemoverTests : GenericRemoverTests { - [Test, AD] - public async Task RemoveOldPartitions_WithoutConfig_ReturnsError( - IConfigurationFinder configurationFinder, - Remover sut) + private readonly Arguments _arguments = new(); + + public RemoverTests() { - A.CallTo(() => configurationFinder.FindClusterConfiguration(A.Ignored, A.Ignored)) - .Returns(ConfigurationReadingResult.Error("")); + _arguments.DelayMilliseconds = 0; // TODO Test this + } - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + [Fact] + public async Task RemoveOldPartitions_WithoutConfig_ReturnsError() + { + ConfigurationReadingReturnsError("error"); - result.IsOk(out var _, out var _).Should().BeFalse(); + await Run(); + + AssertRunFailed(); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithEnoughSpace_ReturnsZeroRemoved( - ISpaceBobApiClient spaceBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithEnoughSpace_ReturnsZeroRemoved() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(2000)); + EnoughFreeSpace(); + + await Run(); - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + AssertRemovedCount(0); + } + + [Fact] + public async Task RemovePartitionsBySpace_WithConnectionError_ReturnsError() + { + NotEnoughFreeSpace(); + PartitionSlimsReturns(BobApiResult>.Unavailable()); - result.IsOk(out var r, out var _).Should().BeTrue(); + await Run(); - r.Should().Be(0); + AssertRunFailed("Unavailable"); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithConnectionError_ReturnsError( - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithConnectionErrorAndErrorSkip_ReturnsOk() { - A.CallTo(() => partitionsBobApiClient.GetPartitions(null, default)) - .WithAnyArguments().Returns(BobApiResult>.Unavailable()); + NotEnoughFreeSpace(); + ContinueOnErrorIs(true); + FreeSpaceReturns(BobApiResult.Unavailable()); + NumberOfReturnedPartitionsIs(1); - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var e).Should().BeFalse(); - e.Should().ContainEquivalentOf("Unavailable"); + AssertRunSucceeded(); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithConnectionErrorAndErrorSkip_ReturnsOk( - Arguments arguments, - ISpaceBobApiClient spaceBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithNotEnoughSpace_RemovesSinglePartition() { - arguments.ContinueOnError = true; - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Unavailable()); + NotEnoughFreeSpace(); + NumberOfReturnedPartitionsIs(1); - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var e).Should().BeTrue(); + AssertDeleteCalledExactTimes(1); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithNotEnoughSpace_RemovesSinglePartition( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithNotEnoughSpace_ReturnsOne() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 100 })); + NotEnoughFreeSpace(); + NumberOfReturnedPartitionsIs(1); - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + await Run(); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(0, 100, A.Ignored)) - .MustHaveHappened(); + AssertRemovedCount(1); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithNotEnoughSpace_ReturnsOne( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithMultiplePartitions_RemovesUntilSpaceIsEnough() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 100 })); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(default, 0, default)).WithAnyArguments() - .Returns(BobApiResult.Ok(true)); - - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); - result.IsOk(out var count, out var _); - - count.Should().Be(1); + FreeSpaceIsEnoughtAfterDeletions(1); + NumberOfReturnedPartitionsIs(2); + + await Run(); + + AssertDeleteCalledExactTimes(1); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithMultiplePartitions_RemovesUntilSpaceIsEnough( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithMultiplePartitions_ReturnsRemovedCount() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .ReturnsNextFromSequence(BobApiResult.Ok(500), BobApiResult.Ok(1500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 200 })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "2", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 100 })); - - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); - - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(0, 200, A.Ignored)) - .MustNotHaveHappened(); + FreeSpaceIsEnoughtAfterDeletions(2); + NumberOfReturnedPartitionsIs(3); + + await Run(); + + AssertRemovedCount(2); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithMultiplePartitions_ReturnsRemovedCount( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithErrorInDeleteCall_ReturnsError() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .ReturnsNextFromSequence(BobApiResult.Ok(500), BobApiResult.Ok(900), BobApiResult.Ok(1500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2", "3" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 200 })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "2", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 150 })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "3", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 100 })); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(default, 0, default)).WithAnyArguments() - .Returns(BobApiResult.Ok(true)); - - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); - result.IsOk(out var count, out var _); - - count.Should().Be(2); + NotEnoughFreeSpace(); + NumberOfReturnedPartitionsIs(1); + DeletePartitionReturns(BobApiResult.Unavailable()); + + await Run(); + + AssertRunFailed("Unavailable"); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithErrorInPartitionsCall_ReturnsError( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithErrorInFirstDeletePartitionCallAndSkipErrors_ReturnsOk() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Unavailable()); + ContinueOnErrorIs(true); + NotEnoughFreeSpace(); + NumberOfReturnedPartitionsIs(2); + DeletePartitionReturns(BobApiResult.Unavailable()); + DeletePartitionReturns(true); - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + await Run(); - result.IsOk(out var _, out var err).Should().BeFalse(); - err.Should().ContainEquivalentOf("Unavailable"); + AssertRunSucceeded(); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithErrorInSinglePartitionCall_ReturnsError( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + [Fact] + public async Task RemovePartitionsBySpace_WithOccupiedSpaceFlag_RemovesUntilSpaceIsSmallerThanThreshold() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Unavailable()); + OccupiedSpaceIsLowAfterDeletions(1); + PartitionSlimsReturns(new PartitionSlim(), new PartitionSlim()); + + await Run(); - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); + AssertDeleteCalledExactTimes(1); + } - result.IsOk(out var _, out var err).Should().BeFalse(); - err.Should().ContainEquivalentOf("Unavailable"); + private async Task Run() + { + var remover = new Remover( + _arguments, + _configurationFinder, + _bobApiClientFactory, + _resultsCombiner, + _removablePartitionsFinder, + _loggerFactory.CreateLogger() + ); + + _result = await remover.RemovePartitionsBySpace(default); + } + + private void EnoughFreeSpace() + { + _arguments.ThresholdString = "1000B"; + _arguments.ThresholdTypeString = "free"; + FreeSpaceReturns(2000); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithErrorInDeleteCall_ReturnsError( - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + private void NotEnoughFreeSpace() { - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 200 })); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(0, 200, A.Ignored)) - .Returns(BobApiResult.Unavailable()); - - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); - - result.IsOk(out var _, out var err).Should().BeFalse(); - err.Should().ContainEquivalentOf("Unavailable"); + _arguments.ThresholdString = "1000B"; + _arguments.ThresholdTypeString = "free"; + FreeSpaceReturns(500); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithErrorInFirstPartitionCallAndSkipErrors_ReturnsOk( - Arguments arguments, - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + private void FreeSpaceIsEnoughtAfterDeletions(int deletionsRequired) { - arguments.ContinueOnError = true; - A.CallTo(() => spaceBobApiClient.GetFreeSpaceBytes(A.Ignored)) - .Returns(BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Unavailable()); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "2", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 300 })); - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(0, 200, A.Ignored)) - .Returns(BobApiResult.Unavailable()); - - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); - - result.IsOk(out var _, out var err).Should().BeTrue(); + _arguments.ThresholdString = "1000B"; + _arguments.ThresholdTypeString = "free"; + var responses = Enumerable + .Range(0, deletionsRequired) + .Select(_ => 500ul) + .Concat(new[] { 2000ul }); + FreeSpaceReturns(responses.ToArray()); } - [Test, AD] - public async Task RemovePartitionsBySpace_WithOccupiedSpaceFlag_RemovesUntilSpaceIsSmallerThanThreshold( - Arguments arguments, - ISpaceBobApiClient spaceBobApiClient, - IPartitionsBobApiClient partitionsBobApiClient, - Remover sut) + private void OccupiedSpaceIsLowAfterDeletions(int deletionsRequired) { - arguments.ThresholdTypeString = "occupied"; - A.CallTo(() => spaceBobApiClient.GetOccupiedSpaceBytes(A.Ignored)) - .ReturnsNextFromSequence(BobApiResult.Ok(1500), BobApiResult.Ok(500)); - A.CallTo(() => partitionsBobApiClient.GetPartitions(A.Ignored, A.Ignored)) - .Returns(BobApiResult>.Ok(new List { "1", "2" })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "1", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 200 })); - A.CallTo(() => partitionsBobApiClient.GetPartition(0, "2", A.Ignored)) - .Returns(BobApiResult.Ok(new Partition { Timestamp = 100 })); - - var result = await sut.RemovePartitionsBySpace(CancellationToken.None); - - A.CallTo(() => partitionsBobApiClient.DeletePartitionsByTimestamp(0, 200, A.Ignored)) - .MustNotHaveHappened(); + _arguments.ThresholdString = "1000B"; + _arguments.ThresholdTypeString = "occupied"; + var responses = Enumerable + .Range(0, deletionsRequired) + .Select(_ => 2000ul) + .Concat(new[] { 500ul }); + OccupiedSpaceReturns(responses.ToArray()); } - private class ADAttribute : AutoDataAttribute + protected override RemoverArguments GetArguments() { - public ADAttribute() : base(() => - { - var fixture = new Fixture(); - fixture.Customize(new FrozenApiClientsCustomization()); - fixture.Customize(new SingleNodeConfigCustomization()); - fixture.Customize(new FrozenArgumentsCustomization(args => - { - args.DelayMilliseconds = 0; - args.ThresholdString = "1000B"; - args.ContinueOnError = false; - args.ThresholdTypeString = "free"; - })); - - return fixture; - }) - { } + return _arguments; } } diff --git a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs new file mode 100644 index 0000000..67b2b0d --- /dev/null +++ b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BobApi; +using BobApi.BobEntities; +using BobApi.Entities; +using BobToolsCli.BobApliClientFactories; +using BobToolsCli.ConfigurationFinding; +using BobToolsCli.ConfigurationReading; +using FakeItEasy; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using OldPartitionsRemover.Entities; +using OldPartitionsRemover.Infrastructure; + +namespace OldPartitionsRemover.UnitTests; + +public abstract class GenericRemoverTests : ResultAssertionsChecker +{ + protected readonly IConfigurationFinder _configurationFinder = A.Fake(); + protected readonly ISpaceBobApiClient _spaceBobApiClient = A.Fake(); + protected readonly IPartitionsBobApiClient _partitionsBobApiClient = + A.Fake(); + protected readonly IBobApiClientFactory _bobApiClientFactory = A.Fake(); + protected readonly ILoggerFactory _loggerFactory = NullLoggerFactory.Instance; + protected readonly ResultsCombiner _resultsCombiner; + protected readonly RemovablePartitionsFinder _removablePartitionsFinder; + + // Prepared state + private readonly List< + ConfigurationReadingResult + > _configurationReadingResults = new(); + private readonly List>> _partitionSlimsResults = new(); + private readonly List> _deletePartitionByIdResults = new(); + private readonly List> _freeSpaceResults = new(); + private readonly List> _occupiedSpaceResults = new(); + + protected abstract RemoverArguments GetArguments(); + + protected Result? _result; + + public GenericRemoverTests() + { + A.CallTo(_partitionsBobApiClient) + .WithReturnType>>>() + .ReturnsLazily(CreateReturner(_partitionSlimsResults, new List())); + A.CallTo(_partitionsBobApiClient) + .WithReturnType>>() + .ReturnsLazily(CreateReturner(_deletePartitionByIdResults, true)); + A.CallTo(_configurationFinder) + .WithReturnType>>() + .ReturnsLazily( + CreateReturner( + _configurationReadingResults, + TestConstants.DefaultClusterConfiguration + ) + ); + A.CallTo(() => _spaceBobApiClient.GetFreeSpaceBytes(default)) + .WithAnyArguments() + .ReturnsLazily(CreateReturner(_freeSpaceResults, 0ul)); + A.CallTo(() => _spaceBobApiClient.GetOccupiedSpaceBytes(default)) + .WithAnyArguments() + .ReturnsLazily(CreateReturner(_occupiedSpaceResults, 0ul)); + + A.CallTo(_bobApiClientFactory) + .WithReturnType() + .Returns(_spaceBobApiClient); + A.CallTo(_bobApiClientFactory) + .WithReturnType() + .Returns(_partitionsBobApiClient); + + _resultsCombiner = new(GetArguments(), _loggerFactory.CreateLogger()); + _removablePartitionsFinder = new(_resultsCombiner, _bobApiClientFactory, GetArguments()); + } + + protected void ContinueOnErrorIs(bool value) => GetArguments().ContinueOnError = value; + + protected void ConfigurationReadingReturnsError(string error) => + _configurationReadingResults.Add( + ConfigurationReadingResult.Error(error) + ); + + protected void FreeSpaceReturns(params ulong[] values) + { + foreach (var v in values) + _freeSpaceResults.Add(v); + } + + protected void OccupiedSpaceReturns(params ulong[] values) + { + foreach (var v in values) + _occupiedSpaceResults.Add(v); + } + + protected void DeletePartitionReturns(BobApiResult response) + { + _deletePartitionByIdResults.Add(response); + } + + protected void NumberOfReturnedPartitionsIs(int count) => + PartitionSlimsReturns( + Enumerable.Range(0, count).Select(_ => new PartitionSlim()).ToArray() + ); + + protected void PartitionSlimsReturns(params PartitionSlim[] partitionSlims) => + PartitionSlimsReturns( + partitionSlims + .Select(p => + { + p.Id ??= Guid.NewGuid().ToString(); + return p; + }) + .ToList() + ); + + protected void PartitionSlimsReturns(BobApiResult> response) => + _partitionSlimsResults.Add(response); + + protected void FreeSpaceReturns(BobApiResult response) => + _freeSpaceResults.Add(response); + + private Func CreateReturner(List sequence, T def) + { + var ind = 0; + return () => + { + if (ind < sequence.Count) + return sequence[ind++]; + else if (sequence.Count > 0) // Return last if any exist + return sequence.Last(); + else + return def; + }; + } + + protected override Result GetResult() + { + return _result!; + } + + protected override IPartitionsBobApiClient GetPartitionsBobApiClientMock() + { + return _partitionsBobApiClient; + } +} diff --git a/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs b/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs new file mode 100644 index 0000000..5eacbb7 --- /dev/null +++ b/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs @@ -0,0 +1,77 @@ +using System.Threading; +using System.Threading.Tasks; +using BobApi; +using BobApi.Entities; +using FakeItEasy; +using FakeItEasy.Configuration; +using FluentAssertions; +using OldPartitionsRemover.Entities; + +namespace OldPartitionsRemover.UnitTests; + +public abstract class ResultAssertionsChecker +{ + protected virtual Result GetResult() => Result.Ok(0); + + protected virtual IPartitionsBobApiClient GetPartitionsBobApiClientMock() => + A.Fake(); + + protected void AssertRunFailed(string? errorContent = null) + { + GetResult().IsOk(out var _, out var e).Should().BeFalse(); + if (errorContent != null) + e.Should().ContainEquivalentOf(errorContent); + } + + protected void AssertRunSucceeded() + { + GetResult() + .IsOk(out var r, out var e) + .Should() + .BeTrue(because: $"error \"{e}\" should not occur"); + } + + protected void AssertRemovedCount(int removedCount) + { + AssertRunSucceeded(); + GetResult().IsOk(out var r, out var _); + r.Should().Be(removedCount); + } + + protected void AssertDeleteCalledExactTimes(int times) => + DeleteCall().MustHaveHappened(times, Times.Exactly); + + protected void AssertDeleteHappened(string? partitionId = null) => + DeleteCall(partitionId: partitionId).MustHaveHappened(); + + protected void AssertDeleteNeverHappened(string? partitionId = null) => + DeleteCall(partitionId: partitionId).MustNotHaveHappened(); + + private IReturnValueArgumentValidationConfiguration>> DeleteCall( + string? partitionId = null + ) + { + if (partitionId != null) + return A.CallTo( + () => + GetPartitionsBobApiClientMock() + .DeletePartitionById( + A.Ignored, + A.Ignored, + partitionId, + A.Ignored + ) + ); + else + return A.CallTo( + () => + GetPartitionsBobApiClientMock() + .DeletePartitionById( + A.Ignored, + A.Ignored, + A.Ignored, + A.Ignored + ) + ); + } +} diff --git a/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs b/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs new file mode 100644 index 0000000..849132f --- /dev/null +++ b/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using BobApi.BobEntities; + +namespace OldPartitionsRemover.UnitTests; + +public static class TestConstants +{ + public static readonly ClusterConfiguration DefaultClusterConfiguration = + new() + { + Nodes = new List + { + new ClusterConfiguration.Node + { + Name = "node1", + Address = "localhost", + Disks = new List + { + new ClusterConfiguration.Node.Disk { Name = "disk1", Path = "/" } + }, + } + }, + VDisks = new List + { + new ClusterConfiguration.VDisk + { + Id = 0, + Replicas = new List + { + new ClusterConfiguration.VDisk.Replica { Disk = "disk1", Node = "node1" } + } + } + } + }; +} diff --git a/test/OldPartitionsRemover.UnitTests/Customizations/FrozenApiClientsCustomization.cs b/test/OldPartitionsRemover.UnitTests/Customizations/FrozenApiClientsCustomization.cs deleted file mode 100644 index 34dd87c..0000000 --- a/test/OldPartitionsRemover.UnitTests/Customizations/FrozenApiClientsCustomization.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using AutoFixture; -using AutoFixture.AutoFakeItEasy; -using BobApi; -using BobApi.BobEntities; -using BobApi.Entities; -using BobToolsCli.BobApliClientFactories; -using FakeItEasy; - -namespace OldPartitionsRemover.UnitTests.Customizations; - -public class FrozenApiClientsCustomization : ICustomization -{ - public void Customize(IFixture fixture) - { - fixture.Customize(new AutoFakeItEasyCustomization()); - - var factory = fixture.Freeze(); - - var partitionsBobApiClient = fixture.Freeze(); - A.CallTo(() => factory.GetPartitionsBobApiClient(A.Ignored)) - .Returns(partitionsBobApiClient); - A.CallTo(partitionsBobApiClient).WithReturnType>>>() - .Returns(BobApiResult>.Ok(new List())); - - var spaceBobApiClient = fixture.Freeze(); - A.CallTo(() => factory.GetSpaceBobApiClient(A.Ignored)) - .Returns(spaceBobApiClient); - - } -} \ No newline at end of file diff --git a/test/OldPartitionsRemover.UnitTests/Customizations/FrozenArgumentsCustomization.cs b/test/OldPartitionsRemover.UnitTests/Customizations/FrozenArgumentsCustomization.cs deleted file mode 100644 index 07811e5..0000000 --- a/test/OldPartitionsRemover.UnitTests/Customizations/FrozenArgumentsCustomization.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using AutoFixture; -using BobToolsCli; - -namespace OldPartitionsRemover.UnitTests.Customizations; - -public class FrozenArgumentsCustomization : ICustomization -where T : CommonArguments -{ - private readonly Action? _setup; - - public FrozenArgumentsCustomization(Action? setup = null) - { - _setup = setup; - } - - public void Customize(IFixture fixture) - { - var arguments = fixture.Freeze(); - _setup?.Invoke(arguments); - fixture.Inject(arguments); - } -} \ No newline at end of file diff --git a/test/OldPartitionsRemover.UnitTests/Customizations/SingleNodeConfigCustomization.cs b/test/OldPartitionsRemover.UnitTests/Customizations/SingleNodeConfigCustomization.cs deleted file mode 100644 index 7c89920..0000000 --- a/test/OldPartitionsRemover.UnitTests/Customizations/SingleNodeConfigCustomization.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using AutoFixture; -using AutoFixture.AutoFakeItEasy; -using BobApi.BobEntities; -using BobToolsCli.ConfigurationFinding; -using BobToolsCli.ConfigurationReading; -using FakeItEasy; - -namespace OldPartitionsRemover.UnitTests.Customizations; - -public class SingleNodeConfigCustomization : ICustomization -{ - private static readonly ClusterConfiguration s_defaultConfiguration = new() - { - Nodes = new List - { - new ClusterConfiguration.Node - { - Name = "node1", - Address = "localhost", - Disks = new List - { - new ClusterConfiguration.Node.Disk { Name = "disk1", Path = "/" } - }, - } - }, - VDisks = new List - { - new ClusterConfiguration.VDisk - { - Id = 0, - Replicas = new List - { - new ClusterConfiguration.VDisk.Replica - { - Disk = "disk1", - Node = "node1" - } - } - } - } - }; - - public void Customize(IFixture fixture) - { - fixture.Customize(new AutoFakeItEasyCustomization()); - - var configurationFinder = fixture.Freeze(); - A.CallTo(configurationFinder).WithReturnType>>() - .Returns(ConfigurationReadingResult.Ok(s_defaultConfiguration)); - } -} \ No newline at end of file diff --git a/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj b/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj index 535a3b2..216db18 100644 --- a/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj +++ b/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj @@ -5,33 +5,22 @@ enable false - true - cobertura - ./test-coverage/ linux-x64 - - - - - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all - - - - From abfa1910fa6f65165406559a5fdf5de8a08b8351 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 10:57:41 +0300 Subject: [PATCH 12/19] OldPartitionsRemover: add alien test --- .../BySpaceRemoving/RemoverTests.cs | 13 ++++ .../Common/GenericRemoverTests.cs | 59 +++++++++++++------ .../Common/ResultAssertionsChecker.cs | 31 ++++++++++ .../Common/TestConstants.cs | 38 ++++++++++++ 4 files changed, 123 insertions(+), 18 deletions(-) diff --git a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs index e4aaa2a..d3f1eac 100644 --- a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs @@ -143,6 +143,19 @@ public async Task RemovePartitionsBySpace_WithOccupiedSpaceFlag_RemovesUntilSpac AssertDeleteCalledExactTimes(1); } + [Fact] + public async Task RemovePartitionsBySpace_WithAlienEnabled_RemovesAlien() + { + AllowAlienIs(true); + ConfigurationReadingReturnsTwoNodes(); + NotEnoughFreeSpace(); + NumberOfReturnedAlienPartitionsIs(1); + + await Run(); + + AssertAlienDeleteHappened(); + } + private async Task Run() { var remover = new Remover( diff --git a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs index 67b2b0d..f3c4e85 100644 --- a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs @@ -32,6 +32,7 @@ private readonly List< ConfigurationReadingResult > _configurationReadingResults = new(); private readonly List>> _partitionSlimsResults = new(); + private readonly List>> _alienPartitionSlimsResults = new(); private readonly List> _deletePartitionByIdResults = new(); private readonly List> _freeSpaceResults = new(); private readonly List> _occupiedSpaceResults = new(); @@ -42,9 +43,12 @@ private readonly List< public GenericRemoverTests() { - A.CallTo(_partitionsBobApiClient) - .WithReturnType>>>() + A.CallTo(() => _partitionsBobApiClient.GetPartitionSlims(default, default, default)) + .WithAnyArguments() .ReturnsLazily(CreateReturner(_partitionSlimsResults, new List())); + A.CallTo(() => _partitionsBobApiClient.GetAlienPartitionSlims(default, default, default)) + .WithAnyArguments() + .ReturnsLazily(CreateReturner(_alienPartitionSlimsResults, new List())); A.CallTo(_partitionsBobApiClient) .WithReturnType>>() .ReturnsLazily(CreateReturner(_deletePartitionByIdResults, true)); @@ -76,11 +80,16 @@ public GenericRemoverTests() protected void ContinueOnErrorIs(bool value) => GetArguments().ContinueOnError = value; + protected void AllowAlienIs(bool value) => GetArguments().AllowAlien = value; + protected void ConfigurationReadingReturnsError(string error) => _configurationReadingResults.Add( ConfigurationReadingResult.Error(error) ); + protected void ConfigurationReadingReturnsTwoNodes() => + _configurationReadingResults.Add(TestConstants.TwoNodesClusterConfiguration); + protected void FreeSpaceReturns(params ulong[] values) { foreach (var v in values) @@ -103,23 +112,36 @@ protected void NumberOfReturnedPartitionsIs(int count) => Enumerable.Range(0, count).Select(_ => new PartitionSlim()).ToArray() ); - protected void PartitionSlimsReturns(params PartitionSlim[] partitionSlims) => - PartitionSlimsReturns( - partitionSlims - .Select(p => - { - p.Id ??= Guid.NewGuid().ToString(); - return p; - }) - .ToList() + protected void NumberOfReturnedAlienPartitionsIs(int count) => + AlienPartitionSlimsReturns( + Enumerable.Range(0, count).Select(_ => new PartitionSlim()).ToArray() ); + protected void PartitionSlimsReturns(params PartitionSlim[] partitionSlims) => + PartitionSlimsReturns(PreprocessPartitionSlims(partitionSlims)); + + protected void AlienPartitionSlimsReturns(params PartitionSlim[] partitionSlims) => + AlienPartitionSlimsReturns(PreprocessPartitionSlims(partitionSlims)); + protected void PartitionSlimsReturns(BobApiResult> response) => _partitionSlimsResults.Add(response); + protected void AlienPartitionSlimsReturns(BobApiResult> response) => + _alienPartitionSlimsResults.Add(response); + protected void FreeSpaceReturns(BobApiResult response) => _freeSpaceResults.Add(response); + protected override Result GetResult() + { + return _result!; + } + + protected override IPartitionsBobApiClient GetPartitionsBobApiClientMock() + { + return _partitionsBobApiClient; + } + private Func CreateReturner(List sequence, T def) { var ind = 0; @@ -134,13 +156,14 @@ private Func CreateReturner(List sequence, T def) }; } - protected override Result GetResult() + private static List PreprocessPartitionSlims(PartitionSlim[] partitionSlims) { - return _result!; - } - - protected override IPartitionsBobApiClient GetPartitionsBobApiClientMock() - { - return _partitionsBobApiClient; + return partitionSlims + .Select(p => + { + p.Id ??= Guid.NewGuid().ToString(); + return p; + }) + .ToList(); } } diff --git a/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs b/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs index 5eacbb7..46524b5 100644 --- a/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs +++ b/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs @@ -47,6 +47,9 @@ protected void AssertDeleteHappened(string? partitionId = null) => protected void AssertDeleteNeverHappened(string? partitionId = null) => DeleteCall(partitionId: partitionId).MustNotHaveHappened(); + protected void AssertAlienDeleteHappened(string? partitionId = null) => + DeleteAlienCall(partitionId: partitionId).MustHaveHappened(); + private IReturnValueArgumentValidationConfiguration>> DeleteCall( string? partitionId = null ) @@ -74,4 +77,32 @@ private IReturnValueArgumentValidationConfiguration>> De ) ); } + + private IReturnValueArgumentValidationConfiguration>> DeleteAlienCall( + string? partitionId = null + ) + { + if (partitionId != null) + return A.CallTo( + () => + GetPartitionsBobApiClientMock() + .DeleteAlienPartitionById( + A.Ignored, + A.Ignored, + partitionId, + A.Ignored + ) + ); + else + return A.CallTo( + () => + GetPartitionsBobApiClientMock() + .DeleteAlienPartitionById( + A.Ignored, + A.Ignored, + A.Ignored, + A.Ignored + ) + ); + } } diff --git a/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs b/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs index 849132f..7a29f14 100644 --- a/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs +++ b/test/OldPartitionsRemover.UnitTests/Common/TestConstants.cs @@ -32,4 +32,42 @@ public static class TestConstants } } }; + + public static readonly ClusterConfiguration TwoNodesClusterConfiguration = + new() + { + Nodes = new List + { + new ClusterConfiguration.Node + { + Name = "node1", + Address = "localhost", + Disks = new List + { + new ClusterConfiguration.Node.Disk { Name = "disk1", Path = "/" } + }, + }, + new ClusterConfiguration.Node + { + Name = "node2", + Address = "localhost", + Disks = new List + { + new ClusterConfiguration.Node.Disk { Name = "disk1", Path = "/" } + }, + } + }, + VDisks = new List + { + new ClusterConfiguration.VDisk + { + Id = 0, + Replicas = new List + { + new ClusterConfiguration.VDisk.Replica { Disk = "disk1", Node = "node1" }, + new ClusterConfiguration.VDisk.Replica { Disk = "disk1", Node = "node2" } + } + } + } + }; } From a2fe7d9c308ce295d5d102de1917aa797e062cfe Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 11:33:38 +0300 Subject: [PATCH 13/19] OldPartitionsRemover: add more test --- .../BySpaceRemoving/Remover.cs | 3 +- .../ByDateRemoving/RemoverTests.cs | 54 +++++++++--- .../BySpaceRemoving/RemoverTests.cs | 30 ++++++- .../Common/GenericRemoverTests.cs | 85 ++++++++++++++----- .../Common/ResultAssertionsChecker.cs | 3 + 5 files changed, 143 insertions(+), 32 deletions(-) diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs index 5983ef0..ae790a6 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using BobApi.BobEntities; @@ -97,7 +98,7 @@ private async Task>> GetRemovalFunctions( ClusterConfiguration clusterConfiguration, ClusterConfiguration.Node node, CancellationToken cancellationToken) { var removableResult = await _removablePartitionsFinder.FindOnNode(clusterConfiguration, node, cancellationToken); - return removableResult; + return removableResult.Map(l => l.OrderBy(p => p.Timestamp).ToList()); } } } diff --git a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs index b449582..82258a9 100644 --- a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs @@ -1,12 +1,7 @@ using System; using System.Collections.Generic; -using System.Threading; using System.Threading.Tasks; -using BobApi; -using BobApi.BobEntities; using BobApi.Entities; -using FakeItEasy; -using FluentAssertions; using Microsoft.Extensions.Logging; using OldPartitionsRemover.ByDateRemoving; using Xunit; @@ -15,6 +10,8 @@ namespace OldPartitionsRemover.UnitTests.ByDateRemoving; public class RemoverTests : GenericRemoverTests { + private static readonly DateTimeOffset s_exampleDateTimeOffset = + new(2000, 01, 01, 0, 0, 0, TimeSpan.Zero); private readonly Arguments _arguments = new(); public RemoverTests() @@ -35,7 +32,7 @@ public async Task RemoveOldPartitions_WithoutConfig_ReturnsError() [Fact] public async Task RemoveOldPartitions_WithoutConnection_ReturnsError() { - PartitionSlimsReturns(BobApiResult>.Unavailable()); + PartitionSlimsReturnsResponse(BobApiResult>.Unavailable()); await Run(); @@ -46,7 +43,7 @@ public async Task RemoveOldPartitions_WithoutConnection_ReturnsError() public async Task RemoveOldPartitions_WithoutConnectionWithContinueOnErrorFlag_ReturnsOk() { ContinueOnErrorIs(true); - PartitionSlimsReturns(BobApiResult>.Unavailable()); + PartitionSlimsReturnsResponse(BobApiResult>.Unavailable()); await Run(); @@ -110,7 +107,7 @@ public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsOk() } [Fact] - public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsTrue() + public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsNumberOfDeletedPartitions() { NumberOfReturnedPartitionsIs(2); EveryPartitionIsOutdated(); @@ -120,6 +117,32 @@ public async Task RemoveOldPartitions_WithSuccessfullDeletion_ReturnsTrue() AssertRemovedCount(2); } + [Fact] + public async Task RemoveOldPartitions_WithAlienEnabled_DeletesAlienPartitions() + { + AllowAlienIs(true); + ConfigurationReadingReturnsTwoNodes(); + NumberOfReturnedAlienPartitionsIs(1); + EveryPartitionIsOutdated(); + + await Run(); + + AssertAlienDeleteHappened(); + } + + [Fact] + public async Task RemoveOldPartitions_WithAlienEnabledAndAliensNotOld_DoesNotDeleteAliens() + { + AllowAlienIs(true); + ConfigurationReadingReturnsTwoNodes(); + NumberOfReturnedAlienPartitionsIs(1); + EveryPartitionIsActual(); + + await Run(); + + AssertAlienDeleteNeverHappened(); + } + private async Task Run() { var remover = new Remover( @@ -133,8 +156,19 @@ private async Task Run() _result = await remover.RemoveOldPartitions(default); } - private void EveryPartitionIsOutdated() => - _arguments.ThresholdString = DateTime.MaxValue.ToString(); + private void EveryPartitionIsOutdated() + { + var ts = s_exampleDateTimeOffset.ToUnixTimeSeconds() - 1; + _arguments.ThresholdString = s_exampleDateTimeOffset.ToString(); + SetAllPartitionsTimestamp(ts); + } + + private void EveryPartitionIsActual() + { + var ts = s_exampleDateTimeOffset.ToUnixTimeSeconds() + 1; + _arguments.ThresholdString = s_exampleDateTimeOffset.ToString(); + SetAllPartitionsTimestamp(ts); + } protected override RemoverArguments GetArguments() { diff --git a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs index d3f1eac..16f13cb 100644 --- a/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/BySpaceRemoving/RemoverTests.cs @@ -42,7 +42,7 @@ public async Task RemovePartitionsBySpace_WithEnoughSpace_ReturnsZeroRemoved() public async Task RemovePartitionsBySpace_WithConnectionError_ReturnsError() { NotEnoughFreeSpace(); - PartitionSlimsReturns(BobApiResult>.Unavailable()); + PartitionSlimsReturnsResponse(BobApiResult>.Unavailable()); await Run(); @@ -156,6 +156,34 @@ public async Task RemovePartitionsBySpace_WithAlienEnabled_RemovesAlien() AssertAlienDeleteHappened(); } + [Fact] + public async Task RemovePartitionsBySpace_WithAlienAndNonAlienPartitions_RemoveOldest() + { + AllowAlienIs(true); + ConfigurationReadingReturnsTwoNodes(); + FreeSpaceIsEnoughtAfterDeletions(1); + PartitionSlimsReturns(new PartitionSlim { Timestamp = 200 }); + AlienPartitionSlimsReturns(new PartitionSlim { Timestamp = 100 }); + + await Run(); + + AssertAlienDeleteHappened(); + AssertDeleteNeverHappened(); + } + + [Fact] + public async Task RemovePartitionsBySpace_WithAlienDisabled_DoesNotRemoveAlien() + { + AllowAlienIs(false); + ConfigurationReadingReturnsTwoNodes(); + NotEnoughFreeSpace(); + NumberOfReturnedAlienPartitionsIs(1); + + await Run(); + + AssertAlienDeleteNeverHappened(); + } + private async Task Run() { var remover = new Remover( diff --git a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs index f3c4e85..26308d1 100644 --- a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs @@ -36,6 +36,7 @@ private readonly List< private readonly List> _deletePartitionByIdResults = new(); private readonly List> _freeSpaceResults = new(); private readonly List> _occupiedSpaceResults = new(); + private long? _allPartitionsDefaultTimestamp = null; protected abstract RemoverArguments GetArguments(); @@ -43,12 +44,26 @@ private readonly List< public GenericRemoverTests() { - A.CallTo(() => _partitionsBobApiClient.GetPartitionSlims(default, default, default)) + A.CallTo>>>( + () => _partitionsBobApiClient.GetPartitionSlims(default, default, default) + ) .WithAnyArguments() - .ReturnsLazily(CreateReturner(_partitionSlimsResults, new List())); + .ReturnsLazily( + CreateReturner( + _partitionSlimsResults, + new List(), + ApplyDefaultValues + ) + ); A.CallTo(() => _partitionsBobApiClient.GetAlienPartitionSlims(default, default, default)) .WithAnyArguments() - .ReturnsLazily(CreateReturner(_alienPartitionSlimsResults, new List())); + .ReturnsLazily( + CreateReturner( + _alienPartitionSlimsResults, + new List(), + ApplyDefaultValues + ) + ); A.CallTo(_partitionsBobApiClient) .WithReturnType>>() .ReturnsLazily(CreateReturner(_deletePartitionByIdResults, true)); @@ -118,20 +133,23 @@ protected void NumberOfReturnedAlienPartitionsIs(int count) => ); protected void PartitionSlimsReturns(params PartitionSlim[] partitionSlims) => - PartitionSlimsReturns(PreprocessPartitionSlims(partitionSlims)); + PartitionSlimsReturnsResponse(partitionSlims.ToList()); protected void AlienPartitionSlimsReturns(params PartitionSlim[] partitionSlims) => - AlienPartitionSlimsReturns(PreprocessPartitionSlims(partitionSlims)); + AlienPartitionSlimsReturnsResponse(partitionSlims.ToList()); - protected void PartitionSlimsReturns(BobApiResult> response) => + protected void PartitionSlimsReturnsResponse(BobApiResult> response) => _partitionSlimsResults.Add(response); - protected void AlienPartitionSlimsReturns(BobApiResult> response) => + protected void AlienPartitionSlimsReturnsResponse(BobApiResult> response) => _alienPartitionSlimsResults.Add(response); protected void FreeSpaceReturns(BobApiResult response) => _freeSpaceResults.Add(response); + protected void SetAllPartitionsTimestamp(long timestamp) => + _allPartitionsDefaultTimestamp = timestamp; + protected override Result GetResult() { return _result!; @@ -142,28 +160,55 @@ protected override IPartitionsBobApiClient GetPartitionsBobApiClientMock() return _partitionsBobApiClient; } - private Func CreateReturner(List sequence, T def) + private Func CreateReturner(List sequence, T def, Func? postprocess = null) { var ind = 0; return () => { + var result = def; + if (ind < sequence.Count) - return sequence[ind++]; + result = sequence[ind++]; else if (sequence.Count > 0) // Return last if any exist - return sequence.Last(); - else - return def; + result = sequence.Last(); + + if (postprocess != null) + result = postprocess(result); + return result; }; } - private static List PreprocessPartitionSlims(PartitionSlim[] partitionSlims) + private BobApiResult> ApplyDefaultValues( + BobApiResult> response + ) { - return partitionSlims - .Select(p => - { - p.Id ??= Guid.NewGuid().ToString(); - return p; - }) - .ToList(); + response = response.Map( + l => + l.All(p => p.Id != null) + ? l + : l.Select( + p => + new PartitionSlim + { + Id = p.Id ?? Guid.NewGuid().ToString(), + Timestamp = p.Timestamp + } + ) + .ToList() + ); + if (_allPartitionsDefaultTimestamp != null) + response = response.Map( + l => + l.Select( + p => + new PartitionSlim + { + Id = p.Id, + Timestamp = _allPartitionsDefaultTimestamp.Value + } + ) + .ToList() + ); + return response; } } diff --git a/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs b/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs index 46524b5..69b093d 100644 --- a/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs +++ b/test/OldPartitionsRemover.UnitTests/Common/ResultAssertionsChecker.cs @@ -50,6 +50,9 @@ protected void AssertDeleteNeverHappened(string? partitionId = null) => protected void AssertAlienDeleteHappened(string? partitionId = null) => DeleteAlienCall(partitionId: partitionId).MustHaveHappened(); + protected void AssertAlienDeleteNeverHappened(string? partitionId = null) => + DeleteAlienCall(partitionId: partitionId).MustNotHaveHappened(); + private IReturnValueArgumentValidationConfiguration>> DeleteCall( string? partitionId = null ) From d8cdb805f3004a4537ad6b3ebd3cf613035f505b Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 13:35:56 +0300 Subject: [PATCH 14/19] OldPartitionsRemover: add more tests --- .../ByDateRemoving/RemoverTests.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs index 82258a9..e0a44e2 100644 --- a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs @@ -130,6 +130,19 @@ public async Task RemoveOldPartitions_WithAlienEnabled_DeletesAlienPartitions() AssertAlienDeleteHappened(); } + [Fact] + public async Task RemoveOldPartitions_WithAlienDisabled_DoesNotDeleteAlienPartitions() + { + AllowAlienIs(false); + ConfigurationReadingReturnsTwoNodes(); + NumberOfReturnedAlienPartitionsIs(1); + EveryPartitionIsOutdated(); + + await Run(); + + AssertAlienDeleteNeverHappened(); + } + [Fact] public async Task RemoveOldPartitions_WithAlienEnabledAndAliensNotOld_DoesNotDeleteAliens() { From b103fb7808d01e75ac6e7b0c0fbade363976643b Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 13:38:39 +0300 Subject: [PATCH 15/19] OldPartitionsRemover: add more tests --- .../ByDateRemoving/RemoverTests.cs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs index e0a44e2..b4e7a61 100644 --- a/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/ByDateRemoving/RemoverTests.cs @@ -10,13 +10,13 @@ namespace OldPartitionsRemover.UnitTests.ByDateRemoving; public class RemoverTests : GenericRemoverTests { - private static readonly DateTimeOffset s_exampleDateTimeOffset = + private static readonly DateTimeOffset s_thresholdString = new(2000, 01, 01, 0, 0, 0, TimeSpan.Zero); private readonly Arguments _arguments = new(); public RemoverTests() { - _arguments.ThresholdString = "01.01.2000"; + _arguments.ThresholdString = s_thresholdString.ToString(); } [Fact] @@ -156,6 +156,21 @@ public async Task RemoveOldPartitions_WithAlienEnabledAndAliensNotOld_DoesNotDel AssertAlienDeleteNeverHappened(); } + [Fact] + public async Task RemoveOldPartitions_WithNormalAndAlienOutdatedPartitions_RemovesBoth() + { + AllowAlienIs(true); + ConfigurationReadingReturnsTwoNodes(); + NumberOfReturnedAlienPartitionsIs(1); + NumberOfReturnedPartitionsIs(1); + EveryPartitionIsOutdated(); + + await Run(); + + AssertAlienDeleteHappened(); + AssertDeleteHappened(); + } + private async Task Run() { var remover = new Remover( @@ -171,15 +186,13 @@ private async Task Run() private void EveryPartitionIsOutdated() { - var ts = s_exampleDateTimeOffset.ToUnixTimeSeconds() - 1; - _arguments.ThresholdString = s_exampleDateTimeOffset.ToString(); + var ts = s_thresholdString.ToUnixTimeSeconds() - 1; SetAllPartitionsTimestamp(ts); } private void EveryPartitionIsActual() { - var ts = s_exampleDateTimeOffset.ToUnixTimeSeconds() + 1; - _arguments.ThresholdString = s_exampleDateTimeOffset.ToString(); + var ts = s_thresholdString.ToUnixTimeSeconds() + 1; SetAllPartitionsTimestamp(ts); } From da133ae0ea7d8423b42c6108a59e110c164a5b1c Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 13:42:34 +0300 Subject: [PATCH 16/19] OldPartitionsRemover: remove dead code --- src/BobApi/IPartitionsBobApiClient.cs | 14 -------------- .../OldPartitionsRemover.UnitTests.csproj | 2 -- 2 files changed, 16 deletions(-) diff --git a/src/BobApi/IPartitionsBobApiClient.cs b/src/BobApi/IPartitionsBobApiClient.cs index 9691233..f42f3e1 100644 --- a/src/BobApi/IPartitionsBobApiClient.cs +++ b/src/BobApi/IPartitionsBobApiClient.cs @@ -8,20 +8,6 @@ namespace BobApi { public interface IPartitionsBobApiClient { - Task> DeletePartitionsByTimestamp( - long vDiskId, - long timestamp, - CancellationToken cancellationToken = default - ); - Task> GetPartition( - long vdiskId, - string partition, - CancellationToken cancellationToken = default - ); - Task>> GetPartitions( - ClusterConfiguration.VDisk vDisk, - CancellationToken cancellationToken = default - ); Task>> GetPartitionSlims( string diskName, long vDiskId, diff --git a/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj b/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj index 216db18..87361dd 100644 --- a/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj +++ b/test/OldPartitionsRemover.UnitTests/OldPartitionsRemover.UnitTests.csproj @@ -3,9 +3,7 @@ net6.0 enable - false - linux-x64 From ca7a6d423dc683692345617a451fbedb451bb05a Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 13:49:35 +0300 Subject: [PATCH 17/19] OldPartitionsRemover: refactor ps update --- .../Common/GenericRemoverTests.cs | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs index 26308d1..d56edfe 100644 --- a/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs +++ b/test/OldPartitionsRemover.UnitTests/Common/GenericRemoverTests.cs @@ -117,19 +117,17 @@ protected void OccupiedSpaceReturns(params ulong[] values) _occupiedSpaceResults.Add(v); } - protected void DeletePartitionReturns(BobApiResult response) - { + protected void DeletePartitionReturns(BobApiResult response) => _deletePartitionByIdResults.Add(response); - } protected void NumberOfReturnedPartitionsIs(int count) => - PartitionSlimsReturns( - Enumerable.Range(0, count).Select(_ => new PartitionSlim()).ToArray() + PartitionSlimsReturnsResponse( + Enumerable.Range(0, count).Select(_ => CreatePartitionSlim()).ToList() ); protected void NumberOfReturnedAlienPartitionsIs(int count) => - AlienPartitionSlimsReturns( - Enumerable.Range(0, count).Select(_ => new PartitionSlim()).ToArray() + AlienPartitionSlimsReturnsResponse( + Enumerable.Range(0, count).Select(_ => CreatePartitionSlim()).ToList() ); protected void PartitionSlimsReturns(params PartitionSlim[] partitionSlims) => @@ -182,33 +180,38 @@ private BobApiResult> ApplyDefaultValues( BobApiResult> response ) { - response = response.Map( - l => - l.All(p => p.Id != null) - ? l - : l.Select( - p => - new PartitionSlim - { - Id = p.Id ?? Guid.NewGuid().ToString(), - Timestamp = p.Timestamp - } - ) - .ToList() + response = UpdatePartitions( + response, + p => UpdatePartitionSlim(p, id: p.Id ?? Guid.NewGuid().ToString()) ); + if (_allPartitionsDefaultTimestamp != null) - response = response.Map( - l => - l.Select( - p => - new PartitionSlim - { - Id = p.Id, - Timestamp = _allPartitionsDefaultTimestamp.Value - } - ) - .ToList() + { + response = UpdatePartitions( + response, + p => UpdatePartitionSlim(p, timestamp: _allPartitionsDefaultTimestamp) ); + } return response; } + + private BobApiResult> UpdatePartitions( + BobApiResult> r, + Func update + ) => r.Map(l => l.Select(update).ToList()); + + private PartitionSlim CreatePartitionSlim() => new PartitionSlim(); + + private PartitionSlim UpdatePartitionSlim( + PartitionSlim p, + string? id = null, + long? timestamp = null + ) + { + return new PartitionSlim + { + Id = id ?? p.Id, + Timestamp = timestamp != null ? timestamp.Value : p.Timestamp + }; + } } From fde76c6199a28f995d25e3604e057da7bd7d21a9 Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 13:52:56 +0300 Subject: [PATCH 18/19] OldPartitionsRemover: refactor names --- src/OldPartitionsRemover/BySpaceRemoving/Remover.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs index ae790a6..72975ee 100644 --- a/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs +++ b/src/OldPartitionsRemover/BySpaceRemoving/Remover.cs @@ -79,9 +79,9 @@ private async Task>> RemoveInClusterWithLessFreeSpace(ClusterCo private async Task> RemoveOnNode(ClusterConfiguration clusterConfiguration, NodeConditionSpecification nodeSpec, CancellationToken cancellationToken) { - var removalFunctionsResult = await GetRemovalFunctions(clusterConfiguration, nodeSpec.Node, cancellationToken); - return await removalFunctionsResult - .Bind(async removalFunctions => await _resultsCombiner.CombineResults(removalFunctions, 0, async (n, rem) => + var removablePartitions = await GetRemovablePartitions(clusterConfiguration, nodeSpec.Node, cancellationToken); + return await removablePartitions + .Bind(async partitions => await _resultsCombiner.CombineResults(partitions, 0, async (n, rem) => { var isDoneRes = await nodeSpec.CheckIsDone(_logger, cancellationToken); return await isDoneRes.Bind(async isDone => @@ -94,7 +94,7 @@ private async Task> RemoveOnNode(ClusterConfiguration clusterConfigu })); } - private async Task>> GetRemovalFunctions( + private async Task>> GetRemovablePartitions( ClusterConfiguration clusterConfiguration, ClusterConfiguration.Node node, CancellationToken cancellationToken) { var removableResult = await _removablePartitionsFinder.FindOnNode(clusterConfiguration, node, cancellationToken); From 854e2dc674297046e805d7ff08bab896e741723f Mon Sep 17 00:00:00 2001 From: Ivan Druzhitskiy Date: Wed, 24 Jan 2024 15:34:13 +0300 Subject: [PATCH 19/19] OldPartitionsRemover: use lambda to get disposable api correctly --- src/BobApi/IPartitionsBobApiClient.cs | 6 +-- .../RemovablePartitionsFinder.cs | 40 +++++++++++++------ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/BobApi/IPartitionsBobApiClient.cs b/src/BobApi/IPartitionsBobApiClient.cs index f42f3e1..ee9a447 100644 --- a/src/BobApi/IPartitionsBobApiClient.cs +++ b/src/BobApi/IPartitionsBobApiClient.cs @@ -1,12 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using BobApi.BobEntities; using BobApi.Entities; namespace BobApi { - public interface IPartitionsBobApiClient + public interface IPartitionsBobApiClient : IDisposable { Task>> GetPartitionSlims( string diskName, diff --git a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs index 71874fb..5292a8e 100644 --- a/src/OldPartitionsRemover/RemovablePartitionsFinder.cs +++ b/src/OldPartitionsRemover/RemovablePartitionsFinder.cs @@ -96,16 +96,15 @@ private async Task>> FindNormalOnNode( CancellationToken cancellationToken ) { - // API is not disposed because it will be captured in removable partitions - var api = _bobApiClientFactory.GetPartitionsBobApiClient(node); if (!vDisksConfiguration.VDisksByDiskByNode.TryGetValue(node.Name, out var vDisksByDisk)) throw new ConfigurationException($"Node {node} is not presented in replicas"); + var getApi = CreateGetApi(node); return await _resultsCombiner.CollectResults( vDisksByDisk, async kv => await _resultsCombiner.CollectResults( kv.Value, - async vDisk => await FindNormal(api, kv.Key, vDisk, cancellationToken) + async vDisk => await FindNormal(getApi, kv.Key, vDisk, cancellationToken) ) ); } @@ -116,14 +115,13 @@ private async Task>> FindAlienOnNode( CancellationToken cancellationToken ) { - // API is not disposed because it will be captured in removable partitions - var api = _bobApiClientFactory.GetPartitionsBobApiClient(node); + var getApi = CreateGetApi(node); return await _resultsCombiner.CollectResults( vDisksConfiguration.VDisksByNode.Where(kv => kv.Key != node.Name), async kv => await _resultsCombiner.CollectResults( kv.Value, - async vDisk => await FindAlien(api, kv.Key, vDisk, cancellationToken) + async vDisk => await FindAlien(getApi, kv.Key, vDisk, cancellationToken) ) ); } @@ -144,12 +142,13 @@ ClusterConfiguration.Node node } private async Task>> FindNormal( - IPartitionsBobApiClient api, + GetApi getApi, string disk, long vDisk, CancellationToken cancellationToken ) { + using var api = getApi(); Result> apiResult = await api.GetPartitionSlims( disk, vDisk, @@ -161,7 +160,8 @@ CancellationToken cancellationToken p => CreateRemovablePartition( p, - async ct => await api.DeletePartitionById(disk, vDisk, p.Id, ct) + getApi, + async (a, ct) => await a.DeletePartitionById(disk, vDisk, p.Id, ct) ) ) .ToList() @@ -169,12 +169,13 @@ CancellationToken cancellationToken } private async Task>> FindAlien( - IPartitionsBobApiClient api, + GetApi getApi, string node, long vDisk, CancellationToken cancellationToken ) { + using var api = getApi(); Result> apiResult = await api.GetAlienPartitionSlims( node, vDisk, @@ -186,8 +187,9 @@ CancellationToken cancellationToken p => CreateRemovablePartition( p, - async ct => - await api.DeleteAlienPartitionById(node, vDisk, p.Id, ct) + getApi, + async (a, ct) => + await a.DeleteAlienPartitionById(node, vDisk, p.Id, ct) ) ) .ToList() @@ -196,18 +198,30 @@ await api.DeleteAlienPartitionById(node, vDisk, p.Id, ct) private RemovablePartition CreateRemovablePartition( PartitionSlim p, - RemoveRemovablePartition remove + GetApi getApi, + Func>> remove ) { return new RemovablePartition( p.Id, DateTimeOffset.FromUnixTimeSeconds(p.Timestamp), - remove + async ct => + { + using var api = getApi(); + return await remove(api, ct); + } ); } + private GetApi CreateGetApi(ClusterConfiguration.Node node) + { + return () => _bobApiClientFactory.GetPartitionsBobApiClient(node); + } + private record struct VDisksConfiguration( Dictionary>> VDisksByDiskByNode, Dictionary> VDisksByNode ); + + private delegate IPartitionsBobApiClient GetApi(); }