From aaab6671e0a887537e0fcb54c5b6d85e53703740 Mon Sep 17 00:00:00 2001 From: Erik Date: Thu, 28 Nov 2024 07:40:51 -0500 Subject: [PATCH] feat!: Smarter cleaning rules for packages with SemVer (#184) BREAKING CHANGE: `MaxVersionsPerPackage` key in config file is now obsolete and has to be replaced by the `Retention` config (see https://www.bagetter.com/docs/configuration#package-auto-deletion for further details). --- docs/docs/configuration.md | 21 +++- .../Configuration/BaGetterOptions.cs | 5 + .../Configuration/RetentionOptions.cs | 33 +++++++ src/BaGetter.Core/Entities/Package.cs | 2 + .../DependencyInjectionExtensions.cs | 1 + .../Indexing/IPackageDeletionService.cs | 16 ++- .../Indexing/PackageDeletionService.cs | 99 +++++++++++++++++-- .../Indexing/PackageIndexingService.cs | 19 +++- .../PackageIndexingServiceInMemoryTests.cs | 97 ++++++++++++------ .../Services/PackageIndexingServiceTests.cs | 5 + .../Support/NullStorageService.cs | 43 ++++++++ 11 files changed, 290 insertions(+), 51 deletions(-) create mode 100644 src/BaGetter.Core/Configuration/RetentionOptions.cs create mode 100644 tests/BaGetter.Core.Tests/Support/NullStorageService.cs diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 363d713a..0cbf25a0 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -175,16 +175,27 @@ downloaded if you know the package's id and version. You can override this behav } ``` -## Enable package auto-deletion +## Package auto-deletion -If your build server generates many nuget packages, your BaGet server can quickly run out of space. To avoid this issue, `MaxVersionsPerPackage` can be configured to auto-delete packages older packages when a new one is uploaded. This will use the `HardDelete` option detailed above and will unlist and delete the files for the older packages. By default this value is not configured and no packages will be deleted automatically. +If your build server generates many nuget packages, your BaGet server can quickly run out of space. Bagetter leverages [SemVer 2](https://semver.org/) and has logic to keep a history of packages based on the version numbering such as `..-.`. + +There is an optional section for `Retention` and the following parameters can be enabled to limit history for each level of the version. If none of these are set, there are no cleaning rules enforced. Each parameter is optional, e.g. if you specify only a `MaxHistoryPerPatch`, the package limit will only enforced for each major and minor version combination. +Packages deleted are always the oldest based on version numbers. + +- MaxHistoryPerMajorVersion: Maximum number of major versions +- MaxHistoryPerMinorVersion: Maximum number of minor versions for each major version +- MaxHistoryPerPatch: Maximum number of patch versions for each major + minor version +- MaxHistoryPerPrerelease: Maximum number of prerelease versions for each major + minor + patch version and prerelease type. if you have `beta` and `alpha` this will keep `MaxHistoryPerPrerelease` versions for both `beta` and `alpha`. ```json { ... - - "MaxVersionsPerPackage ": 5, - + "Retention": { + "MaxHistoryPerMajorVersion": 5, + "MaxHistoryPerMinorVersion": 5, + "MaxHistoryPerPatch": 5, + "MaxHistoryPerPrerelease": 5, + } ... } ``` diff --git a/src/BaGetter.Core/Configuration/BaGetterOptions.cs b/src/BaGetter.Core/Configuration/BaGetterOptions.cs index 9caa850c..8b2e9464 100644 --- a/src/BaGetter.Core/Configuration/BaGetterOptions.cs +++ b/src/BaGetter.Core/Configuration/BaGetterOptions.cs @@ -1,3 +1,4 @@ +using System; using BaGetter.Core.Configuration; namespace BaGetter.Core; @@ -52,9 +53,13 @@ public class BaGetterOptions /// /// If this is set to a value, it will limit the number of versions that can be pushed for a package. /// the older versions will be deleted. + /// This setting is not used anymore and is deprecated. /// + [Obsolete("MaxVersionsPerPackage is deprecated. Please configure RetentionOptions parameters instead.")] public uint? MaxVersionsPerPackage { get; set; } = null; + public RetentionOptions Retention { get; set; } + public DatabaseOptions Database { get; set; } public StorageOptions Storage { get; set; } diff --git a/src/BaGetter.Core/Configuration/RetentionOptions.cs b/src/BaGetter.Core/Configuration/RetentionOptions.cs new file mode 100644 index 00000000..08c4e7a6 --- /dev/null +++ b/src/BaGetter.Core/Configuration/RetentionOptions.cs @@ -0,0 +1,33 @@ +namespace BaGetter.Core; + +public class RetentionOptions +{ + /// + /// If this is set to a value, it will limit the number of versions that can be pushed for a package. + /// The limit is applied to each major version of the package, and if the limit is exceeded, + /// the older versions will be deleted. + /// + public uint? MaxHistoryPerMajorVersion { get; set; } = null; + + /// + /// This corresponds to the maximum number of minor versions for each major version. + /// If this is set to a value, it will limit the number of versions that can be pushed for a package. + /// The limit is applied to each minor version of the package, and if the limit is exceeded, + /// the older versions will be deleted. + /// + public uint? MaxHistoryPerMinorVersion { get; set; } + + /// + /// If this is set to a value, it will limit the number of versions that can be pushed for a package. + /// The limit is applied to each patch number of the package, and if the limit is exceeded, + /// the older versions will be deleted. + /// + public uint? MaxHistoryPerPatch { get; set; } + + /// + /// If this is set to a value, it will limit the number of versions that can be pushed for a package. + /// The limit is applied to each pre-release of the package, and if the limit is exceeded, + /// the older versions will be deleted. + /// + public uint? MaxHistoryPerPrerelease { get; set; } +} diff --git a/src/BaGetter.Core/Entities/Package.cs b/src/BaGetter.Core/Entities/Package.cs index d681e9f9..d3c44ebd 100644 --- a/src/BaGetter.Core/Entities/Package.cs +++ b/src/BaGetter.Core/Entities/Package.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using NuGet.Versioning; namespace BaGetter.Core; // See NuGetGallery's: https://github.com/NuGet/NuGetGallery/blob/master/src/NuGetGallery.Core/Entities/Package.cs +[DebuggerDisplay("{Id} {Version}")] public class Package { public int Key { get; set; } diff --git a/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs b/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs index 9869767a..e4d71e85 100644 --- a/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs +++ b/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs @@ -68,6 +68,7 @@ private static void AddConfiguration(this IServiceCollection services) services.AddBaGetterOptions(nameof(BaGetterOptions.Database)); services.AddBaGetterOptions(nameof(BaGetterOptions.Storage)); services.AddBaGetterOptions(nameof(BaGetterOptions.Mirror)); + services.AddBaGetterOptions(nameof(BaGetterOptions.Retention)); services.AddBaGetterOptions(nameof(BaGetterOptions.Search)); services.AddBaGetterOptions(nameof(BaGetterOptions.Storage)); services.AddBaGetterOptions(nameof(BaGetterOptions.Statistics)); diff --git a/src/BaGetter.Core/Indexing/IPackageDeletionService.cs b/src/BaGetter.Core/Indexing/IPackageDeletionService.cs index 1f48c9b3..3934873f 100644 --- a/src/BaGetter.Core/Indexing/IPackageDeletionService.cs +++ b/src/BaGetter.Core/Indexing/IPackageDeletionService.cs @@ -7,13 +7,19 @@ namespace BaGetter.Core; public interface IPackageDeletionService { /// - /// Delete old versions of packages + /// This method deletes old versions of a package. + /// This leverages semver 2.0 - and assume a package is major.minor.patch-prerelease.build + /// It can leverage the to list all versions of a package and then delete all but the last versions. + /// It also takes into account the , and parameters to further filter the versions to delete. /// - /// Current package object to clean - /// Maximum number of packages to keep - /// + /// Package name + /// Maximum of major versions to keep (optional) + /// Maximum of minor versions to keep (optional) + /// Maximum of patch versions to keep (optional) + /// Maximum of pre-release versions (optional) + /// Cancel the operation /// Number of packages deleted - Task DeleteOldVersionsAsync(Package package, uint maxPackagesToKeep, CancellationToken cancellationToken); + Task DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, CancellationToken cancellationToken); /// /// Attempt to delete a package. diff --git a/src/BaGetter.Core/Indexing/PackageDeletionService.cs b/src/BaGetter.Core/Indexing/PackageDeletionService.cs index 8b3fe35b..51dbfbb3 100644 --- a/src/BaGetter.Core/Indexing/PackageDeletionService.cs +++ b/src/BaGetter.Core/Indexing/PackageDeletionService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -90,16 +91,76 @@ private async Task TryHardDeletePackageAsync(string id, NuGetVersion versi return found; } - public async Task DeleteOldVersionsAsync(Package package, uint maxPackages, CancellationToken cancellationToken) + private static IList GetValidVersions(IEnumerable versions, Func getParent, Func getSelector, int versionsToKeep) + where S : IComparable, IEquatable + where T : IComparable, IEquatable + { + var validVersions = versions + // for each parent group + .GroupBy(v => getParent(v)) + // get all versions by selector + .SelectMany(g => g.Select(k => (parent: g.Key, selector: getSelector(k))) + .Distinct() + .OrderByDescending(k => k.selector) + .Take(versionsToKeep)) + .ToList(); + return versions.Where(k => validVersions.Any(v => getParent(k).Equals(v.parent) && getSelector(k).Equals(v.selector))).ToList(); + } + + public async Task DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, CancellationToken cancellationToken) { // list all versions of the package - var versions = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken); - if (versions is null || versions.Count <= maxPackages) return 0; + var packages = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken); + if (packages is null || packages.Count <= maxMajor) return 0; + + var goodVersions = new HashSet(); + + if (maxMajor.HasValue) + { + goodVersions = GetValidVersions(packages.Select(t => t.Version), v => 0, v => v.Major, (int)maxMajor).ToHashSet(); + } + else + { + goodVersions = packages.Select(p => p.Version).ToHashSet(); + } + + if (maxMinor.HasValue) + { + goodVersions.IntersectWith(GetValidVersions(goodVersions, v => (v.Major), v => v.Minor, (int)maxMinor)); + } + + if (maxPatch.HasValue) + { + goodVersions.IntersectWith(GetValidVersions(goodVersions, v => (v.Major, v.Minor), v => v.Patch, (int)maxPatch)); + } + + if (maxPrerelease.HasValue) + { + // this assume we have something like 1.1.1-alpha.1 - alpha is the release type + var preReleases = packages.Select(p => p.Version).Where(p => p.IsPrerelease).ToList(); + // this will give us 'alpha' or 'beta' etc + var prereleaseTypes = preReleases + .Select(v => v.ReleaseLabels?.FirstOrDefault()) + .Where(lb => lb is not null) + .Distinct(); + + var allPreReleaseValidVersions = new HashSet(); + foreach (var preReleaseType in prereleaseTypes) + { + var preReleaseVersions = preReleases.Where(p => p.ReleaseLabels!.FirstOrDefault() == preReleaseType + && GetPreReleaseBuild(p) is not null).ToList(); + + allPreReleaseValidVersions.UnionWith + (GetValidVersions(preReleaseVersions, + v => (v.Major, v.Minor, v.Patch), v => GetPreReleaseBuild(v).Value, (int)maxPrerelease)); + + } + goodVersions.IntersectWith(allPreReleaseValidVersions); + } + // sort by version and take everything except the last maxPackages - var versionsToDelete = versions - .OrderByDescending(p => p.Version) - .Skip((int)maxPackages) - .ToList(); + var versionsToDelete = packages.Where(p => !goodVersions.Contains(p.Version)).ToList(); + var deleted = 0; foreach (var version in versionsToDelete) { @@ -107,4 +168,28 @@ public async Task DeleteOldVersionsAsync(Package package, uint maxPackages, } return deleted; } + + /// + /// Tries to get the version number of a pre-release build.
+ /// If we have 1.1.1-alpha.1 , this will return 1 or null if not valid. + /// + /// The version as int or null if not found. + private int? GetPreReleaseBuild(NuGetVersion nuGetVersion) + { + if (nuGetVersion.IsPrerelease && nuGetVersion.ReleaseLabels != null) + { + // Assuming the last part of the release label is the build number + var lastLabel = nuGetVersion.ReleaseLabels.LastOrDefault(); + if (int.TryParse(lastLabel, out var buildNumber)) + { + return buildNumber; + } + else + { + _logger.LogWarning("Could not parse build number from prerelease label {PrereleaseLabel} - prerelease number is expected to be like 2.3.4-alpha.1 where 1 is prerelease", nuGetVersion); + } + } + return null; + } + } diff --git a/src/BaGetter.Core/Indexing/PackageIndexingService.cs b/src/BaGetter.Core/Indexing/PackageIndexingService.cs index 0fd3573d..87aa677b 100644 --- a/src/BaGetter.Core/Indexing/PackageIndexingService.cs +++ b/src/BaGetter.Core/Indexing/PackageIndexingService.cs @@ -15,6 +15,7 @@ public class PackageIndexingService : IPackageIndexingService private readonly ISearchIndexer _search; private readonly SystemTime _time; private readonly IOptionsSnapshot _options; + private readonly IOptionsSnapshot _retentionOptions; private readonly ILogger _logger; private readonly IPackageDeletionService _packageDeletionService; @@ -25,6 +26,7 @@ public PackageIndexingService( ISearchIndexer search, SystemTime time, IOptionsSnapshot options, + IOptionsSnapshot retentionOptions, ILogger logger) { _packages = packages ?? throw new ArgumentNullException(nameof(packages)); @@ -32,8 +34,15 @@ public PackageIndexingService( _search = search ?? throw new ArgumentNullException(nameof(search)); _time = time ?? throw new ArgumentNullException(nameof(time)); _options = options ?? throw new ArgumentNullException(nameof(options)); + _retentionOptions = retentionOptions ?? throw new ArgumentNullException(nameof(retentionOptions)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _packageDeletionService = packageDeletionService ?? throw new ArgumentNullException(nameof(packageDeletionService)); +#pragma warning disable CS0618 // Type or member is obsolete + if (_options.Value.MaxVersionsPerPackage > 0) + { + _logger.LogError("MaxVersionsPerPackage is deprecated and is not used. Please use MaxHistoryPerMajorVersion, MaxHistoryPerMinorVersion, MaxHistoryPerPatch, and MaxHistoryPerPrerelease instead."); + } +#pragma warning restore CS0618 // Type or member is obsolete } public async Task IndexAsync(Stream packageStream, CancellationToken cancellationToken) @@ -156,14 +165,20 @@ await _storage.SavePackageContentAsync( await _search.IndexAsync(package, cancellationToken); - if (_options.Value.MaxVersionsPerPackage.HasValue) + if (_retentionOptions.Value.MaxHistoryPerMajorVersion.HasValue) { try { _logger.LogInformation( "Deleting older packages for package {PackageId} {PackageVersion}", package.Id, package.NormalizedVersionString); - var deleted = await _packageDeletionService.DeleteOldVersionsAsync(package, _options.Value.MaxVersionsPerPackage.Value, cancellationToken); + var deleted = await _packageDeletionService.DeleteOldVersionsAsync( + package, + _retentionOptions.Value.MaxHistoryPerMajorVersion, + _retentionOptions.Value.MaxHistoryPerMinorVersion, + _retentionOptions.Value.MaxHistoryPerPatch, + _retentionOptions.Value.MaxHistoryPerPrerelease, + cancellationToken); if (deleted > 0) { _logger.LogInformation( diff --git a/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceInMemoryTests.cs b/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceInMemoryTests.cs index c27609d0..a218d28c 100644 --- a/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceInMemoryTests.cs +++ b/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceInMemoryTests.cs @@ -1,5 +1,7 @@ +using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using BaGetter.Core.Tests.Support; using Microsoft.Extensions.Logging; @@ -17,39 +19,46 @@ namespace BaGetter.Core.Tests.Services; public class PackageIndexingServiceInMemoryTests { private readonly IPackageDatabase _packages; - private readonly Mock _storage; + private readonly IPackageStorageService _storage; private readonly Mock _search; private readonly IPackageDeletionService _deleter; private readonly Mock _time; private readonly PackageIndexingService _target; private readonly BaGetterOptions _options; + private readonly RetentionOptions _retentionOptions; public PackageIndexingServiceInMemoryTests() { _packages = new InMemoryPackageDatabase(); - _storage = new Mock(MockBehavior.Strict); + var storageService = new NullStorageService(); + _storage = new PackageStorageService(storageService, Mock.Of>()); + _search = new Mock(MockBehavior.Strict); _options = new(); + _retentionOptions = new(); var optionsSnapshot = new Mock>(); optionsSnapshot.Setup(o => o.Value).Returns(_options); _deleter = new PackageDeletionService( _packages, - _storage.Object, + _storage, optionsSnapshot.Object, Mock.Of>()); _time = new Mock(MockBehavior.Loose); var options = new Mock>(MockBehavior.Strict); options.Setup(o => o.Value).Returns(_options); + var retentionOptions = new Mock>(MockBehavior.Strict); + retentionOptions.Setup(o => o.Value).Returns(_retentionOptions); _target = new PackageIndexingService( _packages, - _storage.Object, + _storage, _deleter, _search.Object, _time.Object, options.Object, + retentionOptions.Object, Mock.Of>()); } @@ -76,7 +85,6 @@ public async Task IndexIMAsync_WhenPackageAlreadyExists_AndOverwriteForbidden_Re }); var stream = new MemoryStream(); builder.Save(stream); - _storage.Setup(s => s.SavePackageContentAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), stream, It.IsAny(), default, default, default)).Returns(Task.CompletedTask); _search.Setup(s => s.IndexAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), default)).Returns(Task.CompletedTask); // Act @@ -117,9 +125,6 @@ public async Task IndexIMAsync_WhenPackageAlreadyExists_AndOverwriteAllowed_Inde var stream = new MemoryStream(); builder.Save(stream); - _storage.Setup(s => s.DeleteAsync(builder.Id, builder.Version, default)).Returns(Task.CompletedTask); - _storage.Setup(s => s.SavePackageContentAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), stream, It.IsAny(), default, default, default)).Returns(Task.CompletedTask); - _search.Setup(s => s.IndexAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), default)).Returns(Task.CompletedTask); // Act @@ -151,9 +156,6 @@ public async Task IndexIMAsync_WhenPrereleasePackageAlreadyExists_AndOverwritePr var stream = new MemoryStream(); builder.Save(stream); - _storage.Setup(s => s.DeleteAsync(builder.Id, builder.Version, default)).Returns(Task.CompletedTask); - _storage.Setup(s => s.SavePackageContentAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), stream, It.IsAny(), default, default, default)).Returns(Task.CompletedTask); - _search.Setup(s => s.IndexAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), default)).Returns(Task.CompletedTask); // Act @@ -185,9 +187,6 @@ public async Task IndexIMAsync_WhenPrereleasePackageAlreadyExists_AndOverwriteFo var stream = new MemoryStream(); builder.Save(stream); - _storage.Setup(s => s.DeleteAsync(builder.Id, builder.Version, default)).Returns(Task.CompletedTask); - _storage.Setup(s => s.SavePackageContentAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), stream, It.IsAny(), default, default, default)).Returns(Task.CompletedTask); - _search.Setup(s => s.IndexAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), default)).Returns(Task.CompletedTask); // Act @@ -226,8 +225,6 @@ public async Task IndexIMAsync_WithValidPackage_ReturnsSuccess() var stream = new MemoryStream(); builder.Save(stream); - _storage.Setup(s => s.SavePackageContentAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), stream, It.IsAny(), default, default, default)).Returns(Task.CompletedTask); - _search.Setup(s => s.IndexAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), default)).Returns(Task.CompletedTask); // Act @@ -242,14 +239,62 @@ public async Task IndexIMAsync_WithValidPackage_CleansOldVersions() { // Arrange _options.AllowPackageOverwrites = PackageOverwriteAllowed.False; - _options.MaxVersionsPerPackage = 5; + + _retentionOptions.MaxHistoryPerMajorVersion = 2; + _retentionOptions.MaxHistoryPerMinorVersion = 2; + _retentionOptions.MaxHistoryPerPatch = 5; + _retentionOptions.MaxHistoryPerPrerelease = 5; // Add 10 packages - for (var i = 0; i < 10; i++) + for (var major = 1; major < 4; major++) + { + for (var minor = 1; minor < 4; minor++) + { + for (var patch = 1; patch < 7; patch++) + { + await StoreVersion(NuGetVersion.Parse($"{major}.{minor}.{patch}")); + for (var prerelease = 1; prerelease < 7; prerelease++) + { + await StoreVersion(NuGetVersion.Parse($"{major}.{minor}.{patch}-staging.{prerelease}")); + + var version = NuGetVersion.Parse($"{major}.{minor}.{patch}-beta.{prerelease}"); + + var builder = await StoreVersion(version); + + var packageVersions = await _packages.FindAsync(builder.Id, true, default); + var majorCount = packageVersions.Select(p => p.Version.Major).Distinct().Count(); + Assert.Equal(majorCount, Math.Min(major, (int)_retentionOptions.MaxHistoryPerMajorVersion)); + Assert.True(majorCount <= _retentionOptions.MaxHistoryPerMajorVersion, $"Major version {major} has {majorCount} packages"); + + // validate maximum number of minor versions for each major version. + var minorVersions = packageVersions.GroupBy(m => m.Version.Major) + .Select(gp => (version: gp.Key, versionCount: gp.Select(p => p.Version.Major + "." + p.Version.Minor).Distinct().Count())).ToList(); + Assert.All(minorVersions, g => Assert.True(g.versionCount <= _retentionOptions.MaxHistoryPerMinorVersion, $"Minor version {g.version} has {g.versionCount} packages")); + + // validate maximum number of minor versions for each major version. + var patches = packageVersions.GroupBy(m => (m.Version.Major, m.Version.Minor)) + .Select(gp => (version: gp.Key, versionCount: gp.Select(p => p.Version.Major + "." + p.Version.Minor + "." + p.Version.Patch).Distinct().Count())).ToList(); + Assert.All(patches, g => Assert.True(g.versionCount <= _retentionOptions.MaxHistoryPerPatch, $"Patch version {g.version} has {g.versionCount} packages")); + + // validate maximum number of beta versions for each major,minor,patch version. + var betaVersions = packageVersions.Where(p => p.IsPrerelease && p.Version.ReleaseLabels.First() == "beta") + .GroupBy(m => (m.Version.Major, m.Version.Minor, m.Version.Patch)) + .Select(gp => (version: gp.Key, versionCount: gp.Select(p => p.Version.Major + "." + p.Version.Minor + "." + p.Version.Patch).Distinct().Count())).ToList(); + Assert.All(betaVersions, g => Assert.True(g.versionCount <= _retentionOptions.MaxHistoryPerPatch, $"Pre-Release version {g.version} has {g.versionCount} packages")); + + + } + } + } + + } + } + + private async Task StoreVersion(NuGetVersion version) { var builder = new PackageBuilder { Id = "bagetter-test", - Version = NuGetVersion.Parse($"1.0.{i}"), + Version = version, Description = "Test Description", }; builder.Authors.Add("Test Author"); @@ -264,8 +309,6 @@ public async Task IndexIMAsync_WithValidPackage_CleansOldVersions() //_packages.Setup(p => p.ExistsAsync(builder.Id, builder.Version, default)).ReturnsAsync(false); //_packages.Setup(p => p.AddAsync(It.Is(p1 => p1.Id == builder.Id && p1.Version.ToString() == builder.Version.ToString()), default)).ReturnsAsync(PackageAddResult.Success); - _storage.Setup(s => s.SavePackageContentAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), stream, It.IsAny(), default, default, default)).Returns(Task.CompletedTask); - _search.Setup(s => s.IndexAsync(It.Is(p => p.Id == builder.Id && p.Version.ToString() == builder.Version.ToString()), default)).Returns(Task.CompletedTask); // Act @@ -273,17 +316,7 @@ public async Task IndexIMAsync_WithValidPackage_CleansOldVersions() // Assert Assert.Equal(PackageIndexingResult.Success, result); - - var packageCount = await _packages.FindAsync(builder.Id, true, default); - if (i < 5) - { - Assert.Equal(i + 1, packageCount.Count); - } - else - { - Assert.Equal(5, packageCount.Count); - } - } + return builder; } } diff --git a/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceTests.cs b/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceTests.cs index d353b819..5cd273c8 100644 --- a/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceTests.cs +++ b/tests/BaGetter.Core.Tests/Services/PackageIndexingServiceTests.cs @@ -19,6 +19,7 @@ public class PackageIndexingServiceTests private readonly Mock _deleter; private readonly PackageIndexingService _target; private readonly BaGetterOptions _mockOptions; + private RetentionOptions _retentionOptions; public PackageIndexingServiceTests() { @@ -28,8 +29,11 @@ public PackageIndexingServiceTests() _time = new Mock(MockBehavior.Loose); _deleter = new Mock(MockBehavior.Strict); _mockOptions = new(); + _retentionOptions = new(); var options = new Mock>(MockBehavior.Strict); options.Setup(o => o.Value).Returns(_mockOptions); + var retentionOptions = new Mock>(MockBehavior.Strict); + retentionOptions.Setup(o => o.Value).Returns(_retentionOptions); _target = new PackageIndexingService( _packages.Object, @@ -38,6 +42,7 @@ public PackageIndexingServiceTests() _search.Object, _time.Object, options.Object, + retentionOptions.Object, Mock.Of>()); } diff --git a/tests/BaGetter.Core.Tests/Support/NullStorageService.cs b/tests/BaGetter.Core.Tests/Support/NullStorageService.cs new file mode 100644 index 00000000..05f4b94e --- /dev/null +++ b/tests/BaGetter.Core.Tests/Support/NullStorageService.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace BaGetter.Core.Tests.Support; +public class NullStorageService : IStorageService +{ + private HashSet _files = new(); + + public NullStorageService() + { + } + + public Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_files.Contains(path)) + { + throw new FileNotFoundException(); + } + _files.Remove(path); + return Task.CompletedTask; + } + + public Task GetAsync(string path, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task GetDownloadUriAsync(string path, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task PutAsync(string path, Stream content, string contentType, CancellationToken cancellationToken = default) + { + _files.Add(path); + return Task.FromResult(StoragePutResult.Success); + } +}