From 69f8145160ff8179309e79d98db25614cd183794 Mon Sep 17 00:00:00 2001 From: cmanu Date: Fri, 6 Oct 2017 10:29:00 -0700 Subject: [PATCH 1/5] Cmanuissue823 (#246) * Issue823: Correct storage logs and add execution times. * PR feedback * PR feedback --- src/Catalog/Persistence/AggregateStorage.cs | 5 ++ src/Catalog/Persistence/AzureStorage.cs | 10 ++++ src/Catalog/Persistence/Storage.cs | 61 +++++++++++++++------ 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/Catalog/Persistence/AggregateStorage.cs b/src/Catalog/Persistence/AggregateStorage.cs index a3acd643c..46e13d2e7 100644 --- a/src/Catalog/Persistence/AggregateStorage.cs +++ b/src/Catalog/Persistence/AggregateStorage.cs @@ -88,5 +88,10 @@ public override Task> List(CancellationToken cancel { return _primaryStorage.List(cancellationToken); } + + public override Uri GetUri(string name) + { + return this._primaryStorage.GetUri(name); + } } } \ No newline at end of file diff --git a/src/Catalog/Persistence/AzureStorage.cs b/src/Catalog/Persistence/AzureStorage.cs index 2c0c72532..7afd4a770 100644 --- a/src/Catalog/Persistence/AzureStorage.cs +++ b/src/Catalog/Persistence/AzureStorage.cs @@ -273,5 +273,15 @@ await blob.DeleteAsync(deleteSnapshotsOption:DeleteSnapshotsOption.IncludeSnapsh operationContext:null, cancellationToken:cancellationToken); } + + /// + /// Returns the uri of the blob based on the Azure cloud directory + /// + /// The blob name. + /// The blob uri. + public override Uri GetUri(string name) + { + return new Uri(_directory.Uri, name); + } } } diff --git a/src/Catalog/Persistence/Storage.cs b/src/Catalog/Persistence/Storage.cs index 8ddf10719..73462b07f 100644 --- a/src/Catalog/Persistence/Storage.cs +++ b/src/Catalog/Persistence/Storage.cs @@ -2,13 +2,13 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Collections.Generic; -using System.Data; using System.Diagnostics; using System.IO; using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; namespace NuGet.Services.Metadata.Catalog.Persistence { @@ -33,7 +33,9 @@ public async Task Save(Uri resourceUri, StorageContent content, CancellationToke { SaveCount++; - TraceMethod("SAVE", resourceUri); + TraceMethod(nameof(Save), resourceUri); + Stopwatch sw = new Stopwatch(); + sw.Start(); try { @@ -41,35 +43,45 @@ public async Task Save(Uri resourceUri, StorageContent content, CancellationToke } catch (Exception e) { - string message = String.Format("SAVE EXCEPTION: {0} {1}", resourceUri, e.Message); - Trace.WriteLine(message); - throw new Exception(message, e); + TraceException(nameof(Save), resourceUri, e); + throw; } + + sw.Stop(); + TraceExecutionTime(nameof(Save), resourceUri, sw.ElapsedMilliseconds); } public async Task Load(Uri resourceUri, CancellationToken cancellationToken) { LoadCount++; + StorageContent storageContent = null; - TraceMethod("LOAD", resourceUri); + TraceMethod(nameof(Load), resourceUri); + Stopwatch sw = new Stopwatch(); + sw.Start(); try { - return await OnLoad(resourceUri, cancellationToken); + storageContent = await OnLoad(resourceUri, cancellationToken); } catch (Exception e) { - string message = String.Format("LOAD EXCEPTION: {0} {1}", resourceUri, e.Message); - Trace.WriteLine(message); - throw new Exception(message, e); + TraceException(nameof(Load), resourceUri, e); + throw; } + + sw.Stop(); + TraceExecutionTime(nameof(Load), resourceUri, sw.ElapsedMilliseconds); + return storageContent; } public async Task Delete(Uri resourceUri, CancellationToken cancellationToken) { DeleteCount++; - TraceMethod("DELETE", resourceUri); + TraceMethod(nameof(Delete), resourceUri); + Stopwatch sw = new Stopwatch(); + sw.Start(); try { @@ -93,10 +105,12 @@ public async Task Delete(Uri resourceUri, CancellationToken cancellationToken) } catch (Exception e) { - string message = String.Format("DELETE EXCEPTION: {0} {1}", resourceUri, e.Message); - Trace.WriteLine(message); - throw new Exception(message, e); + TraceException(nameof(Delete), resourceUri, e); + throw; } + + sw.Stop(); + TraceExecutionTime(nameof(Delete), resourceUri, sw.ElapsedMilliseconds); } public async Task LoadString(Uri resourceUri, CancellationToken cancellationToken) @@ -175,7 +189,7 @@ protected string GetName(Uri uri) return name; } - protected Uri GetUri(string name) + public virtual Uri GetUri(string name) { string address = BaseAddress.ToString(); if (!address.EndsWith("/")) @@ -191,8 +205,23 @@ protected void TraceMethod(string method, Uri resourceUri) { if (Verbose) { - Trace.WriteLine(String.Format("{0} {1}", method, resourceUri)); + //The Uri depends on the storage implementation. + Uri storageUri = GetUri(GetName(resourceUri)); + Trace.WriteLine(String.Format("{0} {1}", method, storageUri)); } } + + private string TraceException(string method, Uri resourceUri, Exception exception) + { + string message = $"{method} EXCEPTION: {GetUri(GetName(resourceUri))} {exception.ToString()}"; + Trace.WriteLine(message); + return message; + } + + private void TraceExecutionTime(string method, Uri resourceUri, long executionTimeInMilliseconds) + { + string message = JsonConvert.SerializeObject(new { MethodName = method, StreamUri = GetUri(GetName(resourceUri)), ExecutionTimeInMilliseconds = executionTimeInMilliseconds }); + Trace.WriteLine(message); + } } } From da8dcdb0497acc757e995f4c5ec12dbf453afdbf Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 6 Oct 2017 12:31:46 -0700 Subject: [PATCH 2/5] V3 monitoring jobs - updates to the framework to support the new queue based infrastructure (#243) --- src/Catalog/AggregateCursor.cs | 2 +- src/Catalog/Helpers/ParallelAsync.cs | 28 +++ .../NuGet.Services.Metadata.Catalog.csproj | 1 + src/Ng/Jobs/EndpointMonitoringJob.cs | 127 ----------- src/Ng/Ng.csproj | 28 +-- src/Ng/Ng.nuspec | 2 - src/Ng/NgJobFactory.cs | 3 +- src/Ng/Scripts/EndpointMonitoring.cmd | 15 -- .../AggregateMonitoringNotificationService.cs | 47 ---- .../IMonitoringNotificationService.cs | 2 +- .../LoggerMonitoringNotificationService.cs | 2 +- ...kageMonitoringStatusNotificationService.cs | 36 --- ...ervices.Metadata.Catalog.Monitoring.csproj | 28 ++- .../Status/IPackageMonitoringStatusService.cs | 5 + .../Status/PackageMonitoringStatusListItem.cs | 24 ++ .../Status/PackageMonitoringStatusService.cs | 210 +++++++++++------- .../Utility/JsonSerializerUtility.cs | 28 +++ .../Validation/Test/PackageValidator.cs | 29 +-- .../Test/PackageValidatorContext.cs | 20 ++ .../Test/PackageValidatorFactory.cs | 41 ++++ .../Validation/Test/ValidatorFactory.cs | 6 +- .../Test/ValidatorFactoryFactory.cs | 51 +++++ .../ValidationCollector.cs | 47 ++-- .../ValidationCollectorFactory.cs | 75 +++---- .../packages.config | 7 + tests/CatalogTests/App.config | 4 + 26 files changed, 440 insertions(+), 428 deletions(-) create mode 100644 src/Catalog/Helpers/ParallelAsync.cs delete mode 100644 src/Ng/Jobs/EndpointMonitoringJob.cs delete mode 100644 src/Ng/Scripts/EndpointMonitoring.cmd delete mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/AggregateMonitoringNotificationService.cs delete mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/PackageMonitoringStatusNotificationService.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactoryFactory.cs diff --git a/src/Catalog/AggregateCursor.cs b/src/Catalog/AggregateCursor.cs index 88c4d17ea..6171d4784 100644 --- a/src/Catalog/AggregateCursor.cs +++ b/src/Catalog/AggregateCursor.cs @@ -16,7 +16,7 @@ public class AggregateCursor : ReadCursor { public AggregateCursor(IEnumerable innerCursors) { - if (innerCursors == null || innerCursors.Count() < 1) + if (innerCursors == null || !innerCursors.Any()) { throw new ArgumentException("Must supply at least one cursor!", nameof(innerCursors)); } diff --git a/src/Catalog/Helpers/ParallelAsync.cs b/src/Catalog/Helpers/ParallelAsync.cs new file mode 100644 index 000000000..34c165b25 --- /dev/null +++ b/src/Catalog/Helpers/ParallelAsync.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public static class ParallelAsync + { + public const int DefaultDegreeOfParallelism = 32; + + /// + /// Creates a number of tasks specified by using and then runs them in parallel. + /// + /// Creates each task to run. + /// The number of tasks to create. + /// A task that completes when all tasks have completed. + public static Task Repeat(Func taskFactory, int degreeOfParallelism = DefaultDegreeOfParallelism) + { + return Task.WhenAll( + Enumerable + .Repeat(taskFactory, degreeOfParallelism) + .Select(f => f())); + } + } +} \ No newline at end of file diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj index c08c9e61f..50c6e0cec 100644 --- a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj @@ -147,6 +147,7 @@ + diff --git a/src/Ng/Jobs/EndpointMonitoringJob.cs b/src/Ng/Jobs/EndpointMonitoringJob.cs deleted file mode 100644 index bc95ea0a1..000000000 --- a/src/Ng/Jobs/EndpointMonitoringJob.cs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using NuGet.Services.Configuration; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Monitoring; -using NuGet.Services.Metadata.Catalog.Persistence; - -namespace Ng.Jobs -{ - public class EndpointMonitoringJob : LoopingNgJob - { - private ValidationCollectorFactory _collectorFactory; - private ValidationCollector _collector; - private ReadWriteCursor _front; - private ReadCursor _back; - - public EndpointMonitoringJob(ILoggerFactory loggerFactory) : base(loggerFactory) - { - _collectorFactory = new ValidationCollectorFactory(loggerFactory); - } - - public override string GetUsage() - { - return "Usage: ng endpointmonitoring " - + $"-{Arguments.Gallery} " - + $"-{Arguments.Source} " - + $"-{Arguments.Index} " - + $"-{Arguments.EndpointsToTest} |;|;..." - + $"-{Arguments.StorageBaseAddress} " - + $"-{Arguments.StorageType} azure " - + $"[-{Arguments.StoragePath} ]" - + $"[-{Arguments.PackageStatusFolder} ]" - + "|" - + $"[-{Arguments.StorageAccountName} " - + $"-{Arguments.StorageKeyValue} " - + $"-{Arguments.StorageContainer} " - + $"-{Arguments.StoragePath} " - + $"[-{Arguments.VaultName} " - + $"-{Arguments.ClientId} " - + $"-{Arguments.CertificateThumbprint} " - + $"[-{Arguments.ValidateCertificate} true|false]]] " - + $"[-{Arguments.StoragePathAuditing} ]" - + "|" - + $"[-{Arguments.StorageAccountNameAuditing} " - + $"-{Arguments.StorageKeyValueAuditing} " - + $"-{Arguments.StorageContainerAuditing} " - + $"-{Arguments.StoragePathAuditing} ] " - + $"[-{Arguments.Verbose} true|false] " - + $"[-{Arguments.Interval} ]"; - } - - protected override void Init(IDictionary arguments, CancellationToken cancellationToken) - { - var gallery = arguments.GetOrThrow(Arguments.Gallery); - var index = arguments.GetOrThrow(Arguments.Index); - var source = arguments.GetOrThrow(Arguments.Source); - var verbose = arguments.GetOrDefault(Arguments.Verbose, false); - - if (arguments.GetOrThrow(Arguments.StorageType) != Arguments.AzureStorageType) - { - throw new ArgumentException("File storage is not supported!"); - } - - var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); - var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose); - - var endpoints = arguments.GetOrThrow(Arguments.EndpointsToTest).Split(';').Select(e => { - var endpointParts = e.Split('|'); - - if (endpointParts.Count() < 2) - { - throw new ArgumentException( - $"\"{e}\" is not a valid endpoint! Each endpoint must follow this format: " + - $"|."); - } - - return new EndpointFactory.Input(endpointParts[0], new Uri(endpointParts[1])); - }); - - var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(verbose); - - var loggerNotificationService = new LoggerMonitoringNotificationService( - LoggerFactory.CreateLogger()); - - var statusService = new PackageMonitoringStatusService( - new NamedStorageFactory(monitoringStorageFactory, arguments.GetOrDefault(Arguments.PackageStatusFolder, Arguments.PackageStatusFolderDefault)), - LoggerFactory.CreateLogger()); - - var statusNotificationService = new PackageMonitoringStatusNotificationService(statusService); - - var aggregateNotificationService = new AggregateNotificationService( - new IMonitoringNotificationService[] { loggerNotificationService, statusNotificationService }); - - var context = _collectorFactory.Create( - gallery, - index, - source, - monitoringStorageFactory, - auditingStorageFactory, - endpoints, - messageHandlerFactory, - aggregateNotificationService, - verbose); - - _collector = context.Collector; - _front = context.Front; - _back = context.Back; - } - - protected override async Task RunInternal(CancellationToken cancellationToken) - { - bool run; - do - { - run = await _collector.Run(_front, _back, cancellationToken); - } - while (run); - } - } -} diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index 19651e832..ff9352d0a 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -323,7 +323,6 @@ - @@ -353,32 +352,13 @@ Designer - + + Designer + Designer - - - - - - - - - - - - - - - - - - - - - - + diff --git a/src/Ng/Ng.nuspec b/src/Ng/Ng.nuspec index d43535821..a625db9bf 100644 --- a/src/Ng/Ng.nuspec +++ b/src/Ng/Ng.nuspec @@ -29,8 +29,6 @@ - - diff --git a/src/Ng/NgJobFactory.cs b/src/Ng/NgJobFactory.cs index 3aec36b99..90683526d 100644 --- a/src/Ng/NgJobFactory.cs +++ b/src/Ng/NgJobFactory.cs @@ -18,8 +18,7 @@ public static class NgJobFactory { "checklucene", typeof(CheckLuceneJob) }, { "clearlucene", typeof(ClearLuceneJob) }, { "db2lucene", typeof(Db2LuceneJob) }, - { "lightning", typeof(LightningJob) }, - { "endpointmonitoring", typeof(EndpointMonitoringJob) } + { "lightning", typeof(LightningJob) } }; public static NgJob GetJob(string jobName, ILoggerFactory loggerFactory) diff --git a/src/Ng/Scripts/EndpointMonitoring.cmd b/src/Ng/Scripts/EndpointMonitoring.cmd deleted file mode 100644 index 8adf98633..000000000 --- a/src/Ng/Scripts/EndpointMonitoring.cmd +++ /dev/null @@ -1,15 +0,0 @@ -@echo OFF - -cd Ng - -:Top - echo "Starting job - #{Jobs.endpointmonitoring.Title}" - - title #{Jobs.endpointmonitoring.Title} - - start /w .\Ng.exe endpointmonitoring -source #{Jobs.common.v3.Source} -index #{Jobs.common.v3.index} -gallery #{Jobs.common.v3.f2c.Gallery} -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" -statusFolder #{Jobs.endpointmonitoring.StatusFolder} -storageType azure -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} -storageTypeAuditing azure -storageAccountNameAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountName} -storageKeyValueAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountKey} -storageContainerAuditing auditing -storagePathAuditing package -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.common.v3.Interval} - - echo "Finished #{Jobs.endpointmonitoring.Title}" - - goto Top - \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/AggregateMonitoringNotificationService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/AggregateMonitoringNotificationService.cs deleted file mode 100644 index 9147de2a2..000000000 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/AggregateMonitoringNotificationService.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; - -namespace NuGet.Services.Metadata.Catalog.Monitoring -{ - public class AggregateNotificationService : IMonitoringNotificationService - { - IEnumerable _notificationServices; - - public AggregateNotificationService(IEnumerable notificationServices) - { - _notificationServices = notificationServices; - } - - public Task OnPackageValidationFinishedAsync(PackageValidationResult result, CancellationToken token) - { - var tasks = new List(); - - foreach (var notificationService in _notificationServices) - { - tasks.Add(Task.Run( - () => notificationService.OnPackageValidationFinishedAsync(result, token))); - } - - return Task.WhenAll(tasks); - } - - public Task OnPackageValidationFailedAsync(string packageId, string packageVersion, IList catalogEntriesJson, Exception e, CancellationToken token) - { - var tasks = new List(); - - foreach (var notificationService in _notificationServices) - { - tasks.Add(Task.Run( - () => notificationService.OnPackageValidationFailedAsync(packageId, packageVersion, catalogEntriesJson, e, token))); - } - - return Task.WhenAll(tasks); - } - } -} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs index 2b98797cd..f7e78bc51 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs @@ -27,6 +27,6 @@ public interface IMonitoringNotificationService /// Version of the package that could not be validated. /// Catalog entries of the package that queued the validation. /// Exception that was thrown while running validation on the package. - Task OnPackageValidationFailedAsync(string packageId, string packageVersion, IList catalogEntriesJson, Exception e, CancellationToken token); + Task OnPackageValidationFailedAsync(string packageId, string packageVersion, Exception e, CancellationToken token); } } diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs index ad653f8a3..baa0bdb31 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs @@ -52,7 +52,7 @@ public Task OnPackageValidationFinishedAsync(PackageValidationResult result, Can return Task.FromResult(0); } - public Task OnPackageValidationFailedAsync(string packageId, string packageVersion, IList catalogEntriesJson, Exception e, CancellationToken token) + public Task OnPackageValidationFailedAsync(string packageId, string packageVersion, Exception e, CancellationToken token) { _logger.LogError(LogEvents.ValidationFailedToRun, e, "Failed to test {PackageId} {PackageVersion}!", packageId, packageVersion); diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/PackageMonitoringStatusNotificationService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/PackageMonitoringStatusNotificationService.cs deleted file mode 100644 index fc8ce1694..000000000 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/PackageMonitoringStatusNotificationService.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Helpers; - -namespace NuGet.Services.Metadata.Catalog.Monitoring -{ - public class PackageMonitoringStatusNotificationService : IMonitoringNotificationService - { - private IPackageMonitoringStatusService _packageMonitoringStatusService; - - public PackageMonitoringStatusNotificationService(IPackageMonitoringStatusService packageMonitoringStatusService) - { - _packageMonitoringStatusService = packageMonitoringStatusService; - } - - public Task OnPackageValidationFinishedAsync(PackageValidationResult result, CancellationToken token) - { - var status = new PackageMonitoringStatus(result); - - return _packageMonitoringStatusService.UpdateAsync(status, token); - } - - public Task OnPackageValidationFailedAsync(string packageId, string packageVersion, IList catalogEntriesJson, Exception e, CancellationToken token) - { - var status = new PackageMonitoringStatus(new FeedPackageIdentity(packageId, packageVersion), e); - - return _packageMonitoringStatusService.UpdateAsync(status, token); - } - } -} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index 38409cca6..a292c57d1 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -33,6 +33,18 @@ ..\..\packages\Microsoft.ApplicationInsights.2.1.0\lib\net45\Microsoft.ApplicationInsights.dll + + ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll + + + ..\..\packages\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll + + + ..\..\packages\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll + + + ..\..\packages\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Extensions.DependencyInjection.Abstractions.1.0.0\lib\netstandard1.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll @@ -42,6 +54,9 @@ ..\..\packages\Microsoft.Extensions.Logging.Abstractions.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.Abstractions.dll + + ..\..\packages\WindowsAzure.Storage.7.1.2\lib\net40\Microsoft.WindowsAzure.Storage.dll + ..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll @@ -66,6 +81,9 @@ ..\..\packages\NuGet.Services.Logging.2.2.2\lib\net452\NuGet.Services.Logging.dll + + ..\..\packages\NuGet.Services.Storage.2.3.1-sb-queue-14800\lib\net452\NuGet.Services.Storage.dll + ..\..\packages\NuGet.Versioning.4.3.0\lib\net45\NuGet.Versioning.dll @@ -97,6 +115,9 @@ + + ..\..\packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll + @@ -109,9 +130,7 @@ - - @@ -125,12 +144,14 @@ + + @@ -152,6 +173,8 @@ + + @@ -163,6 +186,7 @@ + diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs index fcaf355fb..caf740d51 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs @@ -13,6 +13,11 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring /// public interface IPackageMonitoringStatusService { + /// + /// Returns a list of every package that has been monitored and its . + /// + Task> ListAsync(CancellationToken token); + /// /// Returns the validation status of a package. /// If validation has not yet been run on the package, returns null. diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs new file mode 100644 index 000000000..b85a9edc2 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Part of the metadata of a as returned by . + /// The full can be returned by calling . + /// + public class PackageMonitoringStatusListItem + { + public FeedPackageIdentity Package { get; } + + public PackageState State { get; } + + public PackageMonitoringStatusListItem(FeedPackageIdentity package, PackageState state) + { + Package = package; + State = state; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs index 9a5513add..0eea668cf 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs @@ -9,11 +9,11 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using NuGet.Protocol; using NuGet.Services.Metadata.Catalog.Helpers; using NuGet.Services.Metadata.Catalog.Persistence; +using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; + namespace NuGet.Services.Metadata.Catalog.Monitoring { /// @@ -21,6 +21,9 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring /// public class PackageMonitoringStatusService : IPackageMonitoringStatusService { + private static string[] _packageStateNames = Enum.GetNames(typeof(PackageState)); + private static Array _packageStateValues = Enum.GetValues(typeof(PackageState)); + private ILogger _logger; /// @@ -28,40 +31,72 @@ public class PackageMonitoringStatusService : IPackageMonitoringStatusService /// private IStorageFactory _storageFactory; - /// - /// The to use to save and load statuses of packages. - /// - private static JsonSerializerSettings SerializerSettings => _serializerSettings.Value; - private static Lazy _serializerSettings = new Lazy(() => + public PackageMonitoringStatusService(IStorageFactory storageFactory, ILogger logger) { - var settings = new JsonSerializerSettings(); + _logger = logger; + _storageFactory = storageFactory; + } - settings.Converters.Add(new NuGetVersionConverter()); - settings.Converters.Add(new StringEnumConverter()); + public async Task> ListAsync(CancellationToken token) + { + var listTasks = + _packageStateNames + .Select(state => GetListItems(state, token)) + .ToList(); - return settings; - }); + return + (await Task.WhenAll(listTasks)) + .Where(list => list != null && list.Any()) + .Aggregate((current, next) => current.Concat(next)) + .Where(i => i != null); + } - public PackageMonitoringStatusService(IStorageFactory storageFactory, ILogger logger) + private async Task> GetListItems(string state, CancellationToken token) { - _logger = logger; - _storageFactory = storageFactory; + var storage = GetStorage(state); + var list = await storage.List(token); + return list.Select(item => + { + try + { + return new PackageMonitoringStatusListItem(ParsePackageUri(item.Uri), (PackageState)Enum.Parse(typeof(PackageState), state)); + } + catch (Exception e) + { + _logger.LogWarning("Failed to parse list item {ItemUri}: {Exception}", item.Uri, e); + return null; + } + }); } public async Task GetAsync(FeedPackageIdentity package, CancellationToken token) { var statusTasks = - Enum.GetNames(typeof(PackageState)) - .Select(state => - Task.Run(async () => - { - return await GetPackageAsync(GetStorage(state), package, token); - }) - ); + _packageStateNames + .Select(state => GetPackageAsync(GetStorage(state), package, token)) + .ToList(); - return + var statuses = (await Task.WhenAll(statusTasks)) - .SingleOrDefault(s => s != null); + .Where(s => s != null); + + if (!statuses.Any()) + { + return null; + } + + // If more than one status exist for a single package, find the status with the latest timestamp. + var statusesWithTimeStamps = statuses.Where(s => s.ValidationResult != null && s.ValidationResult.CatalogEntries != null && s.ValidationResult.CatalogEntries.Any()); + if (statusesWithTimeStamps.Any()) + { + return statusesWithTimeStamps.OrderByDescending(s => s.ValidationResult.CatalogEntries.Max(c => c.CommitTimeStamp)).First(); + } + else + { + // No statuses have timestamps (they all failed to process). + // Because they are all in a bad state, choose an arbitrary one. + return statuses.First(); + } } public async Task> GetAsync(PackageState state, CancellationToken token) @@ -70,60 +105,67 @@ public async Task> GetAsync(PackageState st var storage = GetStorage(state); - var statusTasks = + var statusTasks = (await storage.List(token)) - .Select(listItem => - Task.Run(async () => - { - return await GetPackageAsync(storage, listItem.Uri, token); - }) - ); + .Select(listItem => GetPackageAsync(storage, listItem.Uri, token)) + .ToList(); - return + return (await Task.WhenAll(statusTasks)) .Where(s => s != null); } - + public async Task UpdateAsync(PackageMonitoringStatus status, CancellationToken token) { - // Guarantee that we never have the same package in multiple states by deleting it first. - await DeleteAsync(status.Package, token); + // Save the new status first. + // If we save it after deleting the existing status, the save could fail and then we'd lose the data. + await SaveAsync(status, token); + foreach (int stateInt in _packageStateValues) + { + var state = (PackageState)stateInt; + if (state != status.State) + { + // Delete the existing status. + await DeleteAsync(status.Package, state, token); + } + } + } + + private async Task SaveAsync(PackageMonitoringStatus status, CancellationToken token) + { var storage = GetStorage(status.State); - - var packageStatusJson = JsonConvert.SerializeObject(status, SerializerSettings); + + var packageStatusJson = JsonConvert.SerializeObject(status, JsonSerializerUtility.SerializerSettings); var storageContent = new StringStorageContent(packageStatusJson, "application/json"); var packageUri = GetPackageUri(storage, status.Package); + await storage.Save(packageUri, storageContent, token); } - - private Task DeleteAsync(FeedPackageIdentity package, CancellationToken token) + + private async Task DeleteAsync(FeedPackageIdentity package, PackageState state, CancellationToken token) { - var tasks = new List(); - - foreach (var state in Enum.GetNames(typeof(PackageState))) + var storage = GetStorage(state); + if (!storage.Exists(GetPackageFileName(package))) { - var storage = GetStorage(state); - - var packageUri = GetPackageUri(storage, package); - tasks.Add(Task.Run(() => storage.Delete(packageUri, token))); + return; } - return Task.WhenAll(tasks); + await storage.Delete(GetPackageUri(storage, package), token); } - private Storage GetStorage(PackageState state) + private CatalogStorage GetStorage(PackageState state) { return GetStorage(state.ToString()); } - private Storage GetStorage(string stateString) + private CatalogStorage GetStorage(string stateString) { return _storageFactory.Create(stateString.ToLowerInvariant()); } - private Uri GetPackageUri(Storage storage, FeedPackageIdentity package) + private Uri GetPackageUri(CatalogStorage storage, FeedPackageIdentity package) { return storage.ResolveUri(GetPackageFileName(package)); } @@ -137,12 +179,32 @@ private string GetPackageFileName(FeedPackageIdentity package) $"{idString}.{versionString}.json"; } - private Task GetPackageAsync(Storage storage, FeedPackageIdentity package, CancellationToken token) + /// + /// Parses a into a . + /// + /// The must end with "/{id}/{id}.{version}.json" + /// + private FeedPackageIdentity ParsePackageUri(Uri packageUri) + { + var uriSegments = packageUri.Segments; + // The second to last segment is the id. + var id = uriSegments[uriSegments.Length - 2].Trim('/'); + + // The last segment is {id}.{version}.json. + // Remove the id and the "." from the beginning. + var version = uriSegments[uriSegments.Length - 1].Substring(id.Length + ".".Length); + // Remove the ".json" from the end. + version = version.Substring(0, version.Length - ".json".Length); + + return new FeedPackageIdentity(id, version); + } + + private Task GetPackageAsync(CatalogStorage storage, FeedPackageIdentity package, CancellationToken token) { return GetPackageAsync(storage, GetPackageFileName(package), token); } - private Task GetPackageAsync(Storage storage, string fileName, CancellationToken token) + private Task GetPackageAsync(CatalogStorage storage, string fileName, CancellationToken token) { if (!storage.Exists(fileName)) { @@ -152,11 +214,23 @@ private Task GetPackageAsync(Storage storage, string fi return GetPackageAsync(storage, storage.ResolveUri(fileName), token); } - private async Task GetPackageAsync(Storage storage, Uri packageUri, CancellationToken token) + private async Task GetPackageAsync(CatalogStorage storage, Uri packageUri, CancellationToken token) { try { - return JsonConvert.DeserializeObject(await GetStorageContentsAsync(storage, packageUri, token), SerializerSettings); + var content = await storage.Load(packageUri, token); + string statusString = null; + using (var stream = content.GetContentStream()) + { + using (var reader = new StreamReader(stream)) + { + statusString = await reader.ReadToEndAsync(); + } + } + + var status = JsonConvert.DeserializeObject(statusString, JsonSerializerUtility.SerializerSettings); + + return status; } catch (Exception deserializationException) { @@ -169,17 +243,7 @@ private async Task GetPackageAsync(Storage storage, Uri try { /// Construct a from the with this as the exception. - var uriSegments = packageUri.Segments; - // The second to last segment is the id. - var id = uriSegments[uriSegments.Length - 2].Trim('/'); - - // The last segment is {id}.{version}.json. - // Remove the id and the "." from the beginning. - var version = uriSegments[uriSegments.Length - 1].Substring(id.Length + ".".Length); - // Remove the ".json" from the end. - version = version.Substring(0, version.Length - ".json".Length); - - return new PackageMonitoringStatus(new FeedPackageIdentity(id, version), new StatusDeserializationException(deserializationException)); + return new PackageMonitoringStatus(ParsePackageUri(packageUri), new StatusDeserializationException(deserializationException)); } catch (Exception uriParsingException) { @@ -193,21 +257,5 @@ private async Task GetPackageAsync(Storage storage, Uri } } } - - private async Task GetStorageContentsAsync(Storage storage, Uri uri, CancellationToken token) - { - var storageContent = await storage.Load(uri, token); - - var stringStorageContent = storageContent as StringStorageContent; - if (stringStorageContent != null) - { - return stringStorageContent.Content; - } - - using (var reader = new StreamReader(storageContent.GetContentStream())) - { - return reader.ReadToEnd(); - } - } } } diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs new file mode 100644 index 000000000..90a30b0d0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using NuGet.Protocol; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class JsonSerializerUtility + { + /// + /// The to use. + /// + public static JsonSerializerSettings SerializerSettings + { + get + { + var settings = new JsonSerializerSettings(); + + settings.Converters.Add(new NuGetVersionConverter()); + settings.Converters.Add(new StringEnumConverter()); + + return settings; + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs index 855fda079..be7477a1f 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs @@ -20,22 +20,23 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring /// public class PackageValidator { - private readonly IEnumerable _endpointValidators; + public IEnumerable AggregateValidators { get; } + private readonly ILogger _logger; private readonly StorageFactory _auditingStorageFactory; public PackageValidator( - IEnumerable endpointValidators, + IEnumerable aggregateValidators, StorageFactory auditingStorageFactory, ILogger logger) { - if (endpointValidators.Count() < 1) + if (aggregateValidators == null || !aggregateValidators.Any()) { - throw new ArgumentException("Must supply at least one endpoint!", nameof(endpointValidators)); + throw new ArgumentException("Must supply at least one endpoint!", nameof(aggregateValidators)); } - _endpointValidators = endpointValidators.ToList(); + AggregateValidators = aggregateValidators.ToList(); _auditingStorageFactory = auditingStorageFactory ?? throw new ArgumentNullException(nameof(auditingStorageFactory)); _logger = logger; } @@ -44,19 +45,19 @@ public PackageValidator( /// Runs s from the s against a package. /// /// A generated from the results of the s. - public async Task ValidateAsync(string packageId, string packageVersion, IList catalogEntriesJson, CollectorHttpClient client, CancellationToken cancellationToken) + public async Task ValidateAsync(PackageValidatorContext context, CollectorHttpClient client, CancellationToken cancellationToken) { - var package = new PackageIdentity(packageId, NuGetVersion.Parse(packageVersion)); - var catalogEntries = catalogEntriesJson.Select(c => new CatalogIndexEntry(c)); + var package = new PackageIdentity(context.Package.Id, NuGetVersion.Parse(context.Package.Version)); + var deletionAuditEntries = await DeletionAuditEntry.GetAsync( - _auditingStorageFactory, - cancellationToken, - package, + _auditingStorageFactory, + cancellationToken, + package, logger: _logger); - var validationContext = new ValidationContext(package, catalogEntries, deletionAuditEntries, client, cancellationToken); - var results = await Task.WhenAll(_endpointValidators.Select(endpoint => endpoint.ValidateAsync(validationContext))); + var validationContext = new ValidationContext(package, context.CatalogEntries, deletionAuditEntries, client, cancellationToken); + var results = await Task.WhenAll(AggregateValidators.Select(endpoint => endpoint.ValidateAsync(validationContext)).ToList()); return new PackageValidationResult(validationContext, results); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs new file mode 100644 index 000000000..b70643731 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageValidatorContext + { + public FeedPackageIdentity Package { get; private set; } + + public IEnumerable CatalogEntries { get; private set; } + + [JsonConstructor] + public PackageValidatorContext(FeedPackageIdentity package, IEnumerable catalogEntries) + { + Package = package; + CatalogEntries = catalogEntries; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs new file mode 100644 index 000000000..f6d9781f0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorFactory.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using Microsoft.Extensions.Logging; +using NuGet.Protocol; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageValidatorFactory + { + private ILoggerFactory _loggerFactory; + private ILogger _logger; + + public PackageValidatorFactory(ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + _logger = _loggerFactory.CreateLogger(); + } + + public PackageValidator Create( + string galleryUrl, + string indexUrl, + StorageFactory auditingStorageFactory, + IEnumerable endpointInputs, + Func messageHandlerFactory, + bool verbose = false) + { + var validatorFactory = new ValidatorFactoryFactory(_loggerFactory).Create(galleryUrl, indexUrl); + + var endpointFactory = new EndpointFactory(validatorFactory, messageHandlerFactory, _loggerFactory); + var endpoints = endpointInputs.Select(e => endpointFactory.Create(e)); + + return new PackageValidator(endpoints, auditingStorageFactory, _loggerFactory.CreateLogger()); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactory.cs index e90240e09..9da4f8351 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactory.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactory.cs @@ -14,6 +14,9 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring /// public class ValidatorFactory { + private readonly IDictionary _feedToSource; + private readonly ILoggerFactory _loggerFactory; + /// Used to map to the to use. public ValidatorFactory(IDictionary feedToSource, ILoggerFactory loggerFactory) { @@ -21,9 +24,6 @@ public ValidatorFactory(IDictionary feedToSource, IL _loggerFactory = loggerFactory; } - private readonly IDictionary _feedToSource; - private readonly ILoggerFactory _loggerFactory; - public IValidator Create(Type validatorType) { var loggerType = typeof(ILogger<>).MakeGenericType(validatorType); diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactoryFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactoryFactory.cs new file mode 100644 index 000000000..b649e251c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorFactoryFactory.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using NuGet.Configuration; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class ValidatorFactoryFactory + { + private readonly ILoggerFactory _loggerFactory; + + public ValidatorFactoryFactory(ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + } + + public ValidatorFactory Create(string galleryUrl, string indexUrl) + { + return new ValidatorFactory(new Dictionary() + { + {FeedType.HttpV2, new SourceRepository(new PackageSource(galleryUrl), GetResourceProviders(ResourceProvidersToInjectV2), FeedType.HttpV2)}, + {FeedType.HttpV3, new SourceRepository(new PackageSource(indexUrl), GetResourceProviders(ResourceProvidersToInjectV3), FeedType.HttpV3)} + }, _loggerFactory); + } + + private IList> ResourceProvidersToInjectV2 => new List> + { + new Lazy(() => new NonhijackableV2HttpHandlerResourceProvider()), + new Lazy(() => new PackageTimestampMetadataResourceV2Provider(_loggerFactory)), + new Lazy(() => new PackageRegistrationMetadataResourceV2FeedProvider()) + }; + + private IList> ResourceProvidersToInjectV3 => new List> + { + new Lazy(() => new PackageRegistrationMetadataResourceV3Provider()) + }; + + private IEnumerable> GetResourceProviders(IList> providersToInject) + { + var resourceProviders = Repository.Provider.GetCoreV3().ToList(); + resourceProviders.AddRange(providersToInject); + return resourceProviders; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs index 235ff4703..b68b8d9b3 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs @@ -7,45 +7,48 @@ using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; -using NuGet.Services.Metadata.Catalog.Persistence; using NuGet.Services.Metadata.Catalog.Helpers; +using System.Linq; +using Microsoft.Extensions.Logging; +using NuGet.Services.Storage; namespace NuGet.Services.Metadata.Catalog.Monitoring { /// - /// Runs a against collected packages. + /// Creates s from Catalog entries and adds them to a s. /// public class ValidationCollector : SortingIdVersionCollector { + private readonly IStorageQueue _queue; + + private ILogger _logger; + public ValidationCollector( - PackageValidator packageValidator, + IStorageQueue queue, Uri index, - IMonitoringNotificationService notificationService, - Func handlerFunc = null) + ILogger logger, + Func handlerFunc = null) : base(index, handlerFunc) { - _packageValidator = packageValidator ?? throw new ArgumentNullException(nameof(packageValidator)); - _notificationService = notificationService ?? throw new ArgumentNullException(nameof(notificationService)); + _queue = queue; + _logger = logger; } - - private readonly PackageValidator _packageValidator; - private readonly IMonitoringNotificationService _notificationService; protected override async Task ProcessSortedBatch(CollectorHttpClient client, KeyValuePair> sortedBatch, JToken context, CancellationToken cancellationToken) { var packageId = sortedBatch.Key.Id; var packageVersion = sortedBatch.Key.Version; - var catalogEntriesJson = sortedBatch.Value; - - try - { - var result = await _packageValidator.ValidateAsync(packageId, packageVersion, catalogEntriesJson, client, cancellationToken); - await _notificationService.OnPackageValidationFinishedAsync(result, cancellationToken); - } - catch (Exception e) - { - await _notificationService.OnPackageValidationFailedAsync(packageId, packageVersion, catalogEntriesJson, e, cancellationToken); - } + var feedPackage = new FeedPackageIdentity(packageId, packageVersion); + + _logger.LogInformation("Processing catalog entries for {PackageId} {PackageVersion}.", packageId, packageVersion); + + var catalogEntries = sortedBatch.Value.Select(c => new CatalogIndexEntry(c)); + + _logger.LogInformation("Adding {MostRecentCatalogEntryUri} to queue.", catalogEntries.OrderByDescending(c => c.CommitTimeStamp).First().Uri); + + await _queue.Add( + new PackageValidatorContext(feedPackage, catalogEntries), + cancellationToken); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollectorFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollectorFactory.cs index e1143548e..55a32e977 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollectorFactory.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollectorFactory.cs @@ -6,10 +6,8 @@ using System.Linq; using System.Net.Http; using Microsoft.Extensions.Logging; -using NuGet.Configuration; -using NuGet.Protocol; -using NuGet.Protocol.Core.Types; using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.Storage; namespace NuGet.Services.Metadata.Catalog.Monitoring { @@ -21,6 +19,10 @@ public class ValidationCollectorFactory private ILoggerFactory _loggerFactory; private ILogger _logger; + /// + /// Context object returned by . + /// Contains a and two cursors to use with it. + /// public class Result { public Result(ValidationCollector collector, ReadWriteCursor front, ReadCursor back) @@ -41,64 +43,37 @@ public ValidationCollectorFactory(ILoggerFactory loggerFactory) _logger = _loggerFactory.CreateLogger(); } + /// + /// Constructs a from inputs and returns a . + /// + /// Queue that the queues packages to. + /// Url of the catalog that the should run on. + /// Storage where the cursors used by the are stored. + /// Endpoints that validations will be run on for queued packages. + /// Used by to construct a . public Result Create( - string galleryUrl, - string indexUrl, + IStorageQueue queue, string catalogIndexUrl, - StorageFactory monitoringStorageFactory, - StorageFactory auditingStorageFactory, - IEnumerable endpointContexts, - Func messageHandlerFactory, - IMonitoringNotificationService notificationService, - bool verbose = false) + Persistence.IStorageFactory monitoringStorageFactory, + IEnumerable endpointInputs, + Func messageHandlerFactory) { - _logger.LogInformation( - "CONFIG gallery: {Gallery} index: {Index} source: {ConfigSource} storage: {Storage} auditingStorage: {AuditingStorage} endpoints: {Endpoints}", - galleryUrl, indexUrl, catalogIndexUrl, monitoringStorageFactory, auditingStorageFactory, string.Join(", ", endpointContexts.Select(e => e.Name))); - - var validationFactory = new ValidatorFactory(new Dictionary() - { - {FeedType.HttpV2, new SourceRepository(new PackageSource(galleryUrl), GetResourceProviders(ResourceProvidersToInjectV2), FeedType.HttpV2)}, - {FeedType.HttpV3, new SourceRepository(new PackageSource(indexUrl), GetResourceProviders(ResourceProvidersToInjectV3), FeedType.HttpV3)} - }, _loggerFactory); - - var endpointFactory = new EndpointFactory( - validationFactory, - messageHandlerFactory, - _loggerFactory); - - var endpoints = endpointContexts.Select(e => endpointFactory.Create(e)).ToList(); - var collector = new ValidationCollector( - new PackageValidator(endpoints, auditingStorageFactory, _loggerFactory.CreateLogger()), + queue, new Uri(catalogIndexUrl), - notificationService, + _loggerFactory.CreateLogger(), messageHandlerFactory); - var storage = monitoringStorageFactory.Create(); - var front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); - var back = new AggregateCursor(endpoints.Select(e => e.Cursor)); + var front = GetFront(monitoringStorageFactory); + var back = new AggregateCursor(endpointInputs.Select(input => new HttpReadCursor(input.CursorUri, messageHandlerFactory))); return new Result(collector, front, back); } - private IList> ResourceProvidersToInjectV2 => new List> - { - new Lazy(() => new NonhijackableV2HttpHandlerResourceProvider()), - new Lazy(() => new PackageTimestampMetadataResourceV2Provider(_loggerFactory)), - new Lazy(() => new PackageRegistrationMetadataResourceV2FeedProvider()) - }; - - private IList> ResourceProvidersToInjectV3 => new List> - { - new Lazy(() => new PackageRegistrationMetadataResourceV3Provider()) - }; - - private IEnumerable> GetResourceProviders(IList> providersToInject) + public static DurableCursor GetFront(Persistence.IStorageFactory storageFactory) { - var resourceProviders = Repository.Provider.GetCoreV3().ToList(); - resourceProviders.AddRange(providersToInject); - return resourceProviders; + var storage = storageFactory.Create(); + return new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); } } -} +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config index cbc697fa3..0d3327383 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config @@ -1,6 +1,10 @@  + + + + @@ -12,6 +16,7 @@ + @@ -32,5 +37,7 @@ + + \ No newline at end of file diff --git a/tests/CatalogTests/App.config b/tests/CatalogTests/App.config index 84a50ce5d..f2c9e4ca5 100644 --- a/tests/CatalogTests/App.config +++ b/tests/CatalogTests/App.config @@ -49,6 +49,10 @@ + + + + From 1753b24e4d7ad2c71020493b91f60712e8fc3c6c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 9 Oct 2017 13:05:28 -0700 Subject: [PATCH 3/5] V3 monitoring jobs - add new queue-based jobs (#244) --- src/Ng/Arguments.cs | 3 +- src/Ng/CommandHelpers.cs | 89 ++++++-- src/Ng/Jobs/Catalog2MonitoringJob.cs | 76 +++++++ src/Ng/Jobs/Monitoring2MonitoringJob.cs | 64 ++++++ src/Ng/Jobs/MonitoringProcessorJob.cs | 207 ++++++++++++++++++ src/Ng/Ng.csproj | 9 + src/Ng/Ng.nuspec | 4 + src/Ng/NgJobFactory.cs | 5 +- src/Ng/Scripts/Catalog2Monitoring.cmd | 14 ++ src/Ng/Scripts/Monitoring2Monitoring.cmd | 14 ++ src/Ng/Scripts/MonitoringProcessor.cmd | 14 ++ src/Ng/packages.config | 4 +- ...ervices.Metadata.Catalog.Monitoring.csproj | 2 +- .../Test/PackageValidatorContext.cs | 14 ++ .../ValidationCollector.cs | 2 +- .../packages.config | 2 +- 16 files changed, 504 insertions(+), 19 deletions(-) create mode 100644 src/Ng/Jobs/Catalog2MonitoringJob.cs create mode 100644 src/Ng/Jobs/Monitoring2MonitoringJob.cs create mode 100644 src/Ng/Jobs/MonitoringProcessorJob.cs create mode 100644 src/Ng/Scripts/Catalog2Monitoring.cmd create mode 100644 src/Ng/Scripts/Monitoring2Monitoring.cmd create mode 100644 src/Ng/Scripts/MonitoringProcessor.cmd diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs index b871d7dca..8dcf232f8 100644 --- a/src/Ng/Arguments.cs +++ b/src/Ng/Arguments.cs @@ -43,6 +43,7 @@ public static class Arguments public const string StorageContainer = "storageContainer"; public const string StorageKeyValue = "storageKeyValue"; public const string StoragePath = "storagePath"; + public const string StorageQueueName = "storageQueueName"; public const string StorageType = "storageType"; public const string Version = "version"; @@ -103,7 +104,7 @@ public static class Arguments public const string StorageTypeAuditing = "storageTypeAuditing"; #endregion - #region EndpointMonitoring + #region Monitoring /// /// The url of the service index. /// diff --git a/src/Ng/CommandHelpers.cs b/src/Ng/CommandHelpers.cs index 0ce3477fe..6ef2532d0 100644 --- a/src/Ng/CommandHelpers.cs +++ b/src/Ng/CommandHelpers.cs @@ -5,17 +5,28 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Security.Cryptography.X509Certificates; using Lucene.Net.Store; using Lucene.Net.Store.Azure; +using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Auth; using NuGet.Services.Configuration; using NuGet.Services.KeyVault; using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.Storage; + +using ICatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.IStorageFactory; +using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; +using CatalogAggregateStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.AggregateStorageFactory; +using CatalogAzureStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.AzureStorageFactory; +using CatalogAzureStorage = NuGet.Services.Metadata.Catalog.Persistence.AzureStorage; +using CatalogFileStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.FileStorageFactory; namespace Ng { @@ -77,9 +88,17 @@ private static ISecretInjector GetSecretInjector(IDictionary arg return new SecretInjector(secretReader); } + public static void AssertAzureStorage(IDictionary arguments) + { + if (arguments.GetOrThrow(Arguments.StorageType) != Arguments.AzureStorageType) + { + throw new ArgumentException("Only Azure storage is supported!"); + } + } + public static RegistrationStorageFactories CreateRegistrationStorageFactories(IDictionary arguments, bool verbose) { - StorageFactory legacyStorageFactory; + CatalogStorageFactory legacyStorageFactory; var semVer2StorageFactory = CreateSemVer2StorageFactory(arguments, verbose); var storageFactory = CreateStorageFactory(arguments, verbose); @@ -92,7 +111,7 @@ public static RegistrationStorageFactories CreateRegistrationStorageFactories(ID new KeyValuePair(storageFactory.BaseAddress.ToString(), compressedStorageFactory.BaseAddress.ToString()) }); - var aggregateStorageFactory = new AggregateStorageFactory( + var aggregateStorageFactory = new CatalogAggregateStorageFactory( storageFactory, new[] { compressedStorageFactory }, secondaryStorageBaseUrlRewriter.Rewrite) @@ -110,7 +129,7 @@ public static RegistrationStorageFactories CreateRegistrationStorageFactories(ID return new RegistrationStorageFactories(legacyStorageFactory, semVer2StorageFactory); } - public static StorageFactory CreateStorageFactory(IDictionary arguments, bool verbose) + public static CatalogStorageFactory CreateStorageFactory(IDictionary arguments, bool verbose) { IDictionary names = new Dictionary { @@ -126,7 +145,7 @@ public static StorageFactory CreateStorageFactory(IDictionary ar return CreateStorageFactoryImpl(arguments, names, verbose, compressed: false); } - public static StorageFactory CreateCompressedStorageFactory(IDictionary arguments, bool verbose) + public static CatalogStorageFactory CreateCompressedStorageFactory(IDictionary arguments, bool verbose) { if (!arguments.GetOrDefault(Arguments.UseCompressedStorage, false)) { @@ -147,7 +166,7 @@ public static StorageFactory CreateCompressedStorageFactory(IDictionary arguments, bool verbose) + public static CatalogStorageFactory CreateSemVer2StorageFactory(IDictionary arguments, bool verbose) { if (!arguments.GetOrDefault(Arguments.UseSemVer2Storage, false)) { @@ -168,7 +187,7 @@ public static StorageFactory CreateSemVer2StorageFactory(IDictionary arguments, bool verbose) + public static CatalogStorageFactory CreateSuffixedStorageFactory(string suffix, IDictionary arguments, bool verbose) { if (string.IsNullOrEmpty(suffix)) { @@ -189,7 +208,7 @@ public static StorageFactory CreateSuffixedStorageFactory(string suffix, IDictio return CreateStorageFactoryImpl(arguments, names, verbose, compressed: false); } - private static StorageFactory CreateStorageFactoryImpl(IDictionary arguments, + private static CatalogStorageFactory CreateStorageFactoryImpl(IDictionary arguments, IDictionary argumentNameMap, bool verbose, bool compressed) @@ -211,7 +230,7 @@ private static StorageFactory CreateStorageFactoryImpl(IDictionary GetHttpMessageHandlerFactory(bool verbose return handlerFunc; } + + public static IEnumerable GetEndpointFactoryInputs(IDictionary arguments) + { + return arguments.GetOrThrow(Arguments.EndpointsToTest).Split(';').Select(e => { + var endpointParts = e.Split('|'); + + if (endpointParts.Count() < 2) + { + throw new ArgumentException( + $"\"{e}\" is not a valid endpoint! Each endpoint must follow this format: " + + $"|."); + } + + return new EndpointFactory.Input(endpointParts[0], new Uri(endpointParts[1])); + }); + } + + public static IPackageMonitoringStatusService GetPackageMonitoringStatusService(IDictionary arguments, ICatalogStorageFactory storageFactory, ILoggerFactory loggerFactory) + { + return new PackageMonitoringStatusService( + new NamedStorageFactory(storageFactory, arguments.GetOrDefault(Arguments.PackageStatusFolder, Arguments.PackageStatusFolderDefault)), + loggerFactory.CreateLogger()); + } + + public static IStorageQueue CreateStorageQueue(IDictionary arguments, int version) + { + var storageType = arguments.GetOrThrow(Arguments.StorageType); + + if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase)) + { + var storageAccountName = arguments.GetOrThrow(Arguments.StorageAccountName); + var storageKeyValue = arguments.GetOrThrow(Arguments.StorageKeyValue); + var storageQueueName = arguments.GetOrDefault(Arguments.StorageQueueName); + + var credentials = new StorageCredentials(storageAccountName, storageKeyValue); + var account = new CloudStorageAccount(credentials, true); + return new StorageQueue(new AzureStorageQueue(account, storageQueueName), + new JsonMessageSerializer(JsonSerializerUtility.SerializerSettings), version); + } + else + { + throw new NotImplementedException("Only Azure storage queues are supported!"); + } + } } -} +} \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2MonitoringJob.cs b/src/Ng/Jobs/Catalog2MonitoringJob.cs new file mode 100644 index 000000000..5959c999a --- /dev/null +++ b/src/Ng/Jobs/Catalog2MonitoringJob.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Configuration; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace Ng.Jobs +{ + /// + /// Runs a on the catalog. + /// The purpose of this job is to queue newly added, modified, or deleted packages for the to run validations on. + /// + public class Catalog2MonitoringJob : LoopingNgJob + { + private ValidationCollectorFactory _collectorFactory; + private ValidationCollector _collector; + private ReadWriteCursor _front; + private ReadCursor _back; + + public Catalog2MonitoringJob(ILoggerFactory loggerFactory) + : base(loggerFactory) + { + _collectorFactory = new ValidationCollectorFactory(LoggerFactory); + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var gallery = arguments.GetOrThrow(Arguments.Gallery); + var index = arguments.GetOrThrow(Arguments.Index); + var source = arguments.GetOrThrow(Arguments.Source); + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + + var endpointInputs = CommandHelpers.GetEndpointFactoryInputs(arguments); + + var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(verbose); + + var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + + var queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + + Logger.LogInformation( + "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} endpoints: {Endpoints}", + gallery, index, monitoringStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name))); + + var context = _collectorFactory.Create( + queue, + source, + monitoringStorageFactory, + endpointInputs, + messageHandlerFactory); + + _collector = context.Collector; + _front = context.Front; + _back = context.Back; + } + + protected override async Task RunInternal(CancellationToken cancellationToken) + { + bool run; + do + { + run = await _collector.Run(_front, _back, cancellationToken); + } + while (run); + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/Monitoring2MonitoringJob.cs b/src/Ng/Jobs/Monitoring2MonitoringJob.cs new file mode 100644 index 000000000..d00ef122e --- /dev/null +++ b/src/Ng/Jobs/Monitoring2MonitoringJob.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Configuration; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Storage; + +namespace Ng.Jobs +{ + /// + /// Gets s that have and requeue them to be processed by the . + /// + public class Monitoring2MonitoringJob : LoopingNgJob + { + private IPackageMonitoringStatusService _statusService; + private IStorageQueue _queue; + + public Monitoring2MonitoringJob(ILoggerFactory loggerFactory) + : base(loggerFactory) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + + _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + + _queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + } + + protected override async Task RunInternal(CancellationToken cancellationToken) + { + var invalidPackages = await _statusService.GetAsync(PackageState.Invalid, cancellationToken); + + await Task.WhenAll(invalidPackages.Select(invalidPackage => + { + try + { + Logger.LogInformation("Requeuing invalid package {PackageId} {PackageVersion}.", + invalidPackage.Package.Id, invalidPackage.Package.Version); + + return _queue.AddAsync( + new PackageValidatorContext(invalidPackage.Package, invalidPackage.ValidationResult?.CatalogEntries), + cancellationToken); + } + catch (Exception e) + { + Logger.LogError("Failed to requeue invalid package {PackageId} {PackageVersion}: {Exception}", + invalidPackage.Package.Id, invalidPackage.Package.Version, e); + + return Task.FromResult(0); + } + })); + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs new file mode 100644 index 000000000..91b7ffe51 --- /dev/null +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -0,0 +1,207 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Configuration; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Storage; +using NuGet.Versioning; + +namespace Ng.Jobs +{ + /// + /// Runs validations on packages from a using a . + /// + public class MonitoringProcessorJob : LoopingNgJob + { + private PackageValidator _packageValidator; + private IStorageQueue _queue; + private IPackageMonitoringStatusService _statusService; + private IMonitoringNotificationService _notificationService; + private RegistrationResourceV3 _regResource; + private NuGet.Common.ILogger CommonLogger; + private CollectorHttpClient _client; + + public MonitoringProcessorJob(ILoggerFactory loggerFactory) + : base(loggerFactory) + { + CommonLogger = Logger.AsCommon(); + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var gallery = arguments.GetOrThrow(Arguments.Gallery); + var index = arguments.GetOrThrow(Arguments.Index); + var source = arguments.GetOrThrow(Arguments.Source); + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose); + + var endpointInputs = CommandHelpers.GetEndpointFactoryInputs(arguments); + + var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(verbose); + + Logger.LogInformation( + "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} endpoints: {Endpoints}", + gallery, index, monitoringStorageFactory, auditingStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name))); + + _packageValidator = new PackageValidatorFactory(LoggerFactory) + .Create(gallery, index, auditingStorageFactory, endpointInputs, messageHandlerFactory, verbose); + + _queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + + _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + + _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger()); + + _regResource = Repository.Factory.GetCoreV3(index).GetResource(cancellationToken); + + _client = new CollectorHttpClient(messageHandlerFactory()); + } + + protected override async Task RunInternal(CancellationToken cancellationToken) + { + await ParallelAsync.Repeat(() => ProcessPackages(cancellationToken)); + } + + private async Task ProcessPackages(CancellationToken token) + { + StorageQueueMessage queueMessage = null; + do + { + Logger.LogInformation("Fetching next queue message."); + queueMessage = await _queue.GetNextAsync(token); + await HandleQueueMessage(queueMessage, token); + } while (queueMessage != null); + + Logger.LogInformation("No messages left in queue."); + } + + private async Task HandleQueueMessage(StorageQueueMessage queueMessage, CancellationToken token) + { + if (queueMessage == null) + { + return; + } + + var queuedContext = queueMessage.Contents; + var messageWasProcessed = false; + + try + { + await RunPackageValidator(queuedContext, token); + // The validations ran successfully and were saved to storage. + // We can remove the message from the queue because it was processed. + messageWasProcessed = true; + } + catch (Exception e) + { + // Validations failed to run! Save this failed status to storage. + await SaveFailedPackageMonitoringStatus(queuedContext, e, token); + // We can then remove the message from the queue because this failed status can be used to requeue the message. + messageWasProcessed = true; + } + + // Note that if both validations fail and saving the failure status fail, we cannot remove the message from the queue. + if (messageWasProcessed) + { + await _queue.RemoveAsync(queueMessage, token); + } + } + + private async Task RunPackageValidator(PackageValidatorContext queuedContext, CancellationToken token) + { + var feedPackage = queuedContext.Package; + Logger.LogInformation("Running PackageValidator on PackageValidatorContext for {PackageId} {PackageVersion}.", feedPackage.Id, feedPackage.Version); + IEnumerable catalogEntries = null; + + if (queuedContext.CatalogEntries != null) + { + catalogEntries = queuedContext.CatalogEntries; + } + else + { + Logger.LogInformation("PackageValidatorContext for {PackageId} {PackageVersion} is missing catalog entries! " + + "Attempting to fetch most recent catalog entry from registration.", + feedPackage.Id, feedPackage.Version); + + catalogEntries = await FetchCatalogIndexEntriesFromRegistration(feedPackage, token); + } + + var existingStatus = await _statusService.GetAsync(feedPackage, token); + + if (existingStatus?.ValidationResult != null && CompareCatalogEntries(catalogEntries, existingStatus.ValidationResult.CatalogEntries)) + { + // A newer catalog entry of this package has already been validated. + Logger.LogInformation("A newer catalog entry of {PackageId} {PackageVersion} has already been processed ({OldCommitTimeStamp} < {NewCommitTimeStamp}).", + feedPackage.Id, feedPackage.Version, + catalogEntries.Max(c => c.CommitTimeStamp), + existingStatus.ValidationResult.CatalogEntries.Max(c => c.CommitTimeStamp)); + + return; + } + + var context = new PackageValidatorContext(feedPackage, catalogEntries); + + var result = await _packageValidator.ValidateAsync(context, _client, token); + + await _notificationService.OnPackageValidationFinishedAsync(result, token); + + var status = new PackageMonitoringStatus(result); + await _statusService.UpdateAsync(status, token); + } + + private async Task> FetchCatalogIndexEntriesFromRegistration(FeedPackageIdentity feedPackage, CancellationToken token) + { + var id = feedPackage.Id; + var version = NuGetVersion.Parse(feedPackage.Version); + var leafBlob = await _regResource.GetPackageMetadata(new PackageIdentity(id, version), Logger.AsCommon(), token); + + if (leafBlob == null) + { + throw new Exception("Package is missing from registration!"); + } + + var catalogPageUri = new Uri(leafBlob["@id"].ToString()); + var catalogPage = await _client.GetJObjectAsync(catalogPageUri, token); + return new CatalogIndexEntry[] + { + new CatalogIndexEntry( + catalogPageUri, + Schema.DataTypes.PackageDetails.ToString(), + catalogPage["catalog:commitId"].ToString(), + DateTime.Parse(catalogPage["catalog:commitTimeStamp"].ToString()), + id, + version) + }; + } + + private async Task SaveFailedPackageMonitoringStatus(PackageValidatorContext queuedContext, Exception exception, CancellationToken token) + { + var feedPackage = new FeedPackageIdentity(queuedContext.Package.Id, queuedContext.Package.Version); + + await _notificationService.OnPackageValidationFailedAsync(feedPackage.Id, feedPackage.Version, exception, token); + + var status = new PackageMonitoringStatus(feedPackage, exception); + await _statusService.UpdateAsync(status, token); + } + + /// + /// Returns if the newest entry in is older than the newest entry in . + /// + private bool CompareCatalogEntries(IEnumerable first, IEnumerable second) + { + return first.Max(c => c.CommitTimeStamp) < second.Max(c => c.CommitTimeStamp); + } + } +} \ No newline at end of file diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index ff9352d0a..606cf13f2 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -247,6 +247,9 @@ ..\..\packages\NuGet.Packaging.Core.4.3.0\lib\net45\NuGet.Packaging.Core.dll + + ..\..\packages\NuGet.Protocol.4.3.0\lib\net45\NuGet.Protocol.dll + ..\..\packages\NuGet.Services.Configuration.2.2.2\lib\net452\NuGet.Services.Configuration.dll @@ -256,6 +259,9 @@ ..\..\packages\NuGet.Services.Logging.2.2.2\lib\net452\NuGet.Services.Logging.dll + + ..\..\packages\NuGet.Services.Storage.2.3.1-master-15426\lib\net452\NuGet.Services.Storage.dll + ..\..\packages\NuGet.Versioning.4.3.0\lib\net45\NuGet.Versioning.dll @@ -323,6 +329,7 @@ + @@ -331,6 +338,8 @@ + + diff --git a/src/Ng/Ng.nuspec b/src/Ng/Ng.nuspec index a625db9bf..ddd5a8da6 100644 --- a/src/Ng/Ng.nuspec +++ b/src/Ng/Ng.nuspec @@ -29,6 +29,10 @@ + + + + diff --git a/src/Ng/NgJobFactory.cs b/src/Ng/NgJobFactory.cs index 90683526d..76e3abf26 100644 --- a/src/Ng/NgJobFactory.cs +++ b/src/Ng/NgJobFactory.cs @@ -18,7 +18,10 @@ public static class NgJobFactory { "checklucene", typeof(CheckLuceneJob) }, { "clearlucene", typeof(ClearLuceneJob) }, { "db2lucene", typeof(Db2LuceneJob) }, - { "lightning", typeof(LightningJob) } + { "lightning", typeof(LightningJob) }, + { "catalog2monitoring", typeof(Catalog2MonitoringJob) }, + { "monitoring2monitoring", typeof(Monitoring2MonitoringJob) }, + { "monitoringprocessor", typeof(MonitoringProcessorJob) } }; public static NgJob GetJob(string jobName, ILoggerFactory loggerFactory) diff --git a/src/Ng/Scripts/Catalog2Monitoring.cmd b/src/Ng/Scripts/Catalog2Monitoring.cmd new file mode 100644 index 000000000..e984385d6 --- /dev/null +++ b/src/Ng/Scripts/Catalog2Monitoring.cmd @@ -0,0 +1,14 @@ +@echo OFF + +cd Ng + +:Top + echo "Starting job - #{Jobs.catalog2monitoring.Title}" + + title #{Jobs.catalog2monitoring.Title} + + start /w .\Ng.exe catalog2monitoring -source #{Jobs.common.v3.Source} -index #{Jobs.common.v3.index} -gallery #{Jobs.common.v3.f2c.Gallery} -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" -statusFolder #{Jobs.endpointmonitoring.StatusFolder} -storageType azure -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.common.v3.Interval} + + echo "Finished #{Jobs.catalog2monitoring.Title}" + + goto Top \ No newline at end of file diff --git a/src/Ng/Scripts/Monitoring2Monitoring.cmd b/src/Ng/Scripts/Monitoring2Monitoring.cmd new file mode 100644 index 000000000..0543a99d8 --- /dev/null +++ b/src/Ng/Scripts/Monitoring2Monitoring.cmd @@ -0,0 +1,14 @@ +@echo OFF + +cd Ng + +:Top + echo "Starting job - #{Jobs.monitoring2monitoring.Title}" + + title #{Jobs.monitoring2monitoring.Title} + + start /w .\Ng.exe monitoring2monitoring -statusFolder #{Jobs.endpointmonitoring.StatusFolder} -storageType azure -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.monitoring2monitoring.Interval} + + echo "Finished #{Jobs.monitoring2monitoring.Title}" + + goto Top \ No newline at end of file diff --git a/src/Ng/Scripts/MonitoringProcessor.cmd b/src/Ng/Scripts/MonitoringProcessor.cmd new file mode 100644 index 000000000..a2ce2c2a1 --- /dev/null +++ b/src/Ng/Scripts/MonitoringProcessor.cmd @@ -0,0 +1,14 @@ +@echo OFF + +cd Ng + +:Top + echo "Starting job - #{Jobs.monitoringprocessor.Title}" + + title #{Jobs.monitoringprocessor.Title} + + start /w .\Ng.exe monitoringprocessor -source #{Jobs.common.v3.Source} -index #{Jobs.common.v3.index} -gallery #{Jobs.common.v3.f2c.Gallery} -endpointsToTest "#{Jobs.endpointmonitoring.EndpointsToTest}" -statusFolder #{Jobs.endpointmonitoring.StatusFolder} -storageType azure -storageAccountName #{Jobs.common.v3.c2r.StorageAccountName} -storageKeyValue #{Jobs.common.v3.c2r.StorageAccountKey} -storageContainer #{Jobs.endpointmonitoring.MonitoringContainer} -storageQueueName #{Jobs.endpointmonitoring.PackageValidatorQueue} -storageTypeAuditing azure -storageAccountNameAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountName} -storageKeyValueAuditing #{Jobs.feed2catalogv3.AuditingStorageAccountKey} -storageContainerAuditing auditing -storagePathAuditing package -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.common.v3.Interval} + + echo "Finished #{Jobs.monitoringprocessor.Title}" + + goto Top \ No newline at end of file diff --git a/src/Ng/packages.config b/src/Ng/packages.config index 9dd3e66d1..db2c53454 100644 --- a/src/Ng/packages.config +++ b/src/Ng/packages.config @@ -39,9 +39,11 @@ + + @@ -67,7 +69,7 @@ - + diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index a292c57d1..e01efcf3e 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -82,7 +82,7 @@ ..\..\packages\NuGet.Services.Logging.2.2.2\lib\net452\NuGet.Services.Logging.dll - ..\..\packages\NuGet.Services.Storage.2.3.1-sb-queue-14800\lib\net452\NuGet.Services.Storage.dll + ..\..\packages\NuGet.Services.Storage.2.3.1-dev-15374\lib\net452\NuGet.Services.Storage.dll ..\..\packages\NuGet.Versioning.4.3.0\lib\net45\NuGet.Versioning.dll diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs index b70643731..c225fa486 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs @@ -4,10 +4,24 @@ namespace NuGet.Services.Metadata.Catalog.Monitoring { + /// + /// The data to be passed to . + /// public class PackageValidatorContext { + /// + /// This should be incremented every time the structure of this class changes. + /// + public const int Version = 1; + + /// + /// The package to run validations on. + /// public FeedPackageIdentity Package { get; private set; } + /// + /// The catalog entries that initiated this request to run validations. + /// public IEnumerable CatalogEntries { get; private set; } [JsonConstructor] diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs index b68b8d9b3..f9a2ced54 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/ValidationCollector.cs @@ -46,7 +46,7 @@ protected override async Task ProcessSortedBatch(CollectorHttpClient client, Key _logger.LogInformation("Adding {MostRecentCatalogEntryUri} to queue.", catalogEntries.OrderByDescending(c => c.CommitTimeStamp).First().Uri); - await _queue.Add( + await _queue.AddAsync( new PackageValidatorContext(feedPackage, catalogEntries), cancellationToken); } diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config index 0d3327383..e3c3eba83 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config @@ -16,7 +16,7 @@ - + From 28b37a0eab68688ba40477ac05b5b8091163b37e Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Tue, 10 Oct 2017 11:54:32 -0700 Subject: [PATCH 4/5] V3 monitoring jobs - if an exception cannot be deserialized, return a default exception instead (#242) --- ...ervices.Metadata.Catalog.Monitoring.csproj | 1 + .../Utility/SafeExceptionConverter.cs | 39 +++++++++++++++++++ .../Validation/Result/ValidationResult.cs | 1 + 3 files changed, 41 insertions(+) create mode 100644 src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index e01efcf3e..251d61118 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -155,6 +155,7 @@ + diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs new file mode 100644 index 000000000..4ce09c37f --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// for converting exceptions safely. + /// If the exception fails to deserialize, returns an instead of failing. + /// + public class SafeExceptionConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return typeof(Exception).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + try + { + return serializer.Deserialize(reader, objectType); + } + catch (Exception e) + { + // When deserializing the exception fails, we don't want to fail deserialization of the entire object. + // Return the exception that was thrown instead. + return e; + } + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs index 89b03fa8c..8933ec2a2 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs @@ -27,6 +27,7 @@ public class ValidationResult /// If the test ed, the exception that was thrown. /// [JsonProperty("exception")] + [JsonConverter(typeof(SafeExceptionConverter))] public Exception Exception { get; } public ValidationResult(IValidator validator, TestResult result) From 7a74eb47148d1b1fc37e70ba000c9f4f3c4af6da Mon Sep 17 00:00:00 2001 From: Andrei Grigorev Date: Wed, 11 Oct 2017 13:29:45 -0700 Subject: [PATCH 5/5] Used proper version of the NuGet.Services.Storage package (#247) --- .../NuGet.Services.Metadata.Catalog.Monitoring.csproj | 2 +- src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index 251d61118..2486cac07 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -82,7 +82,7 @@ ..\..\packages\NuGet.Services.Logging.2.2.2\lib\net452\NuGet.Services.Logging.dll - ..\..\packages\NuGet.Services.Storage.2.3.1-dev-15374\lib\net452\NuGet.Services.Storage.dll + ..\..\packages\NuGet.Services.Storage.2.3.1\lib\net452\NuGet.Services.Storage.dll ..\..\packages\NuGet.Versioning.4.3.0\lib\net45\NuGet.Versioning.dll diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config index e3c3eba83..e34c7f958 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/packages.config @@ -16,7 +16,7 @@ - +