diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln index a8694c576..3f7683649 100644 --- a/NuGet.Jobs.sln +++ b/NuGet.Jobs.sln @@ -151,6 +151,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Tests", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core.Tests", "tests\Validation.Symbols.Core.Tests\Validation.Symbols.Core.Tests.csproj", "{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Common.Tests", "tests\NuGet.Jobs.Common.Tests\NuGet.Jobs.Common.Tests.csproj", "{CE96428B-8138-4914-9999-2B391797FFF8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -399,6 +401,10 @@ Global {9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.Build.0 = Release|Any CPU + {CE96428B-8138-4914-9999-2B391797FFF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE96428B-8138-4914-9999-2B391797FFF8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE96428B-8138-4914-9999-2B391797FFF8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE96428B-8138-4914-9999-2B391797FFF8}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -464,6 +470,7 @@ Global {2DD07A73-8C88-4429-BB24-C2813586EF92} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} {640D29AB-4D1B-4FC7-AE67-AD12EE5AC503} = {6A776396-02B1-475D-A104-26940ADB04AB} {9ED642DF-4623-4EB2-8B72-52C6489BF9D6} = {6A776396-02B1-475D-A104-26940ADB04AB} + {CE96428B-8138-4914-9999-2B391797FFF8} = {6A776396-02B1-475D-A104-26940ADB04AB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B} diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 97f7528e6..58ae8a3d8 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -4,20 +4,23 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Diagnostics.Tracing; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; using Dapper; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; namespace ArchivePackages { - public class Job : JobBase + public class Job : JsonConfigurationJob { private readonly JobEventSource JobEventSourceLog = JobEventSource.Log; private const string ContentTypeJson = "application/json"; @@ -27,6 +30,8 @@ public class Job : JobBase private const string DefaultPackagesArchiveContainerName = "ng-backups"; private const string DefaultCursorBlobName = "cursor.json"; + private InitializationConfiguration Configuration { get; set; } + /// /// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs /// @@ -44,6 +49,7 @@ public class Job : JobBase /// DestinationContainerName should be same as the primary destination /// public CloudStorageAccount SecondaryDestination { get; set; } + /// /// Destination Container name for both Primary and Secondary destinations. Also, for the cursor blob /// @@ -53,8 +59,11 @@ public class Job : JobBase /// Blob containing the cursor data. Cursor data comprises of cursorDateTime /// public string CursorBlobName { get; set; } - - private ISqlConnectionFactory _packageDbConnectionFactory; + + /// + /// Gallery database registration, for diagnostics. + /// + private SqlConnectionStringBuilder GalleryDatabase { get; set; } protected CloudBlobContainer SourceContainer { get; private set; } @@ -66,33 +75,34 @@ public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); - Source = CloudStorageAccount.Parse( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source)); + Configuration = _serviceProvider.GetRequiredService>().Value; - PrimaryDestination = CloudStorageAccount.Parse( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination)); + GalleryDatabase = GetDatabaseRegistration(); - var secondaryDestinationCstr = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SecondaryDestination); - SecondaryDestination = string.IsNullOrEmpty(secondaryDestinationCstr) ? null : CloudStorageAccount.Parse(secondaryDestinationCstr); + Source = CloudStorageAccount.Parse(Configuration.Source); - SourceContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SourceContainerName) ?? DefaultPackagesContainerName; + PrimaryDestination = CloudStorageAccount.Parse(Configuration.PrimaryDestination); - DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? DefaultPackagesArchiveContainerName; + if (!string.IsNullOrEmpty(Configuration.SecondaryDestination)) + { + SecondaryDestination = CloudStorageAccount.Parse(Configuration.SecondaryDestination); + } + + SourceContainerName = Configuration.SourceContainerName ?? DefaultPackagesContainerName; + DestinationContainerName = Configuration.DestinationContainerName ?? DefaultPackagesArchiveContainerName; SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName); PrimaryDestinationContainer = PrimaryDestination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName); SecondaryDestinationContainer = SecondaryDestination?.CreateCloudBlobClient().GetContainerReference(DestinationContainerName); - CursorBlobName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.CursorBlob) ?? DefaultCursorBlobName; + CursorBlobName = Configuration.CursorBlob ?? DefaultCursorBlobName; } public override async Task Run() { - JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); await Archive(PrimaryDestinationContainer); // todo: consider reusing package query for primary and secondary archives @@ -124,9 +134,9 @@ private async Task Archive(CloudBlobContainer destinationContainer) JobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier)); - JobEventSourceLog.GatheringPackagesToArchiveFromDb(_packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + JobEventSourceLog.GatheringPackagesToArchiveFromDb(GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); List packages; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { packages = (await connection.QueryAsync(@" SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published @@ -135,7 +145,7 @@ FROM Packages p WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime })) .ToList(); } - JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); var archiveSet = packages .AsParallel() @@ -193,129 +203,14 @@ private async Task ArchivePackage(string sourceBlobName, string destinationBlobN JobEventSourceLog.StartedCopy(sourceBlob.Name, destBlob.Name); } } - } - [EventSource(Name = "Outercurve-NuGet-Jobs-ArchivePackages")] - public class JobEventSource : EventSource - { - public static readonly JobEventSource Log = new JobEventSource(); - - private JobEventSource() { } - - [Event( - eventId: 1, - Level = EventLevel.Informational, - Message = "Preparing to archive packages from {0}/{1} to primary destination {2}/{3} using package data from {4}/{5}")] - public void PreparingToArchive(string sourceAccount, string sourceContainer, string destAccount, string destContainer, string dbServer, string dbName) { WriteEvent(1, sourceAccount, sourceContainer, destAccount, destContainer, dbServer, dbName); } - - [Event( - eventId: 2, - Level = EventLevel.Informational, - Message = "Preparing to archive packages to secondary destination {0}/{1}")] - public void PreparingToArchive2(string destAccount, string destContainer) { WriteEvent(2, destAccount, destContainer); } - - [Event( - eventId: 3, - Level = EventLevel.Informational, - Message = "Cursor data: CursorDateTime is {0}")] - public void CursorData(string cursorDateTime) { WriteEvent(3, cursorDateTime); } - - [Event( - eventId: 4, - Level = EventLevel.Informational, - Task = Tasks.GatheringDbPackages, - Opcode = EventOpcode.Start, - Message = "Gathering list of packages to archive from {0}/{1}")] - public void GatheringPackagesToArchiveFromDb(string dbServer, string dbName) { WriteEvent(4, dbServer, dbName); } - - [Event( - eventId: 5, - Level = EventLevel.Informational, - Task = Tasks.GatheringDbPackages, - Opcode = EventOpcode.Stop, - Message = "Gathered {0} packages to archive from {1}/{2}")] - public void GatheredPackagesToArchiveFromDb(int gathered, string dbServer, string dbName) { WriteEvent(5, gathered, dbServer, dbName); } - - [Event( - eventId: 6, - Level = EventLevel.Informational, - Task = Tasks.ArchivingPackages, - Opcode = EventOpcode.Start, - Message = "Starting archive of {0} packages.")] - public void StartingArchive(int count) { WriteEvent(6, count); } - - [Event( - eventId: 7, - Level = EventLevel.Informational, - Task = Tasks.ArchivingPackages, - Opcode = EventOpcode.Stop, - Message = "Started archive.")] - public void StartedArchive() { WriteEvent(7); } - - [Event( - eventId: 8, - Level = EventLevel.Informational, - Message = "Archive already exists: {0}")] - public void ArchiveExists(string blobName) { WriteEvent(8, blobName); } - - [Event( - eventId: 9, - Level = EventLevel.Warning, - Message = "Source Blob does not exist: {0}")] - public void SourceBlobMissing(string blobName) { WriteEvent(9, blobName); } - - [Event( - eventId: 12, - Level = EventLevel.Informational, - Task = Tasks.StartingPackageCopy, - Opcode = EventOpcode.Start, - Message = "Starting copy of {0} to {1}.")] - public void StartingCopy(string source, string dest) { WriteEvent(12, source, dest); } - - [Event( - eventId: 13, - Level = EventLevel.Informational, - Task = Tasks.StartingPackageCopy, - Opcode = EventOpcode.Stop, - Message = "Started copy of {0} to {1}.")] - public void StartedCopy(string source, string dest) { WriteEvent(13, source, dest); } - - [Event( - eventId: 14, - Level = EventLevel.Informational, - Message = "NewCursor data: CursorDateTime is {0}")] - public void NewCursorData(string cursorDateTime) { WriteEvent(14, cursorDateTime); } - } - - public static class Tasks - { - public const EventTask GatheringDbPackages = (EventTask)0x1; - public const EventTask ArchivingPackages = (EventTask)0x2; - public const EventTask StartingPackageCopy = (EventTask)0x3; - } - - public class PackageRef - { - public PackageRef(string id, string version, string hash) - { - Id = id; - Version = version; - Hash = hash; - } - public PackageRef(string id, string version, string hash, DateTime lastEdited) - : this(id, version, hash) + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) { - LastEdited = lastEdited; } - public PackageRef(string id, string version, string hash, DateTime lastEdited, DateTime published) - : this(id, version, hash, lastEdited) + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { - Published = published; + ConfigureInitializationSection(services, configurationRoot); } - public string Id { get; set; } - public string Version { get; set; } - public string Hash { get; set; } - public DateTime? LastEdited { get; set; } - public DateTime? Published { get; set; } } } diff --git a/src/ArchivePackages/ArchivePackages.csproj b/src/ArchivePackages/ArchivePackages.csproj index efb41445d..0b4d2e91d 100644 --- a/src/ArchivePackages/ArchivePackages.csproj +++ b/src/ArchivePackages/ArchivePackages.csproj @@ -44,14 +44,19 @@ + + + + - - Designer - + + + + @@ -75,9 +80,6 @@ 9.0.1 - - 2.25.0-master-30453 - 4.3.3 @@ -85,6 +87,7 @@ 7.1.2 + \ No newline at end of file diff --git a/src/ArchivePackages/ArchivePackages.nuspec b/src/ArchivePackages/ArchivePackages.nuspec index f67729e31..0e75a87bc 100644 --- a/src/ArchivePackages/ArchivePackages.nuspec +++ b/src/ArchivePackages/ArchivePackages.nuspec @@ -11,12 +11,14 @@ - + - + + + \ No newline at end of file diff --git a/src/ArchivePackages/Configuration/InitializationConfiguration.cs b/src/ArchivePackages/Configuration/InitializationConfiguration.cs new file mode 100644 index 000000000..4e988bfa1 --- /dev/null +++ b/src/ArchivePackages/Configuration/InitializationConfiguration.cs @@ -0,0 +1,38 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace ArchivePackages +{ + public class InitializationConfiguration + { + /// + /// Source storage account. + /// + public string Source { get; set; } + + /// + /// Source storage container name (defaults to "packages"). + /// + public string SourceContainerName { get; set; } + + /// + /// Primary archive destination. + /// + public string PrimaryDestination { get; set; } + + /// + /// Secondary archive destination (optional). + /// + public string SecondaryDestination { get; set; } + + /// + /// Destination storage container name (defaults to "ng-backups"). + /// + public string DestinationContainerName { get; set; } + + /// + /// Cursor blob name (defaults to "cursor.json"). + /// + public string CursorBlob { get; set; } + } +} diff --git a/src/ArchivePackages/JobEventSource.cs b/src/ArchivePackages/JobEventSource.cs new file mode 100644 index 000000000..b5c8a6c0f --- /dev/null +++ b/src/ArchivePackages/JobEventSource.cs @@ -0,0 +1,99 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics.Tracing; + +namespace ArchivePackages +{ + [EventSource(Name = "Outercurve-NuGet-Jobs-ArchivePackages")] + public class JobEventSource : EventSource + { + public static readonly JobEventSource Log = new JobEventSource(); + + private JobEventSource() { } + + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Preparing to archive packages from {0}/{1} to primary destination {2}/{3} using package data from {4}/{5}")] + public void PreparingToArchive(string sourceAccount, string sourceContainer, string destAccount, string destContainer, string dbServer, string dbName) { WriteEvent(1, sourceAccount, sourceContainer, destAccount, destContainer, dbServer, dbName); } + + [Event( + eventId: 2, + Level = EventLevel.Informational, + Message = "Preparing to archive packages to secondary destination {0}/{1}")] + public void PreparingToArchive2(string destAccount, string destContainer) { WriteEvent(2, destAccount, destContainer); } + + [Event( + eventId: 3, + Level = EventLevel.Informational, + Message = "Cursor data: CursorDateTime is {0}")] + public void CursorData(string cursorDateTime) { WriteEvent(3, cursorDateTime); } + + [Event( + eventId: 4, + Level = EventLevel.Informational, + Task = JobTasks.GatheringDbPackages, + Opcode = EventOpcode.Start, + Message = "Gathering list of packages to archive from {0}/{1}")] + public void GatheringPackagesToArchiveFromDb(string dbServer, string dbName) { WriteEvent(4, dbServer, dbName); } + + [Event( + eventId: 5, + Level = EventLevel.Informational, + Task = JobTasks.GatheringDbPackages, + Opcode = EventOpcode.Stop, + Message = "Gathered {0} packages to archive from {1}/{2}")] + public void GatheredPackagesToArchiveFromDb(int gathered, string dbServer, string dbName) { WriteEvent(5, gathered, dbServer, dbName); } + + [Event( + eventId: 6, + Level = EventLevel.Informational, + Task = JobTasks.ArchivingPackages, + Opcode = EventOpcode.Start, + Message = "Starting archive of {0} packages.")] + public void StartingArchive(int count) { WriteEvent(6, count); } + + [Event( + eventId: 7, + Level = EventLevel.Informational, + Task = JobTasks.ArchivingPackages, + Opcode = EventOpcode.Stop, + Message = "Started archive.")] + public void StartedArchive() { WriteEvent(7); } + + [Event( + eventId: 8, + Level = EventLevel.Informational, + Message = "Archive already exists: {0}")] + public void ArchiveExists(string blobName) { WriteEvent(8, blobName); } + + [Event( + eventId: 9, + Level = EventLevel.Warning, + Message = "Source Blob does not exist: {0}")] + public void SourceBlobMissing(string blobName) { WriteEvent(9, blobName); } + + [Event( + eventId: 12, + Level = EventLevel.Informational, + Task = JobTasks.StartingPackageCopy, + Opcode = EventOpcode.Start, + Message = "Starting copy of {0} to {1}.")] + public void StartingCopy(string source, string dest) { WriteEvent(12, source, dest); } + + [Event( + eventId: 13, + Level = EventLevel.Informational, + Task = JobTasks.StartingPackageCopy, + Opcode = EventOpcode.Stop, + Message = "Started copy of {0} to {1}.")] + public void StartedCopy(string source, string dest) { WriteEvent(13, source, dest); } + + [Event( + eventId: 14, + Level = EventLevel.Informational, + Message = "NewCursor data: CursorDateTime is {0}")] + public void NewCursorData(string cursorDateTime) { WriteEvent(14, cursorDateTime); } + } +} \ No newline at end of file diff --git a/src/ArchivePackages/JobTasks.cs b/src/ArchivePackages/JobTasks.cs new file mode 100644 index 000000000..2af93d804 --- /dev/null +++ b/src/ArchivePackages/JobTasks.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics.Tracing; + +namespace ArchivePackages +{ + public static class JobTasks + { + public const EventTask GatheringDbPackages = (EventTask)0x1; + + public const EventTask ArchivingPackages = (EventTask)0x2; + + public const EventTask StartingPackageCopy = (EventTask)0x3; + } +} diff --git a/src/ArchivePackages/PackageRef.cs b/src/ArchivePackages/PackageRef.cs new file mode 100644 index 000000000..448105da3 --- /dev/null +++ b/src/ArchivePackages/PackageRef.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace ArchivePackages +{ + public class PackageRef + { + public PackageRef(string id, string version, string hash) + { + Id = id; + Version = version; + Hash = hash; + } + + public PackageRef(string id, string version, string hash, DateTime lastEdited) + : this(id, version, hash) + { + LastEdited = lastEdited; + } + + public PackageRef(string id, string version, string hash, DateTime lastEdited, DateTime published) + : this(id, version, hash, lastEdited) + { + Published = published; + } + + public string Id { get; set; } + + public string Version { get; set; } + + public string Hash { get; set; } + + public DateTime? LastEdited { get; set; } + + public DateTime? Published { get; set; } + } +} diff --git a/src/ArchivePackages/Scripts/ArchivePackages.cmd b/src/ArchivePackages/Scripts/ArchivePackages.cmd index 864f297e1..876bbb35f 100644 --- a/src/ArchivePackages/Scripts/ArchivePackages.cmd +++ b/src/ArchivePackages/Scripts/ArchivePackages.cmd @@ -7,7 +7,7 @@ cd bin title #{Jobs.archivepackages.Title} - start /w archivepackages.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PackageDatabase "#{Jobs.archivepackages.PackageDatabase}" -Source "#{Jobs.archivepackages.Source}" -PrimaryDestination "#{Jobs.archivepackages.PrimaryDestination}" -SecondaryDestination "#{Jobs.archivepackages.SecondaryDestination}" -Sleep "#{Jobs.archivepackages.Sleep}" -InstrumentationKey "#{Jobs.archivepackages.ApplicationInsightsInstrumentationKey}" + start /w archivepackages.exe -Configuration "#{Jobs.archivepackages.Configuration}" -Sleep "#{Jobs.archivepackages.Sleep}" -InstrumentationKey "#{Jobs.archivepackages.ApplicationInsightsInstrumentationKey}" echo "Finished #{Jobs.archivepackages.Title}" diff --git a/src/ArchivePackages/Settings/dev.json b/src/ArchivePackages/Settings/dev.json new file mode 100644 index 000000000..77b36515e --- /dev/null +++ b/src/ArchivePackages/Settings/dev.json @@ -0,0 +1,18 @@ +{ + "Initialization": { + "Source": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "SecondaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdev1;AccountKey=$$Dev-NuGetDev1Storage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=ArchivePackages;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/ArchivePackages/Settings/int.json b/src/ArchivePackages/Settings/int.json new file mode 100644 index 000000000..3bce8378c --- /dev/null +++ b/src/ArchivePackages/Settings/int.json @@ -0,0 +1,18 @@ +{ + "Initialization": { + "Source": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "SecondaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetint1;AccountKey=$$Int-NuGetInt1Storage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/ArchivePackages/Settings/prod.json b/src/ArchivePackages/Settings/prod.json new file mode 100644 index 000000000..13c26609c --- /dev/null +++ b/src/ArchivePackages/Settings/prod.json @@ -0,0 +1,18 @@ +{ + "Initialization": { + "Source": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "SecondaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetprod1;AccountKey=$$Prod-NuGetProd1Storage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Gallery.CredentialExpiration/Configuration/InitializationConfiguration.cs b/src/Gallery.CredentialExpiration/Configuration/InitializationConfiguration.cs new file mode 100644 index 000000000..1fbaeb90e --- /dev/null +++ b/src/Gallery.CredentialExpiration/Configuration/InitializationConfiguration.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Gallery.CredentialExpiration +{ + public class InitializationConfiguration + { + public int AllowEmailResendAfterDays { get; set; } + + public string ContainerName { get; set; } + + public string DataStorageAccount { get; set; } + + public string GalleryAccountUrl { get; set; } + + public string GalleryBrand { get; set; } + + public string MailFrom { get; set; } + + public string SmtpUri { get; set; } + + public int WarnDaysBeforeExpiration { get; set; } + + public bool WhatIf { get; set; } + } +} diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj index 696c6c793..24be2dc28 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj @@ -41,6 +41,7 @@ + @@ -65,6 +66,9 @@ Designer + + + @@ -92,9 +96,6 @@ 9.0.1 - - 2.25.0-master-30263 - 2.1.3 diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.nuspec b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.nuspec index 365c85b4d..e1abdf65f 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.nuspec +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs index e1021e836..0fe1e7ff3 100644 --- a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs +++ b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs @@ -6,20 +6,20 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; -using NuGet.Services.Sql; using Gallery.CredentialExpiration.Models; +using NuGet.Jobs.Configuration; namespace Gallery.CredentialExpiration { public class GalleryCredentialExpiration : ICredentialExpirationExporter { + private readonly Job _job; private readonly CredentialExpirationJobMetadata _jobMetadata; - private readonly ISqlConnectionFactory _galleryDatabase; - public GalleryCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, ISqlConnectionFactory galleryDatabase) + public GalleryCredentialExpiration(Job job, CredentialExpirationJobMetadata jobMetadata) { + _job = job; _jobMetadata = jobMetadata; - _galleryDatabase = galleryDatabase; } /// @@ -51,7 +51,7 @@ public async Task> GetCredentialsAsync(TimeSpan time var minNotificationDate = ConvertToString(GetMinNotificationDate()); // Connect to database - using (var galleryConnection = await _galleryDatabase.CreateAsync()) + using (var galleryConnection = await _job.OpenSqlConnectionAsync()) { // Fetch credentials that expire in _warnDaysBeforeExpiration days // + the user's e-mail address @@ -84,7 +84,10 @@ public List GetExpiringCredentials(List _jobMetadata.JobRunTime) ? _jobMetadata.JobCursor.MaxProcessedCredentialsTime : _jobMetadata.JobRunTime; + var sendEmailsDateLeftBoundary = (_jobMetadata.JobCursor.MaxProcessedCredentialsTime > _jobMetadata.JobRunTime) + ? _jobMetadata.JobCursor.MaxProcessedCredentialsTime + : _jobMetadata.JobRunTime; + return credentialSet.Where( x => x.Expires > sendEmailsDateLeftBoundary).ToList(); } diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index 4290dba1b..dbe7ad1cb 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -2,73 +2,49 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Linq; using System.Net; using System.Net.Mail; using System.Threading; using System.Threading.Tasks; +using Autofac; using Gallery.CredentialExpiration.Models; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.Services.Storage; namespace Gallery.CredentialExpiration { - public class Job : JobBase + public class Job : JsonConfigurationJob { private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); private readonly string _cursorFile = "cursorv2.json"; - private bool _whatIf = false; + private InitializationConfiguration Configuration { get; set; } - private string _galleryBrand; - private string _galleryAccountUrl; + private Storage Storage { get; set; } - private ISqlConnectionFactory _galleryDatabase; - - private string _mailFrom; - private SmtpClient _smtpClient; - - private int _warnDaysBeforeExpiration = 10; - - private Storage _storage; + private SmtpClient SmtpClient { get; set; } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - _whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf); - - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); - _galleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); - - _galleryBrand = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand); - _galleryAccountUrl = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryAccountUrl); + base.Init(serviceContainer, jobArgsDictionary); - _mailFrom = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.MailFrom); + Configuration = _serviceProvider.GetRequiredService>().Value; - var smtpConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SmtpUri); - var smtpUri = new SmtpUri(new Uri(smtpConnectionString)); - _smtpClient = CreateSmtpClient(smtpUri); - - _warnDaysBeforeExpiration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.WarnDaysBeforeExpiration) - ?? _warnDaysBeforeExpiration; - - var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); - var storageContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName); - - var storageAccount = CloudStorageAccount.Parse(storageConnectionString); - var storageFactory = new AzureStorageFactory(storageAccount, storageContainerName, LoggerFactory); - _storage = storageFactory.Create(); + SmtpClient = CreateSmtpClient(Configuration.SmtpUri); + + var storageAccount = CloudStorageAccount.Parse(Configuration.DataStorageAccount); + var storageFactory = new AzureStorageFactory(storageAccount, Configuration.ContainerName, LoggerFactory); + Storage = storageFactory.Create(); } public override async Task Run() @@ -76,20 +52,25 @@ public override async Task Run() var jobRunTime = DateTimeOffset.UtcNow; // Default values var jobCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: jobRunTime ); - var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + var galleryCredentialExpiration = new GalleryCredentialExpiration(this, + new CredentialExpirationJobMetadata(jobRunTime, Configuration.WarnDaysBeforeExpiration, jobCursor)); try { List credentialsInRange = null; // Get the most recent date for the emails being sent - if (_storage.Exists(_cursorFile)) + if (Storage.Exists(_cursorFile)) { - string content = await _storage.LoadString(_storage.ResolveUri(_cursorFile), CancellationToken.None); + string content = await Storage.LoadString(Storage.ResolveUri(_cursorFile), CancellationToken.None); // Load from cursor // Throw if the schema is not correct to ensure that not-intended emails are sent. - jobCursor = JsonConvert.DeserializeObject(content, new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error }); - galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + jobCursor = JsonConvert.DeserializeObject( + content, + new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error }); + + galleryCredentialExpiration = new GalleryCredentialExpiration(this, + new CredentialExpirationJobMetadata(jobRunTime, Configuration.WarnDaysBeforeExpiration, jobCursor)); } // Connect to database @@ -126,10 +107,13 @@ public override async Task Run() } finally { - JobRunTimeCursor newCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: galleryCredentialExpiration.GetMaxNotificationDate()); + JobRunTimeCursor newCursor = new JobRunTimeCursor( + jobCursorTime: jobRunTime, + maxProcessedCredentialsTime: galleryCredentialExpiration.GetMaxNotificationDate()); + string json = JsonConvert.SerializeObject(newCursor); var content = new StringStorageContent(json, "application/json"); - await _storage.Save(_storage.ResolveUri(_cursorFile), content, CancellationToken.None); + await Storage.Save(Storage.ResolveUri(_cursorFile), content, CancellationToken.None); } } @@ -146,7 +130,7 @@ private async Task HandleExpiredCredentialEmail(string username, List BuildApiKeyExpiryMessage(x.Description, x.Expires, jobRunTime)) @@ -156,21 +140,21 @@ private async Task HandleExpiredCredentialEmail(string username, List(services, configurationRoot); + } } } \ No newline at end of file diff --git a/src/Gallery.CredentialExpiration/Scripts/Gallery.CredentialExpiration.cmd b/src/Gallery.CredentialExpiration/Scripts/Gallery.CredentialExpiration.cmd index 45741ff39..3d6a98472 100644 --- a/src/Gallery.CredentialExpiration/Scripts/Gallery.CredentialExpiration.cmd +++ b/src/Gallery.CredentialExpiration/Scripts/Gallery.CredentialExpiration.cmd @@ -9,7 +9,7 @@ cd bin REM SmtpUri is expected to be of the format: smtps://username:password@host:port. Note that if username contains an "@", you need to URI encode it! - start /w gallery.credentialexpiration.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -WhatIf #{Jobs.gallery.credentialexpiration.WhatIf} -WarnDaysBeforeExpiration #{Jobs.gallery.credentialexpiration.WarnDaysBeforeExpiration} -MailFrom "#{Jobs.gallery.credentialexpiration.MailFrom}" -GalleryBrand "#{Jobs.gallery.credentialexpiration.GalleryBrand}" -GalleryAccountUrl "#{Jobs.gallery.credentialexpiration.GalleryAccountUrl}" -SmtpUri "#{Jobs.gallery.credentialexpiration.SmtpUri}" -GalleryDatabase "#{Jobs.gallery.credentialexpiration.GalleryDatabase}" -InstrumentationKey "#{Jobs.gallery.credentialexpiration.InstrumentationKey}" -verbose true -Interval #{Jobs.gallery.credentialexpiration.Interval} -DataStorageAccount "#{Jobs.gallery.credentialexpiration.Storage.Primary}" -ContainerName "#{Jobs.gallery.credentialexpiration.ContainerName}" + start /w gallery.credentialexpiration.exe -Configuration "#{Jobs.gallery.credentialexpiration.Configuration}" -InstrumentationKey "#{Jobs.gallery.credentialexpiration.InstrumentationKey}" -verbose true -Interval #{Jobs.gallery.credentialexpiration.Interval} echo "Finished #{Jobs.gallery.credentialexpiration.Title}" diff --git a/src/Gallery.CredentialExpiration/Settings/dev.json b/src/Gallery.CredentialExpiration/Settings/dev.json new file mode 100644 index 000000000..555705805 --- /dev/null +++ b/src/Gallery.CredentialExpiration/Settings/dev.json @@ -0,0 +1,24 @@ +{ + "Initialization": { + "AllowEmailResendAfterDays": 7, + "ContainerName": "credentialexpiration", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevuse2gallery;AccountKey=$$Dev-NuGetDevUse2Gallery-StorageKey$$", + "GalleryAccountUrl": "https://dev.nugettest.org/account/ApiKeys", + "GalleryBrand": "NuGet Gallery", + "MailFrom": "support@nuget.org", + "SmtpUri": "#{Jobs.gallery.credentialexpiration.SmtpUri}", + "WarnDaysBeforeExpiration": 7, + "WhatIf": true + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Gallery.CredentialExpiration;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Gallery.CredentialExpiration/Settings/int.json b/src/Gallery.CredentialExpiration/Settings/int.json new file mode 100644 index 000000000..c75a86f1f --- /dev/null +++ b/src/Gallery.CredentialExpiration/Settings/int.json @@ -0,0 +1,24 @@ +{ + "Initialization": { + "AllowEmailResendAfterDays": 7, + "ContainerName": "credentialexpiration", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetintusncgallery;AccountKey=$$Int-NuGetIntUsncGallery-StorageKey$$", + "GalleryAccountUrl": "https://int.nugettest.org/account/ApiKeys", + "GalleryBrand": "NuGet Gallery", + "MailFrom": "support@nuget.org", + "SmtpUri": "#{Jobs.gallery.credentialexpiration.SmtpUri}", + "WarnDaysBeforeExpiration": 7, + "WhatIf": true + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Gallery.CredentialExpiration/Settings/prod.json b/src/Gallery.CredentialExpiration/Settings/prod.json new file mode 100644 index 000000000..496b166a4 --- /dev/null +++ b/src/Gallery.CredentialExpiration/Settings/prod.json @@ -0,0 +1,24 @@ +{ + "Initialization": { + "AllowEmailResendAfterDays": 7, + "ContainerName": "credentialexpiration", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "GalleryAccountUrl": "https://www.nuget.org/account/ApiKeys", + "GalleryBrand": "NuGet Gallery", + "MailFrom": "support@nuget.org", + "SmtpUri": "#{Jobs.gallery.credentialexpiration.SmtpUri}", + "WarnDaysBeforeExpiration": 7, + "WhatIf": false + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs index dc0d07ca5..8b0e91721 100644 --- a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs +++ b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Gallery.Maintenance.Models; using Microsoft.Extensions.Logging; +using NuGet.Jobs.Configuration; namespace Gallery.Maintenance { @@ -37,7 +38,7 @@ public override async Task RunAsync(Job job) { IEnumerable expiredKeys; - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync()) { expiredKeys = await connection.QueryWithRetryAsync( SelectQuery, @@ -59,7 +60,7 @@ public override async Task RunAsync(Job job) if (expectedRowCount > 0) { - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync()) { using (var transaction = connection.BeginTransaction()) { diff --git a/src/Gallery.Maintenance/Gallery.Maintenance.csproj b/src/Gallery.Maintenance/Gallery.Maintenance.csproj index b11f12415..8f1c347cb 100644 --- a/src/Gallery.Maintenance/Gallery.Maintenance.csproj +++ b/src/Gallery.Maintenance/Gallery.Maintenance.csproj @@ -53,6 +53,9 @@ Designer + + + @@ -67,9 +70,6 @@ 9.0.1 - - 2.25.0-master-30263 - 4.3.3 diff --git a/src/Gallery.Maintenance/Gallery.Maintenance.nuspec b/src/Gallery.Maintenance/Gallery.Maintenance.nuspec index 6440c9435..f494b5e85 100644 --- a/src/Gallery.Maintenance/Gallery.Maintenance.nuspec +++ b/src/Gallery.Maintenance/Gallery.Maintenance.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/Gallery.Maintenance/Job.cs b/src/Gallery.Maintenance/Job.cs index 488a49b92..cb2edb77a 100644 --- a/src/Gallery.Maintenance/Job.cs +++ b/src/Gallery.Maintenance/Job.cs @@ -3,33 +3,21 @@ using System; using System.Collections.Generic; -using System.ComponentModel.Design; -using System.Data; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Gallery.Maintenance { /// /// Runs all s against the Gallery database. /// - public class Job : JobBase + public class Job : JsonConfigurationJob { - - public ISqlConnectionFactory GalleryDatabase { get; private set; } - - public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) - { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); - - GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); - } - public override async Task Run() { var failedTasks = new List(); @@ -88,5 +76,13 @@ public ILogger CreateTypedLogger(Type type) .MakeGenericMethod(type) .Invoke(null, new object[] { LoggerFactory }) as ILogger; } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + } } } diff --git a/src/Gallery.Maintenance/Scripts/Gallery.Maintenance.cmd b/src/Gallery.Maintenance/Scripts/Gallery.Maintenance.cmd index 2f35679a1..8bea3c4e0 100644 --- a/src/Gallery.Maintenance/Scripts/Gallery.Maintenance.cmd +++ b/src/Gallery.Maintenance/Scripts/Gallery.Maintenance.cmd @@ -7,7 +7,7 @@ cd bin title #{Jobs.Gallery.Maintenance.Title} - start /w Gallery.Maintenance.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -GalleryDatabase "#{Jobs.Gallery.Maintenance.GalleryDatabase}" -InstrumentationKey "#{Jobs.Gallery.Maintenance.InstrumentationKey}" -verbose true -Interval #{Jobs.Gallery.Maintenance.Interval} + start /w Gallery.Maintenance.exe -Configuration "#{Jobs.gallery.maintenance.Configuration}" -InstrumentationKey "#{Jobs.gallery.maintenance.InstrumentationKey}" -Interval #{Jobs.gallery.maintenance.Interval} echo "Finished #{Jobs.Gallery.Maintenance.Title}" diff --git a/src/Gallery.Maintenance/Settings/dev.json b/src/Gallery.Maintenance/Settings/dev.json new file mode 100644 index 000000000..2355c0905 --- /dev/null +++ b/src/Gallery.Maintenance/Settings/dev.json @@ -0,0 +1,12 @@ +{ + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Gallery.Maintenance;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbWriter.ClientId};AadCertificate=$$dev-gallerydb-writer$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Gallery.Maintenance/Settings/int.json b/src/Gallery.Maintenance/Settings/int.json new file mode 100644 index 000000000..70bb344e5 --- /dev/null +++ b/src/Gallery.Maintenance/Settings/int.json @@ -0,0 +1,12 @@ +{ + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBWriter-UserName$$;Password=$$Int-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Gallery.Maintenance/Settings/prod.json b/src/Gallery.Maintenance/Settings/prod.json new file mode 100644 index 000000000..bdd2203e2 --- /dev/null +++ b/src/Gallery.Maintenance/Settings/prod.json @@ -0,0 +1,12 @@ +{ + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBWriter-UserName$$;Password=$$Prod-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Job.cs b/src/Monitoring.RebootSearchInstance/Job.cs index 3bbe5643c..b177b123e 100644 --- a/src/Monitoring.RebootSearchInstance/Job.cs +++ b/src/Monitoring.RebootSearchInstance/Job.cs @@ -17,7 +17,7 @@ namespace NuGet.Monitoring.RebootSearchInstance { - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string AzureManagementSectionName = "AzureManagement"; private const string MonitorConfigurationSectionName = "MonitorConfiguration"; diff --git a/src/NuGet.Jobs.Common/Configuration/StatisticsDbConfiguration.cs b/src/NuGet.Jobs.Common/Configuration/StatisticsDbConfiguration.cs new file mode 100644 index 000000000..d6aad7554 --- /dev/null +++ b/src/NuGet.Jobs.Common/Configuration/StatisticsDbConfiguration.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs.Configuration +{ + public class StatisticsDbConfiguration : IDbConfiguration + { + public string ConnectionString { get; set; } + } +} diff --git a/src/NuGet.Jobs.Common/DelegateSqlConnectionFactory.cs b/src/NuGet.Jobs.Common/DelegateSqlConnectionFactory.cs new file mode 100644 index 000000000..4625b5e40 --- /dev/null +++ b/src/NuGet.Jobs.Common/DelegateSqlConnectionFactory.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Data.SqlClient; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Configuration; + +namespace NuGet.Jobs +{ + public class DelegateSqlConnectionFactory : ISqlConnectionFactory + where TbDbConfiguration : IDbConfiguration + { + private readonly Func> _connectionFunc; + private readonly ILogger> _logger; + + public DelegateSqlConnectionFactory(Func> connectionFunc, ILogger> logger) + { + _connectionFunc = connectionFunc ?? throw new ArgumentNullException(nameof(connectionFunc)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task CreateAsync() => _connectionFunc(); + + public async Task OpenAsync() + { + SqlConnection connection = null; + + try + { + _logger.LogDebug("Opening SQL connection..."); + + connection = await _connectionFunc(); + + await connection.OpenAsync(); + + _logger.LogDebug("Opened SQL connection"); + + return connection; + } + catch (Exception e) + { + _logger.LogError(0, e, "Unable to open SQL connection due to exception"); + + connection?.Dispose(); + + throw; + } + } + } +} diff --git a/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs b/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs index f6f9628b5..08c0c12c4 100644 --- a/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs +++ b/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Dapper; diff --git a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs deleted file mode 100644 index 608481d2c..000000000 --- a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Threading.Tasks; - -// ReSharper disable once CheckNamespace -namespace System.Data.SqlClient -{ - public static class SqlConnectionStringBuilderExtensions - { - public static Task ConnectTo(this SqlConnectionStringBuilder self) - { - return ConnectTo(self.ConnectionString); - } - - private static async Task ConnectTo(string connection) - { - var c = new SqlConnection(connection); - await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false); - return c; - } - } -} \ No newline at end of file diff --git a/src/NuGet.Jobs.Common/ISqlConnectionFactory.cs b/src/NuGet.Jobs.Common/ISqlConnectionFactory.cs new file mode 100644 index 000000000..9c44058c9 --- /dev/null +++ b/src/NuGet.Jobs.Common/ISqlConnectionFactory.cs @@ -0,0 +1,38 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Data.SqlClient; +using System.Threading.Tasks; +using NuGet.Jobs.Configuration; + +namespace NuGet.Jobs +{ + /// + /// A factory to create and open s. + /// + public interface ISqlConnectionFactory + { + /// + /// Create an unopened SQL connection. + /// + /// The unopened SQL connection. + Task CreateAsync(); + + /// + /// Create and then open a SQL connection. + /// + /// A task that creates and then opens a SQL connection. + Task OpenAsync(); + } + + /// + /// A factory to create and open s for a specific + /// . This type can be used to avoid Dependency + /// Injection key bindings. + /// + /// The configuration used to create the connection. + public interface ISqlConnectionFactory : ISqlConnectionFactory + where TDbConfiguration : IDbConfiguration + { + } +} diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index e16184854..52889fd93 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -1,14 +1,23 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace NuGet.Jobs { + using ICoreSqlConnectionFactory = NuGet.Services.Sql.ISqlConnectionFactory; + public abstract class JobBase { private readonly EventSource _jobEventSource; @@ -22,6 +31,7 @@ protected JobBase(EventSource jobEventSource) { JobName = GetType().ToString(); _jobEventSource = jobEventSource; + SqlConnectionFactories = new Dictionary(); } public string JobName { get; private set; } @@ -30,14 +40,193 @@ protected JobBase(EventSource jobEventSource) protected ILogger Logger { get; private set; } + private Dictionary SqlConnectionFactories { get; } + public void SetLogger(ILoggerFactory loggerFactory, ILogger logger) { LoggerFactory = loggerFactory; Logger = logger; } + /// + /// Initialize the job, provided the service container and configuration. + /// public abstract void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary); + /// + /// Run the job. + /// public abstract Task Run(); + + /// + /// Test connection early to fail fast, and log connection diagnostics. + /// + private async Task TestConnection(string name, ICoreSqlConnectionFactory connectionFactory) + { + try + { + using (var connection = await connectionFactory.OpenAsync()) + using (var cmd = new SqlCommand("SELECT CONCAT(CURRENT_USER, '/', SYSTEM_USER)", connection)) + { + var result = cmd.ExecuteScalar(); + var user = result.ToString(); + Logger.LogInformation("Verified CreateSqlConnectionAsync({name}) connects to database {DataSource}/{InitialCatalog} as {User}", + name, connectionFactory.DataSource, connectionFactory.InitialCatalog, user); + } + } + catch (Exception e) + { + Logger.LogError(0, e, "Failed to connect to database {DataSource}/{InitialCatalog}", + connectionFactory.DataSource, connectionFactory.InitialCatalog); + + throw; + } + } + + public SqlConnectionStringBuilder GetDatabaseRegistration() + where T : IDbConfiguration + { + if (SqlConnectionFactories.TryGetValue(GetDatabaseKey(), out var connectionFactory)) + { + return ((AzureSqlConnectionFactory)connectionFactory).SqlConnectionStringBuilder; + } + + return null; + } + + /// + /// Initializes an , for use by validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase( + IServiceProvider services, + bool testConnection = true) + where T : IDbConfiguration + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + + var secretInjector = services.GetRequiredService(); + var connectionString = services.GetRequiredService>().Value.ConnectionString; + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector); + + return RegisterDatabase(GetDatabaseKey(), connectionString, testConnection, secretInjector); + } + + /// + /// Initializes an , for use by non-validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase( + IServiceContainer serviceContainer, + IDictionary jobArgsDictionary, + string connectionStringArgName, + bool testConnection = true) + { + if (serviceContainer == null) + { + throw new ArgumentNullException(nameof(serviceContainer)); + } + + if (jobArgsDictionary == null) + { + throw new ArgumentNullException(nameof(jobArgsDictionary)); + } + + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var connectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, connectionStringArgName); + + return RegisterDatabase(connectionStringArgName, connectionString, testConnection, secretInjector); + } + + /// + /// Register a job database at initialization time. Each call should overwrite any existing + /// registration because calls on every iteration. + /// + /// ConnectionStringBuilder, used for diagnostics. + private SqlConnectionStringBuilder RegisterDatabase( + string name, + string connectionString, + bool testConnection, + ISecretInjector secretInjector) + { + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector, Logger); + SqlConnectionFactories[name] = connectionFactory; + + if (testConnection) + { + Task.Run(() => TestConnection(name, connectionFactory)).Wait(); + } + + return connectionFactory.SqlConnectionStringBuilder; + } + + private ICoreSqlConnectionFactory GetSqlConnectionFactory() + where T : IDbConfiguration + { + return GetSqlConnectionFactory(GetDatabaseKey()); + } + + private ICoreSqlConnectionFactory GetSqlConnectionFactory(string name) + { + if (!SqlConnectionFactories.ContainsKey(name)) + { + throw new InvalidOperationException($"Database {name} has not been registered."); + } + + return SqlConnectionFactories[name]; + } + + private static string GetDatabaseKey() + { + return typeof(T).Name; + } + + /// + /// Create a SqlConnection, for use by jobs that use an EF context. + /// + public Task CreateSqlConnectionAsync() + where T : IDbConfiguration + { + return GetSqlConnectionFactory().CreateAsync(); + } + + /// + /// Synchronous creation of a SqlConnection, for use by jobs that use an EF context. + /// + public SqlConnection CreateSqlConnection() + where T : IDbConfiguration + { + return Task.Run(() => CreateSqlConnectionAsync()).Result; + } + + /// + /// Open a SqlConnection, for use by jobs that do NOT use an EF context. + /// + public Task OpenSqlConnectionAsync() + where T : IDbConfiguration + { + return GetSqlConnectionFactory().OpenAsync(); + } + + /// + /// Opens a SqlConnection, for use by jobs that do NOT use an EF context. + /// + public Task OpenSqlConnectionAsync(string connectionStringArgName) + { + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + return GetSqlConnectionFactory(connectionStringArgName).OpenAsync(); + } } } diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/NuGet.Jobs.Common/JsonConfigurationJob.cs similarity index 63% rename from src/Validation.Common.Job/JsonConfigurationJob.cs rename to src/NuGet.Jobs.Common/JsonConfigurationJob.cs index 9355f050c..02d79c085 100644 --- a/src/Validation.Common.Job/JsonConfigurationJob.cs +++ b/src/NuGet.Jobs.Common/JsonConfigurationJob.cs @@ -1,15 +1,12 @@ // Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; +using System.Diagnostics.Tracing; using System.IO; -using System.Net; -using System.Net.Http; -using System.Reflection; -using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.ApplicationInsights; @@ -21,26 +18,27 @@ using NuGet.Services.Configuration; using NuGet.Services.KeyVault; using NuGet.Services.Logging; -using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; -using NuGet.Services.Validation; -using NuGetGallery; -using NuGetGallery.Diagnostics; -namespace NuGet.Jobs.Validation +namespace NuGet.Jobs { public abstract class JsonConfigurationJob : JobBase { + private const string InitializationConfigurationSectionName = "Initialization"; private const string GalleryDbConfigurationSectionName = "GalleryDb"; + private const string StatisticsDbConfigurationSectionName = "StatisticsDb"; private const string ValidationDbConfigurationSectionName = "ValidationDb"; private const string ServiceBusConfigurationSectionName = "ServiceBus"; private const string ValidationStorageConfigurationSectionName = "ValidationStorage"; - private const string PackageDownloadTimeoutName = "PackageDownloadTimeout"; - /// - /// The maximum number of concurrent connections that can be established to a single server. - /// - private const int MaximumConnectionsPerServer = 64; + public JsonConfigurationJob() + : this(null) + { + } + + public JsonConfigurationJob(EventSource jobEventSource) + : base(jobEventSource) + { + } /// /// The argument this job uses to determine the configuration file's path. @@ -64,7 +62,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - - private void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + protected virtual void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(GalleryDbConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(StatisticsDbConfigurationSectionName)); services.Configure(configurationRoot.GetSection(ValidationDbConfigurationSectionName)); services.Configure(configurationRoot.GetSection(ServiceBusConfigurationSectionName)); services.Configure(configurationRoot.GetSection(ValidationStorageConfigurationSectionName)); services.AddSingleton(new TelemetryClient()); services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.AddTransient(c => - { - var configurationAccessor = c.GetRequiredService>(); - return new CloudBlobClientWrapper( - configurationAccessor.Value.ConnectionString, - readAccessGeoRedundant: false); - }); - services.AddTransient(); - - services.AddScoped(p => - { - return new ValidationEntitiesContext(CreateDbConnection(p)); - }); - services.AddScoped(p => + services.AddScoped>(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: true); + return new DelegateSqlConnectionFactory( + CreateSqlConnectionAsync, + p.GetRequiredService>>()); }); - services.AddTransient(p => + services.AddScoped>(p => { - var config = p.GetRequiredService>().Value; - - return new SubscriptionClientWrapper(config.ConnectionString, config.TopicPath, config.SubscriptionName); + return new DelegateSqlConnectionFactory( + CreateSqlConnectionAsync, + p.GetRequiredService>>()); }); - services.AddSingleton(p => + services.AddScoped>(p => { - var assembly = Assembly.GetEntryAssembly(); - var assemblyName = assembly.GetName().Name; - var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; - - var client = new HttpClient(new WebRequestHandler - { - AllowPipelining = true, - AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), - }); - - client.Timeout = configurationRoot.GetValue(PackageDownloadTimeoutName); - client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); - - return client; + return new DelegateSqlConnectionFactory( + CreateSqlConnectionAsync, + p.GetRequiredService>>()); }); } @@ -184,6 +148,35 @@ private void ConfigureLibraries(IServiceCollection services) services.AddLogging(); } + protected virtual void RegisterDatabases(IServiceProvider serviceProvider) + { + var galleryDb = serviceProvider.GetRequiredService>(); + if (!string.IsNullOrEmpty(galleryDb.Value?.ConnectionString)) + { + RegisterDatabase(serviceProvider); + } + + var statisticsDb = serviceProvider.GetRequiredService>(); + if (!string.IsNullOrEmpty(statisticsDb.Value?.ConnectionString)) + { + RegisterDatabase(serviceProvider); + } + + var validationDb = serviceProvider.GetRequiredService>(); + if (!string.IsNullOrEmpty(validationDb.Value?.ConnectionString)) + { + RegisterDatabase(serviceProvider); + } + } + + protected virtual void ConfigureInitializationSection( + IServiceCollection services, + IConfigurationRoot configurationRoot) + where TConfiguration : class + { + services.Configure(configurationRoot.GetSection(InitializationConfigurationSectionName)); + } + /// /// Method to be implemented in derived classes to provide Autofac-specific configuration for /// that specific job (like setting up keyed resolution). diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index a33533479..4450e2a24 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -47,11 +47,14 @@ + + - + + @@ -71,9 +74,21 @@ + + 4.6.2 + + + 4.2.0 + 1.50.2 + + 1.1.1 + + + 1.1.2 + 2.27.0 @@ -83,6 +98,9 @@ 2.27.0 + + 2.27.0 + 4.3.3 diff --git a/src/NuGet.Services.Revalidate/Initialization/IPackageFinder.cs b/src/NuGet.Services.Revalidate/Initialization/IPackageFinder.cs index ac4366675..aaa55048c 100644 --- a/src/NuGet.Services.Revalidate/Initialization/IPackageFinder.cs +++ b/src/NuGet.Services.Revalidate/Initialization/IPackageFinder.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Collections.Generic; +using System.Threading.Tasks; using NuGet.Versioning; namespace NuGet.Services.Revalidate @@ -41,7 +42,7 @@ public interface IPackageFinder /// The name of this set of packages. /// The set of package registration keys. /// Information about each package registration, if it exists in the database. - List FindPackageRegistrationInformation(string setName, HashSet packageRegistrationKeys); + Task> FindPackageRegistrationInformationAsync(string setName, HashSet packageRegistrationKeys); /// /// Find versions that are appropriate for revalidations. diff --git a/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs b/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs index 0a818cb88..a272fc255 100644 --- a/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs +++ b/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NuGet.Services.Validation; using NuGet.Versioning; @@ -18,6 +19,7 @@ public class InitializationManager private readonly IRevalidationJobStateService _jobState; private readonly IPackageRevalidationStateService _packageState; private readonly IPackageFinder _packageFinder; + private readonly IServiceScopeFactory _scopeFactory; private readonly InitializationConfiguration _config; private readonly ILogger _logger; @@ -25,12 +27,14 @@ public InitializationManager( IRevalidationJobStateService jobState, IPackageRevalidationStateService packageState, IPackageFinder packageFinder, + IServiceScopeFactory scopeFactory, InitializationConfiguration config, ILogger logger) { _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _packageFinder = packageFinder ?? throw new ArgumentNullException(nameof(packageFinder)); + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -122,59 +126,83 @@ private async Task ClearPackageRevalidationStateAsync() private async Task InitializePackageSetAsync(string setName, HashSet packageRegistrationKeys) { - var packageInformations = _packageFinder.FindPackageRegistrationInformation(setName, packageRegistrationKeys); + using (var scope = _scopeFactory.CreateScope()) + { + var scopedPackageFinder = scope.ServiceProvider.GetRequiredService(); + var scopedJobState = scope.ServiceProvider.GetRequiredService(); + var scopedScopeFactory = scope.ServiceProvider.GetRequiredService(); - var chunks = packageInformations - .OrderByDescending(p => p.Downloads) - .WeightedBatch(BatchSize, p => p.Versions); + var packageInformations = await scopedPackageFinder.FindPackageRegistrationInformationAsync(setName, packageRegistrationKeys); + var chunks = packageInformations + .OrderByDescending(p => p.Downloads) + .WeightedBatch(BatchSize, p => p.Versions); - for (var chunkIndex = 0; chunkIndex < chunks.Count; chunkIndex++) - { - while (await _jobState.IsKillswitchActiveAsync()) + for (var chunkIndex = 0; chunkIndex < chunks.Count; chunkIndex++) { - _logger.LogInformation( - "Delaying initialization of chunk {Chunk} of {Chunks} for package set {SetName} due to active killswitch", - chunkIndex + 1, - chunks.Count, - setName); + while (await scopedJobState.IsKillswitchActiveAsync()) + { + _logger.LogInformation( + "Delaying initialization of chunk {Chunk} of {Chunks} for package set {SetName} due to active killswitch", + chunkIndex + 1, + chunks.Count, + setName); - await Task.Delay(_config.SleepDurationBetweenBatches); + await Task.Delay(_config.SleepDurationBetweenBatches); + } + + await InitializePackageSetChunkAsync(setName, chunks, chunkIndex, scopedScopeFactory, _logger); + + // Sleep if this is not the last chunk to prevent overloading the database. + if (chunkIndex < chunks.Count - 1) + { + _logger.LogInformation( + "Sleeping for {SleepDuration} before initializing the next chunk...", + _config.SleepDurationBetweenBatches); + + await Task.Delay(_config.SleepDurationBetweenBatches); + } } - _logger.LogInformation( - "Initializing chunk {Chunk} of {Chunks} for package set {SetName}...", - chunkIndex + 1, - chunks.Count, - setName); + _logger.LogInformation("Finished initializing package set {SetName}", setName); + } + } + + private static async Task InitializePackageSetChunkAsync( + string setName, + List> chunks, + int chunkIndex, + IServiceScopeFactory scopeFactory, + ILogger logger) + { + logger.LogInformation( + "Initializing chunk {Chunk} of {Chunks} for package set {SetName}...", + chunkIndex + 1, + chunks.Count, + setName); + + using (var scope = scopeFactory.CreateScope()) + { + var scopedPackageState = scope.ServiceProvider.GetRequiredService(); + var scopedPackageFinder = scope.ServiceProvider.GetRequiredService(); var chunk = chunks[chunkIndex]; - var versions = _packageFinder.FindAppropriateVersions(chunk); + var versions = scopedPackageFinder.FindAppropriateVersions(chunk); - await InitializeRevalidationsAsync(chunk, versions); + await InitializeRevalidationsAsync(chunk, versions, scopedPackageState, logger); - _logger.LogInformation( + logger.LogInformation( "Initialized chunk {Chunk} of {Chunks} for package set {SetName}", chunkIndex + 1, chunks.Count, setName); - - // Sleep if this is not the last chunk to prevent overloading the database. - if (chunkIndex < chunks.Count - 1) - { - _logger.LogInformation( - "Sleeping for {SleepDuration} before initializing the next chunk...", - _config.SleepDurationBetweenBatches); - - await Task.Delay(_config.SleepDurationBetweenBatches); - } } - - _logger.LogInformation("Finished initializing package set {SetName}", setName); } - private async Task InitializeRevalidationsAsync( + private static async Task InitializeRevalidationsAsync( List packageRegistrations, - Dictionary> versions) + Dictionary> versions, + IPackageRevalidationStateService packageState, + ILogger logger) { var revalidations = new List(); @@ -184,7 +212,7 @@ private async Task InitializeRevalidationsAsync( if (!versions.ContainsKey(packageRegistration.Key) || versions[packageRegistration.Key].Count == 0) { - _logger.LogWarning("Could not find any versions of package {PackageId} to revalidate", packageId); + logger.LogWarning("Could not find any versions of package {PackageId} to revalidate", packageId); continue; } @@ -205,7 +233,7 @@ private async Task InitializeRevalidationsAsync( } } - await _packageState.AddPackageRevalidationsAsync(revalidations); + await packageState.AddPackageRevalidationsAsync(revalidations); } } } diff --git a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs index b6cff5662..8b4791589 100644 --- a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs +++ b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs @@ -7,6 +7,8 @@ using System.Linq; using System.Linq.Expressions; using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using NuGet.Versioning; @@ -29,15 +31,18 @@ public class PackageFinder : IPackageFinder private static string MicrosoftAccountName = "Microsoft"; private readonly IGalleryContext _galleryContext; + private readonly IServiceScopeFactory _scopeFactory; private readonly InitializationConfiguration _config; private readonly ILogger _logger; public PackageFinder( IGalleryContext galleryContext, + IServiceScopeFactory scopeFactory, InitializationConfiguration config, ILogger logger) { _galleryContext = galleryContext ?? throw new ArgumentNullException(nameof(galleryContext)); + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -105,7 +110,7 @@ public HashSet FindAllPackages(HashSet except) return FindRegistrationKeys(RemainingSetName, r => !except.Contains(r.Key)); } - public List FindPackageRegistrationInformation(string setName, HashSet packageRegistrationKeys) + public async Task> FindPackageRegistrationInformationAsync(string setName, HashSet packageRegistrationKeys) { // Fetch the packages' information in batches. var batches = packageRegistrationKeys.Batch(BatchSize); @@ -121,23 +126,38 @@ public List FindPackageRegistrationInformation(s var batch = batches[batchIndex]; - var packages = _galleryContext.PackageRegistrations - .Where(r => batch.Contains(r.Key)) - .Select(r => new PackageRegistrationInformation - { - Key = r.Key, - Id = r.Id, - Downloads = r.DownloadCount, - Versions = r.Packages.Count(), - }); + using (var scope = _scopeFactory.CreateScope()) + { + var scopedContext = scope.ServiceProvider.GetRequiredService(); + var packages = scopedContext + .PackageRegistrations + .Where(r => batch.Contains(r.Key)) + .Select(r => new PackageRegistrationInformation + { + Key = r.Key, + Id = r.Id, + Downloads = r.DownloadCount, + Versions = r.Packages.Count(), + }); - result.AddRange(packages); + result.AddRange(packages); + } _logger.LogInformation( "Fetched batch {Batch} of {BatchesCount} of package informations for package set {SetName}", batchIndex + 1, batches.Count, setName); + + if (batchIndex < batches.Count - 1) + { + _logger.LogInformation( + "Sleeping for {SleepDuration} before fetching the next batch for package set {SetName}...", + _config.SleepDurationBetweenBatches, + setName); + + await Task.Delay(_config.SleepDurationBetweenBatches); + } } return result; diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 0dd289326..c80b7b812 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -26,7 +27,7 @@ namespace NuGet.Services.Revalidate using GalleryContext = EntitiesContext; using IGalleryContext = IEntitiesContext; - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet"; private const string InitializeArgumentName = "Initialize"; @@ -46,16 +47,6 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 15bd808c4..d09410f29 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -58,12 +58,14 @@ + + @@ -85,6 +87,7 @@ + diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.nuspec b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.nuspec index d8a409dd3..be017564f 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.nuspec +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.nuspec @@ -13,6 +13,7 @@ + diff --git a/src/NuGet.Services.Revalidate/Scripts/NuGet.Services.Revalidate.Initialize.cmd b/src/NuGet.Services.Revalidate/Scripts/NuGet.Services.Revalidate.Initialize.cmd new file mode 100644 index 000000000..e86f63d07 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Scripts/NuGet.Services.Revalidate.Initialize.cmd @@ -0,0 +1,16 @@ +@echo OFF + +cd bin + +:Top +echo "Initializing job - #{Jobs.nuget.services.revalidate.Title}" + +title #{Jobs.nuget.services.revalidate.Title} + +start /w NuGet.Services.Revalidate.exe ^ + -Configuration #{Jobs.nuget.services.revalidate.Configuration} ^ + -InstrumentationKey "#{Jobs.nuget.services.revalidate.InstrumentationKey}" ^ + -Initialize ^ + -VerifyInitialization + +echo "Initialized #{Jobs.nuget.services.revalidate.Title}" diff --git a/src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs new file mode 100644 index 000000000..f5d52aa38 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Validation; + +namespace NuGet.Services.Revalidate +{ + public interface IPackageRevalidationInserter + { + Task AddPackageRevalidationsAsync(IReadOnlyList revalidations); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs b/src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs new file mode 100644 index 000000000..7ecc685e7 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs @@ -0,0 +1,95 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs; +using NuGet.Jobs.Configuration; +using NuGet.Jobs.Validation; +using NuGet.Services.Validation; + +namespace NuGet.Services.Revalidate +{ + public class PackageRevalidationInserter : IPackageRevalidationInserter + { + private const string TableName = "[dbo].[PackageRevalidations]"; + + private const string PackageIdColumn = "PackageId"; + private const string PackageNormalizedVersionColumn = "PackageNormalizedVersion"; + private const string EnqueuedColumn = "Enqueued"; + private const string ValidationTrackingIdColumn = "ValidationTrackingId"; + private const string CompletedColumn = "Completed"; + + private readonly ISqlConnectionFactory _connectionFactory; + private readonly ILogger _logger; + + public PackageRevalidationInserter( + ISqlConnectionFactory connectionFactory, + ILogger logger) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) + { + _logger.LogDebug("Persisting package revalidations to database..."); + + var table = PrepareTable(revalidations); + + using (var connection = await _connectionFactory.OpenAsync()) + { + var bulkCopy = new SqlBulkCopy( + connection, + SqlBulkCopyOptions.TableLock | SqlBulkCopyOptions.FireTriggers | SqlBulkCopyOptions.UseInternalTransaction, + externalTransaction: null); + + foreach (DataColumn column in table.Columns) + { + bulkCopy.ColumnMappings.Add(column.ColumnName, column.ColumnName); + } + + bulkCopy.DestinationTableName = TableName; + bulkCopy.WriteToServer(table); + } + + _logger.LogDebug("Finished persisting package revalidations to database..."); + } + + private DataTable PrepareTable(IReadOnlyList revalidations) + { + // Prepare the table. + var table = new DataTable(); + + table.Columns.Add(PackageIdColumn, typeof(string)); + table.Columns.Add(PackageNormalizedVersionColumn, typeof(string)); + table.Columns.Add(CompletedColumn, typeof(bool)); + + var enqueued = table.Columns.Add(EnqueuedColumn, typeof(DateTime)); + var trackingId = table.Columns.Add(ValidationTrackingIdColumn, typeof(Guid)); + + enqueued.AllowDBNull = true; + trackingId.AllowDBNull = true; + + // Populate the table. + foreach (var revalidation in revalidations) + { + var row = table.NewRow(); + + row[PackageIdColumn] = revalidation.PackageId; + row[PackageNormalizedVersionColumn] = revalidation.PackageNormalizedVersion; + row[EnqueuedColumn] = ((object)revalidation.Enqueued) ?? DBNull.Value; + row[ValidationTrackingIdColumn] = ((object)revalidation.ValidationTrackingId) ?? DBNull.Value; + row[CompletedColumn] = revalidation.Completed; + + table.Rows.Add(row); + } + + return table; + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs index 8f7a79c1a..fb90b8497 100644 --- a/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Data; using System.Data.Entity; using System.Data.Entity.Infrastructure; using System.Linq; @@ -15,36 +16,22 @@ namespace NuGet.Services.Revalidate public class PackageRevalidationStateService : IPackageRevalidationStateService { private readonly IValidationEntitiesContext _context; + private readonly IPackageRevalidationInserter _inserter; private readonly ILogger _logger; - public PackageRevalidationStateService(IValidationEntitiesContext context, ILogger logger) + public PackageRevalidationStateService( + IValidationEntitiesContext context, + IPackageRevalidationInserter inserter, + ILogger logger) { _context = context ?? throw new ArgumentNullException(nameof(context)); + _inserter = inserter ?? throw new ArgumentNullException(nameof(inserter)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) + public Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) { - var validationContext = _context as ValidationEntitiesContext; - - if (validationContext != null) - { - validationContext.Configuration.AutoDetectChangesEnabled = false; - validationContext.Configuration.ValidateOnSaveEnabled = false; - } - - foreach (var revalidation in revalidations) - { - _context.PackageRevalidations.Add(revalidation); - } - - await _context.SaveChangesAsync(); - - if (validationContext != null) - { - validationContext.Configuration.AutoDetectChangesEnabled = true; - validationContext.Configuration.ValidateOnSaveEnabled = true; - } + return _inserter.AddPackageRevalidationsAsync(revalidations); } public async Task RemovePackageRevalidationsAsync(int max) diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index 2612db0dc..a8e2fe715 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -6,7 +6,7 @@ "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], - "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled + "MaxPackageCreationDate": "2018-08-08T23:59:59.0000000Z", "SleepDurationBetweenBatches": "00:00:30" }, @@ -17,7 +17,7 @@ }, "MinPackageEventRate": 120, - "MaxPackageEventRate": 500, + "MaxPackageEventRate": 200, "AppInsights": { "AppId": "46f13c7d-635f-42c3-8120-593edeaad426", @@ -31,10 +31,10 @@ }, "GalleryDb": { - "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" + "ConnectionString": "Data Source=tcp:#{Jobs.nuget.services.revalidate.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationDb": { - "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-dev-validation;Integrated Security=False;User ID=$$Dev-ValidationDBWriter-UserName$$;Password=$$Dev-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" + "ConnectionString": "Data Source=tcp:#{Jobs.nuget.services.revalidate.DatabaseAddress};Initial Catalog=nuget-dev-validation;Integrated Security=False;User ID=$$Dev-ValidationDBWriter-UserName$$;Password=$$Dev-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationStorage": { "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$" diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index e721eb379..6507e2200 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -6,7 +6,7 @@ "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], - "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled + "MaxPackageCreationDate": "2018-08-08T23:59:59.0000000Z", "SleepDurationBetweenBatches": "00:00:30" }, @@ -17,7 +17,7 @@ }, "MinPackageEventRate": 120, - "MaxPackageEventRate": 500, + "MaxPackageEventRate": 200, "AppInsights": { "AppId": "718e0c81-9132-4bf2-b24b-aa625dafd800", @@ -31,10 +31,10 @@ }, "GalleryDb": { - "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;Integrated Security=False;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" + "ConnectionString": "Data Source=tcp:#{Jobs.nuget.services.revalidate.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;Integrated Security=False;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationDb": { - "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-int-validation;Integrated Security=False;User ID=$$Int-ValidationDBWriter-UserName$$;Password=$$Int-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" + "ConnectionString": "Data Source=tcp:#{Jobs.nuget.services.revalidate.DatabaseAddress};Initial Catalog=nuget-int-validation;Integrated Security=False;User ID=$$Int-ValidationDBWriter-UserName$$;Password=$$Int-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationStorage": { "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$" diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index 7a97b2039..f8eb66934 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -6,7 +6,7 @@ "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], - "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled + "MaxPackageCreationDate": "2018-08-08T23:59:59.0000000Z", "SleepDurationBetweenBatches": "00:00:30" }, @@ -17,7 +17,7 @@ }, "MinPackageEventRate": 120, - "MaxPackageEventRate": 500, + "MaxPackageEventRate": 200, "AppInsights": { "AppId": "338f6804-b1a9-4fe3-bba7-c93064e7ae7b", @@ -31,10 +31,10 @@ }, "GalleryDb": { - "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;Integrated Security=False;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" + "ConnectionString": "Data Source=tcp:#{Jobs.nuget.services.revalidate.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;Integrated Security=False;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationDb": { - "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-prod-validation;Integrated Security=False;User ID=$$Prod-ValidationDBWriter-UserName$$;Password=$$Prod-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" + "ConnectionString": "Data Source=tcp:#{Jobs.nuget.services.revalidate.DatabaseAddress};Initial Catalog=nuget-prod-validation;Integrated Security=False;User ID=$$Prod-ValidationDBWriter-UserName$$;Password=$$Prod-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationStorage": { "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$" diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index 890124bb7..551f0a5d3 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -127,7 +127,7 @@ 1.2.0 - 2.26.0-master-34394 + 2.27.0 2.27.0-master-35351 diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj index a72a38207..c506ac961 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj @@ -107,7 +107,7 @@ 9.0.1 - 2.25.0-master-30453 + 2.27.0 diff --git a/src/PackageHash/Job.cs b/src/PackageHash/Job.cs index 58cdcfb5d..714990328 100644 --- a/src/PackageHash/Job.cs +++ b/src/PackageHash/Job.cs @@ -18,7 +18,7 @@ namespace NuGet.Services.PackageHash { - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string PackageHashConfigurationSectionName = "PackageHash"; diff --git a/src/Search.GenerateAuxiliaryData/Configuration/InitializationConfiguration.cs b/src/Search.GenerateAuxiliaryData/Configuration/InitializationConfiguration.cs new file mode 100644 index 000000000..1ff142b10 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Configuration/InitializationConfiguration.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Search.GenerateAuxiliaryData +{ + public class InitializationConfiguration + { + public string AzureCdnCloudStorageAccount { get; set; } + + public string AzureCdnCloudStorageContainerName { get; set; } + + public string PrimaryDestination { get; set; } + + public string DestinationContainerName { get; set; } + } +} diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 4d0971f80..87f40476a 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -4,20 +4,20 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; namespace Search.GenerateAuxiliaryData { - internal class Job - : JobBase + public class Job : JsonConfigurationJob { private const string DefaultContainerName = "ng-search-data"; @@ -40,40 +40,62 @@ internal class Job private const string StatisticsReportName = "downloads.v1.json"; private List _exportersToRun; - private CloudBlobContainer _destContainer; - private CloudBlobContainer _statisticsContainer; + + private InitializationConfiguration Configuration { get; set; } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - var statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); - var statisticsStorageAccount = CloudStorageAccount.Parse( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount)); + Configuration = _serviceProvider.GetRequiredService>().Value; - var statisticsReportsContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName); + var destinationContainer = CloudStorageAccount.Parse(Configuration.PrimaryDestination) + .CreateCloudBlobClient() + .GetContainerReference(Configuration.DestinationContainerName ?? DefaultContainerName); - var destination = CloudStorageAccount.Parse( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination)); - - var destinationContainerName = - JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) - ?? DefaultContainerName; - - _destContainer = destination.CreateCloudBlobClient().GetContainerReference(destinationContainerName); - _statisticsContainer = statisticsStorageAccount.CreateCloudBlobClient().GetContainerReference(statisticsReportsContainerName); + var statisticsContainer = CloudStorageAccount.Parse(Configuration.AzureCdnCloudStorageAccount) + .CreateCloudBlobClient() + .GetContainerReference(Configuration.AzureCdnCloudStorageContainerName); _exportersToRun = new List { - new VerifiedPackagesExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), - new RankingsExporter(LoggerFactory.CreateLogger(), statisticsDbConnectionFactory, _destContainer, ScriptRankingsTotal, OutputNameRankings), - new BlobStorageExporter(LoggerFactory.CreateLogger(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) + new VerifiedPackagesExporter( + LoggerFactory.CreateLogger(), + OpenSqlConnectionAsync, + destinationContainer, + ScriptVerifiedPackages, + OutputNameVerifiedPackages), + + new NestedJArrayExporter( + LoggerFactory.CreateLogger(), + OpenSqlConnectionAsync, + destinationContainer, + ScriptCuratedFeed, + OutputNameCuratedFeed, + Col0CuratedFeed, + Col1CuratedFeed), + + new NestedJArrayExporter( + LoggerFactory.CreateLogger(), + OpenSqlConnectionAsync, + destinationContainer, + ScriptOwners, + OutputNameOwners, + Col0Owners, + Col1Owners), + + new RankingsExporter( + LoggerFactory.CreateLogger(), + OpenSqlConnectionAsync, + destinationContainer, + ScriptRankingsTotal, + OutputNameRankings), + + new BlobStorageExporter( + LoggerFactory.CreateLogger(), + statisticsContainer, + StatisticsReportName, + destinationContainer, + StatisticsReportName) }; } @@ -100,5 +122,14 @@ public override async Task Run() throw new ExporterException($"{failedExporters.Count()} tasks failed: {string.Join(", ", failedExporters)}"); } } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } } } \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs index 83824718a..a7946d393 100644 --- a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs +++ b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs @@ -6,10 +6,10 @@ using System.Data; using System.Data.SqlClient; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,8 +20,12 @@ class NestedJArrayExporter public string Col1 { get; } public string SqlScript { get; } - public NestedJArrayExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + public NestedJArrayExporter( + ILogger logger, + Func> openSqlConnectionAsync, + CloudBlobContainer defaultDestinationContainer, + string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) + : base(logger, openSqlConnectionAsync, defaultDestinationContainer, defaultName) { Col0 = defaultCol0; Col1 = defaultCol1; diff --git a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs index 520812ff7..3b4db07a2 100644 --- a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs +++ b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -21,11 +21,11 @@ public sealed class RankingsExporter : SqlExporter public RankingsExporter( ILogger logger, - ISqlConnectionFactory connectionFactory, + Func> openSqlConnectionAsync, CloudBlobContainer defaultDestinationContainer, string defaultRankingsScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(logger, openSqlConnectionAsync, defaultDestinationContainer, defaultName) { _rankingsTotalScript = defaultRankingsScript; } diff --git a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd index 7e346f31e..90b165dc9 100644 --- a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd +++ b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd @@ -1,13 +1,15 @@ @echo OFF - + +REM This script is the same as Search.GenerateAuxillaryData.cmd. However, this copy is required until "Jobs.ServiceNames" deployment config is consolidated. + cd bin :Top - echo "Starting job - #{Jobs.Asia.search.generateauxiliarydata.Title}" + echo "Starting job - #{Jobs.search.generateauxiliarydata.Title}" - title #{Jobs.Asia.search.generateauxiliarydata.Title} + title #{Jobs.search.generateauxiliarydata.Title} - start /w search.generateauxiliarydata.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PrimaryDestination #{Jobs.Asia.search.generateauxiliarydata.Storage.Primary} -PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" -StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" -AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" + start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" echo "Finished #{Jobs.search.generateauxiliarydata.Title}" diff --git a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd index 48ca5b3cb..0ce435c6e 100644 --- a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd +++ b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd @@ -7,7 +7,7 @@ cd bin title #{Jobs.search.generateauxiliarydata.Title} - start /w search.generateauxiliarydata.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PrimaryDestination #{Jobs.search.generateauxiliarydata.Storage.Primary} -PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" -StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" -AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" + start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" echo "Finished #{Jobs.search.generateauxiliarydata.Title}" diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj index e9d6dddac..4f7d20863 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj @@ -43,6 +43,7 @@ + @@ -57,6 +58,7 @@ + @@ -92,9 +94,6 @@ 9.0.1 - - 2.25.0-master-30453 - 7.1.2 diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec index ec7e45e2b..7352e9c37 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec @@ -17,5 +17,7 @@ + + \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/dev-asia.json b/src/Search.GenerateAuxiliaryData/Settings/dev-asia.json new file mode 100644 index 000000000..73ffc03a2 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/dev-asia.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdevgalleryeastasia;AccountKey=$$Dev-NuGetDevEastAsiaGallery-StorageKey$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData.Asia;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData.Asia;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbReader.ClientId};AadCertificate=$$dev-statisticsdb-reader$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/dev.json b/src/Search.GenerateAuxiliaryData/Settings/dev.json new file mode 100644 index 000000000..e44f78bca --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/dev.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdev0;AccountKey=$$Dev-NuGetDev0Storage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbReader.ClientId};AadCertificate=$$dev-statisticsdb-reader$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/int.json b/src/Search.GenerateAuxiliaryData/Settings/int.json new file mode 100644 index 000000000..4dcc87c73 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/int.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBReadOnly-UserName$$;Password=$$Int-StatisticsDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/prod-asia.json b/src/Search.GenerateAuxiliaryData/Settings/prod-asia.json new file mode 100644 index 000000000..e2e574ea6 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/prod-asia.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetgalleryeastasia;AccountKey=$$Prod-NuGetGalleryEastAsiaStorage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBReadOnly-UserName$$;Password=$$Prod-StatisticsDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/prod.json b/src/Search.GenerateAuxiliaryData/Settings/prod.json new file mode 100644 index 000000000..5d4c844c6 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/prod.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetprod0;AccountKey=$$Prod-NuGetProd0Storage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBReadOnly-UserName$$;Password=$$Prod-StatisticsDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/SqlExporter.cs b/src/Search.GenerateAuxiliaryData/SqlExporter.cs index fedaa8227..ace4960c6 100644 --- a/src/Search.GenerateAuxiliaryData/SqlExporter.cs +++ b/src/Search.GenerateAuxiliaryData/SqlExporter.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -11,7 +12,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,14 +20,18 @@ public abstract class SqlExporter : Exporter { private static Assembly _executingAssembly = Assembly.GetExecutingAssembly(); private static string _assemblyName = _executingAssembly.GetName().Name; - - public ISqlConnectionFactory ConnectionFactory { get; } - public SqlExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultName) + private Func> OpenSqlConnectionAsync { get; } + + public SqlExporter( + ILogger logger, + Func> openSqlConnectionAsync, + CloudBlobContainer defaultDestinationContainer, + string defaultName) : base(logger, defaultDestinationContainer, defaultName) { _logger = logger; - ConnectionFactory = connectionFactory; + OpenSqlConnectionAsync = openSqlConnectionAsync; } protected static string GetEmbeddedSqlScript(string resourceName) @@ -38,12 +42,12 @@ protected static string GetEmbeddedSqlScript(string resourceName) public override async Task ExportAsync() { - _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", - _name, ConnectionFactory.DataSource, ConnectionFactory.InitialCatalog); - JContainer result; - using (var connection = await ConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { + _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", + _name, connection.DataSource, connection.Database); + result = GetResultOfQuery(connection); } diff --git a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs index 20acab67b..dc7ee4e08 100644 --- a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs +++ b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -18,11 +18,11 @@ internal sealed class VerifiedPackagesExporter : SqlExporter public VerifiedPackagesExporter( ILogger logger, - ISqlConnectionFactory connectionFactory, + Func> openSqlConnectionAsync, CloudBlobContainer defaultDestinationContainer, string defaultVerifiedPackagesScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(logger, openSqlConnectionAsync, defaultDestinationContainer, defaultName) { _verifiedPackagesScript = defaultVerifiedPackagesScript; } diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs similarity index 85% rename from src/Stats.AggregateCdnDownloadsInGallery/Job.cs rename to src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs index 976b2e208..7b29d5550 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs @@ -9,16 +9,18 @@ using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; using IPackageIdGroup = System.Linq.IGrouping; namespace Stats.AggregateCdnDownloadsInGallery { - public class Job - : JobBase + public class AggregateCdnDownloadsJob : JsonConfigurationJob { private const int _defaultBatchSize = 5000; private const int _defaultBatchSleepSeconds = 10; @@ -55,40 +57,35 @@ GROUP BY Stats.[PackageRegistrationKey] DROP TABLE #AggregateCdnDownloadsInGallery"; private const string _storedProcedureName = "[dbo].[SelectTotalDownloadCountsPerPackageVersion]"; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; - private int _batchSize; - private int _batchSleepSeconds; + + private AggregateCdnDownloadsConfiguration _configuration; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); - - var galleryDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); - _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize; - _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds; + _configuration = _serviceProvider.GetRequiredService>().Value; } public override async Task Run() { // Gather download counts data from statistics warehouse IReadOnlyList downloadData; - Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", _batchSize, _batchSleepSeconds); - Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog); + Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", + _configuration.BatchSize, + _configuration.BatchSleepSeconds); + var stopwatch = Stopwatch.StartNew(); - using (var statisticsDatabase = await _statisticsDbConnectionFactory.CreateAsync()) - using (var statisticsDatabaseTransaction = statisticsDatabase.BeginTransaction(IsolationLevel.Snapshot)) + using (var connection = await OpenSqlConnectionAsync()) + using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", connection.DataSource, connection.Database); + downloadData = ( - await statisticsDatabase.QueryWithRetryAsync( + await connection.QueryWithRetryAsync( _storedProcedureName, - transaction: statisticsDatabaseTransaction, + transaction: transaction, commandType: CommandType.StoredProcedure, commandTimeout: TimeSpan.FromMinutes(15), maxRetries: 3)) @@ -106,10 +103,10 @@ await statisticsDatabase.QueryWithRetryAsync( return; } - using (var destinationDatabase = await _galleryDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { // Fetch package registrations so we can match package ID to package registration key. - var packageRegistrationLookup = await GetPackageRegistrations(destinationDatabase); + var packageRegistrationLookup = await GetPackageRegistrations(connection); // Group based on package ID and store in a stack for easy incremental processing. var allGroups = downloadData.GroupBy(p => p.PackageId).ToList(); @@ -126,9 +123,9 @@ await statisticsDatabase.QueryWithRetryAsync( while (remainingGroups.Any()) { // Create a batch of one or more package registrations to update. - var batch = PopGroupBatch(remainingGroups, _batchSize); + var batch = PopGroupBatch(remainingGroups, _configuration.BatchSize); - await ProcessBatch(batch, destinationDatabase, packageRegistrationLookup); + await ProcessBatch(batch, connection, packageRegistrationLookup); Logger.LogInformation( "There are {GroupCount} package registration groups remaining.", @@ -136,8 +133,8 @@ await statisticsDatabase.QueryWithRetryAsync( if (remainingGroups.Any()) { - Logger.LogInformation("Sleeping for {BatchSleepSeconds} seconds before continuing.", _batchSleepSeconds); - await Task.Delay(TimeSpan.FromSeconds(_batchSleepSeconds)); + Logger.LogInformation("Sleeping for {BatchSleepSeconds} seconds before continuing.", _configuration.BatchSleepSeconds); + await Task.Delay(TimeSpan.FromSeconds(_configuration.BatchSleepSeconds)); } } @@ -305,5 +302,14 @@ private async Task> GetPackageRegistrations(SqlConne return packageRegistrationDictionary; } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } } } \ No newline at end of file diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Configuration/AggregateCdnDownloadsConfiguration.cs b/src/Stats.AggregateCdnDownloadsInGallery/Configuration/AggregateCdnDownloadsConfiguration.cs new file mode 100644 index 000000000..25ffb5d10 --- /dev/null +++ b/src/Stats.AggregateCdnDownloadsInGallery/Configuration/AggregateCdnDownloadsConfiguration.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Stats.AggregateCdnDownloadsInGallery +{ + public class AggregateCdnDownloadsConfiguration + { + public int BatchSize { get; set; } + + public int BatchSleepSeconds { get; set; } + } +} diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Program.cs b/src/Stats.AggregateCdnDownloadsInGallery/Program.cs index 813a3279a..6ceac30d4 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Program.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Program.cs @@ -9,7 +9,7 @@ public class Program { public static void Main(string[] args) { - var job = new Job(); + var job = new AggregateCdnDownloadsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd b/src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd index 61710bc56..66147c4b0 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd +++ b/src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd @@ -8,14 +8,8 @@ echo "Starting job - #{Jobs.stats.aggregatecdndownloadsingallery.Title}" title #{Jobs.stats.aggregatecdndownloadsingallery.Title} start /w stats.aggregatecdndownloadsingallery.exe ^ - -VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^ - -ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^ - -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^ - -StatisticsDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.StatisticsDatabase}" ^ - -DestinationDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.DestinationDatabase}" ^ + -Configuration "#{Jobs.stats.aggregatecdndownloadsingallery.Configuration}" ^ -InstrumentationKey "#{Jobs.stats.aggregatecdndownloadsingallery.InstrumentationKey}" ^ - -BatchSize "#{Jobs.stats.aggregatecdndownloadsingallery.BatchSize}" ^ - -BatchSleepSeconds "#{Jobs.stats.aggregatecdndownloadsingallery.BatchSleepSeconds}" ^ -verbose true ^ -Interval #{Jobs.stats.aggregatecdndownloadsingallery.Interval} diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Settings/dev.json b/src/Stats.AggregateCdnDownloadsInGallery/Settings/dev.json new file mode 100644 index 000000000..b663af59e --- /dev/null +++ b/src/Stats.AggregateCdnDownloadsInGallery/Settings/dev.json @@ -0,0 +1,21 @@ +{ + "Initialization": { + "BatchSize": 5000, + "BatchSleepSeconds": 10 + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Stats.AggregateCdnDownloads;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbWriter.ClientId};AadCertificate=$$dev-gallerydb-writer$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Stats.AggregateCdnDownloads;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbWriter.ClientId};AadCertificate=$$dev-statisticsdb-writer$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Settings/int.json b/src/Stats.AggregateCdnDownloadsInGallery/Settings/int.json new file mode 100644 index 000000000..984ba6029 --- /dev/null +++ b/src/Stats.AggregateCdnDownloadsInGallery/Settings/int.json @@ -0,0 +1,21 @@ +{ + "Initialization": { + "BatchSize": 5000, + "BatchSleepSeconds": 10 + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBWriter-UserName$$;Password=$$Int-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Settings/prod.json b/src/Stats.AggregateCdnDownloadsInGallery/Settings/prod.json new file mode 100644 index 000000000..1a0c9b313 --- /dev/null +++ b/src/Stats.AggregateCdnDownloadsInGallery/Settings/prod.json @@ -0,0 +1,21 @@ +{ + "Initialization": { + "BatchSize": 5000, + "BatchSleepSeconds": 10 + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBWriter-UserName$$;Password=$$Prod-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj index 3d2de7505..2c7114d35 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj +++ b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj @@ -42,8 +42,9 @@ + - + @@ -51,6 +52,7 @@ + @@ -75,9 +77,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.nuspec b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.nuspec index 01e314f0d..2d53f89c3 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.nuspec +++ b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs b/src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs new file mode 100644 index 000000000..642eeaa96 --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Stats.CreateAzureCdnWarehouseReports +{ + public class CreateAzureCdnWarehouseReportsConfiguration + { + public string AzureCdnCloudStorageAccount { get; set; } + + public string AzureCdnCloudStorageContainerName { get; set; } + + public string DataStorageAccount { get; set; } + + public string DataContainerName { get; set; } + + public int? CommandTimeOut { get; set; } + + public int? PerPackageReportDegreeOfParallelism { get; set; } + + public string ReportName { get; set; } + } +} diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs similarity index 61% rename from src/Stats.CreateAzureCdnWarehouseReports/Job.cs rename to src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs index 611390958..636d1a7eb 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs @@ -4,29 +4,31 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports { - public class Job - : JobBase + public class CreateAzureCdnWarehouseReportsJob : JsonConfigurationJob { private const int DefaultPerPackageReportDegreeOfParallelism = 8; // Generate private const int DefaultSqlCommandTimeoutSeconds = 1800; // 30 minute SQL command timeout by default private const string _recentPopularityDetailByPackageReportBaseName = "recentpopularitydetail_"; + private CloudStorageAccount _cloudStorageAccount; private CloudStorageAccount _dataStorageAccount; private string _statisticsContainerName; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private string _reportName; private string[] _dataContainerNames; private int _sqlCommandTimeoutSeconds = DefaultSqlCommandTimeoutSeconds; @@ -34,45 +36,51 @@ public class Job private static readonly IDictionary _storedProcedures = new Dictionary { - {ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, - {ReportNames.Last6Weeks, "[dbo].[DownloadReportLast6Weeks]" }, - {ReportNames.RecentCommunityPopularity, "[dbo].[DownloadReportRecentCommunityPopularity]" }, - {ReportNames.RecentCommunityPopularityDetail, "[dbo].[DownloadReportRecentCommunityPopularityDetail]" }, - {ReportNames.RecentPopularity, "[dbo].[DownloadReportRecentPopularity]" }, - {ReportNames.RecentPopularityDetail, "[dbo].[DownloadReportRecentPopularityDetail]" }, + { ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, + { ReportNames.Last6Weeks, "[dbo].[DownloadReportLast6Weeks]" }, + { ReportNames.RecentCommunityPopularity, "[dbo].[DownloadReportRecentCommunityPopularity]" }, + { ReportNames.RecentCommunityPopularityDetail, "[dbo].[DownloadReportRecentCommunityPopularityDetail]" }, + { ReportNames.RecentPopularity, "[dbo].[DownloadReportRecentPopularity]" }, + { ReportNames.RecentPopularityDetail, "[dbo].[DownloadReportRecentPopularityDetail]" }, }; private static readonly IDictionary _storedProceduresPerPackageId = new Dictionary { - {ReportNames.RecentPopularityDetailByPackageId, "[dbo].[DownloadReportRecentPopularityDetailByPackage]" } + { ReportNames.RecentPopularityDetailByPackageId, "[dbo].[DownloadReportRecentPopularityDetailByPackage]" } }; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; + base.Init(serviceContainer, jobArgsDictionary); + + var configuration = _serviceProvider.GetRequiredService>().Value; + + _sqlCommandTimeoutSeconds = configuration.CommandTimeOut ?? DefaultSqlCommandTimeoutSeconds; + + _perPackageReportDegreeOfParallelism = configuration.PerPackageReportDegreeOfParallelism ?? DefaultPerPackageReportDegreeOfParallelism; - var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDatabaseConnectionString, secretInjector); + _cloudStorageAccount = ValidateAzureCloudStorageAccount( + configuration.AzureCdnCloudStorageAccount, + nameof(configuration.AzureCdnCloudStorageAccount)); - var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDatabaseConnectionString, secretInjector); + _statisticsContainerName = ValidateAzureContainerName( + configuration.AzureCdnCloudStorageContainerName, + nameof(configuration.AzureCdnCloudStorageContainerName)); - var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); - var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); - _perPackageReportDegreeOfParallelism = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.PerPackageReportDegreeOfParallelism) ?? DefaultPerPackageReportDegreeOfParallelism; + _dataStorageAccount = ValidateAzureCloudStorageAccount( + configuration.DataStorageAccount, + nameof(configuration.DataStorageAccount)); - _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString, JobArgumentNames.AzureCdnCloudStorageAccount); - _statisticsContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName), JobArgumentNames.AzureCdnCloudStorageContainerName); - _dataStorageAccount = ValidateAzureCloudStorageAccount(dataStorageAccountConnectionString, JobArgumentNames.DataStorageAccount); - _reportName = ValidateReportName(JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.WarehouseReportName)); + _reportName = ValidateReportName( + configuration.ReportName, + nameof(configuration.ReportName)); - var containerNames = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataContainerName) + var containerNames = configuration.DataContainerName .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries); foreach (var containerName in containerNames) { - ValidateAzureContainerName(containerName, JobArgumentNames.DataContainerName); + ValidateAzureContainerName(containerName, nameof(configuration.DataContainerName)); } _dataContainerNames = containerNames; @@ -80,10 +88,13 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); + var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); - Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger(); var reportCollectorLogger = LoggerFactory.CreateLogger(); @@ -92,13 +103,35 @@ public override async Task Run() { // generate all reports var reportGenerators = new Dictionary + { + { + new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + { - { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } + new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) @@ -114,12 +147,13 @@ public override async Task Run() { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); - var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } - Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); @@ -131,7 +165,12 @@ public override async Task Run() { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } - var downloadCountReport = new DownloadCountReport(LoggerFactory.CreateLogger(), targets, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + + var downloadCountReport = new DownloadCountReport( + LoggerFactory.CreateLogger(), + targets, + OpenSqlConnectionAsync, + OpenSqlConnectionAsync); await downloadCountReport.Run(); stopwatch.Stop(); @@ -140,7 +179,12 @@ public override async Task Run() stopwatch.Restart(); // build stats-totals.json - var galleryTotalsReport = new GalleryTotalsReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var galleryTotalsReport = new GalleryTotalsReport( + LoggerFactory.CreateLogger(), + _cloudStorageAccount, + _statisticsContainerName, + OpenSqlConnectionAsync, + OpenSqlConnectionAsync); await galleryTotalsReport.Run(); stopwatch.Stop(); @@ -149,7 +193,12 @@ public override async Task Run() // build tools.v1.json - var toolsReport = new DownloadsPerToolVersionReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var toolsReport = new DownloadsPerToolVersionReport( + LoggerFactory.CreateLogger(), + _cloudStorageAccount, + _statisticsContainerName, + OpenSqlConnectionAsync, + OpenSqlConnectionAsync); await toolsReport.Run(); stopwatch.Stop(); @@ -158,7 +207,8 @@ public override async Task Run() stopwatch.Restart(); } - private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple[] parameters) + private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, + ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple[] parameters) { var dataTable = await reportDataCollector.CollectAsync(reportGenerationTime, parameters); if (dataTable.Rows.Count == 0) @@ -174,18 +224,32 @@ private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobC private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { - var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(LoggerFactory.CreateLogger(), _statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds( + LoggerFactory.CreateLogger(), + OpenSqlConnectionAsync, + reportGenerationTime, + _sqlCommandTimeoutSeconds); if (!dirtyPackageIds.Any()) + { return; + } // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); - var reportDataCollector = new ReportDataCollector(LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector( + LoggerFactory.CreateLogger(), + _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], + OpenSqlConnectionAsync, + _sqlCommandTimeoutSeconds); + var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = _perPackageReportDegreeOfParallelism }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); - var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(LoggerFactory.CreateLogger(), ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId); + var reportBuilder = new RecentPopularityDetailByPackageReportBuilder( + LoggerFactory.CreateLogger(), + ReportNames.RecentPopularityDetailByPackageId, + "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId); ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsightsHelper.TrackReportProcessed(reportBuilder.ReportName + " report", packageId); @@ -211,7 +275,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer new ReportDataCollector( LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], - _statisticsDbConnectionFactory, + OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) } }; @@ -228,7 +292,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; - await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDbConnectionFactory, runToCursor, _sqlCommandTimeoutSeconds); + await ReportDataCollector.UpdateDirtyPackageIdCursor(OpenSqlConnectionAsync, runToCursor, _sqlCommandTimeoutSeconds); } } } @@ -236,7 +300,11 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { Logger.LogDebug("Getting list of inactive packages."); - var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var packageIds = await ReportDataCollector.ListInactivePackageIdReports( + OpenSqlConnectionAsync, + reportGenerationTime, + _sqlCommandTimeoutSeconds); + Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count); // Collect the list of reports @@ -265,11 +333,11 @@ private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlob }); } - private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount, string parameterName) + private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount, string configurationName) { if (string.IsNullOrEmpty(cloudStorageAccount)) { - throw new ArgumentException($"Job parameter {parameterName} is not defined."); + throw new ArgumentException($"Job configuration {configurationName} is not defined."); } CloudStorageAccount account; @@ -278,20 +346,20 @@ private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloud return account; } - throw new ArgumentException($"Job parameter {parameterName} is invalid."); + throw new ArgumentException($"Job configuration {configurationName} is invalid."); } - private static string ValidateAzureContainerName(string containerName, string parameterName) + private static string ValidateAzureContainerName(string containerName, string configurationName) { if (string.IsNullOrWhiteSpace(containerName)) { - throw new ArgumentException($"Job parameter {parameterName} is not defined."); + throw new ArgumentException($"Job configuration {configurationName} is not defined."); } return containerName; } - private static string ValidateReportName(string reportName) + private static string ValidateReportName(string reportName, string configurationName) { if (string.IsNullOrWhiteSpace(reportName)) { @@ -300,12 +368,21 @@ private static string ValidateReportName(string reportName) if (!_storedProcedures.ContainsKey(reportName.ToLowerInvariant())) { - throw new ArgumentException("Job parameter ReportName contains unknown report name."); + throw new ArgumentException($"Job configuration {configurationName} contains unknown report name."); } return reportName; } + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } + private static class ReportNames { public const string NuGetClientVersion = "nugetclientversion"; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs index 85ae20a62..bc9deae7e 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; using NuGet.Versioning; namespace Stats.CreateAzureCdnWarehouseReports @@ -25,9 +24,9 @@ public class DownloadCountReport public DownloadCountReport( ILogger logger, IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) + : base(logger, targets, openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync) { } @@ -35,12 +34,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection downloadData; - _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + downloadData = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs index 7411526f1..df9e0ccc9 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs @@ -11,7 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -26,10 +25,10 @@ public DownloadsPerToolVersionReport( ILogger logger, CloudStorageAccount cloudStorageAccount, string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync) { } @@ -37,12 +36,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection data; - _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + data = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs index cdccbb09d..92919f973 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -27,10 +26,10 @@ public GalleryTotalsReport( ILogger logger, CloudStorageAccount cloudStorageAccount, string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync) { } @@ -38,12 +37,13 @@ public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; - _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", - GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog); - using (var connection = await GalleryDbConnectionFactory.CreateAsync()) + using (var connection = await OpenGallerySqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData = (await connection.QueryWithRetryAsync( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } @@ -52,12 +52,12 @@ public async Task Run() _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse - _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Program.cs b/src/Stats.CreateAzureCdnWarehouseReports/Program.cs index f16d9e031..e5c658e9e 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Program.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Program.cs @@ -9,7 +9,7 @@ class Program { static void Main(string[] args) { - var job = new Job(); + var job = new CreateAzureCdnWarehouseReportsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs index faef0cce4..b01f5b186 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs @@ -3,12 +3,12 @@ using System; using System.Collections.Generic; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -18,20 +18,20 @@ public abstract class ReportBase protected readonly IReadOnlyCollection Targets; - protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory; + protected readonly Func> OpenStatisticsSqlConnectionAsync; - protected ISqlConnectionFactory GalleryDbConnectionFactory; + protected Func> OpenGallerySqlConnectionAsync; protected ReportBase( ILogger logger, IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) { _logger = logger; Targets = targets.ToList().AsReadOnly(); - StatisticsDbConnectionFactory = statisticsDbConnectionFactory; - GalleryDbConnectionFactory = galleryDbConnectionFactory; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; } protected async Task GetBlobContainer(StorageContainerTarget target) diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs index be58465b4..7472f515c 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -16,19 +15,19 @@ internal class ReportDataCollector { private int _commandTimeoutSeconds; private readonly string _procedureName; - private readonly ISqlConnectionFactory _sourceDbConnectionFactory; + private readonly Func> _openGallerySqlConnectionAsync; private ILogger _logger; public ReportDataCollector( ILogger logger, string procedureName, - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, int timeout) { _logger = logger; _procedureName = procedureName; - _sourceDbConnectionFactory = sourceDbConnectionFactory; + _openGallerySqlConnectionAsync = openGallerySqlConnectionAsync; _commandTimeoutSeconds = timeout; } @@ -48,7 +47,7 @@ public async Task CollectAsync(DateTime reportGenerationTime, params public static async Task> GetDirtyPackageIds( ILogger logger, - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { @@ -57,7 +56,7 @@ public static async Task> GetDirtyPackageIds IReadOnlyCollection packageIds = new List(); // Get the data - await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger); + await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(openGallerySqlConnectionAsync, reportGenerationTime, commandTimeout), logger); logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count); @@ -65,11 +64,11 @@ public static async Task> GetDirtyPackageIds } public static async Task> ListInactivePackageIdReports( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openGallerySqlConnectionAsync()) { var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection); command.CommandType = CommandType.StoredProcedure; @@ -122,7 +121,7 @@ private static async Task WithRetry(Func action, ILogger logger) private async Task ExecuteSql(DateTime reportGenerationTime, params Tuple[] parameters) { - using (var connection = await _sourceDbConnectionFactory.CreateAsync()) + using (var connection = await _openGallerySqlConnectionAsync()) { var command = new SqlCommand(_procedureName, connection); command.CommandType = CommandType.StoredProcedure; @@ -146,11 +145,11 @@ private async Task ExecuteSql(DateTime reportGenerationTime, params T } private static async Task> GetDirtyPackageIdsFromWarehouse( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openGallerySqlConnectionAsync()) { var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection); command.CommandType = CommandType.StoredProcedure; @@ -172,11 +171,11 @@ private static async Task> GetDirtyPackageId } public static async Task UpdateDirtyPackageIdCursor( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime runToCursor, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openGallerySqlConnectionAsync()) { var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection); command.CommandType = CommandType.StoredProcedure; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd b/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd index b4fe1d9e7..bf9c90725 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd +++ b/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd @@ -3,26 +3,16 @@ cd bin :Top - echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}" +echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}" - title #{Jobs.stats.createazurecdnwarehousereports.Title} +title #{Jobs.stats.createazurecdnwarehousereports.Title} - start /w stats.createazurecdnwarehousereports.exe ^ - -VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^ - -ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^ - -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^ - -AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" ^ - -AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" ^ - -StatisticsDatabase "#{Jobs.stats.createazurecdnwarehousereports.StatisticsDatabase}" ^ - -SourceDatabase "#{Jobs.stats.createazurecdnwarehousereports.SourceDatabase}" ^ - -DataStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.DataStorageAccount}" ^ - -InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^ - -DataContainerName "#{Jobs.stats.createazurecdnwarehousereports.DataContainerName}" ^ - -CommandTimeOut "#{Jobs.stats.createazurecdnwarehousereports.CommandTimeOut}" ^ - -PerPackageReportDegreeOfParallelism "#{Jobs.stats.createazurecdnwarehousereports.PerPackageReportDegreeOfParallelism}" ^ - -verbose true ^ - -Interval #{Jobs.stats.createazurecdnwarehousereports.Interval} +start /w stats.createazurecdnwarehousereports.exe ^ + -Configuration "#{Jobs.stats.createazurecdnwarehousereports.Configuration}" + -InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^ + -Interval #{Jobs.stats.createazurecdnwarehousereports.Interval} ^ + -verbose true - echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}" +echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}" - goto Top +goto Top diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json new file mode 100644 index 000000000..4eddfbc13 --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json @@ -0,0 +1,25 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdev0;AccountKey=$$Dev-NuGetDev0Storage-Key$$", + "DataContainerName": "ng-search-data", + "CommandTimeOut": "7200", + "PerPackageReportDegreeOfParallelism": "64" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Stats.CreateAzureCdnWarehouseReports;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Stats.CreateAzureCdnWarehouseReports;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbWriter.ClientId};AadCertificate=$$dev-statisticsdb-writer$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json new file mode 100644 index 000000000..fe47aa6ba --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json @@ -0,0 +1,25 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "DataContainerName": "ng-search-data", + "CommandTimeOut": "7200", + "PerPackageReportDegreeOfParallelism": "64" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/prod.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/prod.json new file mode 100644 index 000000000..47b14d6e8 --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Settings/prod.json @@ -0,0 +1,25 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetprod0;AccountKey=$$Prod-NuGetProd0Storage-Key$$", + "DataContainerName": "ng-search-data", + "CommandTimeOut": "7200", + "PerPackageReportDegreeOfParallelism": "64" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj index 0b09db277..a559d7432 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj @@ -41,6 +41,7 @@ + @@ -48,7 +49,7 @@ - + @@ -62,6 +63,9 @@ + + + @@ -96,12 +100,6 @@ 9.0.1 - - 2.25.0 - - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec index 850276ed1..851685278 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/Configuration/ImportAzureCdnStatisticsConfiguration.cs b/src/Stats.ImportAzureCdnStatistics/Configuration/ImportAzureCdnStatisticsConfiguration.cs new file mode 100644 index 000000000..5bbf199ba --- /dev/null +++ b/src/Stats.ImportAzureCdnStatistics/Configuration/ImportAzureCdnStatisticsConfiguration.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Stats.ImportAzureCdnStatistics +{ + public class ImportAzureCdnStatisticsConfiguration + { + public string AzureCdnCloudStorageAccount { get; set; } + + public string AzureCdnCloudStorageContainerName { get; set; } + + public string AzureCdnPlatform { get; set; } + + public string AzureCdnAccountNumber { get; set; } + + public bool AggregatesOnly { get; set; } + } +} diff --git a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs index 8f0a9251b..f1594f17d 100644 --- a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs +++ b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs @@ -1,21 +1,21 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Data; using System.Data.SqlClient; using System.Threading.Tasks; -using NuGet.Services.Sql; namespace Stats.ImportAzureCdnStatistics { internal class DataImporter { - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; + private readonly Func> _openStatisticsSqlConnectionAsync; private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]"; - public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory) + public DataImporter(Func> openStatisticsSqlConnectionAsync) { - _statisticsDbConnectionFactory = statisticsDbConnectionFactory; + _openStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task GetDataTableAsync(string tableName) @@ -23,7 +23,7 @@ public async Task GetDataTableAsync(string tableName) var dataTable = new DataTable(); var query = string.Format(_sqlSelectTop1FromTable, tableName); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) { var tableAdapter = new SqlDataAdapter(query, connection) { diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/ImportAzureCdnStatisticsJob.cs similarity index 64% rename from src/Stats.ImportAzureCdnStatistics/Job.cs rename to src/Stats.ImportAzureCdnStatistics/ImportAzureCdnStatisticsJob.cs index 6b0b18119..e84411e76 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/ImportAzureCdnStatisticsJob.cs @@ -6,66 +6,57 @@ using System.ComponentModel.Design; using System.Globalization; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; using Stats.AzureCdnLogs.Common; namespace Stats.ImportAzureCdnStatistics { - public class Job - : JobBase + public class ImportAzureCdnStatisticsJob : JsonConfigurationJob { - private bool _aggregatesOnly; - private string _azureCdnAccountNumber; - private string _cloudStorageContainerName; + private ImportAzureCdnStatisticsConfiguration _configuration; private AzureCdnPlatform _azureCdnPlatform; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private CloudStorageAccount _cloudStorageAccount; private CloudBlobClient _cloudBlobClient; private LogFileProvider _blobLeaseManager; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); - var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); - var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); - _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString); + _configuration = _serviceProvider.GetRequiredService>().Value; - _azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber); - _azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform); - _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName)); + _azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform); - _aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly); - - // construct a cloud blob client for the configured storage account - _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient(); + var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount); + _cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient(); _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5); - // Get the source blob container (containing compressed log files) - // and construct a log source (fetching raw logs from the source blob container) - var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName); - _blobLeaseManager = new LogFileProvider(sourceBlobContainer, LoggerFactory); + _blobLeaseManager = new LogFileProvider( + _cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName), + LoggerFactory); } public override async Task Run() { // Get the target blob container (for archiving decompressed log files) - var targetBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive"); + var targetBlobContainer = _cloudBlobClient.GetContainerReference( + _configuration.AzureCdnCloudStorageContainerName + "-archive"); await targetBlobContainer.CreateIfNotExistsAsync(); // Get the dead-letter table (corrupted or failed blobs will end up there) - var deadLetterBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter"); + var deadLetterBlobContainer = _cloudBlobClient.GetContainerReference( + _configuration.AzureCdnCloudStorageContainerName + "-deadletter"); await deadLetterBlobContainer.CreateIfNotExistsAsync(); // Create a parser - var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory); + var warehouse = new Warehouse(LoggerFactory, OpenSqlConnectionAsync); var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility( targetBlobContainer, deadLetterBlobContainer, @@ -74,11 +65,13 @@ public override async Task Run() var logProcessor = new LogFileProcessor(statisticsBlobContainerUtility, LoggerFactory, warehouse); // Get the next to-be-processed raw log file using the cdn raw log file name prefix - var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber); + var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", + _azureCdnPlatform.GetRawLogFilePrefix(), + _configuration.AzureCdnAccountNumber); // Get next raw log file to be processed IReadOnlyCollection alreadyAggregatedLogFiles = null; - if (_aggregatesOnly) + if (_configuration.AggregatesOnly) { // We only want to process aggregates for the log files. // Get the list of files we already processed so we can skip them. @@ -90,9 +83,9 @@ public override async Task Run() { var packageTranslator = new PackageTranslator("packagetranslations.json"); var packageStatisticsParser = new PackageStatisticsParser(packageTranslator, LoggerFactory); - await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _aggregatesOnly); + await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _configuration.AggregatesOnly); - if (_aggregatesOnly) + if (_configuration.AggregatesOnly) { _blobLeaseManager.TrackLastProcessedBlobUri(leasedLogFile.Uri); } @@ -139,5 +132,14 @@ private static string ValidateAzureContainerName(string containerName) } return containerName; } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } } } \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs b/src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs index e871dba7f..5ff38fb4d 100644 --- a/src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs +++ b/src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs @@ -31,7 +31,7 @@ public LogFileProcessor( _warehouse = warehouse ?? throw new ArgumentNullException(nameof(warehouse)); _statisticsBlobContainerUtility = statisticsBlobContainerUtility ?? throw new ArgumentNullException(nameof(statisticsBlobContainerUtility)); - _logger = loggerFactory.CreateLogger(); + _logger = loggerFactory.CreateLogger(); } public async Task ProcessLogFileAsync(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, bool aggregatesOnly = false) diff --git a/src/Stats.ImportAzureCdnStatistics/Program.cs b/src/Stats.ImportAzureCdnStatistics/Program.cs index 0c181880b..1fac1a81b 100644 --- a/src/Stats.ImportAzureCdnStatistics/Program.cs +++ b/src/Stats.ImportAzureCdnStatistics/Program.cs @@ -9,7 +9,7 @@ public class Program { public static void Main(string[] args) { - var job = new Job(); + var job = new ImportAzureCdnStatisticsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd b/src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd index 6b7e42ff8..4e0abc0a0 100644 --- a/src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd +++ b/src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd @@ -3,12 +3,16 @@ cd bin :Top - echo "Starting job - #{Jobs.stats.importazurecdnstatistics.Title}" +echo "Starting job - #{Jobs.stats.importazurecdnstatistics.Title}" - title #{Jobs.stats.importazurecdnstatistics.Title} +title #{Jobs.stats.importazurecdnstatistics.Title} - start /w stats.importazurecdnstatistics.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -AzureCdnCloudStorageAccount "#{Jobs.stats.importazurecdnstatistics.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.importazurecdnstatistics.AzureCdn.CloudStorageContainerName}" -AzureCdnPlatform "#{Jobs.stats.importazurecdnstatistics.AzureCdn.Platform}" -AzureCdnAccountNumber "#{Jobs.stats.importazurecdnstatistics.AzureCdn.AccountNumber}" -StatisticsDatabase "#{Jobs.stats.importazurecdnstatistics.StatisticsDatabase}" -InstrumentationKey "#{Jobs.stats.importazurecdnstatistics.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.importazurecdnstatistics.Interval} +start /w stats.importazurecdnstatistics.exe ^ + -Configuration "#{Jobs.stats.importazurecdnstatistics.Configuration}" ^ + -InstrumentationKey "#{Jobs.stats.importazurecdnstatistics.InstrumentationKey}" ^ + -Interval #{Jobs.stats.importazurecdnstatistics.Interval} ^ + -verbose true - echo "Finished #{Jobs.stats.importazurecdnstatistics.Title}" +echo "Finished #{Jobs.stats.importazurecdnstatistics.Title}" - goto Top \ No newline at end of file +goto Top \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/Settings/dev.json b/src/Stats.ImportAzureCdnStatistics/Settings/dev.json new file mode 100644 index 000000000..f9969dd0a --- /dev/null +++ b/src/Stats.ImportAzureCdnStatistics/Settings/dev.json @@ -0,0 +1,19 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw", + "AzureCdnPlatform": "HttpLargeObject", + "AzureCdnAccountNumber": "$$Dev-AzureCdn-AccountNumber$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Stats.ImportAzureCdnStatistics;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbWriter.ClientId};AadCertificate=$$dev-statisticsdb-writer$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/Settings/int.json b/src/Stats.ImportAzureCdnStatistics/Settings/int.json new file mode 100644 index 000000000..9dc142b22 --- /dev/null +++ b/src/Stats.ImportAzureCdnStatistics/Settings/int.json @@ -0,0 +1,19 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw", + "AzureCdnPlatform": "HttpLargeObject", + "AzureCdnAccountNumber": "$$Int-AzureCdn-AccountNumber$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/Settings/prod.json b/src/Stats.ImportAzureCdnStatistics/Settings/prod.json new file mode 100644 index 000000000..68c142df0 --- /dev/null +++ b/src/Stats.ImportAzureCdnStatistics/Settings/prod.json @@ -0,0 +1,19 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw", + "AzureCdnPlatform": "HttpLargeObject", + "AzureCdnAccountNumber": "$$Prod-AzureCdn-AccountNumber$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj index 6ba8e18cd..ebb18cae0 100644 --- a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj +++ b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj @@ -49,6 +49,7 @@ + @@ -70,7 +71,7 @@ - + @@ -91,6 +92,9 @@ Always + + + @@ -122,12 +126,6 @@ 1.0.0 - - 2.25.0 - - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.nuspec b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.nuspec index 00568f14c..e992b30c6 100644 --- a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.nuspec +++ b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs index 6e4c7bcae..c1f9f3f62 100644 --- a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs +++ b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; using Stopwatch = System.Diagnostics.Stopwatch; @@ -21,7 +20,7 @@ internal class Warehouse private const int _maxRetryCount = 3; private readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(5); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; + private readonly Func> _openStatisticsSqlConnectionAsync; private readonly IDictionary _cachedPackageDimensions = new Dictionary(); private readonly IList _cachedToolDimensions = new List(); private readonly IDictionary _cachedClientDimensions = new Dictionary(); @@ -31,7 +30,7 @@ internal class Warehouse private readonly IDictionary _cachedIpAddressFacts = new Dictionary(); private IReadOnlyCollection _times; - public Warehouse(ILoggerFactory loggerFactory, ISqlConnectionFactory statisticsDbConnectionFactory) + public Warehouse(ILoggerFactory loggerFactory, Func> openStatisticsSqlConnectionAsync) { if (loggerFactory == null) { @@ -39,7 +38,9 @@ public Warehouse(ILoggerFactory loggerFactory, ISqlConnectionFactory statisticsD } _logger = loggerFactory.CreateLogger(); - _statisticsDbConnectionFactory = statisticsDbConnectionFactory ?? throw new ArgumentNullException(nameof(statisticsDbConnectionFactory)); + + _openStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync + ?? throw new ArgumentNullException(nameof(openStatisticsSqlConnectionAsync)); } public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, string logFileName) @@ -47,7 +48,7 @@ public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, str _logger.LogDebug("Inserting into facts table..."); var stopwatch = Stopwatch.StartNew(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { try @@ -124,7 +125,7 @@ public async Task CreateAsync(IReadOnlyCollection var packages = packagesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(_openStatisticsSqlConnectionAsync); var factsDataTable = await dataImporter.GetDataTableAsync("Fact_Download"); var knownOperationsAvailable = operations.Any(); @@ -245,7 +246,7 @@ public async Task CreateAsync(IReadOnlyCollection sou var ipAddresses = ipAddressesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(_openStatisticsSqlConnectionAsync); var dataTable = await dataImporter.GetDataTableAsync("Fact_Dist_Download"); var knownClientsAvailable = clients.Any(); @@ -341,7 +342,7 @@ public async Task StoreLogFileAggregatesAsync(LogFileAggregates logFileAggregate { _logger.LogDebug("Storing log file aggregates..."); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) { try { @@ -375,7 +376,7 @@ public async Task> GetAlreadyAggregatedLogFilesAsync _logger.LogDebug("Retrieving already processed log files..."); var alreadyAggregatedLogFiles = new List(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) { try { @@ -433,7 +434,7 @@ private async Task HasImportedStatisticsAsync(string logFileName, string c try { - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) { var command = connection.CreateCommand(); command.CommandText = commandText; @@ -474,7 +475,7 @@ private async Task> GetDimension(string dimension, stri _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IDictionary dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } @@ -546,7 +547,7 @@ private async Task> GetDimension(string dimension, str _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IReadOnlyCollection dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await _openStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index a10b412c3..e843aa3b8 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -14,9 +14,11 @@ namespace Stats.RefreshClientDimension { + using ICoreSqlConnectionFactory = NuGet.Services.Sql.ISqlConnectionFactory; + public class RefreshClientDimensionJob : JobBase { - private static ISqlConnectionFactory _statisticsDbConnectionFactory; + private static ICoreSqlConnectionFactory _statisticsDbConnectionFactory; private static string _targetClientName; private static string _userAgentFilter; diff --git a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj index 980174366..66606dd9c 100644 --- a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj +++ b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj @@ -84,7 +84,7 @@ 9.0.1 - 2.25.0-master-30453 + 2.27.0 1.2.0 diff --git a/src/Stats.RollUpDownloadFacts/Configuration/RollUpDownloadFactsConfiguration.cs b/src/Stats.RollUpDownloadFacts/Configuration/RollUpDownloadFactsConfiguration.cs new file mode 100644 index 000000000..ac9d0c82d --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Configuration/RollUpDownloadFactsConfiguration.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Stats.RollUpDownloadFacts +{ + public class RollUpDownloadFactsConfiguration + { + public int? MinAgeInDays { get; set; } + } +} diff --git a/src/Stats.RollUpDownloadFacts/Program.cs b/src/Stats.RollUpDownloadFacts/Program.cs index 1550c66ef..10b96b312 100644 --- a/src/Stats.RollUpDownloadFacts/Program.cs +++ b/src/Stats.RollUpDownloadFacts/Program.cs @@ -9,7 +9,7 @@ public class Program { public static void Main(string[] args) { - var job = new Job(); + var job = new RollUpDownloadFactsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/RollUpDownloadFactsJob.cs similarity index 70% rename from src/Stats.RollUpDownloadFacts/Job.cs rename to src/Stats.RollUpDownloadFacts/RollUpDownloadFactsJob.cs index 76cd61979..ce2727b88 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/RollUpDownloadFactsJob.cs @@ -7,35 +7,38 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; namespace Stats.RollUpDownloadFacts { - public class Job - : JobBase + public class RollUpDownloadFactsJob : JsonConfigurationJob { private const string _startTemplateRecordsDeletion = "Package Dimension ID "; private const string _endTemplateFactDownloadDeletion = " records from [dbo].[Fact_Download]"; private const int DefaultMinAgeInDays = 43; - private static int _minAgeInDays; - private static ISqlConnectionFactory _statisticsDbConnectionFactory; + + private RollUpDownloadFactsConfiguration _configuration; + private int _minAgeInDays; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); + + _configuration = _serviceProvider.GetRequiredService>().Value; - _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; + _minAgeInDays = _configuration.MinAgeInDays ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); } public override async Task Run() { - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { connection.InfoMessage -= OnSqlConnectionInfoMessage; connection.InfoMessage += OnSqlConnectionInfoMessage; @@ -72,5 +75,14 @@ private void OnSqlConnectionInfoMessage(object sender, SqlInfoMessageEventArgs e ApplicationInsightsHelper.TrackRollUpMetric("Download Facts Deleted", value, packageDimensionId); } } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } } } \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd b/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd index c49025f34..a67f091c4 100644 --- a/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd +++ b/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd @@ -3,12 +3,16 @@ cd bin :Top - echo "Starting job - #{Jobs.stats.rollupdownloadfacts.Title}" +echo "Starting job - #{Jobs.stats.rollupdownloadfacts.Title}" - title #{Jobs.stats.rollupdownloadfacts.Title} +title #{Jobs.stats.rollupdownloadfacts.Title} - start /w stats.rollupdownloadfacts.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -MinAgeInDays "#{Jobs.stats.rollupdownloadfacts.MinAgeInDays}" -StatisticsDatabase "#{Jobs.stats.rollupdownloadfacts.StatisticsDatabase}" -InstrumentationKey "#{Jobs.stats.rollupdownloadfacts.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.rollupdownloadfacts.Interval} +start /w stats.rollupdownloadfacts.exe ^ + -Configuration "#{Jobs.stats.rollupdownloadfacts.Configuration}" ^ + -InstrumentationKey "#{Jobs.stats.rollupdownloadfacts.InstrumentationKey}" ^ + -Interval #{Jobs.stats.rollupdownloadfacts.Interval} ^ + -verbose true - echo "Finished #{Jobs.stats.rollupdownloadfacts.Title}" +echo "Finished #{Jobs.stats.rollupdownloadfacts.Title}" - goto Top \ No newline at end of file +goto Top \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Settings/dev.json b/src/Stats.RollUpDownloadFacts/Settings/dev.json new file mode 100644 index 000000000..144111637 --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Settings/dev.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "MinAgeInDays": "43" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Stats.RollUpDownloadFacts;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbWriter.ClientId};AadCertificate=$$dev-statisticsdb-writer$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Settings/int.json b/src/Stats.RollUpDownloadFacts/Settings/int.json new file mode 100644 index 000000000..7a920ed55 --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Settings/int.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "MinAgeInDays": "43" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Settings/prod.json b/src/Stats.RollUpDownloadFacts/Settings/prod.json new file mode 100644 index 000000000..23e6bba77 --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Settings/prod.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "MinAgeInDays": "43" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj index 8189d5c56..54a710af6 100644 --- a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj +++ b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj @@ -43,13 +43,17 @@ - + + + + + @@ -81,12 +85,6 @@ 1.0.0 - - 2.25.0 - - - 2.25.0-master-30453 - diff --git a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec index 3df47101c..1dbd510b8 100644 --- a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec +++ b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/UpdateLicenseReports/Configuration/UpdateLicenseReportsConfiguration.cs b/src/UpdateLicenseReports/Configuration/UpdateLicenseReportsConfiguration.cs new file mode 100644 index 000000000..b13715eab --- /dev/null +++ b/src/UpdateLicenseReports/Configuration/UpdateLicenseReportsConfiguration.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace UpdateLicenseReports +{ + public class UpdateLicenseReportsConfiguration + { + public string LicenseReportService { get; set; } + + public string LicenseReportUser { get; set; } + + public string LicenseReportPassword { get; set; } + + public int? RetryCount { get; set; } + + public bool Test { get; set; } + } +} diff --git a/src/UpdateLicenseReports/PackageLicenseReport.cs b/src/UpdateLicenseReports/PackageLicenseReport.cs new file mode 100644 index 000000000..d8db72972 --- /dev/null +++ b/src/UpdateLicenseReports/PackageLicenseReport.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace UpdateLicenseReports +{ + internal class PackageLicenseReport + { + public int Sequence { get; set; } + + public string PackageId { get; set; } + + public string Version { get; set; } + + public string ReportUrl { get; set; } + + public string Comment { get; set; } + + public ICollection Licenses { get; private set; } + + public PackageLicenseReport(int sequence) + { + Sequence = sequence; + PackageId = null; + Version = null; + ReportUrl = null; + Comment = null; + Licenses = new LinkedList(); + } + + public override string ToString() + { + return string.Format("{{ {0}, {1}, [ {2} ] }}", Sequence, string.Join(", ", PackageId, Version, ReportUrl, Comment), string.Join(", ", Licenses)); + } + } +} diff --git a/src/UpdateLicenseReports/Scripts/UpdateLicenseReports.cmd b/src/UpdateLicenseReports/Scripts/UpdateLicenseReports.cmd index 58cf524e7..bfd0e6842 100644 --- a/src/UpdateLicenseReports/Scripts/UpdateLicenseReports.cmd +++ b/src/UpdateLicenseReports/Scripts/UpdateLicenseReports.cmd @@ -3,12 +3,15 @@ cd bin :Top - echo "Starting job - #{Jobs.updatelicensereports.Title}" +echo "Starting job - #{Jobs.updatelicensereports.Title}" - title #{Jobs.updatelicensereports.Title} +title #{Jobs.updatelicensereports.Title} - start /w updatelicensereports.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -LicenseReportService "#{Jobs.updatelicensereports.LicenseReportServiceUri}" -LicenseReportUser "#{Jobs.updatelicensereports.LicenseReportUser}" -LicenseReportPassword "#{Jobs.updatelicensereports.LicenseReportPassword}" -PackageDatabase "#{Jobs.updatelicensereports.PackageDatabase}" -verbose true -Sleep #{Jobs.updatelicensereports.Sleep} -InstrumentationKey "#{Jobs.updatelicensereports.ApplicationInsightsInstrumentationKey}" +start /w updatelicensereports.exe -Configuration #{Jobs.updatelicensereports.Configuration} ^ + -verbose true ^ + -Sleep #{Jobs.updatelicensereports.Sleep} ^ + -InstrumentationKey "#{Jobs.updatelicensereports.ApplicationInsightsInstrumentationKey}" - echo "Finished #{Jobs.updatelicensereports.Title}" +echo "Finished #{Jobs.updatelicensereports.Title}" - goto Top \ No newline at end of file +goto Top \ No newline at end of file diff --git a/src/UpdateLicenseReports/Settings/dev.json b/src/UpdateLicenseReports/Settings/dev.json new file mode 100644 index 000000000..9ba4af8e7 --- /dev/null +++ b/src/UpdateLicenseReports/Settings/dev.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "Test": true + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=UpdateLicenseReports;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbWriter.ClientId};AadCertificate=$$dev-gallerydb-writer$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/UpdateLicenseReports/Settings/int.json b/src/UpdateLicenseReports/Settings/int.json new file mode 100644 index 000000000..2a5d8f284 --- /dev/null +++ b/src/UpdateLicenseReports/Settings/int.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "Test": true + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;Integrated Security=False;User ID=$$Int-GalleryDBWriter-UserName$$;Password=$$Int-GalleryDBWriter-Password$$;Connect Timeout=30;Encrypt=True" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/UpdateLicenseReports/Settings/prod.json b/src/UpdateLicenseReports/Settings/prod.json new file mode 100644 index 000000000..4b6d7d958 --- /dev/null +++ b/src/UpdateLicenseReports/Settings/prod.json @@ -0,0 +1,18 @@ +{ + "Initialization": { + "LicenseReportService": "https://clm.sonatype.com/nuget/feed/licenses", + "LicenseReportUser": "$$Prod-Sonatype-UserName$$", + "LicenseReportPassword": "$$Prod-Sonatype-Password$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;Integrated Security=False;User ID=$$Prod-GalleryDBWriter-UserName$$;Password=$$Prod-GalleryDBWriter-Password$$;Connect Timeout=30;Encrypt=True" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/UpdateLicenseReports/TestData/LicenseServiceSampleResponse.json b/src/UpdateLicenseReports/TestData/LicenseServiceSampleResponse.json new file mode 100644 index 000000000..5f684bafb --- /dev/null +++ b/src/UpdateLicenseReports/TestData/LicenseServiceSampleResponse.json @@ -0,0 +1,13 @@ +{ + "events": [ + { + "sequence": 1, + "packageId": "Example", + "version": "1.0", + "licenses": [ "Apache 2.0", "MIT" ], + "reportUrl": "http://sample-license-service/...", + "comment": "Example package published to nuget.org" + } + ], + "next": "" +} diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Program.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Program.cs index ecd913335..a7c7c48e4 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Program.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Program.cs @@ -6,7 +6,7 @@ class Program { static void Main(string[] args) { - var job = new Job(); + var job = new UpdateLicenseReportsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.csproj b/src/UpdateLicenseReports/UpdateLicenseReports.csproj index 0d9b7210c..dfd557e1f 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.csproj +++ b/src/UpdateLicenseReports/UpdateLicenseReports.csproj @@ -46,13 +46,21 @@ - + + + + + + + + PreserveNewest + @@ -74,9 +82,6 @@ 2.0.10 - - 2.25.0-master-30453 - diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.nuspec b/src/UpdateLicenseReports/UpdateLicenseReports.nuspec index 1297be7f6..af427fb3e 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.nuspec +++ b/src/UpdateLicenseReports/UpdateLicenseReports.nuspec @@ -18,5 +18,9 @@ + + + + \ No newline at end of file diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReportsJob.cs similarity index 52% rename from src/UpdateLicenseReports/UpdateLicenseReports.Job.cs rename to src/UpdateLicenseReports/UpdateLicenseReportsJob.cs index 0a3fb2fad..7a99f4aea 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReportsJob.cs @@ -9,19 +9,25 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using Autofac; using Dapper; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; namespace UpdateLicenseReports { - internal class Job : JobBase + internal class UpdateLicenseReportsJob : JsonConfigurationJob { - private const int _defaultRetryCount = 4; + private const string LicenseServiceSampleResponseFile = @"TestData\LicenseServiceSampleResponse.json"; + + private const int DefaultRetryCount = 4; + private static readonly JSchema _sonatypeSchema = JSchema.Parse(@"{ 'type': 'object', 'properties': { 'next' : { 'type' : 'string' }, @@ -38,13 +44,6 @@ internal class Job : JobBase 'comment' : { 'type' : 'string' } } } } } }"); - private Uri _licenseReportService; - private string _licenseReportUser; - private string _licenseReportPassword; - private ISqlConnectionFactory _packageDbConnectionFactory; - private int? _retryCount; - private NetworkCredential _licenseReportCredentials; - private static PackageLicenseReport CreateReport(JObject messageEvent) { PackageLicenseReport report = new PackageLicenseReport(messageEvent["sequence"].Value()); @@ -59,42 +58,27 @@ private static PackageLicenseReport CreateReport(JObject messageEvent) return report; } + private UpdateLicenseReportsConfiguration _configuration; + private Uri _licenseService; + private NetworkCredential _licenseServiceCredentials; + private int _retryCount; + public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var dbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(dbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); - var retryCountString = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.RetryCount); - if (string.IsNullOrEmpty(retryCountString)) - { - _retryCount = _defaultRetryCount; - } - else - { - _retryCount = Convert.ToInt32(retryCountString); - } - - _licenseReportService = new Uri(JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.LicenseReportService)); - _licenseReportUser = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.LicenseReportUser); - _licenseReportPassword = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.LicenseReportPassword); + _configuration = _serviceProvider.GetRequiredService>().Value; - // Build credentials - if (!string.IsNullOrEmpty(_licenseReportUser)) - { - if (!string.IsNullOrEmpty(_licenseReportPassword)) - { - _licenseReportCredentials = new NetworkCredential(_licenseReportUser, _licenseReportPassword); - } - else - { - _licenseReportCredentials = new NetworkCredential(_licenseReportUser, string.Empty); - } - } - else if (!string.IsNullOrEmpty(_licenseReportPassword)) + if (!_configuration.Test) { - _licenseReportCredentials = new NetworkCredential(string.Empty, _licenseReportPassword); + _licenseService = new Uri(_configuration.LicenseReportService); + + _licenseServiceCredentials = new NetworkCredential( + _configuration.LicenseReportUser ?? string.Empty, + _configuration.LicenseReportPassword ?? string.Empty); } + + _retryCount = _configuration.RetryCount ?? DefaultRetryCount; } public override async Task Run() @@ -111,12 +95,12 @@ public override async Task Run() private async Task FetchNextReportUrlAsync() { - Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); - Uri nextLicenseReport = null; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { + Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", + connection.DataSource, connection.Database); + var nextReportUrl = (await connection.QueryAsync( @"SELECT TOP 1 NextLicenseReport FROM GallerySettings")).SingleOrDefault(); @@ -129,29 +113,40 @@ private async Task FetchNextReportUrlAsync() Logger.LogInformation("Next Report URL '{NextReportUrl}' is invalid. Using default", nextReportUrl); } - nextLicenseReport = nextLicenseReport ?? _licenseReportService; - } + nextLicenseReport = nextLicenseReport ?? _licenseService; - Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", - (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", + (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), + connection.DataSource, connection.Database); + } return nextLicenseReport; } + private Task DownloadNextReportAsync(Uri nextLicenseReport) + { + return _configuration.Test + ? Task.FromResult(GetExampleReportAsync()) + : DownloadNextReportFromServiceAsync(nextLicenseReport); + } - private async Task ProcessReportsAsync(Uri nextLicenseReport) + private string GetExampleReportAsync() + { + return File.ReadAllText(LicenseServiceSampleResponseFile); + } + + private async Task DownloadNextReportFromServiceAsync(Uri nextLicenseReport) { HttpWebResponse response = null; var tries = 0; Logger.LogInformation("Downloading license report {ReportUrl}", nextLicenseReport.AbsoluteUri); - while (tries < _retryCount.Value && response == null) + while (tries < _retryCount && response == null) { var request = (HttpWebRequest)WebRequest.Create(nextLicenseReport); - if (_licenseReportCredentials != null) + if (_licenseServiceCredentials != null) { - request.Credentials = _licenseReportCredentials; + request.Credentials = _licenseServiceCredentials; } WebException thrown = null; @@ -166,7 +161,7 @@ private async Task ProcessReportsAsync(Uri nextLicenseReport) { // Try again in 10 seconds tries++; - if (tries < _retryCount.Value) + if (tries < _retryCount) { thrown = ex; } @@ -197,84 +192,101 @@ private async Task ProcessReportsAsync(Uri nextLicenseReport) { Logger.LogInformation("Reading license report {ReportUrl}", nextLicenseReport.AbsoluteUri); - string content; using (var reader = new StreamReader(response.GetResponseStream())) { - content = await reader.ReadToEndAsync(); + return await reader.ReadToEndAsync(); } + } + else if (response.StatusCode != HttpStatusCode.NoContent) + { + Logger.LogInformation("No report for {NextReportUrl} yet.", nextLicenseReport.AbsoluteUri); + } + else + { + Logger.LogInformation("HTTP {StatusCode} error requesting {NextReportUrl}: {StatusDescription}", response.StatusCode, nextLicenseReport.AbsoluteUri, response.StatusDescription); + } + } - Logger.LogInformation("Read license report {ReportUrl}", nextLicenseReport.AbsoluteUri); - - var sonatypeMessage = JObject.Parse(content); - if (!sonatypeMessage.IsValid(_sonatypeSchema)) - { - Logger.LogInformation("Invalid license report in {ReportUrl}. {Error}", nextLicenseReport.AbsoluteUri, Strings.UpdateLicenseReportsJob_JsonDoesNotMatchSchema); - return false; - } + return null; + } - var events = sonatypeMessage["events"].Cast().ToList(); - foreach (var messageEvent in events) - { - var report = CreateReport(messageEvent); + private async Task ProcessReportsAsync(Uri nextLicenseReport) + { + var content = await DownloadNextReportAsync(nextLicenseReport); - Logger.LogInformation("Storing license report for {PackageId} {PackageVersion}", report.PackageId, report.Version); + if (content == null) + { + return false; + } - if (await StoreReportAsync(report) == -1) - { - Logger.LogInformation("Unable to store report for {PackageId} {PackageVersion}. Package does not exist in database.", report.PackageId, report.Version); - } - else - { - Logger.LogInformation("Stored license report for {PackageId} {PackageVersion}", report.PackageId, report.Version); - } - } + Logger.LogInformation("Read license report {ReportUrl}", nextLicenseReport.AbsoluteUri); - // Store the next URL - if (sonatypeMessage["next"].Value().Length > 0) - { - var nextReportUrl = sonatypeMessage["next"].Value(); - if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport)) - { - Logger.LogInformation("Invalid next report URL: {NextReportUrl}", nextReportUrl); - return false; - } + var sonatypeMessage = JObject.Parse(content); + if (!sonatypeMessage.IsValid(_sonatypeSchema)) + { + Logger.LogInformation("Invalid license report in {ReportUrl}. {Error}", nextLicenseReport.AbsoluteUri, Strings.UpdateLicenseReportsJob_JsonDoesNotMatchSchema); + return false; + } - Logger.LogInformation("Storing next license report URL: {NextReportUrl}", nextLicenseReport.AbsoluteUri); + var events = sonatypeMessage["events"].Cast().ToList(); + foreach (var messageEvent in events) + { + var report = CreateReport(messageEvent); - // Record the next report to the database so we can check it again if we get aborted before finishing. - using (var connection = await _packageDbConnectionFactory.CreateAsync()) - { - await connection.QueryAsync(@" - UPDATE GallerySettings - SET NextLicenseReport = @nextLicenseReport", - new { nextLicenseReport = nextLicenseReport.AbsoluteUri }); - } + if (_configuration.Test) + { + Logger.LogInformation("Test complete for {PackageId} {PackageVersion}.", report.PackageId, report.Version); + return false; + } - return true; // Continue and read the next report later - } - else - { - nextLicenseReport = null; - } + Logger.LogInformation("Storing license report for {PackageId} {PackageVersion}", report.PackageId, report.Version); - Logger.LogInformation("Processing license report {NextReportUrl}", nextLicenseReport.AbsoluteUri); - } - else if (response.StatusCode != HttpStatusCode.NoContent) + if (await StoreReportAsync(report) == -1) { - Logger.LogInformation("No report for {NextReportUrl} yet.", nextLicenseReport.AbsoluteUri); + Logger.LogInformation("Unable to store report for {PackageId} {PackageVersion}. Package does not exist in database.", report.PackageId, report.Version); } else { - Logger.LogInformation("HTTP {StatusCode} error requesting {NextReportUrl}: {StatusDescription}", response.StatusCode, nextLicenseReport.AbsoluteUri, response.StatusDescription); + Logger.LogInformation("Stored license report for {PackageId} {PackageVersion}", report.PackageId, report.Version); } + } - return false; + // Store the next URL + if (sonatypeMessage["next"].Value().Length > 0) + { + var nextReportUrl = sonatypeMessage["next"].Value(); + if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport)) + { + Logger.LogInformation("Invalid next report URL: {NextReportUrl}", nextReportUrl); + return false; + } + + Logger.LogInformation("Storing next license report URL: {NextReportUrl}", nextLicenseReport.AbsoluteUri); + + // Record the next report to the database so we can check it again if we get aborted before finishing. + using (var connection = await OpenSqlConnectionAsync()) + { + await connection.QueryAsync(@" + UPDATE GallerySettings + SET NextLicenseReport = @nextLicenseReport", + new { nextLicenseReport = nextLicenseReport.AbsoluteUri }); + } + + return true; // Continue and read the next report later } + else + { + nextLicenseReport = null; + } + + Logger.LogInformation("Processing license report {NextReportUrl}", nextLicenseReport.AbsoluteUri); + + return true; } private async Task StoreReportAsync(PackageLicenseReport report) { - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) using (var command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport2"; @@ -299,35 +311,13 @@ private async Task StoreReportAsync(PackageLicenseReport report) } } - private class PackageLicenseReport + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) { - public int Sequence { get; set; } - - public string PackageId { get; set; } - - public string Version { get; set; } - - public string ReportUrl { get; set; } - - public string Comment { get; set; } - - public ICollection Licenses { get; private set; } - - public PackageLicenseReport(int sequence) - { - Sequence = sequence; - PackageId = null; - Version = null; - ReportUrl = null; - Comment = null; - Licenses = new LinkedList(); - } - - public override string ToString() - { - return string.Format("{{ {0}, {1}, [ {2} ] }}", Sequence, string.Join(", ", PackageId, Version, ReportUrl, Comment), string.Join(", ", Licenses)); - } } + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } } } diff --git a/src/Validation.Common.Job/SubcriptionProcessorJob.cs b/src/Validation.Common.Job/SubcriptionProcessorJob.cs index 38d24c525..f05231b56 100644 --- a/src/Validation.Common.Job/SubcriptionProcessorJob.cs +++ b/src/Validation.Common.Job/SubcriptionProcessorJob.cs @@ -12,7 +12,7 @@ namespace NuGet.Jobs.Validation { - public abstract class SubcriptionProcessorJob : JsonConfigurationJob + public abstract class SubcriptionProcessorJob : ValidationJobBase { private const string SubscriptionProcessorConfigurationSectionName = "ServiceBus"; diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index ccf242593..af94ffb50 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -61,7 +61,7 @@ - + diff --git a/src/Validation.Common.Job/ValidationJobBase.cs b/src/Validation.Common.Job/ValidationJobBase.cs new file mode 100644 index 000000000..f6681f350 --- /dev/null +++ b/src/Validation.Common.Job/ValidationJobBase.cs @@ -0,0 +1,102 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Net; +using System.Net.Http; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Configuration; +using NuGet.Services.ServiceBus; +using NuGet.Services.Validation; +using NuGetGallery; +using NuGetGallery.Diagnostics; + +namespace NuGet.Jobs.Validation +{ + public abstract class ValidationJobBase : JsonConfigurationJob + { + private const string PackageDownloadTimeoutName = "PackageDownloadTimeout"; + + /// + /// The maximum number of concurrent connections that can be established to a single server. + /// + private const int MaximumConnectionsPerServer = 64; + + public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) + { + base.Init(serviceContainer, jobArgsDictionary); + + ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer; + } + + protected override void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + base.ConfigureDefaultJobServices(services, configurationRoot); + + ConfigureDatabaseServices(services); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddTransient(c => + { + var configurationAccessor = c.GetRequiredService>(); + return new CloudBlobClientWrapper( + configurationAccessor.Value.ConnectionString, + readAccessGeoRedundant: false); + }); + services.AddTransient(); + + services.AddTransient(p => + { + var config = p.GetRequiredService>().Value; + + return new SubscriptionClientWrapper(config.ConnectionString, config.TopicPath, config.SubscriptionName); + }); + + services.AddSingleton(p => + { + var assembly = Assembly.GetEntryAssembly(); + var assemblyName = assembly.GetName().Name; + var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; + + var client = new HttpClient(new WebRequestHandler + { + AllowPipelining = true, + AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), + }); + + client.Timeout = configurationRoot.GetValue(PackageDownloadTimeoutName); + client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); + + return client; + }); + } + + private void ConfigureDatabaseServices(IServiceCollection services) + { + services.AddScoped(p => + { + var connectionFactory = p.GetRequiredService>(); + var connection = connectionFactory.CreateAsync().GetAwaiter().GetResult(); + + return new ValidationEntitiesContext(connection); + }); + + services.AddScoped(p => + { + var connectionFactory = p.GetRequiredService>(); + var connection = connectionFactory.CreateAsync().GetAwaiter().GetResult(); + + return new EntitiesContext(connection, readOnly: true); + }); + } + } +} diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 16997f935..14f9e5bdc 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -35,7 +35,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: false); + return new EntitiesContext(CreateSqlConnection(), readOnly: false); }); services.Add(ServiceDescriptor.Transient(typeof(IEntityRepository<>), typeof(EntityRepository<>))); diff --git a/src/Validation.PackageSigning.RevalidateCertificate/Job.cs b/src/Validation.PackageSigning.RevalidateCertificate/Job.cs index 44d49fb6e..ec5e49107 100644 --- a/src/Validation.PackageSigning.RevalidateCertificate/Job.cs +++ b/src/Validation.PackageSigning.RevalidateCertificate/Job.cs @@ -15,7 +15,7 @@ namespace Validation.PackageSigning.RevalidateCertificate { - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string RevalidationConfigurationSectionName = "RevalidateJob"; diff --git a/tests/NuGet.Jobs.Common.Tests/JobBaseFacts.cs b/tests/NuGet.Jobs.Common.Tests/JobBaseFacts.cs new file mode 100644 index 000000000..9b5e58fe8 --- /dev/null +++ b/tests/NuGet.Jobs.Common.Tests/JobBaseFacts.cs @@ -0,0 +1,157 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Jobs.Configuration; +using NuGet.Services.KeyVault; +using Xunit; + +namespace NuGet.Jobs.Common.Tests +{ + public class JobBaseFacts + { + private const string DefaultGalleryDbConnectionString = + "Data Source=(localdb)\\mssqllocaldb; Initial Catalog=NuGetGallery; Integrated Security=True; MultipleActiveResultSets=True"; + + private const string DefaultValidationDbConnectionString = + "Data Source=(localdb)\\mssqllocaldb; Initial Catalog=Validation; Integrated Security=True; MultipleActiveResultSets=True"; + + public class TheRegisterDatabaseMethod + { + [Fact] + private void ReturnsConnectionStringBuilder_GalleryDb() + { + // Arrange + var job = new TestJob(); + + // Act + var csBuilder = job.RegisterDatabase( + job.ServiceContainer, + testConnection: false); + + // Assert + Assert.Equal("(localdb)\\mssqllocaldb", csBuilder.DataSource); + Assert.Equal("NuGetGallery", csBuilder.InitialCatalog); + } + + [Fact] + private void ReturnsConnectionStringBuilder_ValidationDb() + { + // Arrange + var job = new TestJob(); + + // Act + var csBuilder = job.RegisterDatabase( + job.ServiceContainer, + testConnection: false); + + // Assert + Assert.Equal("(localdb)\\mssqllocaldb", csBuilder.DataSource); + Assert.Equal("Validation", csBuilder.InitialCatalog); + } + + [Fact] + private void DoesNotOverwriteRegistrations() + { + // Arrange + var job = new TestJob(); + + // Act + job.RegisterDatabase( + job.ServiceContainer, + testConnection: false); + + job.RegisterDatabase( + job.ServiceContainer, + testConnection: false); + + // Assert + var galleryDb = job.GetDatabaseRegistration(); + var validationDb = job.GetDatabaseRegistration(); + + Assert.NotNull(galleryDb); + Assert.Equal("NuGetGallery", galleryDb.InitialCatalog); + + Assert.NotNull(validationDb); + Assert.Equal("Validation", validationDb.InitialCatalog); + } + } + + public class TestJob : JobBase + { + public IServiceContainer ServiceContainer + { + get => MockServiceContainer.Object; + } + + public Mock MockServiceContainer { get; } + + public TestJob() + { + var mockSecretInjector = new Mock(); + + var galleryOptionsSnapshot = CreateMockOptionsSnapshot( + new GalleryDbConfiguration { + ConnectionString = DefaultGalleryDbConnectionString + }); + + var validationOptionsSnapshot = CreateMockOptionsSnapshot( + new ValidationDbConfiguration + { + ConnectionString = DefaultValidationDbConnectionString + }); + + MockServiceContainer = new Mock(); + + MockServiceContainer + .Setup(x => x.GetService(It.IsAny())) + .Returns(serviceType => + { + if (serviceType == typeof(ISecretInjector)) + { + return mockSecretInjector.Object; + } + else if (serviceType == typeof(IOptionsSnapshot)) + { + return galleryOptionsSnapshot.Object; + } + else if (serviceType == typeof(IOptionsSnapshot)) + { + return validationOptionsSnapshot.Object; + } + else + { + throw new InvalidOperationException($"Unexpected service lookup: {serviceType.Name}"); + } + }); + } + + private Mock> CreateMockOptionsSnapshot(TDbConfiguration configuration) + { + var mockOptionsSnapshot = new Mock>(); + + mockOptionsSnapshot + .Setup(x => x.Value) + .Returns(configuration); + + return mockOptionsSnapshot; + } + + public override void Init( + IServiceContainer serviceContainer, + IDictionary jobArgsDictionary) + { + } + + public override Task Run() + { + return Task.CompletedTask; + } + } + } +} diff --git a/tests/NuGet.Jobs.Common.Tests/NuGet.Jobs.Common.Tests.csproj b/tests/NuGet.Jobs.Common.Tests/NuGet.Jobs.Common.Tests.csproj new file mode 100644 index 000000000..68cf4d3a2 --- /dev/null +++ b/tests/NuGet.Jobs.Common.Tests/NuGet.Jobs.Common.Tests.csproj @@ -0,0 +1,67 @@ + + + + + Debug + AnyCPU + {CE96428B-8138-4914-9999-2B391797FFF8} + Library + Properties + NuGet.Jobs.Common.Tests + NuGet.Jobs.Common.Tests + v4.6.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + 4.7.145 + + + 2.3.1 + + + 2.3.1 + runtime; build; native; contentfiles; analyzers + all + + + + + {4b4b1efb-8f33-42e6-b79f-54e7f3293d31} + NuGet.Jobs.Common + + + + \ No newline at end of file diff --git a/tests/NuGet.Jobs.Common.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Jobs.Common.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..d37ace1ca --- /dev/null +++ b/tests/NuGet.Jobs.Common.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Jobs.Common.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("ce96428b-8138-4914-9999-2b391797fff8")] \ No newline at end of file diff --git a/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs index 8594f9cda..821f26610 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Moq; using NuGet.Services.Validation; @@ -22,7 +23,7 @@ public class TheInitializeAsyncMethod : FactsBase public async Task ThrowsIfAlreadyInitialized() { // Arrange - _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); + _jobState.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); // Act & Assert var e = await Assert.ThrowsAsync(() => _target.InitializeAsync()); @@ -39,8 +40,8 @@ public async Task RemovesPreviousRevalidations() _packageFinder.Setup(f => f.FindDependencyPackages(It.IsAny>())).Returns(new HashSet()); _packageFinder.Setup(f => f.FindAllPackages(It.IsAny>())).Returns(new HashSet()); - _packageFinder.Setup(f => f.FindPackageRegistrationInformation(It.IsAny(), It.IsAny>())) - .Returns(new List()); + _packageFinder.Setup(f => f.FindPackageRegistrationInformationAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(new List()); var firstRemove = true; @@ -223,6 +224,10 @@ public async Task PartitionsPackagesIntoBatchesOf1000OrLessVersions(int[] packag _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Exactly(expectedBatches)); + + // A scope should be created for each package set. Also, a scope should be created + // for each batch. + _scopeFactory.Verify(f => f.CreateScope(), Times.Exactly(4 + expectedBatches)); } public static IEnumerable PartitionsPackagesIntoBatchesOf1000OrLessVersionsData() @@ -230,32 +235,32 @@ public static IEnumerable PartitionsPackagesIntoBatchesOf1000OrLessVer yield return new object[] { new[] { 1001 }, - 1 + 1, }; yield return new object[] { new[] { 1, 1001, 1 }, - 3 + 3, }; // Should be batched into two batches of 501 items. yield return new object[] { new[] { 1, 500, 500, 1 }, - 2 + 2, }; yield return new object[] { new[] { 500, 500 }, - 1 + 1, }; yield return new object[] { Enumerable.Repeat(1, 1000).ToArray(), - 1 + 1, }; } @@ -277,7 +282,7 @@ public async Task MarksAsInitializedAfterAddingRevalidations() .Callback(() => addRevalidationOrder = order++) .Returns(Task.CompletedTask); - _settings + _jobState .Setup(s => s.MarkAsInitializedAsync()) .Callback(() => markAsInitializedOrder = order++) .Returns(Task.CompletedTask); @@ -285,7 +290,7 @@ public async Task MarksAsInitializedAfterAddingRevalidations() // Act & Assert await _target.InitializeAsync(); - _settings.Verify(s => s.MarkAsInitializedAsync(), Times.Once); + _jobState.Verify(s => s.MarkAsInitializedAsync(), Times.Once); Assert.True(markAsInitializedOrder > addRevalidationOrder); } @@ -339,20 +344,20 @@ List RegistrationInformation(IEnumerable f.FindPackageRegistrationInformation(PackageFinder.MicrosoftSetName, It.IsAny>())) - .Returns(RegistrationInformation(microsoftPackages)); + .Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.MicrosoftSetName, It.IsAny>())) + .ReturnsAsync(RegistrationInformation(microsoftPackages)); _packageFinder - .Setup(f => f.FindPackageRegistrationInformation(PackageFinder.PreinstalledSetName, It.IsAny>())) - .Returns(RegistrationInformation(preinstalledPackages)); + .Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.PreinstalledSetName, It.IsAny>())) + .ReturnsAsync(RegistrationInformation(preinstalledPackages)); _packageFinder - .Setup(f => f.FindPackageRegistrationInformation(PackageFinder.DependencySetName, It.IsAny>())) - .Returns(RegistrationInformation(dependencyPackages)); + .Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.DependencySetName, It.IsAny>())) + .ReturnsAsync(RegistrationInformation(dependencyPackages)); _packageFinder - .Setup(f => f.FindPackageRegistrationInformation(PackageFinder.RemainingSetName, It.IsAny>())) - .Returns(RegistrationInformation(remainingPackages)); + .Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.RemainingSetName, It.IsAny>())) + .ReturnsAsync(RegistrationInformation(remainingPackages)); // Build the list of versions for each version of packages. var versions = microsoftPackages @@ -396,7 +401,7 @@ public class TheVerifyAsyncMethod : FactsBase [Fact] public async Task ThrowsIfNotInitialized() { - _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(false); + _jobState.Setup(s => s.IsInitializedAsync()).ReturnsAsync(false); var e = await Assert.ThrowsAsync(() => _target.VerifyInitializationAsync()); @@ -406,7 +411,7 @@ public async Task ThrowsIfNotInitialized() [Fact] public async Task ThrowsIfAppropriatePackageCountDoesNotMatchRevalidationCount() { - _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); + _jobState.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); _packageFinder.Setup(f => f.AppropriatePackageCount()).Returns(100); _packageState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(50); @@ -418,7 +423,7 @@ public async Task ThrowsIfAppropriatePackageCountDoesNotMatchRevalidationCount() [Fact] public async Task DoesNotThrowIfCountsMatch() { - _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); + _jobState.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); _packageFinder.Setup(f => f.AppropriatePackageCount()).Returns(100); _packageState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(100); @@ -428,25 +433,39 @@ public async Task DoesNotThrowIfCountsMatch() public class FactsBase { - public readonly Mock _settings; + public readonly Mock _jobState; public readonly Mock _packageState; public readonly Mock _packageFinder; + public readonly Mock _scopeFactory; public readonly InitializationConfiguration _config; public readonly InitializationManager _target; public FactsBase() { - _settings = new Mock(); + _jobState = new Mock(); _packageState = new Mock(); _packageFinder = new Mock(); + _scopeFactory = new Mock(); + + var scope = new Mock(); + var serviceProvider = new Mock(); + + serviceProvider.Setup(p => p.GetService(typeof(IRevalidationJobStateService))).Returns(_jobState.Object); + serviceProvider.Setup(p => p.GetService(typeof(IPackageRevalidationStateService))).Returns(_packageState.Object); + serviceProvider.Setup(p => p.GetService(typeof(IPackageFinder))).Returns(_packageFinder.Object); + serviceProvider.Setup(p => p.GetService(typeof(IServiceScopeFactory))).Returns(_scopeFactory.Object); + + scope.Setup(s => s.ServiceProvider).Returns(serviceProvider.Object); + _scopeFactory.Setup(s => s.CreateScope()).Returns(scope.Object); _config = new InitializationConfiguration(); _target = new InitializationManager( - _settings.Object, + _jobState.Object, _packageState.Object, _packageFinder.Object, + _scopeFactory.Object, _config, Mock.Of>()); } diff --git a/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs index 7127d9d14..01b567e39 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Moq; using NuGet.Versioning; @@ -207,7 +209,7 @@ public void ExcludesIdsFromExceptParameter() public class TheFindPackageRegistrationInformationMethod : FactsBase { [Fact] - public void FindsRegistrationInformation() + public async Task FindsRegistrationInformation() { // Arrange // Package registration 3 isn't in the input set and should be ignored. @@ -249,7 +251,7 @@ public void FindsRegistrationInformation() }); // Act and assert - var actual = _target.FindPackageRegistrationInformation("Name", new HashSet { 1, 2 }); + var actual = await _target.FindPackageRegistrationInformationAsync("Name", new HashSet { 1, 2 }); Assert.Equal(2, actual.Count); Assert.Equal(1, actual[0].Key); @@ -264,7 +266,7 @@ public void FindsRegistrationInformation() } [Fact] - public void PackageCountDoesntFilterPackagesThatWontBeRevalidated() + public async Task PackageCountDoesntFilterPackagesThatWontBeRevalidated() { // Arrange // Package registration 3 isn't in the input set and should be ignored. @@ -294,7 +296,7 @@ public void PackageCountDoesntFilterPackagesThatWontBeRevalidated() }); // Act & Assert - var actual = _target.FindPackageRegistrationInformation("Name", new HashSet { 1 }); + var actual = await _target.FindPackageRegistrationInformationAsync("Name", new HashSet { 1 }); Assert.Single(actual); Assert.Equal(1, actual[0].Key); @@ -464,6 +466,7 @@ public void SkipsPackagesThatAreNotAvailableOrDeleted() public class FactsBase { public readonly Mock _context; + public readonly Mock _scopeFactory; public readonly InitializationConfiguration _config; public readonly PackageFinder _target; @@ -471,10 +474,19 @@ public class FactsBase public FactsBase() { _context = new Mock(); + _scopeFactory = new Mock(); _config = new InitializationConfiguration(); + var scope = new Mock(); + var serviceProvider = new Mock(); + + _scopeFactory.Setup(s => s.CreateScope()).Returns(scope.Object); + scope.Setup(s => s.ServiceProvider).Returns(serviceProvider.Object); + serviceProvider.Setup(p => p.GetService(typeof(IEntitiesContext))).Returns(_context.Object); + _target = new PackageFinder( _context.Object, + _scopeFactory.Object, _config, Mock.Of>()); } diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs index 69e976fb4..4e6f23d75 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs @@ -15,29 +15,6 @@ namespace NuGet.Services.Revalidate.Tests.Services { public class PackageRevalidationStateServiceFacts { - public class TheAddPackageRevalidationsAsyncMethod : FactsBase - { - [Fact] - public async Task AddsRevalidations() - { - // Arrange - var revalidations = new List - { - new PackageRevalidation { PackageId = "A" }, - new PackageRevalidation { PackageId = "B" } - }; - - _context.Mock(); - - // Act & Assert - await _target.AddPackageRevalidationsAsync(revalidations); - - Assert.Equal(2, _context.Object.PackageRevalidations.Count()); - - _context.Verify(c => c.SaveChangesAsync(), Times.Once); - } - } - public class TheRemoveRevalidationsAsyncMethod : FactsBase { [Fact] @@ -124,6 +101,7 @@ public FactsBase() _target = new PackageRevalidationStateService( _context.Object, + Mock.Of(), Mock.Of>()); } } diff --git a/tests/Tests.CredentialExpiration/TestCredentialExpiration.cs b/tests/Tests.CredentialExpiration/TestCredentialExpiration.cs index 97a837017..5670d54b8 100644 --- a/tests/Tests.CredentialExpiration/TestCredentialExpiration.cs +++ b/tests/Tests.CredentialExpiration/TestCredentialExpiration.cs @@ -23,7 +23,7 @@ public TestCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, Lis { _jobMetadata = jobMetadata; _credentialSet = credentialSet; - _galleryCredentialsExpiration = new GalleryCredentialExpiration(jobMetadata, null); + _galleryCredentialsExpiration = new GalleryCredentialExpiration(null, jobMetadata); _maxNotificationDate = _galleryCredentialsExpiration.GetMaxNotificationDate(); _minNotificationDate = _galleryCredentialsExpiration.GetMinNotificationDate(); } diff --git a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs index 9a202115a..b3386ccad 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs @@ -7,7 +7,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -45,7 +44,7 @@ private static RankingsExporter CreateExporter() { return new RankingsExporter( new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, + openSqlConnectionAsync: () => null, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultRankingsScript: "b", defaultName: "c"); diff --git a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs index 50b8aea23..a16001b10 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs @@ -7,7 +7,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -44,7 +43,7 @@ private static VerifiedPackagesExporter CreateExporter() { return new VerifiedPackagesExporter( new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, + openSqlConnectionAsync: () => null, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultVerifiedPackagesScript: "b", defaultName: "c"); diff --git a/tests/Tests.Stats.AggregateCdnDownloadsInGallery/JobTests.cs b/tests/Tests.Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJobTests.cs similarity index 95% rename from tests/Tests.Stats.AggregateCdnDownloadsInGallery/JobTests.cs rename to tests/Tests.Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJobTests.cs index e6f1e7326..fc381ad40 100644 --- a/tests/Tests.Stats.AggregateCdnDownloadsInGallery/JobTests.cs +++ b/tests/Tests.Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJobTests.cs @@ -8,7 +8,7 @@ namespace Stats.AggregateCdnDownloadsInGallery { - public class JobTests + public class AggregateCdnDownloadsJobTests { public class PopGroupBatch @@ -55,7 +55,7 @@ public void ReturnsCorrectNumberOfPackageRegistrationGroups(int batchSize, int e }.GroupBy(x => x.PackageId).OrderByDescending(x => x.Key)); // Act - var groupBatch = Job.PopGroupBatch(data, batchSize); + var groupBatch = AggregateCdnDownloadsJob.PopGroupBatch(data, batchSize); // Assert Assert.Equal(expectedGroupCount, groupBatch.Count); diff --git a/tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj b/tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj index c6d27e4ac..7d08e4cf0 100644 --- a/tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj +++ b/tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj @@ -45,7 +45,7 @@ - + diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs index 0c3bdd2f0..c5682c7af 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs @@ -311,9 +311,12 @@ public async Task EnqueuesScanAndSignEvenIfRepositorySigningIsDisabled() _request.NupkgUrl, _config.V3ServiceIndexUrl, It.Is>(l => - l.Count() == 2 && - l.Contains("Billy") && - l.Contains("Bob"))), + // Ensure that the owners are lexicographically ordered. + l.Count() == 4 && + l[0] == "Annie" && + l[1] == "Bob" && + l[2] == "zack" && + l[3] == "Zorro")), Times.Once); _validatorStateServiceMock