Merge pull request #168 from NuGet/dev

[ReleasePrep][2017.04.28]RI of dev into master

src/Catalog/CollectorHttpClient.cs

@@ -1,13 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using Newtonsoft.Json.Linq;
 using System;
 using System.IO;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using VDS.RDF;
 
 namespace NuGet.Services.Metadata.Catalog
@@ -72,17 +72,17 @@ private JObject ParseJObject(string json)
 
         public virtual Task<IGraph> GetGraphAsync(Uri address)
         {
-            return GetGraphAsync(address, CancellationToken.None);
+            return GetGraphAsync(address, readOnly: false, token: CancellationToken.None);
         }
 
-        public virtual Task<IGraph> GetGraphAsync(Uri address, CancellationToken token)
+        public virtual Task<IGraph> GetGraphAsync(Uri address, bool readOnly, CancellationToken token)
         {
             var task = GetJObjectAsync(address, token);
             return task.ContinueWith((t) =>
             {
                 try
                 {
-                    return Utils.CreateGraph(t.Result);
+                    return Utils.CreateGraph(t.Result, readOnly);
                 }
                 catch (Exception e)
                 {

src/Catalog/Helpers/Utils.cs

@@ -1,5 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
@@ -243,13 +244,23 @@ public static IGraph CreateGraph(string json)
         }
 
         public static IGraph CreateGraph(JToken compacted)
+        {
+            return CreateGraph(compacted, readOnly: false);
+        }
+
+        public static IGraph CreateGraph(JToken compacted, bool readOnly)
         {
             JToken flattened = JsonLdProcessor.Flatten(compacted, new JsonLdOptions());
 
             IRdfReader rdfReader = new JsonLdReader();
             IGraph graph = new Graph();
             rdfReader.Load(graph, new StringReader(flattened.ToString(Newtonsoft.Json.Formatting.None, new Newtonsoft.Json.JsonConverter[0])));
 
+            if (readOnly)
+            {
+                graph = new ReadOnlyGraph(graph);
+            }
+
             return graph;
         }
 

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -115,6 +115,7 @@
     <Compile Include="CommitCollector.cs" />
     <Compile Include="CommitMetadata.cs" />
     <Compile Include="Helpers\NuGetVersionUtility.cs" />
+    <Compile Include="ReadOnlyGraph.cs" />
     <Compile Include="Persistence\StorageListItem.cs" />
     <Compile Include="StringInterner.cs" />
     <Compile Include="Utilities.cs" />

src/Catalog/ReadOnlyGraph.cs

@@ -0,0 +1,69 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using VDS.RDF;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// An immutable graph supporting multi-threaded read-only access. According to dotNetRDF documentation, the
+    /// <see cref="Graph"/> class is thread-safe when used in a read-only manner.
+    /// <see cref="http://www.dotnetrdf.org/api/html/T_VDS_RDF_Graph.htm"/>
+    /// </summary>
+    public class ReadOnlyGraph : Graph
+    {
+        private readonly bool _isReadOnly;
+
+        public ReadOnlyGraph(IGraph graph)
+        {
+            Merge(graph);
+            _isReadOnly = true;
+        }
+
+        public override bool Assert(IEnumerable<Triple> ts)
+        {
+            ThrowIfReadOnly();
+            return base.Assert(ts);
+        }
+
+        public override bool Assert(Triple t)
+        {
+            ThrowIfReadOnly();
+            return base.Assert(t);
+        }
+
+        public override bool Retract(IEnumerable<Triple> ts)
+        {
+            ThrowIfReadOnly();
+            return base.Retract(ts);
+        }
+
+        public override bool Retract(Triple t)
+        {
+            ThrowIfReadOnly();
+            return base.Retract(t);
+        }
+
+        public override void Merge(IGraph g)
+        {
+            ThrowIfReadOnly();
+            base.Merge(g);
+        }
+
+        public override void Merge(IGraph g, bool keepOriginalGraphUri)
+        {
+            ThrowIfReadOnly();
+            base.Merge(g, keepOriginalGraphUri);
+        }
+
+        private void ThrowIfReadOnly()
+        {
+            if (_isReadOnly)
+            {
+                throw new NotSupportedException("This RDF graph cannot be modified.");
+            }
+        }
+    }
+}

src/Catalog/Registration/RegistrationCollector.cs

@@ -85,7 +85,9 @@ protected override async Task ProcessGraphs(
             KeyValuePair<string, IDictionary<string, IGraph>> sortedGraphs,
             CancellationToken cancellationToken)
         {
-            await RegistrationMaker.Process(
+            var tasks = new List<Task>();
+
+            var legacyTask = RegistrationMaker.Process(
                 registrationKey: new RegistrationKey(sortedGraphs.Key),
                 newItems: sortedGraphs.Value,
                 shouldInclude: _shouldIncludeSemVer2,
@@ -94,18 +96,22 @@ await RegistrationMaker.Process(
                 partitionSize: PartitionSize,
                 packageCountThreshold: PackageCountThreshold,
                 cancellationToken: cancellationToken);
+            tasks.Add(legacyTask);
 
             if (_semVer2StorageFactory != null)
             {
-                await RegistrationMaker.Process(
+                var semVer2Task = RegistrationMaker.Process(
                    registrationKey: new RegistrationKey(sortedGraphs.Key),
                    newItems: sortedGraphs.Value,
                    storageFactory: _semVer2StorageFactory,
                    contentBaseAddress: ContentBaseAddress,
                    partitionSize: PartitionSize,
                    packageCountThreshold: PackageCountThreshold,
                    cancellationToken: cancellationToken);
+                tasks.Add(semVer2Task);
             }
+
+            await Task.WhenAll(tasks);
         }
 
         public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPackage(StorageFactory semVer2StorageFactory)

src/Catalog/Registration/RegistrationEntryKey.cs

@@ -1,18 +1,23 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using NuGet.Services.Metadata.Catalog.Helpers;
+
 namespace NuGet.Services.Metadata.Catalog.Registration
 {
     public class RegistrationEntryKey
     {
+        private readonly string _normalizedVersion;
+
         public RegistrationEntryKey(RegistrationKey registrationKey, string version)
         {
             RegistrationKey = registrationKey;
             Version = version;
+            _normalizedVersion = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant();
         }
 
-        public RegistrationKey RegistrationKey { get; set; }
-        public string Version { get; set; }
+        public RegistrationKey RegistrationKey { get; }
+        public string Version { get; }
 
         public override string ToString()
         {
@@ -21,7 +26,7 @@ public override string ToString()
 
         public override int GetHashCode()
         {
-            return ToString().GetHashCode();
+            return $"{RegistrationKey}/{_normalizedVersion}".GetHashCode();
         }
 
         public override bool Equals(object obj)
@@ -33,7 +38,8 @@ public override bool Equals(object obj)
                 return false;
             }
 
-            return (RegistrationKey.Equals(rhs.RegistrationKey)) && (Version == rhs.Version); 
+            return RegistrationKey.Equals(rhs.RegistrationKey) &&
+                   _normalizedVersion == rhs._normalizedVersion; 
         }
     }
 }

src/Catalog/Registration/RegistrationKey.cs

@@ -13,7 +13,7 @@ public RegistrationKey(string id)
             Id = id;
         }
 
-        public string Id { get; set; }
+        public string Id { get; }
 
         public override string ToString()
         {
@@ -34,7 +34,7 @@ public override bool Equals(object obj)
                 return false;
             }
 
-            return (Id == rhs.Id); 
+            return StringComparer.OrdinalIgnoreCase.Equals(Id, rhs.Id); 
         }
 
         public static RegistrationKey Promote(string resourceUri, IGraph graph)

src/Catalog/SortingGraphCollector.cs

@@ -1,13 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-using Newtonsoft.Json.Linq;
+
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
 using VDS.RDF;
 
 namespace NuGet.Services.Metadata.Catalog
@@ -36,7 +36,10 @@ protected override async Task ProcessSortedBatch(
                 if (Utils.IsType((JObject)context, item, _types))
                 {
                     var itemUri = item["@id"].ToString();
-                    var task = client.GetGraphAsync(new Uri(itemUri), cancellationToken);
+
+                    // Download the graph to a read-only container. This allows operations on each graph to be safely
+                    // parallelized.
+                    var task = client.GetGraphAsync(new Uri(itemUri), readOnly: true, token: cancellationToken);
 
                     graphTasks.Add(itemUri, task);
 

src/Ng/Jobs/Feed2CatalogJob.cs

@@ -3,21 +3,22 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
-using NuGet.Services.Metadata.Catalog.Persistence;
-using NuGet.Services.Metadata.Catalog;
-using System.Linq;
-using System.Net.Http;
-using Newtonsoft.Json.Linq;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using NuGet.Services.Configuration;
+using NuGet.Services.Metadata.Catalog;
+using NuGet.Services.Metadata.Catalog.Persistence;
 
 namespace Ng.Jobs
 {
     public class Feed2CatalogJob : LoopingNgJob
     {
+        private const string FeedUrlFormatSuffix = "&$top={2}&$select=Created,LastEdited,Published,LicenseNames,LicenseReportUrl&semVerLevel=2.0.0";
         private static readonly DateTime DateTimeMinValueUtc = new DateTime(0L, DateTimeKind.Utc);
 
         protected bool Verbose;
@@ -179,32 +180,32 @@ protected virtual HttpClient CreateHttpClient(bool verbose)
             return CatalogUtility.CreateHttpClient(verbose);
         }
 
-        private static Uri MakeCreatedUri(string source, DateTime since, int top = 100)
+        private static Uri MakeCreatedUri(string source, DateTime since, int top)
         {
-            var address = string.Format("{0}/Packages?$filter=Created gt DateTime'{1}'&$top={2}&$orderby=Created&$select=Created,LastEdited,Published,LicenseNames,LicenseReportUrl",
+            var address = string.Format("{0}/Packages?$filter=Created gt DateTime'{1}'&$orderby=Created" + FeedUrlFormatSuffix,
                 source.Trim('/'),
                 since.ToString("o"),
                 top);
 
             return new Uri(address);
         }
 
-        private static Uri MakeLastEditedUri(string source, DateTime since, int top = 100)
+        private static Uri MakeLastEditedUri(string source, DateTime since, int top)
         {
-            var address = string.Format("{0}/Packages?$filter=LastEdited gt DateTime'{1}'&$top={2}&$orderby=LastEdited&$select=Created,LastEdited,Published,LicenseNames,LicenseReportUrl",
+            var address = string.Format("{0}/Packages?$filter=LastEdited gt DateTime'{1}'&$orderby=LastEdited" + FeedUrlFormatSuffix,
                 source.Trim('/'),
                 since.ToString("o"),
                 top);
 
             return new Uri(address);
         }
 
-        private static Task<SortedList<DateTime, IList<CatalogUtility.PackageDetails>>> GetCreatedPackages(HttpClient client, string source, DateTime since, int top = 100)
+        private static Task<SortedList<DateTime, IList<CatalogUtility.PackageDetails>>> GetCreatedPackages(HttpClient client, string source, DateTime since, int top)
         {
             return CatalogUtility.GetPackages(client, MakeCreatedUri(source, since, top), "Created");
         }
 
-        private static Task<SortedList<DateTime, IList<CatalogUtility.PackageDetails>>> GetEditedPackages(HttpClient client, string source, DateTime since, int top = 100)
+        private static Task<SortedList<DateTime, IList<CatalogUtility.PackageDetails>>> GetEditedPackages(HttpClient client, string source, DateTime since, int top)
         {
             return CatalogUtility.GetPackages(client, MakeLastEditedUri(source, since, top), "LastEdited");
         }

src/Ng/Ng.csproj

@@ -343,10 +343,14 @@
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
+    <None Include="Scripts\Catalog2LuceneV3-Reg2.cmd" />
+    <None Include="Scripts\Catalog2LuceneV3-Reg2-Secondary.cmd" />
     <None Include="Scripts\Catalog2LuceneV3-Reg1-Secondary.cmd" />
     <None Include="Scripts\Catalog2LuceneV3-Reg1.cmd" />
     <None Include="Scripts\Catalog2LuceneV3.cmd" />
     <None Include="Scripts\Catalog2DnxV3.cmd" />
+    <None Include="Scripts\Catalog2RegistrationV3-Reg2.cmd" />
+    <None Include="Scripts\Catalog2RegistrationV3-Reg2-Secondary.cmd" />
     <None Include="Scripts\Catalog2RegistrationV3-Reg1-Secondary.cmd" />
     <None Include="Scripts\Catalog2RegistrationV3-Reg1.cmd" />
     <None Include="Scripts\Catalog2RegistrationV3.cmd" />

src/Ng/Ng.nuspec

@@ -13,13 +13,18 @@
     <file src="bin\$configuration$\*.*" target="Ng"/>
 
     <file src="Scripts\Catalog2DnxV3.cmd" />
+    <file src="Scripts\Catalog2DnxV3Secondary.cmd" />
     <file src="Scripts\Catalog2LuceneV3.cmd" />
     <file src="Scripts\Catalog2LuceneV3-Reg1.cmd" />
     <file src="Scripts\Catalog2LuceneV3-Reg1-Secondary.cmd" />
+    <file src="Scripts\Catalog2LuceneV3-Reg2.cmd" />
+    <file src="Scripts\Catalog2LuceneV3-Reg2-Secondary.cmd" />
 
     <file src="Scripts\Catalog2RegistrationV3.cmd" />
     <file src="Scripts\Catalog2RegistrationV3-Reg1.cmd" />
     <file src="Scripts\Catalog2RegistrationV3-Reg1-Secondary.cmd" />
+    <file src="Scripts\Catalog2RegistrationV3-Reg2.cmd" />
+    <file src="Scripts\Catalog2RegistrationV3-Reg2-Secondary.cmd" />
 
     <file src="Scripts\DB2Catalog.cmd" />
 

src/Ng/Scripts/Catalog2DnxV3Secondary.cmd

@@ -0,0 +1,14 @@
+@echo OFF
+
+cd Ng
+
+:Top
+	echo "Starting job - #{Jobs.ngcatalog2dnx.Title.Secondary}"
+
+	title #{Jobs.ngcatalog2dnx.Title.Secondary}
+
+    start /w Ng.exe catalog2dnx -source #{Jobs.ngcatalog2dnx.Catalog.Source.Secondary} -contentBaseAddress #{Jobs.ngcatalog2dnx.ContentBaseAddress.Secondary} -storageBaseAddress #{Jobs.ngcatalog2dnx.StorageBaseAddress.Secondary} -storageType azure -storageAccountName #{Jobs.common.v3.Storage.Secondary.Name} -storageKeyValue #{Jobs.common.v3.Storage.Secondary.Key} -storageContainer #{Jobs.ngcatalog2dnx.Container} -instrumentationkey #{Jobs.common.v3.Logging.InstrumentationKey} -vaultName #{Deployment.Azure.KeyVault.VaultName} -clientId #{Deployment.Azure.KeyVault.ClientId} -certificateThumbprint #{Deployment.Azure.KeyVault.CertificateThumbprint} -verbose true -interval #{Jobs.ngcatalog2dnx.Interval}
+
+	echo "Finished #{Jobs.ngcatalog2dnx.Title.Secondary}"
+
+	goto Top