Merge pull request #248 from NuGet/dev

[ReleasePrep][2017.10.16]RI of dev into master

src/Catalog/AggregateCursor.cs

@@ -16,7 +16,7 @@ public class AggregateCursor : ReadCursor
     {
         public AggregateCursor(IEnumerable<ReadCursor> innerCursors)
         {
-            if (innerCursors == null || innerCursors.Count() < 1)
+            if (innerCursors == null || !innerCursors.Any())
             {
                 throw new ArgumentException("Must supply at least one cursor!", nameof(innerCursors));
             }

src/Catalog/Helpers/ParallelAsync.cs

@@ -0,0 +1,28 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+    public static class ParallelAsync
+    {
+        public const int DefaultDegreeOfParallelism = 32;
+
+        /// <summary>
+        /// Creates a number of tasks specified by <paramref name="degreeOfParallelism"/> using <paramref name="taskFactory"/> and then runs them in parallel.
+        /// </summary>
+        /// <param name="taskFactory">Creates each task to run.</param>
+        /// <param name="degreeOfParallelism">The number of tasks to create.</param>
+        /// <returns>A task that completes when all tasks have completed.</returns>
+        public static Task Repeat(Func<Task> taskFactory, int degreeOfParallelism = DefaultDegreeOfParallelism)
+        {
+            return Task.WhenAll(
+                Enumerable
+                    .Repeat(taskFactory, degreeOfParallelism)
+                    .Select(f => f()));
+        }
+    }
+}

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -147,6 +147,7 @@
     <Compile Include="Helpers\FeedHelpers.cs" />
     <Compile Include="Helpers\FeedPackageDetails.cs" />
     <Compile Include="Helpers\FeedPackageIdentity.cs" />
+    <Compile Include="Helpers\ParallelAsync.cs" />
     <Compile Include="Helpers\UriUtils.cs" />
     <Compile Include="LogEvents.cs" />
     <Compile Include="Helpers\NuGetVersionUtility.cs" />

src/Catalog/Persistence/AggregateStorage.cs

@@ -88,5 +88,10 @@ public override Task<IEnumerable<StorageListItem>> List(CancellationToken cancel
         {
             return _primaryStorage.List(cancellationToken);
         }
+
+        public override Uri GetUri(string name)
+        {
+            return this._primaryStorage.GetUri(name);
+        }
     }
 }

src/Catalog/Persistence/AzureStorage.cs

@@ -273,5 +273,15 @@ await blob.DeleteAsync(deleteSnapshotsOption:DeleteSnapshotsOption.IncludeSnapsh
                                    operationContext:null,
                                    cancellationToken:cancellationToken);
         }
+
+        /// <summary>
+        /// Returns the uri of the blob based on the Azure cloud directory
+        /// </summary>
+        /// <param name="name">The blob name.</param>
+        /// <returns>The blob uri.</returns>
+        public override Uri GetUri(string name)
+        {
+            return new Uri(_directory.Uri, name);
+        }
     }
 }

src/Catalog/Persistence/Storage.cs

@@ -2,13 +2,13 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 using System;
 using System.Collections.Generic;
-using System.Data;
 using System.Diagnostics;
 using System.IO;
 using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.WindowsAzure.Storage;
+using Newtonsoft.Json;
 
 namespace NuGet.Services.Metadata.Catalog.Persistence
 {
@@ -33,43 +33,55 @@ public async Task Save(Uri resourceUri, StorageContent content, CancellationToke
         {
             SaveCount++;
 
-            TraceMethod("SAVE", resourceUri);
+            TraceMethod(nameof(Save), resourceUri);
+            Stopwatch sw = new Stopwatch();
+            sw.Start();
 
             try
             {
                 await OnSave(resourceUri, content, cancellationToken);
             }
             catch (Exception e)
             {
-                string message = String.Format("SAVE EXCEPTION: {0} {1}", resourceUri, e.Message);
-                Trace.WriteLine(message);
-                throw new Exception(message, e);
+                TraceException(nameof(Save), resourceUri, e);
+                throw;
             }
+
+            sw.Stop();
+            TraceExecutionTime(nameof(Save), resourceUri, sw.ElapsedMilliseconds);
         }
 
         public async Task<StorageContent> Load(Uri resourceUri, CancellationToken cancellationToken)
         {
             LoadCount++;
+            StorageContent storageContent = null;
 
-            TraceMethod("LOAD", resourceUri);
+            TraceMethod(nameof(Load), resourceUri);
+            Stopwatch sw = new Stopwatch();
+            sw.Start();
 
             try
             {
-                return await OnLoad(resourceUri, cancellationToken);
+                storageContent = await OnLoad(resourceUri, cancellationToken);
             }
             catch (Exception e)
             {
-                string message = String.Format("LOAD EXCEPTION: {0} {1}", resourceUri, e.Message);
-                Trace.WriteLine(message);
-                throw new Exception(message, e);
+                TraceException(nameof(Load), resourceUri, e);
+                throw;
             }
+
+            sw.Stop();
+            TraceExecutionTime(nameof(Load), resourceUri, sw.ElapsedMilliseconds);
+            return storageContent;
         }
 
         public async Task Delete(Uri resourceUri, CancellationToken cancellationToken)
         {
             DeleteCount++;
 
-            TraceMethod("DELETE", resourceUri);
+            TraceMethod(nameof(Delete), resourceUri);
+            Stopwatch sw = new Stopwatch();
+            sw.Start();
 
             try
             {
@@ -93,10 +105,12 @@ public async Task Delete(Uri resourceUri, CancellationToken cancellationToken)
             }
             catch (Exception e)
             {
-                string message = String.Format("DELETE EXCEPTION: {0} {1}", resourceUri, e.Message);
-                Trace.WriteLine(message);
-                throw new Exception(message, e);
+                TraceException(nameof(Delete), resourceUri, e);
+                throw;
             }
+
+            sw.Stop();
+            TraceExecutionTime(nameof(Delete), resourceUri, sw.ElapsedMilliseconds);
         }
 
         public async Task<string> LoadString(Uri resourceUri, CancellationToken cancellationToken)
@@ -175,7 +189,7 @@ protected string GetName(Uri uri)
             return name;
         }
 
-        protected Uri GetUri(string name)
+        public virtual Uri GetUri(string name)
         {
             string address = BaseAddress.ToString();
             if (!address.EndsWith("/"))
@@ -191,8 +205,23 @@ protected void TraceMethod(string method, Uri resourceUri)
         {
             if (Verbose)
             {
-                Trace.WriteLine(String.Format("{0} {1}", method, resourceUri));
+                //The Uri depends on the storage implementation.
+                Uri storageUri = GetUri(GetName(resourceUri));
+                Trace.WriteLine(String.Format("{0} {1}", method, storageUri));
             }
         }
+
+        private string TraceException(string method, Uri resourceUri, Exception exception)
+        {
+            string message = $"{method} EXCEPTION: {GetUri(GetName(resourceUri))} {exception.ToString()}";
+            Trace.WriteLine(message);
+            return message;
+        }
+
+        private void TraceExecutionTime(string method, Uri resourceUri, long executionTimeInMilliseconds)
+        {
+            string message = JsonConvert.SerializeObject(new { MethodName = method, StreamUri = GetUri(GetName(resourceUri)), ExecutionTimeInMilliseconds = executionTimeInMilliseconds });
+            Trace.WriteLine(message);
+        }
     }
 }

src/Ng/Arguments.cs

@@ -43,6 +43,7 @@ public static class Arguments
         public const string StorageContainer = "storageContainer";
         public const string StorageKeyValue = "storageKeyValue";
         public const string StoragePath = "storagePath";
+        public const string StorageQueueName = "storageQueueName";
         public const string StorageType = "storageType";
         public const string Version = "version";
 
@@ -103,7 +104,7 @@ public static class Arguments
         public const string StorageTypeAuditing = "storageTypeAuditing";
         #endregion
 
-        #region EndpointMonitoring
+        #region Monitoring
         /// <summary>
         /// The url of the service index.
         /// </summary>