Fix for bug #3341

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -119,6 +119,7 @@
     <Compile Include="CloudBlobStorageExtensions.cs" />
     <Compile Include="CommitCollector.cs" />
     <Compile Include="CommitMetadata.cs" />
+    <Compile Include="Persistence\StorageListItem.cs" />
     <Compile Include="Utilities.cs" />
     <Compile Include="DeleteCatalogItem.cs" />
     <Compile Include="Dnx\DnxCatalogCollector.cs" />

src/Catalog/Persistence/AggregateStorage.cs

@@ -84,7 +84,7 @@ public override bool Exists(string fileName)
             return _primaryStorage.Exists(fileName);
         }
 
-        public override Task<IEnumerable<Uri>> List(CancellationToken cancellationToken)
+        public override Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
         {
             return _primaryStorage.List(cancellationToken);
         }

src/Catalog/Persistence/AzureStorage.cs

@@ -72,11 +72,18 @@ public override bool Exists(string fileName)
             return false;
         }
 
-        public override async Task<IEnumerable<Uri>> List(CancellationToken cancellationToken)
+        public override async Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
         {
             var files = await _directory.ListBlobsAsync(cancellationToken);
 
-            return files.Select(file => file.Uri).AsEnumerable();
+            return files.Select(GetStorageListItem).AsEnumerable();
+        }
+
+        private StorageListItem GetStorageListItem(IListBlobItem listBlobItem)
+        {
+            var lastModified = (listBlobItem as CloudBlockBlob)?.Properties.LastModified?.UtcDateTime;
+
+            return new StorageListItem(listBlobItem.Uri, lastModified);
         }
 
         //  save

src/Catalog/Persistence/FileStorage.cs

@@ -31,14 +31,15 @@ public FileStorage(Uri baseAddress, string path)
         //File exists
         public override bool Exists(string fileName)
         {
-            return System.IO.File.Exists(fileName);
+            return File.Exists(fileName);
         }
 
-        public override Task<IEnumerable<Uri>> List(CancellationToken cancellationToken)
+        public override Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
         {
             DirectoryInfo directoryInfo = new DirectoryInfo(Path);
             var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories)
-                .Select(file => GetUri(file.FullName.Replace(Path, string.Empty)));
+                .Select(file => 
+                    new StorageListItem(GetUri(file.FullName.Replace(Path, string.Empty)), file.LastWriteTimeUtc));
 
             return Task.FromResult(files.AsEnumerable());
         }

src/Catalog/Persistence/IStorage.cs

@@ -16,6 +16,6 @@ public interface IStorage
         Task<string> LoadString(Uri resourceUri, CancellationToken cancellationToken);
         Uri BaseAddress { get; }
         Uri ResolveUri(string relativeUri);
-        Task<IEnumerable<Uri>> List(CancellationToken cancellationToken);
+        Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken);
     }
 }

src/Catalog/Persistence/Storage.cs

@@ -118,7 +118,7 @@ public async Task<string> LoadString(Uri resourceUri, CancellationToken cancella
 
         public Uri BaseAddress { get; protected set; }
         public abstract bool Exists(string fileName);
-        public abstract Task<IEnumerable<Uri>> List(CancellationToken cancellationToken);
+        public abstract Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken);
 
         public bool Verbose
         {

src/Catalog/Persistence/StorageListItem.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Metadata.Catalog.Persistence
+{
+    public class StorageListItem
+    {
+        public Uri Uri { get; set; }
+
+        public DateTime? LastModifiedUtc { get; set; }
+
+        public StorageListItem(Uri uri, DateTime? lastModifiedUtc)
+        {
+            Uri = uri;
+            LastModifiedUtc = lastModifiedUtc;
+        }
+    }
+}

src/Catalog/Registration/RecordingStorage.cs

@@ -59,7 +59,7 @@ public Uri ResolveUri(string relativeUri)
             return _innerStorage.ResolveUri(relativeUri);
         }
 
-        public Task<IEnumerable<Uri>> List(CancellationToken cancellationToken)
+        public Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
         {
             return _innerStorage.List(cancellationToken);
         }

src/Ng/Jobs/Feed2CatalogJob.cs

@@ -217,7 +217,7 @@ private static Uri MakeLastEditedUri(string source, DateTime since, int top = 10
             // NOTE we're getting more files than needed (to account for a time difference between servers)
             var minimumFileTime = since.AddMinutes(-15);
             var auditRecordUris = (await auditingStorage.List(CancellationToken.None))
-                .Where(recordUri => FilterDeletedPackage(minimumFileTime, recordUri));
+                 .Where(record => FilterDeletedPackage(minimumFileTime, record)).Select(record => record.Uri);
 
             foreach (var auditRecordUri in auditRecordUris)
             {
@@ -267,26 +267,23 @@ private static Uri MakeLastEditedUri(string source, DateTime since, int top = 10
             return result;
         }
 
-        private bool FilterDeletedPackage(DateTime minimumFileTime, Uri auditRecordUri)
+        private bool FilterDeletedPackage(DateTime minimumFileTime, StorageListItem auditRecord)
         {
-            var fileName = GetFileName(auditRecordUri);
-
-            // over time, we have had three "deleted" file names. Try working with them all.
-            if (fileName.EndsWith("-Deleted.audit.v1.json") || fileName.EndsWith("-deleted.audit.v1.json") || fileName.EndsWith("-softdeleted.audit.v1.json"))
+            // Nothing we can do if the last modified time in not available
+            if (auditRecord.LastModifiedUtc == null)
+            {
+                Logger.LogWarning("Could not get date for filename in FilterDeletedPackage. Uri: {AuditRecordUri}", auditRecord.Uri);
+            }
+            else
             {
-                var deletedDateTimeString = fileName
-                    .Replace("-Deleted.audit.v1.json", string.Empty)
-                    .Replace("-deleted.audit.v1.json", string.Empty)
-                    .Replace("-softdeleted.audit.v1.json", string.Empty)
-                    .Replace("_", ":");
-
-                DateTime recordTime;
-                if (DateTime.TryParse(deletedDateTimeString, out recordTime))
+                var fileName = GetFileName(auditRecord.Uri);
+
+                // over time, we have had three "deleted" file names. Try working with them all.
+                if (fileName.EndsWith("-Deleted.audit.v1.json") || fileName.EndsWith("-deleted.audit.v1.json") || fileName.EndsWith("-softdeleted.audit.v1.json")
+                    || fileName.EndsWith("-softdelete.audit.v1.json") || fileName.EndsWith("-delete.audit.v1.json"))
                 {
-                    return recordTime >= minimumFileTime;
+                    return auditRecord.LastModifiedUtc.Value >= minimumFileTime;
                 }
-
-                Logger.LogWarning("Could not parse date from filename in FilterDeletedPackage. Uri: {AuditRecordUri}", auditRecordUri);
             }
 
             return false;

tests/NgTests/Feed2CatalogTests.cs

@@ -102,9 +102,13 @@ public async Task AppendsDeleteToExistingCatalog()
             // Arrange
             var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages();
             var auditingStorage = new MemoryStorage();
-            auditingStorage.Content.Add(
-                new Uri(auditingStorage.BaseAddress, "2015-01-01T00:01:01-deleted.audit.v1.json"),
-                new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100));
+
+            var firstAuditingRecord = new Uri(auditingStorage.BaseAddress, $"{Guid.NewGuid()}-deleted.audit.v1.json");
+            var secondAuditingRecord = new Uri(auditingStorage.BaseAddress, $"{Guid.NewGuid()}-deleted.audit.v1.json");
+
+            auditingStorage.Content.Add(firstAuditingRecord, new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100));
+            auditingStorage.Content.Add(secondAuditingRecord, new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100.Replace("OtherPackage", "AnotherPackage")));
+            auditingStorage.ListMock.Add(secondAuditingRecord, new StorageListItem(secondAuditingRecord, new DateTime(2010, 1, 1)));
 
             var mockServer = new MockServerHttpClientHandler();
 

tests/NgTests/Infrastructure/MemoryStorage.cs

@@ -13,7 +13,9 @@ namespace NgTests.Infrastructure
     public class MemoryStorage 
         : Storage
     {
-        public Dictionary<Uri, StorageContent> Content { get; private set;  }
+        public Dictionary<Uri, StorageContent> Content { get; }
+
+        public Dictionary<Uri, StorageListItem> ListMock { get; }
 
         public MemoryStorage()
             : this(new Uri("http://tempuri.org"))
@@ -24,6 +26,7 @@ public MemoryStorage(Uri baseAddress)
           : base(baseAddress)
         {
             Content = new Dictionary<Uri, StorageContent>();
+            ListMock = new Dictionary<Uri, StorageListItem>();
         }
 
         private MemoryStorage(Uri baseAddress, Dictionary<Uri, StorageContent> content)
@@ -64,9 +67,10 @@ public override bool Exists(string fileName)
             return Content.Keys.Any(k => k.PathAndQuery.EndsWith(fileName));
         }
 
-        public override Task<IEnumerable<Uri>> List(CancellationToken cancellationToken)
+        public override Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
         {
-            return Task.FromResult(Content.Keys.AsEnumerable());
+            return Task.FromResult(Content.Keys.AsEnumerable().Select(x => 
+                ListMock.ContainsKey(x) ? ListMock[x] : new StorageListItem(x, DateTime.UtcNow)));
         }
     }
 }

tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
@@ -14,7 +15,8 @@ public static class MockServerHttpClientHandlerExtensions
     {
         public static async Task AddStorage(this MockServerHttpClientHandler handler, IStorage storage)
         {
-            var files = await storage.List(CancellationToken.None);
+            var files = (await storage.List(CancellationToken.None)).Select(x => x.Uri);
+
             foreach (var file in files)
             {
                 var storageFileUrl = file;