Improve listing performance for transfer whose source is Azure File D…

…irectory.
Azure · May 16, 2019 · b2947b8 · b2947b8
1 parent 576ec65
commit b2947b8
Show file tree

Hide file tree

Showing 31 changed files with 2,124 additions and 400 deletions.
diff --git a/DataMovement.sln b/DataMovement.sln
@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 14
-VisualStudioVersion = 14.0.25420.1
+# Visual Studio 15
+VisualStudioVersion = 15.0.26228.10
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataMovement", "lib\DataMovement.csproj", "{B821E031-09CC-48F0-BDC6-2793228D4027}"
 EndProject

diff --git a/lib/DataMovement.csproj b/lib/DataMovement.csproj
@@ -85,6 +85,7 @@
     </Compile>
     <Compile Include="AssemblyInfo.cs" />
     <Compile Include="ChunkedMemoryStream.cs" />
+    <Compile Include="DirectoryListingScheduler.cs" />
     <Compile Include="Extensions\StorageCopyState.cs" />
     <Compile Include="FileNativeMethods.cs" />
     <Compile Include="LongPathFile.cs" />
@@ -128,6 +129,7 @@
     <Compile Include="TransferControllers\TransferWriters\RangeBasedWriter.cs" />
     <Compile Include="TransferControllers\TransferWriters\StreamedWriter.cs" />
     <Compile Include="TransferEnumerators\AzureBlobEntry.cs" />
+    <Compile Include="TransferEnumerators\AzureFileDirectoryEntry.cs" />
     <Compile Include="TransferEnumerators\AzureBlobEnumerator.cs" />
     <Compile Include="TransferEnumerators\AzureFileEntry.cs" />
     <Compile Include="TransferEnumerators\AzureFileEnumerator.cs" />
@@ -160,11 +162,13 @@
     <Compile Include="TransferJobs\DirectoryLocation.cs" />
     <Compile Include="TransferJobs\DirectoryTransfer.cs" />
     <Compile Include="TransferJobs\FileLocation.cs" />
-    <Compile Include="TransferJobs\MultipleObjectsTransfer.cs" />
+    <Compile Include="TransferJobs\FlatDirectoryTransfer.cs" />
+    <Compile Include="TransferJobs\HierarchyDirectoryTransfer.cs" />
     <Compile Include="TransferJobs\SerializableTransferLocation.cs" />
     <Compile Include="TransferJobs\SingleObjectCheckpoint.cs" />
     <Compile Include="TransferJobs\SingleObjectTransfer.cs" />
     <Compile Include="TransferJobs\StreamLocation.cs" />
+    <Compile Include="TransferJobs\SubDirectoryTransfer.cs" />
     <Compile Include="TransferJobs\Transfer.cs" />
     <Compile Include="TransferJobs\TransferJob.cs" />
     <Compile Include="Constants.cs" />

diff --git a/lib/DirectoryListingScheduler.cs b/lib/DirectoryListingScheduler.cs
@@ -0,0 +1,84 @@
+//------------------------------------------------------------------------------
+// <copyright file="DirectoryListingScheduler.cs" company="Microsoft">
+//    Copyright (c) Microsoft Corporation
+// </copyright>
+//------------------------------------------------------------------------------
+
+
+namespace Microsoft.Azure.Storage.DataMovement
+{
+    using System;
+    using System.Collections.Concurrent;
+    using System.Collections.Generic;
+    using System.Text;
+    using System.Threading;
+    using System.Threading.Tasks;
+
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable")]
+    class DirectoryListingScheduler
+    {
+        SemaphoreSlim semaphore = null;
+        private static DirectoryListingScheduler SchedulerInstance = null;
+        private static object SingletonLock = new object();
+
+        private DirectoryListingScheduler()
+        {
+            int parallelLevel = TransferManager.Configurations.ParallelOperations;
+            parallelLevel = (int)Math.Ceiling(((double)parallelLevel) / 8);
+            semaphore = new SemaphoreSlim(parallelLevel, parallelLevel);
+        }
+
+        public static DirectoryListingScheduler Instance()
+        {
+            if (null == SchedulerInstance)
+            {
+                lock (SingletonLock)
+                {
+                    if (null == SchedulerInstance)
+                    {
+                        SchedulerInstance = new DirectoryListingScheduler();
+                    }
+                }
+            }
+
+            return SchedulerInstance;
+        }
+
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")]
+        public Task Schedule(
+            SubDirectoryTransfer directoryTransfer,
+            CancellationToken cancellationToken,
+            Action persistDirTransfer,
+            int timeOut)
+        {
+            if (this.semaphore.Wait(timeOut, cancellationToken))
+            {
+                try
+                {
+                    if (null != persistDirTransfer)
+                    {
+                        persistDirTransfer();
+                    }
+                }
+                catch
+                {
+                    this.semaphore.Release();
+                    throw;
+                }
+
+                Task task = directoryTransfer.ExecuteAsync(null, cancellationToken);
+                task.ContinueWith((sourceTask) =>
+                {
+                    this.semaphore.Release();
+                    return sourceTask;
+                });
+
+                return task;
+            }
+            else
+            {
+                return null;
+            }
+        }
+    }
+}
diff --git a/lib/Exceptions/TransferErrorCode.cs b/lib/Exceptions/TransferErrorCode.cs
@@ -116,6 +116,11 @@ public enum TransferErrorCode
         /// </summary>
         UnsupportedDummyTransfer = 21,
 
+        /// <summary>
+        /// Failed when trying in ShouldTransferCallbackAsync.
+        /// </summary>
+        FailedCheckingShouldTransfer = 22,
+
         /// <summary>
         /// Uncategorized transfer error.
         /// </summary>

diff --git a/lib/Extensions/StorageCopyState.cs b/lib/Extensions/StorageCopyState.cs
@@ -7,6 +7,7 @@
 namespace Microsoft.Azure.Storage.DataMovement.Extensions
 {
     using System;
+    using Microsoft.Azure.Storage.Blob;
 
     internal enum StorageCopyStatus
     {

diff --git a/lib/Extensions/StorageExtensions.cs b/lib/Extensions/StorageExtensions.cs
@@ -144,6 +144,12 @@ internal static string ConvertToString(this object instance)
                 return file.SnapshotQualifiedUri.AbsoluteUri;
             }
 
+            CloudFileDirectory cloudFileDirectory = instance as CloudFileDirectory;
+            if (null != cloudFileDirectory)
+            {
+                return cloudFileDirectory.SnapshotQualifiedUri.AbsoluteUri;
+            }
+
             return instance.ToString();
         }