Microsoft Azure Storage Data Movement Library v0.3.0

.gitignore

@@ -5,6 +5,7 @@
 *.suo
 *.user
 *.sln.docstates
+.vs/
 
 # Build results
 [Dd]ebug/
@@ -176,6 +177,9 @@ UpgradeLog*.htm
 # Microsoft Fakes
 FakesAssemblies/
 
-
 # local build nupkg
-*.nupkg
+*.nupkg
+
+# DNX
+project.lock.json
+artifacts/

BreakingChanges.txt

@@ -0,0 +1,7 @@
+Tracking Breaking Changes since 0.2.0
+
+  - Removed directory downloading/copying interfaces without DownloadOptions/CopyOptions
+  - Changed the return value for directory transferring interfaces from Task to Task<TransferStatus>. 
+    This will no longer throw an exception if any of the subtransfers fail. 
+    In addition, you can now get the count for transferred files, failed files, and skipped files. 
+    If you'd like to see the detailed reason for a subtransfer failure, please use the FileFailed event in TransferContext.

DataMovement.sln

@@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMLibTestCodeGen", "test\DM
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MsTestLib", "test\MsTestLib\MsTestLib.csproj", "{AC39B50F-DC27-4411-9ED4-A4A137190ACB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DMTestLib", "test\DMTestLib\DMTestLib.csproj", "{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -63,6 +65,16 @@ Global
 		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{AC39B50F-DC27-4411-9ED4-A4A137190ACB}.Release|Win32.ActiveCfg = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Debug|Win32.ActiveCfg = Debug|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5}.Release|Win32.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -71,5 +83,6 @@ Global
 		{2A4656A4-F744-4653-A9D6-15112E9AB352} = {4353D299-C4E9-41FF-BB35-6769BACA424A}
 		{7018EE4E-D389-424E-A8DD-F9B4FFDA5194} = {4353D299-C4E9-41FF-BB35-6769BACA424A}
 		{AC39B50F-DC27-4411-9ED4-A4A137190ACB} = {4353D299-C4E9-41FF-BB35-6769BACA424A}
+		{2C79E154-EFD2-4FFD-8A73-5A62A61BC6E5} = {4353D299-C4E9-41FF-BB35-6769BACA424A}
 	EndGlobalSection
 EndGlobal

DataMovement_k.sln

@@ -0,0 +1,55 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25123.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{AFBDCFD6-5A7C-467E-85E8-EF1860C899C6}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{93CD4A89-45FD-4F41-AF42-8C28764B8B73}"
+	ProjectSection(SolutionItems) = preProject
+		global.json = global.json
+	EndProjectSection
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "DataMovement", "netcore\Microsoft.WindowsAzure.Storage.DataMovement\DataMovement.xproj", "{43F9D67B-160F-4CEA-81D1-671FE917E789}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{5FCFE2F3-9199-4CF6-8C7C-5A83AF83AC55}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "DMTestLib", "netcore\DMTestLib\DMTestLib.xproj", "{8140456D-26F4-43CE-8D29-00A96AB732CF}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "DMLibTest", "netcore\DMLibTest\DMLibTest.xproj", "{008D045A-1E29-4BB6-8D20-F465EB0F6C18}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "MsTestLib", "netcore\MsTestLib\MsTestLib.xproj", "{AF47B5FF-B528-4855-AA9F-D3B31169D6A5}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{43F9D67B-160F-4CEA-81D1-671FE917E789}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43F9D67B-160F-4CEA-81D1-671FE917E789}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43F9D67B-160F-4CEA-81D1-671FE917E789}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43F9D67B-160F-4CEA-81D1-671FE917E789}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8140456D-26F4-43CE-8D29-00A96AB732CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8140456D-26F4-43CE-8D29-00A96AB732CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8140456D-26F4-43CE-8D29-00A96AB732CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8140456D-26F4-43CE-8D29-00A96AB732CF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{008D045A-1E29-4BB6-8D20-F465EB0F6C18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{008D045A-1E29-4BB6-8D20-F465EB0F6C18}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{008D045A-1E29-4BB6-8D20-F465EB0F6C18}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{008D045A-1E29-4BB6-8D20-F465EB0F6C18}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AF47B5FF-B528-4855-AA9F-D3B31169D6A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AF47B5FF-B528-4855-AA9F-D3B31169D6A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AF47B5FF-B528-4855-AA9F-D3B31169D6A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AF47B5FF-B528-4855-AA9F-D3B31169D6A5}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{43F9D67B-160F-4CEA-81D1-671FE917E789} = {AFBDCFD6-5A7C-467E-85E8-EF1860C899C6}
+		{8140456D-26F4-43CE-8D29-00A96AB732CF} = {5FCFE2F3-9199-4CF6-8C7C-5A83AF83AC55}
+		{008D045A-1E29-4BB6-8D20-F465EB0F6C18} = {5FCFE2F3-9199-4CF6-8C7C-5A83AF83AC55}
+		{AF47B5FF-B528-4855-AA9F-D3B31169D6A5} = {5FCFE2F3-9199-4CF6-8C7C-5A83AF83AC55}
+	EndGlobalSection
+EndGlobal

changelog.txt

@@ -1,3 +1,11 @@
+2016.08.26 Version 0.3.0
+ * Upgrade azure storage client library to 7.2.0
+ * Fixed a bug that ParallelOperations setting doesn't take effect
+ * Fixed a bug of randomly throwing exception when transferring a directory
+ * Removed directory downloading/copying interfaces without DownloadOptions/CopyOptions 
+ * Changed the return value for directory transferring interfaces from Task to Task<TransferStatus> 
+ * Support to work with the RTM release of .Net Core 1.0 
+
 2015.12.28 Version 0.2.0
  * Support download, upload, copy directory (local file directory, blob virtual directory, file share directory) 
  * Support transfer directory in recursive mode or flat mode 

global.json

@@ -0,0 +1,12 @@
+{
+    "projects": [
+        "src",
+        "test",
+        "wrap"
+    ],
+  "sdk": {
+    "version": "1.0.0-preview2-003121",
+    "runtime": "coreclr",
+    "architecture": "x64"
+  }
+}

lib/AssemblyInfo.cs

@@ -1,16 +1,12 @@
-
 //------------------------------------------------------------------------------
 // <copyright file="AssemblyInfo.cs" company="Microsoft">
 //    Copyright (c) Microsoft Corporation
 // </copyright>
 //------------------------------------------------------------------------------
-using System;
 using System.Reflection;
-using System.Resources;
-using System.Runtime.InteropServices;
 
 // General Information about an assembly is controlled through the following 
 // set of attributes. Change these attribute values to modify the information
 // associated with an assembly.
 [assembly: AssemblyTitle("Microsoft.WindowsAzure.Storage.DataMovement.dll")]
-[assembly: AssemblyDescription("")]
+[assembly: AssemblyDescription("")]

lib/Constants.cs

@@ -7,6 +7,7 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement
 {
     using System;
     using System.Reflection;
+    using System.Runtime.InteropServices;
 
     /// <summary>
     /// Constants for use with the transfer classes.
@@ -76,7 +77,6 @@ public static class Constants
         /// TODO: update this number when doc for cloud file is available.
         /// </summary>
         internal const long FileRangeSpanSize = 148 * 1024 * 1024;
-
         /// <summary>
         /// Percentage of available we'll try to use for our memory cache.
         /// </summary>
@@ -141,19 +141,35 @@ public static class Constants
         /// <returns>UserAgent string.</returns>
         private static string GetUserAgent()
         {
+#if DOTNET5_4
+
+            AssemblyName assemblyName = typeof(Constants).GetTypeInfo().Assembly.GetName();
+#else
+
             AssemblyName assemblyName = Assembly.GetExecutingAssembly().GetName();
+#endif
             return UserAgentProductName + "/" + assemblyName.Version.ToString();
         }
 
         private static string GetFormatVersion()
         {
+#if DOTNET5_4
+
+            AssemblyName assemblyName = typeof(Constants).GetTypeInfo().Assembly.GetName();
+#else
+
             AssemblyName assemblyName = Assembly.GetExecutingAssembly().GetName();
+#endif
             return assemblyName.Name + "/" + assemblyName.Version.ToString();
         }
 
         private static long GetMemoryCacheMaximum()
         {
+#if DOTNET5_4
+            return 8 == Marshal.SizeOf(new IntPtr()) ? (long)2 * 1024 * 1024 * 1024 : (long)512 * 1024 * 1024;
+#else
             return Environment.Is64BitProcess ? (long)2 * 1024 * 1024 * 1024 : (long)512 * 1024 * 1024;
+#endif
         }
     }
 }

lib/DataMovement.csproj

@@ -13,14 +13,16 @@
     <FileAlignment>512</FileAlignment>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
     <TargetFrameworkProfile />
+    <DefineConstants>CODE_ACCESS_SECURITY;BINARY_SERIALIZATION;REQUEST_EVENT_ARGS_EXPOSES_REQUEST;$(DefineConstants)</DefineConstants>
     <RestorePackages>true</RestorePackages>
+    <WINDOWSDESKTOP />
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
     <OutputPath>bin\Debug\</OutputPath>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DefineConstants>TRACE;DEBUG;$(DefineConstants)</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
     <Prefer32Bit>false</Prefer32Bit>
@@ -33,7 +35,7 @@
     <DebugType>pdbonly</DebugType>
     <Optimize>true</Optimize>
     <OutputPath>bin\Release\</OutputPath>
-    <DefineConstants>TRACE</DefineConstants>
+    <DefineConstants>TRACE;$(DefineConstants)</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
     <Prefer32Bit>false</Prefer32Bit>
@@ -68,8 +70,8 @@
       <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Microsoft.WindowsAzure.Storage, Version=6.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\packages\WindowsAzure.Storage.6.1.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
+    <Reference Include="Microsoft.WindowsAzure.Storage, Version=7.2.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\WindowsAzure.Storage.7.2.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
@@ -80,6 +82,7 @@
     <Reference Include="System.Core" />
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System.Data" />
+    <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.Spatial, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll</HintPath>
       <Private>True</Private>
@@ -91,6 +94,12 @@
       <Link>SharedAssemblyInfo.cs</Link>
     </Compile>
     <Compile Include="AssemblyInfo.cs" />
+    <Compile Include="Interop\Interop.cs" />
+    <Compile Include="Interop\Interop.Linux.cs" />
+    <Compile Include="Interop\Interop.OSX.cs" />
+    <Compile Include="Interop\Interop.Windows.cs" />
+    <Compile Include="MD5Wrapper.cs" />
+    <Compile Include="Exceptions\TransferSkippedException.cs" />
     <Compile Include="NativeMD5.cs" />
     <Compile Include="OverwriteCallback.cs" />
     <Compile Include="SerializationHelper\SerializableCloudBlobDirectory.cs" />
@@ -133,9 +142,10 @@
     <Compile Include="TransferEnumerators\FileEnumerator.cs" />
     <Compile Include="TransferEnumerators\FileListContinuationToken.cs" />
     <Compile Include="TransferEnumerators\AzureBlobToAzureBlobNameResolver.cs" />
+    <Compile Include="TransferEnumerators\FileToAzureFileNameResolver.cs" />
     <Compile Include="TransferEnumerators\ITransferEnumerator.cs" />
     <Compile Include="TransferEnumerators\NameResolverHelper.cs" />
-    <Compile Include="TransferEnumerators\FileToAzureNameResolver.cs" />
+    <Compile Include="TransferEnumerators\FileToAzureBlobNameResolver.cs" />
     <Compile Include="TransferEnumerators\INameResolver.cs" />
     <Compile Include="TransferEnumerators\ListContinuationToken.cs" />
     <Compile Include="TransferEnumerators\SerializableListContinuationToken.cs" />
@@ -183,7 +193,7 @@
     <Compile Include="TransferOptions\DirectoryOptions.cs" />
     <Compile Include="TransferOptions\DownloadDirectoryOptions.cs" />
     <Compile Include="TransferOptions\UploadDirectoryOptions.cs" />
-    <Compile Include="TransferProgress.cs" />
+    <Compile Include="TransferStatus.cs" />
     <Compile Include="TransferScheduler.cs" />
     <Compile Include="TransferConfigurations.cs" />
     <Compile Include="TransferOptions\CopyOptions.cs" />

lib/Exceptions/TransferException.cs

@@ -12,8 +12,12 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement
     /// <summary>
     /// Base exception class for exceptions thrown by Blob/FileTransferJobs.
     /// </summary>
+#if BINARY_SERIALIZATION
     [Serializable]
-    public sealed class TransferException : Exception
+#else
+    [DataContract]
+#endif // BINARY_SERIALIZATION
+    public class TransferException : Exception
     {
         /// <summary>
         /// Version of current TransferException serialization format.
@@ -33,6 +37,9 @@ public sealed class TransferException : Exception
         /// <summary>
         /// Transfer error code.
         /// </summary>
+#if !BINARY_SERIALIZATION
+        [DataMember]
+#endif
         private TransferErrorCode errorCode;
 
         /// <summary>
@@ -99,12 +106,13 @@ public TransferException(
             this.errorCode = errorCode;
         }
 
+#if BINARY_SERIALIZATION
         /// <summary>
         /// Initializes a new instance of the <see cref="TransferException" /> class.
         /// </summary>
         /// <param name="info">Serialization information.</param>
         /// <param name="context">Streaming context.</param>
-        private TransferException(
+        protected TransferException(
             SerializationInfo info, 
             StreamingContext context)
             : base(info, context)
@@ -116,6 +124,7 @@ private TransferException(
                 this.errorCode = (TransferErrorCode)info.GetInt32(ErrorCodeFieldName);
             }
         }
+#endif // BINARY_SERIALIZATION
 
         /// <summary>
         /// Gets the detailed error code.
@@ -129,6 +138,7 @@ public TransferErrorCode ErrorCode
             }
         }
 
+#if BINARY_SERIALIZATION
         /// <summary>
         /// Serializes the exception.
         /// </summary>
@@ -148,5 +158,6 @@ public override void GetObjectData(
 
             base.GetObjectData(info, context);
         }
+#endif //BINARY_SERIALIZATION
     }
 }