Merge branch 'dev' into nytian/http_payload_worker

eng/ci/publish.yml

@@ -0,0 +1,99 @@
+# This is our package-publishing pipeline.
+# When executed, it automatically publishes the output of the 'official pipeline' (the nupkgs) to our internal ADO feed.
+# It may optionally also publish the packages to NuGet, but that is gated behind a manual approval.
+
+trigger: none # only trigger is manual
+pr: none # only trigger is manual
+
+# We include to this variable group to be able to access the NuGet API key
+variables:
+- group: durabletask_config
+
+resources:
+    repositories:
+        - repository: 1es
+          type: git
+          name: 1ESPipelineTemplates/1ESPipelineTemplates
+          ref: refs/tags/release
+        - repository: eng
+          type: git
+          name: engineering
+          ref: refs/tags/release
+
+    pipelines:
+    - pipeline: officialPipeline # Reference to the pipeline to be used as an artifact source
+      source: 'durabletask-extension.official'
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+        name: 1es-pool-azfunc
+        image: 1es-windows-2022
+        os: windows
+
+    stages:
+    - stage: release
+      jobs:
+
+      # ADO release
+      - job: adoRelease
+        displayName: ADO Release
+        templateContext:
+          inputs:
+          - input: pipelineArtifact
+            pipeline: officialPipeline  # Pipeline reference, as defined in the resources section
+            artifactName: drop
+            targetPath: $(System.DefaultWorkingDirectory)/drop
+
+          # The preferred method of release on 1ES is by populating the 'output' section of a 1ES template.
+          # We use this method to release to ADO, but not to release to NuGet; this is explained in the 'nugetRelease' job.
+          # To read more about the 'output syntax', see: 
+          # - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs
+          # - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages
+          outputs:
+          - output: nuget # 'nuget' is an output "type" for pushing to NuGet
+            displayName: 'Push to durabletask ADO feed'
+            packageParentPath: $(System.DefaultWorkingDirectory) # This needs to be set to some prefix of the `packagesToPush` parameter. Apparently it helps with SDL tooling
+            packagesToPush: '$(System.DefaultWorkingDirectory)/**/*.nupkg;!$(System.DefaultWorkingDirectory)/**/*.symbols.nupkg'
+            publishVstsFeed: '3f99e810-c336-441f-8892-84983093ad7f/c895696b-ce37-4fe7-b7ce-74333a04f8bf'
+            allowPackageConflicts: true
+
+      # NuGet approval gate
+      - job: nugetApproval
+        displayName: NuGetApproval
+        pool: server # This task only works when executed on serverl pools, so this needs to be specified
+        steps:
+          # Wait for manual approval. 
+          - task: ManualValidation@1
+            inputs:
+              instructions: Confirm you want to push to NuGet
+              onTimeout: 'reject'
+
+      # NuGet release
+      - job: nugetRelease
+        displayName: NuGet Release
+        dependsOn:
+        - nugetApproval
+        - adoRelease
+        condition: succeeded('nugetApproval', 'adoRelease')
+        templateContext:
+          inputs:
+          - input: pipelineArtifact
+            pipeline: officialPipeline  # Pipeline reference as defined in the resources section
+            artifactName: drop
+            targetPath: $(System.DefaultWorkingDirectory)/drop
+        # Ideally, we would push to NuGet using the 1ES "template output" syntax, like we do for ADO.
+        # Unfortunately, that syntax does not allow for skipping duplicates when pushing to NuGet feeds
+        # (i.e; not failing the job when trying to push a package version that already exists on NuGet).
+        # This is a problem for us because our pipelines often produce multiple packages, and we want to be able to
+        # perform a 'nuget push *.nupkg' that skips packages already on NuGet while pushing the rest.
+        # Therefore, we use a regular .NET Core ADO Task to publish the packages until that usability gap is addressed.
+        steps:
+        - task: DotNetCoreCLI@2
+          displayName: 'Push to nuget.org'
+          inputs:
+            command: custom
+            custom: nuget
+            arguments: 'push "*.nupkg" --api-key $(nuget_api_key) --skip-duplicate --source https://api.nuget.org/v3/index.json'
+            workingDirectory: '$(System.DefaultWorkingDirectory)/drop'

eng/templates/build.yml

@@ -124,3 +124,14 @@ jobs:
             SourceFolder: '$(System.DefaultWorkingDirectory)/test/PerfTests/DFPerfTests/Output/'
             Contents: '**'
             TargetFolder: '$(System.DefaultWorkingDirectory)/azure-functions-durable-extension/'
+
+      # We also need to build the Java smoke test, for CodeQL compliance
+      # We don't need to build the other smoke tests, because they can be analyzed without being compiled,
+      # as they're interpreted languages.
+      # This could be a separate pipeline, but the task is so small that it's paired with the .NET code build
+      # for convenience.
+      - pwsh: |
+          cd ./test/SmokeTests/OOProcSmokeTests/durableJava/
+          gradle build
+          ls
+        displayName: 'Build Java OOProc test (for CodeQL compliance)'

release_notes.md

@@ -6,6 +6,8 @@
 
 ### Bug Fixes
 
+- Fix custom connection name not working when using IDurableClientFactory.CreateClient() - contributed by [@hctan](https://github.com/hctan)
+
 ### Breaking Changes
 
 ### Dependency Updates

src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx

@@ -306,10 +306,10 @@ https://docs.microsoft.com/azure/azure-functions/durable/durable-functions-check
     <value>I/O operations are not allowed inside an orchestrator function.</value>
   </data>
   <data name="MethodAnalyzerMessageFormat" xml:space="preserve">
-    <value>Method call '{0}' violates the orchestrator deterministic code constraint. Methods definied in source code that are used in an orchestrator must be deterministic.</value>
+    <value>Method call '{0}' violates the orchestrator deterministic code constraint. Methods defined in source code that are used in an orchestrator must be deterministic.</value>
   </data>
   <data name="MethodAnalyzerTitle" xml:space="preserve">
-    <value>Methods definied in source code that are used in an orchestrator must be deterministic.</value>
+    <value>Methods defined in source code that are used in an orchestrator must be deterministic.</value>
   </data>
   <data name="SignalEntityAnalyzerDescription" xml:space="preserve">
     <value>SignalEntityAsync must use an Entity Interface.</value>

src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs

@@ -159,7 +159,7 @@ private AzureStorageDurabilityProvider GetAzureStorageStorageProvider(DurableCli
             // Need to check this.defaultStorageProvider != null for external clients that call GetDurabilityProvider(attribute)
             // which never initializes the defaultStorageProvider.
             if (string.Equals(this.defaultSettings?.TaskHubName, settings.TaskHubName, StringComparison.OrdinalIgnoreCase) &&
-                string.Equals(this.defaultSettings?.StorageConnectionString, settings.StorageConnectionString, StringComparison.OrdinalIgnoreCase) &&
+                string.Equals(this.defaultSettings?.StorageAccountDetails?.ConnectionString, settings.StorageAccountDetails?.ConnectionString, StringComparison.OrdinalIgnoreCase) &&
                 this.defaultStorageProvider != null)
             {
                 // It's important that clients use the same AzureStorageOrchestrationService instance

src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj

@@ -45,6 +45,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.*" PrivateAssets="All" />
+    <PackageReference Include="System.Text.Json" Version="6.0.10" />
   </ItemGroup>
 
   <!-- Embed the SBOM manifest, which is generated as part of the "official" build -->

test/Common/CustomTestStorageAccountProvider.cs

@@ -0,0 +1,33 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System;
+using DurableTask.AzureStorage;
+using Microsoft.WindowsAzure.Storage;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests
+{
+    internal class CustomTestStorageAccountProvider : IStorageAccountProvider
+    {
+        private readonly string customConnectionString;
+        private readonly string customConnectionName;
+
+        public CustomTestStorageAccountProvider(string connectionName)
+        {
+            this.customConnectionName = connectionName;
+            this.customConnectionString = $"DefaultEndpointsProtocol=https;AccountName=test;AccountKey={GenerateRandomKey()};EndpointSuffix=core.windows.net";
+        }
+
+        public CloudStorageAccount GetCloudStorageAccount(string name) =>
+            CloudStorageAccount.Parse(name != this.customConnectionName ? TestHelpers.GetStorageConnectionString() : this.customConnectionString);
+
+        public StorageAccountDetails GetStorageAccountDetails(string name) =>
+            new StorageAccountDetails { ConnectionString = name != this.customConnectionName ? TestHelpers.GetStorageConnectionString() : this.customConnectionString };
+
+        private static string GenerateRandomKey()
+        {
+            string key = Guid.NewGuid().ToString();
+            return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(key));
+        }
+    }
+}

test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs

@@ -154,5 +154,46 @@ public void EnvironmentIsVMSS_WorkerIdFromEnvironmentVariables()
 
             Assert.Equal("waws-prod-euapbn1-003:dw0SmallDedicatedWebWorkerRole_hr0HostRole-3-VM-13", settings.WorkerId);
         }
+
+        [Fact]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
+        public void CustomConnectionNameIsResolved()
+        {
+            var storageAccountProvider = new CustomTestStorageAccountProvider("CustomConnection");
+            var mockOptions = new OptionsWrapper<DurableTaskOptions>(new DurableTaskOptions());
+            var nameResolver = new Mock<INameResolver>().Object;
+
+            var factory = new AzureStorageDurabilityProviderFactory(
+                mockOptions,
+                storageAccountProvider,
+                nameResolver,
+                NullLoggerFactory.Instance,
+                TestHelpers.GetMockPlatformInformationService());
+
+            factory.GetDurabilityProvider(); // This will initialize the default connection string
+            var provider = factory.GetDurabilityProvider(new DurableClientAttribute() { ConnectionName = "CustomConnection", TaskHub = "TestHubName" });
+
+            Assert.Equal("CustomConnection", provider.ConnectionName);
+        }
+
+        [Fact]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
+        public void DefaultConnectionNameIsResolved()
+        {
+            var storageAccountProvider = new CustomTestStorageAccountProvider("CustomConnection");
+            var mockOptions = new OptionsWrapper<DurableTaskOptions>(new DurableTaskOptions());
+            var nameResolver = new Mock<INameResolver>().Object;
+
+            var factory = new AzureStorageDurabilityProviderFactory(
+                mockOptions,
+                storageAccountProvider,
+                nameResolver,
+                NullLoggerFactory.Instance,
+                TestHelpers.GetMockPlatformInformationService());
+
+            var provider = factory.GetDurabilityProvider();
+
+            Assert.Equal("Storage", provider.ConnectionName);
+        }
     }
 }