diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index 1db525394..46a8a871a 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -7,74 +7,73 @@ using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; -namespace Microsoft.Azure.Durable.Tests.E2E +namespace Microsoft.Azure.Durable.Tests.E2E; + +public static class HelloCities { - public static class HelloCities + [Function(nameof(HelloCities))] + public static async Task> RunOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) { - [Function(nameof(HelloCities))] - public static async Task> RunOrchestrator( - [OrchestrationTrigger] TaskOrchestrationContext context) - { - ILogger logger = context.CreateReplaySafeLogger(nameof(HelloCities)); - logger.LogInformation("Saying hello."); - var outputs = new List(); + ILogger logger = context.CreateReplaySafeLogger(nameof(HelloCities)); + logger.LogInformation("Saying hello."); + var outputs = new List(); - // Replace name and input with values relevant for your Durable Functions Activity - outputs.Add(await context.CallActivityAsync(nameof(SayHello), "Tokyo")); - outputs.Add(await context.CallActivityAsync(nameof(SayHello), "Seattle")); - outputs.Add(await context.CallActivityAsync(nameof(SayHello), "London")); + // Replace name and input with values relevant for your Durable Functions Activity + outputs.Add(await context.CallActivityAsync(nameof(SayHello), "Tokyo")); + outputs.Add(await context.CallActivityAsync(nameof(SayHello), "Seattle")); + outputs.Add(await context.CallActivityAsync(nameof(SayHello), "London")); - // returns ["Hello Tokyo!", "Hello Seattle!", "Hello London!"] - return outputs; - } + // returns ["Hello Tokyo!", "Hello Seattle!", "Hello London!"] + return outputs; + } - [Function(nameof(SayHello))] - public static string SayHello([ActivityTrigger] string name, FunctionContext executionContext) - { - ILogger logger = executionContext.GetLogger("SayHello"); - logger.LogInformation("Saying hello to {name}.", name); - return $"Hello {name}!"; - } + [Function(nameof(SayHello))] + public static string SayHello([ActivityTrigger] string name, FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("SayHello"); + logger.LogInformation("Saying hello to {name}.", name); + return $"Hello {name}!"; + } - [Function("HelloCities_HttpStart")] - public static async Task HttpStart( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, - [DurableClient] DurableTaskClient client, - FunctionContext executionContext) - { - ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); + [Function("HelloCities_HttpStart")] + public static async Task HttpStart( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); - // Function input comes from the request content. - string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( - nameof(HelloCities)); + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(HelloCities)); - logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); - // Returns an HTTP 202 response with an instance management payload. - // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration - return await client.CreateCheckStatusResponseAsync(req, instanceId); - } + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } - [Function("HelloCities_HttpStart_Scheduled")] - public static async Task HttpStartScheduled( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, - [DurableClient] DurableTaskClient client, - FunctionContext executionContext, - DateTime scheduledStartTime) - { - ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); + [Function("HelloCities_HttpStart_Scheduled")] + public static async Task HttpStartScheduled( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext, + DateTime scheduledStartTime) + { + ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); - var startOptions = new StartOrchestrationOptions(StartAt: scheduledStartTime); + var startOptions = new StartOrchestrationOptions(StartAt: scheduledStartTime); - // Function input comes from the request content. - string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( - nameof(HelloCities), startOptions); + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(HelloCities), startOptions); - logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); - // Returns an HTTP 202 response with an instance management payload. - // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration - return await client.CreateCheckStatusResponseAsync(req, instanceId); - } + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); } } diff --git a/test/e2e/Apps/BasicDotNetIsolated/OrchestrationQuery.cs b/test/e2e/Apps/BasicDotNetIsolated/OrchestrationQuery.cs index 81101156e..ed984d1b0 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/OrchestrationQuery.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/OrchestrationQuery.cs @@ -8,6 +8,7 @@ using Microsoft.DurableTask.Client; namespace Microsoft.Azure.Durable.Tests.E2E; + public static class OrchestrationQueryFunctions { [Function(nameof(GetAllInstances))] diff --git a/test/e2e/Apps/BasicDotNetIsolated/PurgeOrchestrationHistory.cs b/test/e2e/Apps/BasicDotNetIsolated/PurgeOrchestrationHistory.cs index 77b111113..a3b7446da 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/PurgeOrchestrationHistory.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/PurgeOrchestrationHistory.cs @@ -8,44 +8,43 @@ using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; -namespace Microsoft.Azure.Durable.Tests.E2E +namespace Microsoft.Azure.Durable.Tests.E2E; + +public static class PurgeOrchestrationHistory { - public static class PurgeOrchestrationHistory + [Function(nameof(PurgeOrchestrationHistory))] + public static async Task PurgeHistory( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext, + DateTime? purgeStartTime=null, + DateTime? purgeEndTime=null) { - [Function(nameof(PurgeOrchestrationHistory))] - public static async Task PurgeHistory( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, - [DurableClient] DurableTaskClient client, - FunctionContext executionContext, - DateTime? purgeStartTime=null, - DateTime? purgeEndTime=null) - { - ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); + ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); - logger.LogInformation("Starting purge all instance history"); - try - { - var requestPurgeResult = await client.PurgeAllInstancesAsync(new PurgeInstancesFilter(purgeStartTime, purgeEndTime, new List{ - OrchestrationRuntimeStatus.Completed, - OrchestrationRuntimeStatus.Failed, - OrchestrationRuntimeStatus.Terminated - })); + logger.LogInformation("Starting purge all instance history"); + try + { + var requestPurgeResult = await client.PurgeAllInstancesAsync(new PurgeInstancesFilter(purgeStartTime, purgeEndTime, new List{ + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + OrchestrationRuntimeStatus.Terminated + })); - logger.LogInformation("Finished purge all instance history"); + logger.LogInformation("Finished purge all instance history"); - var response = req.CreateResponse(HttpStatusCode.OK); - response.Headers.Add("Content-Type", "text/plain"); - await response.WriteStringAsync($"Purged {requestPurgeResult.PurgedInstanceCount} records"); - return response; - } - catch (RpcException ex) - { - logger.LogError(ex, "Failed to purge all instance history"); - var response = req.CreateResponse(HttpStatusCode.InternalServerError); - response.Headers.Add("Content-Type", "text/plain"); - await response.WriteStringAsync($"Failed to purge all instance history: {ex.Message}"); - return response; - } + var response = req.CreateResponse(HttpStatusCode.OK); + response.Headers.Add("Content-Type", "text/plain"); + await response.WriteStringAsync($"Purged {requestPurgeResult.PurgedInstanceCount} records"); + return response; + } + catch (RpcException ex) + { + logger.LogError(ex, "Failed to purge all instance history"); + var response = req.CreateResponse(HttpStatusCode.InternalServerError); + response.Headers.Add("Content-Type", "text/plain"); + await response.WriteStringAsync($"Failed to purge all instance history: {ex.Message}"); + return response; } } } diff --git a/test/e2e/Apps/BasicDotNetIsolated/SuspendResumeOrchestration.cs b/test/e2e/Apps/BasicDotNetIsolated/SuspendResumeOrchestration.cs new file mode 100644 index 000000000..52f2e7c61 --- /dev/null +++ b/test/e2e/Apps/BasicDotNetIsolated/SuspendResumeOrchestration.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Net; +using Grpc.Core; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.DurableTask.Client; + +namespace Microsoft.Azure.Durable.Tests.E2E; + +public static class SuspendResumeOrchestration +{ + [Function("SuspendInstance")] + public static async Task Suspend( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + string instanceId) + { + string suspendReason = "Suspending the instance for test."; + try + { + await client.SuspendInstanceAsync(instanceId, suspendReason); + return req.CreateResponse(HttpStatusCode.OK); + } + catch (RpcException ex) + { + var response = req.CreateResponse(HttpStatusCode.BadRequest); + response.Headers.Add("Content-Type", "text/plain"); + await response.WriteStringAsync(ex.Message); + return response; + } + } + + [Function("ResumeInstance")] + public static async Task Resume( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + string instanceId) + { + string resumeReason = "Resuming the instance for test."; + try + { + await client.ResumeInstanceAsync(instanceId, resumeReason); + return req.CreateResponse(HttpStatusCode.OK); + } + catch (RpcException ex) + { + var response = req.CreateResponse(HttpStatusCode.BadRequest); + response.Headers.Add("Content-Type", "text/plain"); + await response.WriteStringAsync(ex.Message); + return response; + } + } +} diff --git a/test/e2e/Apps/BasicDotNetIsolated/TerminateOrchestration.cs b/test/e2e/Apps/BasicDotNetIsolated/TerminateOrchestration.cs index 35ddd9ff2..48c5fd4bd 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/TerminateOrchestration.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/TerminateOrchestration.cs @@ -9,72 +9,71 @@ using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; -namespace Microsoft.Azure.Durable.Tests.E2E +namespace Microsoft.Azure.Durable.Tests.E2E; + +public static class LongRunningOrchestration { - public static class LongRunningOrchestration + [Function(nameof(LongRunningOrchestrator))] + public static async Task> LongRunningOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) { - [Function(nameof(LongRunningOrchestrator))] - public static async Task> LongRunningOrchestrator( - [OrchestrationTrigger] TaskOrchestrationContext context) + ILogger logger = context.CreateReplaySafeLogger(nameof(HelloCities)); + logger.LogInformation("Starting long-running orchestration."); + var outputs = new List(); + + // Call our fake activity 100,000 times to simulate an orchestration that might run for >= 10,000s (2.7 hours) + for (int i = 0; i < 100000; i++) { - ILogger logger = context.CreateReplaySafeLogger(nameof(HelloCities)); - logger.LogInformation("Starting long-running orchestration."); - var outputs = new List(); + outputs.Add(await context.CallActivityAsync(nameof(SimulatedWorkActivity), 100)); + } - // Call our fake activity 100,000 times to simulate an orchestration that might run for >= 10,000s (2.7 hours) - for (int i = 0; i < 100000; i++) - { - outputs.Add(await context.CallActivityAsync(nameof(SimulatedWorkActivity), 100)); - } + return outputs; + } - return outputs; - } + [Function(nameof(SimulatedWorkActivity))] + public static string SimulatedWorkActivity([ActivityTrigger]int sleepMs, FunctionContext executionContext) + { + // Sleep the provided number of ms to simulate a long-running activity operation + ILogger logger = executionContext.GetLogger("SimulatedWorkActivity"); + logger.LogInformation("Sleeping for {sleepMs}ms.", sleepMs); + Thread.Sleep(sleepMs); + return $"Slept for {sleepMs}ms."; + } - [Function(nameof(SimulatedWorkActivity))] - public static string SimulatedWorkActivity([ActivityTrigger]int sleepMs, FunctionContext executionContext) - { - // Sleep the provided number of ms to simulate a long-running activity operation - ILogger logger = executionContext.GetLogger("SimulatedWorkActivity"); - logger.LogInformation("Sleeping for {sleepMs}ms.", sleepMs); - Thread.Sleep(sleepMs); - return $"Slept for {sleepMs}ms."; - } + [Function("LongOrchestrator_HttpStart")] + public static async Task HttpStart( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("LongOrchestrator_HttpStart"); - [Function("LongOrchestrator_HttpStart")] - public static async Task HttpStart( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, - [DurableClient] DurableTaskClient client, - FunctionContext executionContext) - { - ILogger logger = executionContext.GetLogger("LongOrchestrator_HttpStart"); + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(LongRunningOrchestrator)); - string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( - nameof(LongRunningOrchestrator)); + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } - logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); - - return await client.CreateCheckStatusResponseAsync(req, instanceId); + [Function("TerminateInstance")] + public static async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + string instanceId) + { + string reason = "Long-running orchestration was terminated early."; + try + { + await client.TerminateInstanceAsync(instanceId, reason); + return req.CreateResponse(HttpStatusCode.OK); } - - [Function("TerminateInstance")] - public static async Task Run( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, - [DurableClient] DurableTaskClient client, - string instanceId) + catch (RpcException ex) { - string reason = "Long-running orchestration was terminated early."; - try - { - await client.TerminateInstanceAsync(instanceId, reason); - return req.CreateResponse(HttpStatusCode.OK); - } - catch (RpcException ex) - { - var response = req.CreateResponse(HttpStatusCode.BadRequest); - response.Headers.Add("Content-Type", "text/plain"); - await response.WriteStringAsync(ex.Message); - return response; - } + var response = req.CreateResponse(HttpStatusCode.BadRequest); + response.Headers.Add("Content-Type", "text/plain"); + await response.WriteStringAsync(ex.Message); + return response; } } } diff --git a/test/e2e/Tests/Helpers/DurableHelpers.cs b/test/e2e/Tests/Helpers/DurableHelpers.cs index b47e95b44..701da99ec 100644 --- a/test/e2e/Tests/Helpers/DurableHelpers.cs +++ b/test/e2e/Tests/Helpers/DurableHelpers.cs @@ -9,6 +9,13 @@ internal class DurableHelpers { static readonly HttpClient _httpClient = new HttpClient(); + static readonly List finalStates = new List() + { + "Completed", + "Terminated", + "Failed" + }; + internal class OrchestrationStatusDetails { public string RuntimeStatus { get; set; } = string.Empty; @@ -52,6 +59,25 @@ internal static async Task GetRunningOrchestrationDe return new OrchestrationStatusDetails(statusQueryResponseString); } + internal static async Task WaitForOrchestrationStateAsync(string statusQueryGetUri, string desiredState, int maxTimeoutSeconds) + { + DateTime timeoutTime = DateTime.Now + TimeSpan.FromSeconds(maxTimeoutSeconds); + while (DateTime.Now < timeoutTime) + { + var currentStatus = await GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + if (currentStatus.RuntimeStatus == desiredState) + { + return; + } + if (finalStates.Contains(currentStatus.RuntimeStatus)) + { + throw new TaskCanceledException($"Orchestration reached {currentStatus.RuntimeStatus} state when test was expecting {desiredState}"); + } + await Task.Delay(100); + } + throw new TimeoutException($"Orchestration did not reach {desiredState} status within {maxTimeoutSeconds} seconds."); + } + private static string TokenizeAndGetValueFromKeyAsString(string? json, string key) { if (string.IsNullOrEmpty(json)) diff --git a/test/e2e/Tests/Tests/HelloCitiesTest.cs b/test/e2e/Tests/Tests/HelloCitiesTest.cs index 98ca2ea30..5fd52e0bb 100644 --- a/test/e2e/Tests/Tests/HelloCitiesTest.cs +++ b/test/e2e/Tests/Tests/HelloCitiesTest.cs @@ -43,14 +43,15 @@ public async Task HttpTriggerTests(string functionName, HttpStatusCode expectedS Assert.Equal(expectedStatusCode, response.StatusCode); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - Thread.Sleep(1000); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); + var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Completed", orchestrationDetails.RuntimeStatus); Assert.Contains(partialExpectedOutput, orchestrationDetails.Output); } [Theory] - [InlineData("HelloCities_HttpStart_Scheduled", 10, HttpStatusCode.Accepted)] + [InlineData("HelloCities_HttpStart_Scheduled", 5, HttpStatusCode.Accepted)] [InlineData("HelloCities_HttpStart_Scheduled", -5, HttpStatusCode.Accepted)] public async Task ScheduledStartTests(string functionName, int startDelaySeconds, HttpStatusCode expectedStatusCode) { @@ -65,28 +66,20 @@ public async Task ScheduledStartTests(string functionName, int startDelaySeconds Assert.Equal(expectedStatusCode, response.StatusCode); - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - while (DateTime.UtcNow < scheduledStartTime + TimeSpan.FromSeconds(-1)) + if (scheduledStartTime > DateTime.UtcNow + TimeSpan.FromSeconds(1)) { - WriteOutput($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}"); - orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Pending", orchestrationDetails.RuntimeStatus); - Thread.Sleep(1000); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Pending", 30); } - // Give a small amount of time for the orchestration to complete, even if scheduled to run immediately - Thread.Sleep(3000); - WriteOutput($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}, looking for completed"); - var finalOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - int retryAttempts = 0; - while (finalOrchestrationDetails.RuntimeStatus != "Completed" && retryAttempts < 10) - { - Thread.Sleep(1000); - finalOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - retryAttempts++; - } - Assert.Equal("Completed", finalOrchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", Math.Max(startDelaySeconds, 0) + 30); - Assert.True(finalOrchestrationDetails.LastUpdatedTime > scheduledStartTime); + // This +1s should not be necessary - however, experimentally the orchestration may run up to one second before the scheduled time. + // It is unclear currently whether this is a bug where orchestrations run early, or a clock difference/error, + // but leaving this logic in for now until further investigation. + Assert.True(DateTime.UtcNow + TimeSpan.FromSeconds(1) >= scheduledStartTime); + + var finalOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + WriteOutput($"Last updated at {finalOrchestrationDetails.LastUpdatedTime}, scheduled to complete at {scheduledStartTime}"); + Assert.True(finalOrchestrationDetails.LastUpdatedTime + TimeSpan.FromSeconds(1) >= scheduledStartTime); } } diff --git a/test/e2e/Tests/Tests/OrchestrationQueryTests.cs b/test/e2e/Tests/Tests/OrchestrationQueryTests.cs index 9b22a024a..4d542373b 100644 --- a/test/e2e/Tests/Tests/OrchestrationQueryTests.cs +++ b/test/e2e/Tests/Tests/OrchestrationQueryTests.cs @@ -46,13 +46,9 @@ public async Task ListRunningOrchestrations_ShouldContainRunningOrchestration() string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - Thread.Sleep(1000); - + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); try { - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Running", orchestrationDetails.RuntimeStatus); - using HttpResponseMessage statusResponse = await HttpHelpers.InvokeHttpTrigger("GetRunningInstances", ""); Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode); diff --git a/test/e2e/Tests/Tests/PurgeInstancesTests.cs b/test/e2e/Tests/Tests/PurgeInstancesTests.cs index 5ffe72751..3b7422601 100644 --- a/test/e2e/Tests/Tests/PurgeInstancesTests.cs +++ b/test/e2e/Tests/Tests/PurgeInstancesTests.cs @@ -76,7 +76,9 @@ public async Task PurgeOrchestrationHistoryAfterInvocation_Succeeds() { using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("HelloCities_HttpStart", ""); Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); - Thread.Sleep(1000); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); DateTime purgeEndTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); using HttpResponseMessage purgeResponse = await HttpHelpers.InvokeHttpTrigger("PurgeOrchestrationHistory", $"?purgeEndTime={purgeEndTime:o}"); diff --git a/test/e2e/Tests/Tests/SuspendResumeTests.cs b/test/e2e/Tests/Tests/SuspendResumeTests.cs new file mode 100644 index 000000000..4abdb9b68 --- /dev/null +++ b/test/e2e/Tests/Tests/SuspendResumeTests.cs @@ -0,0 +1,176 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Net; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E; + +[Collection(Constants.FunctionAppCollectionName)] +public class SuspendResumeTests +{ + private readonly FunctionAppFixture _fixture; + private readonly ITestOutputHelper _output; + + public SuspendResumeTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper) + { + _fixture = fixture; + _fixture.TestLogs.UseTestLogger(testOutputHelper); + _output = testOutputHelper; + } + + + [Fact] + public async Task SuspendAndResumeRunningOrchestration_ShouldSucceed() + { + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("LongOrchestrator_HttpStart", ""); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 5); + try + { + using HttpResponseMessage suspendResponse = await HttpHelpers.InvokeHttpTrigger("SuspendInstance", $"?instanceId={instanceId}"); + await AssertRequestSucceedsAsync(suspendResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Suspended", 5); + + using HttpResponseMessage resumeResponse = await HttpHelpers.InvokeHttpTrigger("ResumeInstance", $"?instanceId={instanceId}"); + await AssertRequestSucceedsAsync(resumeResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 5); + } + finally + { + await TryTerminateInstanceAsync(instanceId); + } + } + + [Fact] + public async Task SuspendSuspendedOrchestration_ShouldFail() + { + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("LongOrchestrator_HttpStart", ""); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 5); + try + { + using HttpResponseMessage suspendResponse = await HttpHelpers.InvokeHttpTrigger("SuspendInstance", $"?instanceId={instanceId}"); + await AssertRequestSucceedsAsync(suspendResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Suspended", 5); + + using HttpResponseMessage resumeResponse = await HttpHelpers.InvokeHttpTrigger("SuspendInstance", $"?instanceId={instanceId}"); + await AssertRequestFailsAsync(resumeResponse); + + // Give some time for Core Tools to write logs out + Thread.Sleep(500); + + Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot suspend orchestration instance in the Suspended state.") && + x.Contains(instanceId)); + } + finally + { + await TryTerminateInstanceAsync(instanceId); + } + } + + + [Fact] + public async Task ResumeRunningOrchestration_ShouldFail() + { + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("LongOrchestrator_HttpStart", ""); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 5); + try + { + using HttpResponseMessage resumeResponse = await HttpHelpers.InvokeHttpTrigger("ResumeInstance", $"?instanceId={instanceId}"); + await AssertRequestFailsAsync(resumeResponse); + + // Give some time for Core Tools to write logs out + Thread.Sleep(500); + + Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot resume orchestration instance in the Running state.") && + x.Contains(instanceId)); + } + finally + { + await TryTerminateInstanceAsync(instanceId); + } + } + + + [Fact] + public async Task SuspendResumeCompletedOrchestration_ShouldFail() + { + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("HelloCities_HttpStart", ""); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 5); + try + { + using HttpResponseMessage suspendResponse = await HttpHelpers.InvokeHttpTrigger("SuspendInstance", $"?instanceId={instanceId}"); + await AssertRequestFailsAsync(suspendResponse); + + using HttpResponseMessage resumeResponse = await HttpHelpers.InvokeHttpTrigger("ResumeInstance", $"?instanceId={instanceId}"); + await AssertRequestFailsAsync(resumeResponse); + + // Give some time for Core Tools to write logs out + Thread.Sleep(500); + + + Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot suspend orchestration instance in the Completed state.") && + x.Contains(instanceId)); + Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot resume orchestration instance in the Completed state.") && + x.Contains(instanceId)); + } + finally + { + await TryTerminateInstanceAsync(instanceId); + } + } + + private static async Task AssertRequestSucceedsAsync(HttpResponseMessage response) + { + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + string? responseMessage = await response.Content.ReadAsStringAsync(); + Assert.NotNull(responseMessage); + Assert.Empty(responseMessage); + } + + private static async Task AssertRequestFailsAsync(HttpResponseMessage response) + { + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + + string? responseMessage = await response.Content.ReadAsStringAsync(); + Assert.NotNull(responseMessage); + // Unclear error message - see https://github.com/Azure/azure-functions-durable-extension/issues/3027, will update this code when that bug is fixed + Assert.Equal("Status(StatusCode=\"Unknown\", Detail=\"Exception was thrown by handler.\")", responseMessage); + } + + private static async Task TryTerminateInstanceAsync(string instanceId) + { + try + { + // Clean up the instance by terminating it - no-op if this fails + using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); + return true; + } + catch (Exception) { } + return false; + } +} diff --git a/test/e2e/Tests/Tests/TerminateOrchestratorTests.cs b/test/e2e/Tests/Tests/TerminateOrchestratorTests.cs index d938d1e5a..bbb527117 100644 --- a/test/e2e/Tests/Tests/TerminateOrchestratorTests.cs +++ b/test/e2e/Tests/Tests/TerminateOrchestratorTests.cs @@ -30,18 +30,12 @@ public async Task TerminateRunningOrchestration_ShouldSucceed() string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - Thread.Sleep(1000); - - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Running", orchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); await AssertTerminateRequestSucceedsAsync(terminateResponse); - Thread.Sleep(1000); - - orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Terminated", orchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Terminated", 30); } @@ -55,18 +49,12 @@ public async Task TerminateScheduledOrchestration_ShouldSucceed() string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - Thread.Sleep(1000); - - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Pending", orchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Pending", 30); using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); await AssertTerminateRequestSucceedsAsync(terminateResponse); - Thread.Sleep(1000); - - orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Terminated", orchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Terminated", 30); } @@ -79,15 +67,13 @@ public async Task TerminateTerminatedOrchestration_ShouldFail() string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - Thread.Sleep(1000); - - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Running", orchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); await AssertTerminateRequestSucceedsAsync(terminateResponse); - Thread.Sleep(1000); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Terminated", 30); + using HttpResponseMessage terminateAgainResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); await AssertTerminateRequestFailsAsync(terminateAgainResponse); @@ -96,9 +82,6 @@ public async Task TerminateTerminatedOrchestration_ShouldFail() Assert.Contains(_fixture.TestLogs.CoreToolsLogs, x => x.Contains("Cannot terminate orchestration instance in the Terminated state.") && x.Contains(instanceId)); - - orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Terminated", orchestrationDetails.RuntimeStatus); } @@ -111,10 +94,7 @@ public async Task TerminateCompletedOrchestration_ShouldFail() string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - Thread.Sleep(1000); - - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - Assert.Equal("Completed", orchestrationDetails.RuntimeStatus); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); await AssertTerminateRequestFailsAsync(terminateResponse);