Ensure all tests are running in CI pipeline (#645)

CONTRIBUTING.md

@@ -39,6 +39,16 @@ The general flow for making a change to the script host is:
 
 3. Run the unit tests via Visual Studio Test Explorer by selecting "Run All"
 
+## Writing tests
+All tests for Durable Functions are found in [test/Common](https://github.com/Azure/azure-functions-durable-extension/tree/dev/test/Common). These tests are written using the [XUnit framework](https://xunit.github.io/). 
+
+NOTE: In order to run any tests you write in our CI pipeline, the test must have one of the following attributes:
+
+ - `[Trait("Category", PlatformSpecificHelpers.TestCategory)]`
+ - `[Trait("Category", PlatformSpecificHelpers.TestCategory + "_BVT")]`
+ - `[Trait("Category", PlatformSpecificHelpers.FlakeyTestCategory)]`
+
+Please avoid writing flakey tests.
 
 ## Testing code changes locally (Visual Studio) 
 

test/Common/DurableTaskLifeCycleNotificationTest.cs

@@ -32,6 +32,7 @@ public DurableTaskLifeCycleNotificationTest(ITestOutputHelper output)
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationStartAndCompleted(bool extendedSessionsEnabled)
@@ -117,6 +118,7 @@ public async Task OrchestrationStartAndCompleted(bool extendedSessionsEnabled)
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationFailed(bool extendedSessionsEnabled)
@@ -201,6 +203,7 @@ public async Task OrchestrationFailed(bool extendedSessionsEnabled)
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationTerminate(bool extendedSessionsEnabled)
@@ -292,6 +295,7 @@ public async Task OrchestrationTerminate(bool extendedSessionsEnabled)
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationStartedOptOutOfEvent(bool extendedSessionsEnabled)
@@ -349,6 +353,7 @@ public async Task OrchestrationStartedOptOutOfEvent(bool extendedSessionsEnabled
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationCompletedOptOutOfEvent(bool extendedSessionsEnabled)
@@ -406,6 +411,7 @@ public async Task OrchestrationCompletedOptOutOfEvent(bool extendedSessionsEnabl
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationFailedOptOutOfEvent(bool extendedSessionsEnabled)
@@ -464,6 +470,7 @@ public async Task OrchestrationFailedOptOutOfEvent(bool extendedSessionsEnabled)
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationTerminatedOptOutOfEvent(bool extendedSessionsEnabled)
@@ -527,6 +534,7 @@ public async Task OrchestrationTerminatedOptOutOfEvent(bool extendedSessionsEnab
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationEventGridApiReturnBadStatus(bool extendedSessionsEnabled)
@@ -792,6 +800,7 @@ private HttpMessageHandler ConfigureEventGridMockHandler(
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationEventGridApiServiceUnavailableRetry(bool extendedSessionsEnabled)
@@ -871,6 +880,7 @@ public async Task OrchestrationEventGridApiServiceUnavailableRetry(bool extended
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationEventGridApiExceptionRetry(bool extendedSessionsEnabled)
@@ -949,6 +959,7 @@ public async Task OrchestrationEventGridApiExceptionRetry(bool extendedSessionsE
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationEventGridApiExceptionNoRetry(bool extendedSessionsEnabled)
@@ -1018,6 +1029,7 @@ public async Task OrchestrationEventGridApiExceptionNoRetry(bool extendedSession
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationEventGridApiExceptionRetryCountOver(bool extendedSessionsEnabled)
@@ -1087,6 +1099,7 @@ public async Task OrchestrationEventGridApiExceptionRetryCountOver(bool extended
         }
 
         [Theory]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         [InlineData(true)]
         [InlineData(false)]
         public async Task OrchestrationEventGridApiRetryStatus(bool extendedSessionsEnabled)

test/Common/HttpApiHandlerTests.cs

@@ -355,6 +355,7 @@ public async Task GetAllStatus_is_Success()
         }
 
         [Fact]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
         public async Task GetQueryStatus_is_Success()
         {
             var list = (IList<DurableOrchestrationStatus>)new List<DurableOrchestrationStatus>

test/Common/TestFrameworkTests.cs

@@ -0,0 +1,48 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using Xunit;
+using Xunit.Sdk;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests
+{
+    public class TestFrameworkTests
+    {
+        [Fact]
+        [Trait("Category", PlatformSpecificHelpers.TestCategory)]
+        public void AllTestsHaveContinuousIntegrationFriendlyCategories()
+        {
+            Assembly currentAssembly = Assembly.GetExecutingAssembly();
+            List<MethodInfo> testsMissingCategoryTrait = currentAssembly.GetTypes()
+                .SelectMany(type => type.GetMethods())
+                .Where(MethodIsTest)
+                .Where(MethodMissingCorrectCategoryTrait)
+                .ToList();
+            Assert.False(testsMissingCategoryTrait.Any());
+        }
+
+        private static bool MethodIsTest(MethodInfo method)
+        {
+            var factAttribute = method.GetCustomAttribute<FactAttribute>(true);
+            var theoryAttribute = method.GetCustomAttribute<TheoryAttribute>(true);
+            return factAttribute != null || theoryAttribute != null;
+        }
+
+        private static bool MethodMissingCorrectCategoryTrait(MethodInfo method)
+        {
+            IReadOnlyList<KeyValuePair<string, string>> methodTraits = TraitHelper.GetTraits(method);
+            return !methodTraits.Any(TraitIsValidCategory);
+        }
+
+        private static bool TraitIsValidCategory(KeyValuePair<string, string> trait)
+        {
+            return trait.Key.Equals("Category") &&
+                (trait.Value.Equals(PlatformSpecificHelpers.TestCategory) ||
+                trait.Value.Equals(PlatformSpecificHelpers.TestCategory + "_BVT") ||
+                trait.Value.Equals(PlatformSpecificHelpers.FlakeyTestCategory));
+        }
+    }
+}