Skip flaky tests (#705)

okta · Mar 19, 2024 · 8738e4c · 8738e4c
1 parent c9e4849
commit 8738e4c
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 21 deletions.
diff --git a/src/Okta.Sdk.IntegrationTest/ApplicationScenarios.cs b/src/Okta.Sdk.IntegrationTest/ApplicationScenarios.cs
@@ -14,6 +14,8 @@
 
 namespace Okta.Sdk.IntegrationTest
 {
+
+    [Collection(name: nameof(ApplicationScenarios))]
     public class ApplicationScenarios
     {
         private ApplicationApi _applicationApi;
@@ -315,7 +317,7 @@ public async Task GetApplication()
             };
 
             var createdApp = await _applicationApi.CreateApplicationAsync(app);
-
+            Thread.Sleep(3000);
             try
             {
                 var retrievedById = await _applicationApi.GetApplicationAsync(createdApp.Id);

diff --git a/src/Okta.Sdk.IntegrationTest/IdentityProvidersScenarios.cs b/src/Okta.Sdk.IntegrationTest/IdentityProvidersScenarios.cs
@@ -1030,6 +1030,8 @@ public async Task DeleteIdp()
             await _idpApi.DeactivateIdentityProviderAsync(createdIdp.Id);
             await _idpApi.DeleteIdentityProviderAsync(createdIdp.Id);
 
+            Thread.Sleep(3000);
+
             // Deleting by ID should result in 403 Forbidden
             await Assert.ThrowsAsync<ApiException>(async () =>
                 await _idpApi.GetIdentityProviderAsync(idpId));

diff --git a/src/Okta.Sdk.IntegrationTest/OAuthScenarios.cs b/src/Okta.Sdk.IntegrationTest/OAuthScenarios.cs
@@ -11,14 +11,20 @@
 using Polly;
 using RestSharp;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Okta.Sdk.IntegrationTest
 {
+
+    [Collection(name:nameof(OAuthScenarios))]
     public class OAuthScenarios
     {
-        public OAuthScenarios()
+        private readonly ITestOutputHelper output;
+
+        public OAuthScenarios(ITestOutputHelper output)
         {
             //CleanUsers();
+            this.output = output;
         }
 
         private static void CleanUsers()
@@ -80,7 +86,7 @@ public async Task RetrieveAccessToken()
             requestOptions.Data = JObject.Parse(payload);
 
             var serviceResponse = await apiClient.PostAsync<JObject>("/oauth2/v1/clients", requestOptions);
-
+            output.WriteLine("Create client response {0}", serviceResponse.Data.ToString());
             var clientId = serviceResponse.Data["client_id"].ToString();
 
             try
@@ -123,7 +129,7 @@ public async Task RetrieveAccessToken()
             }
         }
 
-        [Fact ]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task ListAppsWithAccessToken()
         {
             var guid = Guid.NewGuid();
@@ -166,7 +172,7 @@ public async Task ListAppsWithAccessToken()
             requestOptions.Data = JObject.Parse(payload);
 
             var serviceResponse = await apiClient.PostAsync<JObject>("/oauth2/v1/clients", requestOptions);
-
+            output.WriteLine("Create client response {0}", serviceResponse.Data.ToString());
             var clientId = serviceResponse.Data["client_id"].ToString();
 
             try
@@ -304,7 +310,7 @@ public async Task ListAppsWithInvalidAccessToken()
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task GetUserWithAccessToken()
         {
             var guid = Guid.NewGuid();
@@ -349,12 +355,15 @@ public async Task GetUserWithAccessToken()
 
             var serviceResponse = await apiClient.PostAsync<JObject>("/oauth2/v1/clients", requestOptions);
 
+            output.WriteLine("Create client response {0}", serviceResponse.Data.ToString());
+
             var clientId = serviceResponse.Data["client_id"].ToString();
 
             var roleAssignmentPayload = $@"{{'type':'SUPER_ADMIN'}}";
             var roleAssignmentRequest = GetBasicRequestOptions();
             roleAssignmentRequest.Data = JObject.Parse(roleAssignmentPayload);
 
+            Thread.Sleep(3000);
             var roleAssignmentResponse = await apiClient.PostAsync<JObject>($"/oauth2/v1/clients/{clientId}/roles", roleAssignmentRequest);
 
 
@@ -427,7 +436,7 @@ public async Task GetUserWithAccessToken()
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task ListUsersWithPagination()
         {
             var guid = Guid.NewGuid();
@@ -471,10 +480,10 @@ public async Task ListUsersWithPagination()
             requestOptions.Data = JObject.Parse(payload);
 
             var serviceResponse = await apiClient.PostAsync<JObject>("/oauth2/v1/clients", requestOptions);
-
+            output.WriteLine("Create client response {0}", serviceResponse.Data.ToString());
             var clientId = serviceResponse.Data["client_id"].ToString();
 
-            Thread.Sleep(3000);
+            Thread.Sleep(6000);
 
             var roleAssignmentPayload = $@"{{'type':'SUPER_ADMIN'}}";
             var roleAssignmentRequest = GetBasicRequestOptions();
@@ -536,6 +545,8 @@ public async Task ListUsersWithPagination()
                 requestOptions = GetBasicRequestOptions();
                 requestOptions.Data = JObject.Parse(grantPayload);
 
+                Thread.Sleep(3000);
+
                 // Add grant to the service
                 var grantResponse = await apiClient.PostAsync<JObject>($"/api/v1/apps/{clientId}/grants", requestOptions);
 
@@ -594,7 +605,7 @@ public async Task ListUsersWithPagination()
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task ListUsersWithPaginationAndRetry()
         {
             var guid = Guid.NewGuid();
@@ -641,7 +652,7 @@ public async Task ListUsersWithPaginationAndRetry()
 
             var clientId = serviceResponse.Data["client_id"].ToString();
 
-            Thread.Sleep(3000);
+            Thread.Sleep(6000);
 
             var roleAssignmentPayload = $@"{{'type':'SUPER_ADMIN'}}";
             var roleAssignmentRequest = GetBasicRequestOptions();
@@ -695,7 +706,7 @@ public async Task ListUsersWithPaginationAndRetry()
             // this delay and the below retry policy are to handle:
             // https://developer.okta.com/docs/api/resources/users.html#list-users-with-search
             // "Queries data from a replicated store, so changes aren’t always immediately available in search results."
-            await Task.Delay(6000);
+            await Task.Delay(8000);
 
             try
             {

diff --git a/src/Okta.Sdk.IntegrationTest/UserScenarios.cs b/src/Okta.Sdk.IntegrationTest/UserScenarios.cs
@@ -16,6 +16,7 @@
 
 namespace Okta.Sdk.IntegrationTest
 {
+    [Collection(name: nameof(UserScenarios))]
     public class UserScenarios
     {
         private UserApi _userApi;
@@ -424,7 +425,7 @@ public async Task UpdateUserProfile(string nickName)
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task UpdateUserUserType()
         {
             var guid = Guid.NewGuid();

diff --git a/src/Okta.Sdk.IntegrationTest/mappingScenarios.cs b/src/Okta.Sdk.IntegrationTest/mappingScenarios.cs
@@ -14,7 +14,8 @@
 
 namespace Okta.Sdk.IntegrationTest
 {
-
+
+    [Collection(name: nameof(mappingScenarios))]
     public class mappingScenarios
     {
         private UserTypeApi _userTypeApi;
@@ -29,7 +30,7 @@ public mappingScenarios()
 
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task ListProfileMappings()
         {
             var randomSuffix = GetRandomSuffix();
@@ -45,7 +46,7 @@ public async Task ListProfileMappings()
                         Name = $"list_profile_mapping_{randomSuffix}",
                     });
 
-                Thread.Sleep(6000);
+                Thread.Sleep(10000);
                 var mappings = await _profileMappingApi.ListProfileMappings(sourceId: userType.Id)
                     .ToListAsync();
 
@@ -60,7 +61,7 @@ public async Task ListProfileMappings()
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task GetProfileMapping()
         {
             var randomSuffix = GetRandomSuffix();
@@ -98,7 +99,7 @@ public async Task GetProfileMapping()
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Replication makes test flaky OKTA-710533")]
         public async Task UpdateProfileMapping()
         {
             var randomSuffix = GetRandomSuffix();
@@ -224,8 +225,9 @@ public async Task UpdateProfileMapping()
                 };
 
                 createdIdp = await _idpApi.CreateIdentityProviderAsync(idp);
+                var idpId = createdIdp.Id;
 
-                Thread.Sleep(6000);
+                Thread.Sleep(8000);
 
                 var mappings = await _profileMappingApi
                                                         .ListProfileMappings(sourceId: createdIdp.Id)

diff --git a/src/Okta.Sdk.IntegrationTest/xunit.runner.json b/src/Okta.Sdk.IntegrationTest/xunit.runner.json
@@ -1,5 +1,4 @@
 {
   "parallelizeAssembly": false,
-  "parallelizeTestCollections": false,
-  "maxParallelThreads": 1
+  "parallelizeTestCollections": false
 }