diff --git a/source/backend/api/Pims.Api.csproj b/source/backend/api/Pims.Api.csproj index e3d29a385e..46901f2028 100644 --- a/source/backend/api/Pims.Api.csproj +++ b/source/backend/api/Pims.Api.csproj @@ -49,6 +49,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs b/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs index d2bf249d70..dd2e87be8f 100644 --- a/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs +++ b/source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs @@ -10,6 +10,7 @@ using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Requests.Http; using Pims.Core.Api.Exceptions; +using Polly.Registry; namespace Pims.Api.Repositories.Cdogs { @@ -28,12 +29,14 @@ public class CdogsAuthRepository : CdogsBaseRepository, IDocumentGenerationAuthR /// Injected Httpclient factory. /// The injected configuration provider. /// The jsonOptions. + /// The polly retry policy. public CdogsAuthRepository( ILogger logger, IHttpClientFactory httpClientFactory, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, configuration, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider) { _currentToken = null; _lastSucessfullRequest = DateTime.UnixEpoch; diff --git a/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs b/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs index 185199adc0..685fbddb7f 100644 --- a/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs +++ b/source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Options; using Pims.Api.Models.Config; using Pims.Core.Api.Repositories.Rest; +using Polly.Registry; namespace Pims.Api.Repositories.Cdogs { @@ -24,12 +25,14 @@ public abstract class CdogsBaseRepository : BaseRestRepository /// Injected Httpclient factory. /// The injected configuration provider. /// The json options. + /// The polly retry policy. protected CdogsBaseRepository( ILogger logger, IHttpClientFactory httpClientFactory, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, jsonOptions, pollyPipelineProvider) { _config = new CdogsConfig(); configuration.Bind(CdogsConfigSectionKey, _config); diff --git a/source/backend/api/Repositories/Cdogs/CdogsRepository.cs b/source/backend/api/Repositories/Cdogs/CdogsRepository.cs index 7d3a36d51e..8ee5fd926a 100644 --- a/source/backend/api/Repositories/Cdogs/CdogsRepository.cs +++ b/source/backend/api/Repositories/Cdogs/CdogsRepository.cs @@ -14,6 +14,7 @@ using Pims.Api.Models.Cdogs; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Requests.Http; +using Polly.Registry; namespace Pims.Api.Repositories.Cdogs { @@ -32,13 +33,15 @@ public class CdogsRepository : CdogsBaseRepository, IDocumentGenerationRepositor /// Injected repository that handles authentication. /// The injected configuration provider. /// The jsonOptions. + /// The polly retry policy. public CdogsRepository( ILogger logger, IHttpClientFactory httpClientFactory, IDocumentGenerationAuthRepository authRepository, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, configuration, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider) { _authRepository = authRepository; } diff --git a/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs b/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs index dd4389abdc..c3bfae24e1 100644 --- a/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanAuthRepository.cs @@ -11,6 +11,7 @@ using Pims.Api.Models.Mayan; using Pims.Api.Models.Requests.Http; using Pims.Core.Api.Exceptions; +using Polly.Registry; namespace Pims.Api.Repositories.Mayan { @@ -28,12 +29,14 @@ public class MayanAuthRepository : MayanBaseRepository, IEdmsAuthRepository /// Injected Httpclient factory. /// The injected configuration provider. /// The jsonOptions. + /// The polly retry policy. public MayanAuthRepository( ILogger logger, IHttpClientFactory httpClientFactory, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, configuration, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider) { _currentToken = string.Empty; } diff --git a/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs b/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs index a57678618a..c9d47b009c 100644 --- a/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanBaseRepository.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Options; using Pims.Api.Models.Config; using Pims.Core.Api.Repositories.Rest; +using Polly.Registry; namespace Pims.Api.Repositories.Mayan { @@ -25,12 +26,14 @@ public abstract class MayanBaseRepository : BaseRestRepository /// Injected Httpclient factory. /// The injected configuration provider. /// The injected json options. + /// The polly retry policy. protected MayanBaseRepository( ILogger logger, IHttpClientFactory httpClientFactory, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, jsonOptions, pollyPipelineProvider) { _config = new MayanConfig(); configuration.Bind(MayanConfigSectionKey, _config); diff --git a/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs b/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs index 2ca2dcaec3..bb3ff93ef7 100644 --- a/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs @@ -19,6 +19,7 @@ using Pims.Api.Models.Mayan.Document; using Pims.Api.Models.Mayan.Metadata; using Pims.Api.Models.Requests.Http; +using Polly.Registry; namespace Pims.Api.Repositories.Mayan { @@ -38,13 +39,15 @@ public class MayanDocumentRepository : MayanBaseRepository, IEdmsDocumentReposit /// Injected repository that handles authentication. /// The injected configuration provider. /// The jsonOptions. + /// The polly retry policy. public MayanDocumentRepository( ILogger logger, IHttpClientFactory httpClientFactory, IEdmsAuthRepository authRepository, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, configuration, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider) { _authRepository = authRepository; } diff --git a/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs b/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs index 9ecaa04300..19adc71b62 100644 --- a/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs +++ b/source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs @@ -11,6 +11,7 @@ using Pims.Api.Models.Mayan; using Pims.Api.Models.Mayan.Metadata; using Pims.Api.Models.Requests.Http; +using Polly.Registry; namespace Pims.Api.Repositories.Mayan { @@ -30,13 +31,15 @@ public class MayanMetadataRepository : MayanBaseRepository, IEdmsMetadataReposit /// Injected repository that handles authentication. /// The injected configuration provider. /// The json options. + /// The polly retry policy. public MayanMetadataRepository( ILogger logger, IHttpClientFactory httpClientFactory, IEdmsAuthRepository authRepository, IConfiguration configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, configuration, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider) { _authRepository = authRepository; } diff --git a/source/backend/api/Startup.cs b/source/backend/api/Startup.cs index d0e9c5b8c2..b5e9049505 100644 --- a/source/backend/api/Startup.cs +++ b/source/backend/api/Startup.cs @@ -5,6 +5,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Net.Http; using System.Reflection; using System.Security.Claims; using System.Text; @@ -44,6 +45,7 @@ using Pims.Core.Api.Exceptions; using Pims.Core.Api.Helpers; using Pims.Core.Api.Middleware; +using Pims.Core.Configuration; using Pims.Core.Converters; using Pims.Core.Http; using Pims.Core.Json; @@ -52,6 +54,7 @@ using Pims.Dal.Repositories; using Pims.Geocoder; using Pims.Ltsa; +using Polly; using Prometheus; namespace Pims.Api @@ -100,6 +103,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSerilogging(this.Configuration); var jsonSerializerOptions = this.Configuration.GenerateJsonSerializerOptions(); var pimsOptions = this.Configuration.GeneratePimsOptions(); + services.AddMapster(jsonSerializerOptions, pimsOptions, options => { options.Default.IgnoreNonMapped(true); @@ -233,6 +237,20 @@ public void ConfigureServices(IServiceCollection services) x.MultipartBodyLengthLimit = maxFileSize; // In case of multipart }); + PollyOptions pollyOptions = new(); + this.Configuration.GetSection("Polly").Bind(pollyOptions); + + services.AddResiliencePipeline("retry-network-policy", (builder) => + { + builder.AddRetry(new() + { + BackoffType = DelayBackoffType.Exponential, + Delay = TimeSpan.FromSeconds(pollyOptions.DelayInSeconds), + MaxRetryAttempts = pollyOptions.MaxRetries, + ShouldHandle = new PredicateBuilder().Handle().HandleResult(response => (int)response.StatusCode >= 500 && (int)response.StatusCode <= 599), + }); + }); + // Export metrics from all HTTP clients registered in services services.UseHttpClientMetrics(); diff --git a/source/backend/api/appsettings.json b/source/backend/api/appsettings.json index 4399742c76..ae3c019b15 100644 --- a/source/backend/api/appsettings.json +++ b/source/backend/api/appsettings.json @@ -147,5 +147,9 @@ "CDogsHost": "[CDOGS_HOST]", "ServiceClientId": "[CLIENT_ID]", "ServiceClientSecret": "[CLIENT_SECRET]" + }, + "Polly": { + "MaxRetries": 3, + "DelayInSeconds": 1 } } diff --git a/source/backend/core.api/Pims.Core.Api.csproj b/source/backend/core.api/Pims.Core.Api.csproj index 51d218ea46..3f7785fb82 100644 --- a/source/backend/core.api/Pims.Core.Api.csproj +++ b/source/backend/core.api/Pims.Core.Api.csproj @@ -14,6 +14,7 @@ + diff --git a/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs b/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs index 48a4a19bbb..9bb6d0a213 100644 --- a/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs +++ b/source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs @@ -14,6 +14,8 @@ using Pims.Api.Models; using Pims.Api.Models.CodeTypes; using Pims.Api.Models.Requests.Http; +using Polly; +using Polly.Registry; namespace Pims.Core.Api.Repositories.Rest { @@ -25,20 +27,25 @@ public abstract class BaseRestRepository : IRestRespository protected readonly IHttpClientFactory _httpClientFactory; protected readonly ILogger _logger; protected readonly IOptions _jsonOptions; + private readonly ResiliencePipeline _resiliencePipeline; /// /// Initializes a new instance of the class. /// /// Injected Logger Provider. /// Injected Httpclient factory. + /// + /// protected BaseRestRepository( ILogger logger, IHttpClientFactory httpClientFactory, - IOptions jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) { _logger = logger; _httpClientFactory = httpClientFactory; _jsonOptions = jsonOptions; + _resiliencePipeline = pollyPipelineProvider.GetPipeline("retry-network-policy"); } public abstract void AddAuthentication(HttpClient client, string authenticationToken = null); @@ -51,7 +58,7 @@ public async Task> GetAsync(Uri endpoint, string authenti client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); try { - HttpResponseMessage response = await client.GetAsync(endpoint).ConfigureAwait(true); + HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.GetAsync(endpoint, cancellation).ConfigureAwait(true)); var result = await ProcessResponse(response); return result; } @@ -74,7 +81,7 @@ public async Task GetRawAsync(Uri endpoint, string authenti client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); try { - HttpResponseMessage response = await client.GetAsync(endpoint).ConfigureAwait(true); + HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.GetAsync(endpoint, cancellation).ConfigureAwait(true)); return response; } catch (Exception e) @@ -96,7 +103,7 @@ public async Task> PostAsync(Uri endpoint, HttpContent co try { - HttpResponseMessage response = await client.PostAsync(endpoint, content).ConfigureAwait(true); + HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.PostAsync(endpoint, content, cancellation).ConfigureAwait(true)); var result = await ProcessResponse(response); return result; } @@ -120,7 +127,7 @@ public async Task> PutAsync(Uri endpoint, HttpContent con try { - HttpResponseMessage response = await client.PutAsync(endpoint, content).ConfigureAwait(true); + HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.PutAsync(endpoint, content, cancellation).ConfigureAwait(true)); var result = await ProcessResponse(response); return result; } @@ -150,7 +157,7 @@ public async Task> DeleteAsync(Uri endpoint, string aut try { - HttpResponseMessage response = await client.DeleteAsync(endpoint).ConfigureAwait(true); + HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.DeleteAsync(endpoint, cancellation).ConfigureAwait(true)); _logger.LogTrace("Response: {response}", response); diff --git a/source/backend/core/Http/HttpRequestClient.cs b/source/backend/core/Http/HttpRequestClient.cs index 4a8a615031..67495351a6 100644 --- a/source/backend/core/Http/HttpRequestClient.cs +++ b/source/backend/core/Http/HttpRequestClient.cs @@ -4,10 +4,13 @@ using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Pims.Core.Exceptions; +using Polly; +using Polly.Registry; namespace Pims.Core.Http { @@ -17,6 +20,7 @@ namespace Pims.Core.Http public class HttpRequestClient : IHttpRequestClient { #region Variables + protected readonly ResiliencePipeline _resiliencePipeline; private readonly JsonSerializerOptions _serializeOptions; private readonly ILogger _logger; #endregion @@ -37,7 +41,8 @@ public class HttpRequestClient : IHttpRequestClient /// /// /// - public HttpRequestClient(IHttpClientFactory clientFactory, IOptionsMonitor options, ILogger logger) + /// The polly retry policy. + public HttpRequestClient(IHttpClientFactory clientFactory, IOptionsMonitor options, ILogger logger, ResiliencePipelineProvider pollyPipelineProvider) { this.Client = clientFactory.CreateClient(); _serializeOptions = options?.CurrentValue ?? new JsonSerializerOptions() @@ -47,6 +52,7 @@ public HttpRequestClient(IHttpClientFactory clientFactory, IOptionsMonitor("retry-network-policy"); } #endregion @@ -563,20 +569,23 @@ private async Task SendInternalAsync(string url, HttpMethod method = HttpMethod.Get; } - using var message = new HttpRequestMessage(method, url); - message.Headers.Add("User-Agent", "Pims.Api"); - message.Content = content; - - if (headers != null) + return await _resiliencePipeline.ExecuteAsync(async cancellation => { - foreach (var header in headers) + using var message = new HttpRequestMessage(method, url); + message.Headers.Add("User-Agent", "Pims.Api"); + message.Content = content; + + if (headers != null) { - message.Headers.Add(header.Key, header.Value); + foreach (var header in headers) + { + message.Headers.Add(header.Key, header.Value); + } } - } - _logger.LogInformation("HTTP request made '{message.RequestUri}'", message.RequestUri); - return await this.Client.SendAsync(message); + _logger.LogInformation("HTTP request made '{message.RequestUri}'", message.RequestUri); + return await this.Client.SendAsync(message, cancellation); + }); } #endregion #endregion diff --git a/source/backend/core/Http/IOpenIdConnectRequestClient.cs b/source/backend/core/Http/IOpenIdConnectRequestClient.cs index 9c91337806..c60feb945c 100644 --- a/source/backend/core/Http/IOpenIdConnectRequestClient.cs +++ b/source/backend/core/Http/IOpenIdConnectRequestClient.cs @@ -1,4 +1,5 @@ using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Pims.Core.Http.Configuration; diff --git a/source/backend/core/Http/OpenIdConnectRequestClient.cs b/source/backend/core/Http/OpenIdConnectRequestClient.cs index 44b5fed4b6..f7eef73b69 100644 --- a/source/backend/core/Http/OpenIdConnectRequestClient.cs +++ b/source/backend/core/Http/OpenIdConnectRequestClient.cs @@ -10,6 +10,7 @@ using Pims.Core.Exceptions; using Pims.Core.Extensions; using Pims.Core.Http.Configuration; +using Polly.Registry; namespace Pims.Core.Http { @@ -58,8 +59,9 @@ public OpenIdConnectRequestClient( IOptionsMonitor authClientOptions, IOptionsMonitor openIdConnectOptions, IOptionsMonitor serializerOptions, - ILogger logger) - : base(clientFactory, serializerOptions, logger) + ILogger logger, + ResiliencePipelineProvider pollyPipelineProvider) + : base(clientFactory, serializerOptions, logger, pollyPipelineProvider) { this.AuthClientOptions = authClientOptions.CurrentValue; this.OpenIdConnectOptions = openIdConnectOptions.CurrentValue; @@ -78,7 +80,7 @@ public OpenIdConnectRequestClient( var authority = this.AuthClientOptions.Authority ?? throw new ConfigurationException($"Configuration 'OpenIdConnect:Authority' is missing or invalid."); - var response = await this.Client.GetAsync($"{authority}/.well-known/openid-configuration"); + var response = await _resiliencePipeline.ExecuteAsync(async cancellation => await this.Client.GetAsync($"{authority}/.well-known/openid-configuration", cancellation)); if (response.IsSuccessStatusCode) { @@ -170,18 +172,21 @@ public async Task RequestToken() keycloakTokenUrl = $"{authority}{keycloakTokenUrl}"; } - using var tokenMessage = new HttpRequestMessage(HttpMethod.Post, keycloakTokenUrl); - var p = new Dictionary + return await _resiliencePipeline.ExecuteAsync(async cancellation => + { + using var tokenMessage = new HttpRequestMessage(HttpMethod.Post, keycloakTokenUrl); + var p = new Dictionary { { "client_id", client }, { "grant_type", "client_credentials" }, { "client_secret", clientSecret }, { "audience", audience }, }; - var form = new FormUrlEncodedContent(p); - form.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/x-www-form-urlencoded"); - tokenMessage.Content = form; - return await this.Client.SendAsync(tokenMessage); + var form = new FormUrlEncodedContent(p); + form.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/x-www-form-urlencoded"); + tokenMessage.Content = form; + return await this.Client.SendAsync(tokenMessage, cancellation); + }); } /// @@ -210,18 +215,21 @@ public async Task RefreshToken(string refreshToken) keycloakTokenUrl = $"{authority}{keycloakTokenUrl}"; } - using var tokenMessage = new HttpRequestMessage(HttpMethod.Post, keycloakTokenUrl); - var p = new Dictionary + return await _resiliencePipeline.ExecuteAsync(async cancellation => + { + using var tokenMessage = new HttpRequestMessage(HttpMethod.Post, keycloakTokenUrl); + var p = new Dictionary { { "client_id", client }, { "grant_type", "refresh_token" }, { "client_secret", clientSecret }, { "refresh_token", refreshToken }, }; - var form = new FormUrlEncodedContent(p); - form.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/x-www-form-urlencoded"); - tokenMessage.Content = form; - return await this.Client.SendAsync(tokenMessage); + var form = new FormUrlEncodedContent(p); + form.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/x-www-form-urlencoded"); + tokenMessage.Content = form; + return await this.Client.SendAsync(tokenMessage, cancellation); + }); } #endregion diff --git a/source/backend/core/Options/PollyOptions.cs b/source/backend/core/Options/PollyOptions.cs new file mode 100644 index 0000000000..e6ba81b78e --- /dev/null +++ b/source/backend/core/Options/PollyOptions.cs @@ -0,0 +1,21 @@ +namespace Pims.Core.Configuration +{ + /// + /// PollyOptions class, provides a way to configure polly resiliency pipelines. + /// + public class PollyOptions + { + #region Properties + + /// + /// get/set - The number of times http requests will be retried. + /// + public int MaxRetries { get; set; } = 3; + + /// + /// get/set - The number of seconds to delay before making a retry, using iterative backoff. + /// + public int DelayInSeconds { get; set; } = 1; + #endregion + } +} diff --git a/source/backend/core/Pims.Core.csproj b/source/backend/core/Pims.Core.csproj index 7186c70c2f..b5380fa65f 100644 --- a/source/backend/core/Pims.Core.csproj +++ b/source/backend/core/Pims.Core.csproj @@ -12,6 +12,7 @@ + diff --git a/source/backend/dal.keycloak/Pims.Dal.Keycloak.csproj b/source/backend/dal.keycloak/Pims.Dal.Keycloak.csproj index 0a8dde6f00..d537c917ca 100644 --- a/source/backend/dal.keycloak/Pims.Dal.Keycloak.csproj +++ b/source/backend/dal.keycloak/Pims.Dal.Keycloak.csproj @@ -11,6 +11,7 @@ + diff --git a/source/backend/dal/Pims.Dal.csproj b/source/backend/dal/Pims.Dal.csproj index 453a8d2c64..3a76841cb6 100644 --- a/source/backend/dal/Pims.Dal.csproj +++ b/source/backend/dal/Pims.Dal.csproj @@ -35,6 +35,7 @@ + diff --git a/source/backend/keycloak/Pims.Keycloak.csproj b/source/backend/keycloak/Pims.Keycloak.csproj index 90f26d1ebb..696230289f 100644 --- a/source/backend/keycloak/Pims.Keycloak.csproj +++ b/source/backend/keycloak/Pims.Keycloak.csproj @@ -11,6 +11,7 @@ + diff --git a/source/backend/ltsa/Pims.Ltsa.csproj b/source/backend/ltsa/Pims.Ltsa.csproj index e471010bca..362ae2924b 100644 --- a/source/backend/ltsa/Pims.Ltsa.csproj +++ b/source/backend/ltsa/Pims.Ltsa.csproj @@ -14,6 +14,7 @@ + diff --git a/source/backend/scheduler/Repositories/PimsBaseRepository.cs b/source/backend/scheduler/Repositories/PimsBaseRepository.cs index 5f822e7eb3..6023069335 100644 --- a/source/backend/scheduler/Repositories/PimsBaseRepository.cs +++ b/source/backend/scheduler/Repositories/PimsBaseRepository.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Pims.Core.Api.Repositories.Rest; +using Polly.Registry; namespace Pims.Scheduler.Repositories { @@ -17,11 +18,13 @@ public abstract class PimsBaseRepository : BaseRestRepository /// Injected Logger Provider. /// Injected Httpclient factory. /// Injected app-wide json options. + /// protected PimsBaseRepository( ILogger logger, IHttpClientFactory httpClientFactory, - IOptions jsonOptions) - : base(logger, httpClientFactory, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, jsonOptions, pollyPipelineProvider) { } diff --git a/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs b/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs index f78ef001f5..41328b5eeb 100644 --- a/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs +++ b/source/backend/scheduler/Repositories/PimsDocumentQueueRepository.cs @@ -13,6 +13,7 @@ using Pims.Core.Http; using Pims.Dal.Entities.Models; using Pims.Scheduler.Http.Configuration; +using Polly.Registry; namespace Pims.Scheduler.Repositories { @@ -34,13 +35,15 @@ public class PimsDocumentQueueRepository : PimsBaseRepository, IPimsDocumentQueu /// Injected repository that handles authentication. /// The injected configuration provider. /// The injected json options. + /// public PimsDocumentQueueRepository( ILogger logger, IHttpClientFactory httpClientFactory, IOpenIdConnectRequestClient authRepository, IOptionsMonitor configuration, - IOptions jsonOptions) - : base(logger, httpClientFactory, jsonOptions) + IOptions jsonOptions, + ResiliencePipelineProvider pollyPipelineProvider) + : base(logger, httpClientFactory, jsonOptions, pollyPipelineProvider) { _authRepository = authRepository; _configuration = configuration; diff --git a/source/backend/tests/unit/api/Repositories/BaseRestRepositoryTest.cs b/source/backend/tests/unit/api/Repositories/BaseRestRepositoryTest.cs index 62feec20d2..de19047f1e 100644 --- a/source/backend/tests/unit/api/Repositories/BaseRestRepositoryTest.cs +++ b/source/backend/tests/unit/api/Repositories/BaseRestRepositoryTest.cs @@ -16,6 +16,8 @@ using Pims.Core.Api.Repositories.Rest; using Pims.Core.Extensions; using Pims.Core.Test.Http; +using Polly; +using Polly.Registry; using Xunit; namespace Pims.Core.Test.Repositories.Rest @@ -26,16 +28,20 @@ public class BaseRestRepositoryTest private readonly Mock _httpClientFactory; private readonly Mock> _jsonOptions; private readonly TestBaseRestRepository _repository; + private readonly Mock> _pipelineProvider; public BaseRestRepositoryTest() { _logger = new Mock>(); _httpClientFactory = new Mock(); _jsonOptions = new Mock>(); + _pipelineProvider = new Mock>(); + _pipelineProvider.Setup(_pipelineProvider => _pipelineProvider.GetPipeline(It.IsAny())).Returns(ResiliencePipeline.Empty); _repository = new TestBaseRestRepository( _logger.Object, _httpClientFactory.Object, - _jsonOptions.Object); + _jsonOptions.Object, + _pipelineProvider.Object); } [Fact] @@ -388,8 +394,8 @@ public async Task DeleteAsync_Exception() // Helper class to test the abstract BaseRestRepository private class TestBaseRestRepository : BaseRestRepository { - public TestBaseRestRepository(ILogger logger, IHttpClientFactory httpClientFactory, IOptions jsonOptions) - : base(logger, httpClientFactory, jsonOptions) + public TestBaseRestRepository(ILogger logger, IHttpClientFactory httpClientFactory, IOptions jsonOptions, ResiliencePipelineProvider provider) + : base(logger, httpClientFactory, jsonOptions, provider) { } diff --git a/source/backend/tests/unit/api/Repositories/MayanAuthRepositoryTest.cs b/source/backend/tests/unit/api/Repositories/MayanAuthRepositoryTest.cs index a5c7ef1936..77b832c67d 100644 --- a/source/backend/tests/unit/api/Repositories/MayanAuthRepositoryTest.cs +++ b/source/backend/tests/unit/api/Repositories/MayanAuthRepositoryTest.cs @@ -16,6 +16,8 @@ using Pims.Api.Models.Requests.Http; using Pims.Api.Repositories.Mayan; using Pims.Core.Api.Exceptions; +using Polly; +using Polly.Registry; using Xunit; namespace Pims.Api.Repositories.Mayan @@ -27,6 +29,7 @@ public class MayanAuthRepositoryTest private readonly IOptions _jsonOptions; private readonly IConfiguration _configuration; private readonly MayanAuthRepository _repository; + private readonly Mock> _pipelineProvider; public MayanAuthRepositoryTest() { @@ -34,11 +37,14 @@ public MayanAuthRepositoryTest() _httpClientFactory = new Mock(); _jsonOptions = Options.Create(new JsonSerializerOptions() { AllowTrailingCommas = true, PropertyNameCaseInsensitive = true }); _configuration = _configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { { "Mayan:BaseUri", "http://mayan" } }).Build(); + _pipelineProvider = new Mock>(); + _pipelineProvider.Setup(_pipelineProvider => _pipelineProvider.GetPipeline(It.IsAny())).Returns(ResiliencePipeline.Empty); _repository = new MayanAuthRepository( _logger.Object, _httpClientFactory.Object, _configuration, - _jsonOptions); + _jsonOptions, + _pipelineProvider.Object); } [Fact] diff --git a/source/backend/tests/unit/api/Repositories/MayanDocumentRepositoryTest.cs b/source/backend/tests/unit/api/Repositories/MayanDocumentRepositoryTest.cs index 3b0757ba01..c67e51a82b 100644 --- a/source/backend/tests/unit/api/Repositories/MayanDocumentRepositoryTest.cs +++ b/source/backend/tests/unit/api/Repositories/MayanDocumentRepositoryTest.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -25,6 +26,8 @@ using Pims.Api.Repositories.Mayan; using Pims.Core.Extensions; using Pims.Core.Test.Http; +using Polly; +using Polly.Registry; using Xunit; using DocumentTypeModel = Pims.Api.Models.Mayan.Document.DocumentTypeModel; @@ -38,6 +41,7 @@ public class MayanDocumentRepositoryTest private readonly Mock> _jsonOptions; private readonly IConfiguration _configuration; private readonly MayanDocumentRepository _repository; + private readonly Mock> _pipelineProvider; public MayanDocumentRepositoryTest() { @@ -46,13 +50,16 @@ public MayanDocumentRepositoryTest() _authRepository = new Mock(); _jsonOptions = new Mock>(); _configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { { "Mayan:BaseUri", "http://mayan" } }).Build(); + _pipelineProvider = new Mock>(); + _pipelineProvider.Setup(_pipelineProvider => _pipelineProvider.GetPipeline(It.IsAny())).Returns(ResiliencePipeline< HttpResponseMessage>.Empty); _repository = new MayanDocumentRepository( _logger.Object, _httpClientFactory.Object, _authRepository.Object, _configuration, - _jsonOptions.Object); + _jsonOptions.Object, + _pipelineProvider.Object); } [Fact] diff --git a/source/backend/tests/unit/api/Repositories/MayanMetadataRepositoryTest.cs b/source/backend/tests/unit/api/Repositories/MayanMetadataRepositoryTest.cs index 40eb4239a9..8b0ac8e5e3 100644 --- a/source/backend/tests/unit/api/Repositories/MayanMetadataRepositoryTest.cs +++ b/source/backend/tests/unit/api/Repositories/MayanMetadataRepositoryTest.cs @@ -22,6 +22,8 @@ namespace Pims.Api.Test.Repositories using Pims.Api.Models.Mayan.Metadata; using Pims.Api.Models.Requests.Http; using Pims.Api.Repositories.Mayan; + using Polly; + using Polly.Registry; using Xunit; public class MayanMetadataRepositoryTest @@ -32,6 +34,7 @@ public class MayanMetadataRepositoryTest private readonly Mock> _jsonOptions; private readonly IConfiguration _configuration; private readonly MayanMetadataRepository _repository; + private readonly Mock> _pipelineProvider; public MayanMetadataRepositoryTest() { @@ -40,12 +43,15 @@ public MayanMetadataRepositoryTest() _authRepository = new Mock(); _jsonOptions = new Mock>(); _configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { { "Mayan:BaseUri", "http://mayan" } }).Build(); + _pipelineProvider = new Mock>(); + _pipelineProvider.Setup(_pipelineProvider => _pipelineProvider.GetPipeline(It.IsAny())).Returns(ResiliencePipeline.Empty); _repository = new MayanMetadataRepository( _logger.Object, _httpClientFactory.Object, _authRepository.Object, _configuration, - _jsonOptions.Object); + _jsonOptions.Object, + _pipelineProvider.Object); } [Fact] diff --git a/source/backend/tests/unit/dal/Core/Http/OpenIdConnectRequestClientTest.cs b/source/backend/tests/unit/dal/Core/Http/OpenIdConnectRequestClientTest.cs index 1e3036975c..71cacfaaf1 100644 --- a/source/backend/tests/unit/dal/Core/Http/OpenIdConnectRequestClientTest.cs +++ b/source/backend/tests/unit/dal/Core/Http/OpenIdConnectRequestClientTest.cs @@ -9,6 +9,8 @@ using Pims.Core.Http; using Pims.Core.Http.Configuration; using Pims.Core.Test; +using Polly; +using Polly.Registry; using Xunit; namespace Pims.Api.Test.Helpers @@ -36,9 +38,10 @@ public void OpenIdConnectRequestClient_Constructor() mockOpenIdConnectOptions.Setup(m => m.CurrentValue).Returns(openIdConnectOptions); var mockJsonSerializeOptions = new Mock>(); var mockLogger = new Mock>(); + var mockPipeline = new Mock>(); // Act - var client = new OpenIdConnectRequestClient(clientFactory, tokenHandler, mockAuthClientOptions.Object, mockOpenIdConnectOptions.Object, mockJsonSerializeOptions.Object, mockLogger.Object); + var client = new OpenIdConnectRequestClient(clientFactory, tokenHandler, mockAuthClientOptions.Object, mockOpenIdConnectOptions.Object, mockJsonSerializeOptions.Object, mockLogger.Object, mockPipeline.Object); // Assert Assert.NotNull(client); diff --git a/source/backend/tests/unit/dal/Libraries/Geocoder/ServiceCollectionExtensionsTest.cs b/source/backend/tests/unit/dal/Libraries/Geocoder/ServiceCollectionExtensionsTest.cs index ff823e3637..ac4ff9c31a 100644 --- a/source/backend/tests/unit/dal/Libraries/Geocoder/ServiceCollectionExtensionsTest.cs +++ b/source/backend/tests/unit/dal/Libraries/Geocoder/ServiceCollectionExtensionsTest.cs @@ -7,6 +7,7 @@ using Moq; using Pims.Core.Http; using Pims.Geocoder; +using Polly.Registry; using Xunit; namespace Pims.Dal.Test.Libraries.Geocoder @@ -28,6 +29,8 @@ public void AddGeocoderService_Success() services.AddScoped((s) => mockHttpClientFactory.Object); var mockLogger = new Mock>(); services.AddScoped((s) => mockLogger.Object); + var mockResiliencyPipeline = new Mock>(); + services.AddScoped((s) => mockResiliencyPipeline.Object); // Act var result = services.AddGeocoderService(mockConfig.Object); @@ -37,7 +40,7 @@ public void AddGeocoderService_Success() // Assert result.Should().NotBeNull(); - result.Count.Should().Be(11); + result.Count.Should().Be(12); provider.Should().NotBeNull(); service.Should().NotBeNull(); client.Should().NotBeNull(); diff --git a/source/backend/tests/unit/dal/Libraries/Ltsa/LtsaServiceCollectionTest.cs b/source/backend/tests/unit/dal/Libraries/Ltsa/LtsaServiceCollectionTest.cs index 6248f26bf9..ae2e532408 100644 --- a/source/backend/tests/unit/dal/Libraries/Ltsa/LtsaServiceCollectionTest.cs +++ b/source/backend/tests/unit/dal/Libraries/Ltsa/LtsaServiceCollectionTest.cs @@ -13,6 +13,7 @@ using Pims.Core.Test; using Pims.Ltsa; using Pims.Ltsa.Configuration; +using Polly.Registry; using Xunit; namespace Pims.Dal.Test.Libraries.Ltsa @@ -55,11 +56,13 @@ public void LtsaServiceCollection_Success() var mockIOptionsMonitor = new Mock>(); var mockIlogger = new Mock>(); var mockILtsaService = new Mock>(); + var mockPipeline = new Mock>(); helper.AddSingleton(mockClientFactory.Object); helper.AddSingleton(mockIOptionsMonitor.Object); helper.AddSingleton(mockIlogger.Object); helper.AddSingleton(mockILtsaService.Object); + helper.AddSingleton(mockPipeline.Object); // Act _ = helper.Services.AddLtsaService(section: ltsaConfig.GetSection("Ltsa")); diff --git a/tools/keycloak/sync/Client/PimsRequestClient.cs b/tools/keycloak/sync/Client/PimsRequestClient.cs index 2df6c4abb0..0d87c2a158 100644 --- a/tools/keycloak/sync/Client/PimsRequestClient.cs +++ b/tools/keycloak/sync/Client/PimsRequestClient.cs @@ -2,11 +2,13 @@ using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Pims.Core.Http.Configuration; using Pims.Keycloak.Configuration; using Pims.Tools.Keycloak.Sync.Configuration; +using Polly.Registry; namespace Pims.Tools.Keycloak.Sync { @@ -33,6 +35,7 @@ public class PimsRequestClient : RequestClient, IPimsRequestClient /// /// /// + /// The polly retry policy. public PimsRequestClient( IHttpClientFactory clientFactory, JwtSecurityTokenHandler tokenHandler, @@ -41,8 +44,9 @@ public PimsRequestClient( IOptionsMonitor options, IOptionsMonitor requestOptions, IOptionsMonitor serializerOptions, - ILogger logger) - : base(clientFactory, tokenHandler, keycloakOptions, openIdConnectOptions, requestOptions, serializerOptions, logger) + ILogger logger, + ResiliencePipelineProvider pollyPipelineProvider) + : base(clientFactory, tokenHandler, keycloakOptions, openIdConnectOptions, requestOptions, serializerOptions, logger, pollyPipelineProvider) { this.OpenIdConnectOptions.Token = $"{keycloakOptions.CurrentValue.Authority}{this.OpenIdConnectOptions.Token}"; _options = options.CurrentValue; diff --git a/tools/keycloak/sync/Client/RequestClient.cs b/tools/keycloak/sync/Client/RequestClient.cs index 31141cbd0a..bb1ce182fa 100644 --- a/tools/keycloak/sync/Client/RequestClient.cs +++ b/tools/keycloak/sync/Client/RequestClient.cs @@ -9,8 +9,7 @@ using Pims.Core.Exceptions; using Pims.Core.Http.Configuration; using Pims.Tools.Keycloak.Sync.Configuration; -using Polly; -using Polly.Retry; +using Polly.Registry; namespace Pims.Tools.Keycloak.Sync { @@ -23,7 +22,6 @@ public class RequestClient : Pims.Core.Http.OpenIdConnectRequestClient, IRequest private readonly RequestOptions _requestOptions; private readonly ILogger _logger; private readonly JsonSerializerOptions _serializerOptions; - private readonly AsyncRetryPolicy _retryPolicy; #endregion #region Constructors @@ -38,6 +36,7 @@ public class RequestClient : Pims.Core.Http.OpenIdConnectRequestClient, IRequest /// /// /// + /// The polly retry policy. public RequestClient( IHttpClientFactory clientFactory, JwtSecurityTokenHandler tokenHandler, @@ -45,8 +44,9 @@ public RequestClient( IOptionsMonitor openIdConnectOptions, IOptionsMonitor requestOptions, IOptionsMonitor serializerOptions, - ILogger logger) - : base(clientFactory, tokenHandler, authClientOptions, openIdConnectOptions, serializerOptions, logger) + ILogger logger, + ResiliencePipelineProvider pollyPipelineProvider) + : base(clientFactory, tokenHandler, authClientOptions, openIdConnectOptions, serializerOptions, logger, pollyPipelineProvider) { _requestOptions = requestOptions.CurrentValue; _logger = logger; @@ -56,9 +56,6 @@ public RequestClient( PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; - _retryPolicy = Policy - .Handle(result => result?.StatusCode != null && (int)result.StatusCode >= 500 && (int)result.StatusCode < 600 && _requestOptions.RetryAfterFailure) - .WaitAndRetryAsync(_requestOptions.RetryAttempts, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))); } #endregion @@ -74,7 +71,7 @@ public RequestClient( public async Task HandleGetAsync(string url, Func onError = null) where TR : class { - var response = await _retryPolicy.ExecuteAsync(async () => await SendAsync(url, HttpMethod.Get)); + var response = await SendAsync(url, HttpMethod.Get); if (response.IsSuccessStatusCode) { @@ -109,7 +106,7 @@ public async Task HandleGetAsync(string url, Func HandleRequestAsync(HttpMethod method, string url, Func onError = null) where TR : class { - var response = await _retryPolicy.ExecuteAsync(async () => await SendAsync(url, method)); + var response = await SendAsync(url, method); if (response.IsSuccessStatusCode) { diff --git a/tools/keycloak/sync/Pims.Tools.Keycloak.Sync.csproj b/tools/keycloak/sync/Pims.Tools.Keycloak.Sync.csproj index 8402fdf911..e657d379ff 100644 --- a/tools/keycloak/sync/Pims.Tools.Keycloak.Sync.csproj +++ b/tools/keycloak/sync/Pims.Tools.Keycloak.Sync.csproj @@ -32,6 +32,7 @@ + diff --git a/tools/keycloak/sync/Program.cs b/tools/keycloak/sync/Program.cs index 270437286b..1ac240c9ad 100644 --- a/tools/keycloak/sync/Program.cs +++ b/tools/keycloak/sync/Program.cs @@ -1,6 +1,7 @@ using System; using System.IdentityModel.Tokens.Jwt; using System.IO; +using System.Net.Http; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; @@ -8,12 +9,15 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Pims.Core.Exceptions; +using Pims.Core.Configuration; using Pims.Core.Http; using Pims.Core.Http.Configuration; + using Pims.Keycloak; using Pims.Keycloak.Configuration; using Pims.Tools.Keycloak.Sync.Configuration; using Pims.Tools.Keycloak.Sync.Configuration.Realm; +using Polly; namespace Pims.Tools.Keycloak.Sync { @@ -85,6 +89,20 @@ public static async Task Main(string[] args) services.AddHttpClient("Pims.Tools.Keycloak.Sync", client => { }); + PollyOptions pollyOptions = new(); + config.GetSection("Polly").Bind(pollyOptions); + + services.AddResiliencePipeline("retry-network-policy", (builder) => + { + builder.AddRetry(new() + { + BackoffType = DelayBackoffType.Exponential, + Delay = TimeSpan.FromSeconds(pollyOptions.DelayInSeconds), + MaxRetryAttempts = pollyOptions.MaxRetries, + ShouldHandle = new PredicateBuilder().Handle().HandleResult(response => (int)response.StatusCode >= 500 && (int)response.StatusCode <= 599), + }); + }); + var provider = services.BuildServiceProvider(); var logger = provider.GetService>(); diff --git a/tools/keycloak/sync/appsettings.json b/tools/keycloak/sync/appsettings.json index 6ca9e201a9..bd8c4b6ab0 100644 --- a/tools/keycloak/sync/appsettings.json +++ b/tools/keycloak/sync/appsettings.json @@ -40,5 +40,9 @@ "PropertyNamingPolicy": "CamelCase", "WriteIndented": true } + }, + "Polly": { + "MaxRetries": 3, + "DelayInSeconds": 1 } }