psp-1362 polly policies DI support.

source/backend/api/Pims.Api.csproj

@@ -49,6 +49,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Polly.Extensions" Version="8.5.1" />
     <PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1" />
     <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="8.2.1" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.9.0" />

source/backend/api/Repositories/Cdogs/CdogsAuthRepository.cs

@@ -10,6 +10,7 @@
 using Pims.Api.Models.CodeTypes;
 using Pims.Api.Models.Requests.Http;
 using Pims.Core.Api.Exceptions;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Cdogs
 {
@@ -28,12 +29,14 @@ public class CdogsAuthRepository : CdogsBaseRepository, IDocumentGenerationAuthR
         /// <param name="httpClientFactory">Injected Httpclient factory.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The jsonOptions.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         public CdogsAuthRepository(
             ILogger<CdogsAuthRepository> logger,
             IHttpClientFactory httpClientFactory,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, configuration, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider)
         {
             _currentToken = null;
             _lastSucessfullRequest = DateTime.UnixEpoch;

source/backend/api/Repositories/Cdogs/CdogsBaseRepository.cs

@@ -6,6 +6,7 @@
 using Microsoft.Extensions.Options;
 using Pims.Api.Models.Config;
 using Pims.Core.Api.Repositories.Rest;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Cdogs
 {
@@ -24,12 +25,14 @@ public abstract class CdogsBaseRepository : BaseRestRepository
         /// <param name="httpClientFactory">Injected Httpclient factory.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The json options.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         protected CdogsBaseRepository(
             ILogger logger,
             IHttpClientFactory httpClientFactory,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, jsonOptions, pollyPipelineProvider)
         {
             _config = new CdogsConfig();
             configuration.Bind(CdogsConfigSectionKey, _config);

source/backend/api/Repositories/Cdogs/CdogsRepository.cs

@@ -14,6 +14,7 @@
 using Pims.Api.Models.Cdogs;
 using Pims.Api.Models.CodeTypes;
 using Pims.Api.Models.Requests.Http;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Cdogs
 {
@@ -32,13 +33,15 @@ public class CdogsRepository : CdogsBaseRepository, IDocumentGenerationRepositor
         /// <param name="authRepository">Injected repository that handles authentication.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The jsonOptions.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         public CdogsRepository(
             ILogger<CdogsRepository> logger,
             IHttpClientFactory httpClientFactory,
             IDocumentGenerationAuthRepository authRepository,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, configuration, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider)
         {
             _authRepository = authRepository;
         }

source/backend/api/Repositories/Mayan/MayanAuthRepository.cs

@@ -11,6 +11,7 @@
 using Pims.Api.Models.Mayan;
 using Pims.Api.Models.Requests.Http;
 using Pims.Core.Api.Exceptions;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Mayan
 {
@@ -28,12 +29,14 @@ public class MayanAuthRepository : MayanBaseRepository, IEdmsAuthRepository
         /// <param name="httpClientFactory">Injected Httpclient factory.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The jsonOptions.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         public MayanAuthRepository(
             ILogger<MayanAuthRepository> logger,
             IHttpClientFactory httpClientFactory,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, configuration, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider)
         {
             _currentToken = string.Empty;
         }

source/backend/api/Repositories/Mayan/MayanBaseRepository.cs

@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Options;
 using Pims.Api.Models.Config;
 using Pims.Core.Api.Repositories.Rest;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Mayan
 {
@@ -25,12 +26,14 @@ public abstract class MayanBaseRepository : BaseRestRepository
         /// <param name="httpClientFactory">Injected Httpclient factory.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The injected json options.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         protected MayanBaseRepository(
             ILogger logger,
             IHttpClientFactory httpClientFactory,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, jsonOptions, pollyPipelineProvider)
         {
             _config = new MayanConfig();
             configuration.Bind(MayanConfigSectionKey, _config);

source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs

@@ -19,6 +19,7 @@
 using Pims.Api.Models.Mayan.Document;
 using Pims.Api.Models.Mayan.Metadata;
 using Pims.Api.Models.Requests.Http;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Mayan
 {
@@ -38,13 +39,15 @@ public class MayanDocumentRepository : MayanBaseRepository, IEdmsDocumentReposit
         /// <param name="authRepository">Injected repository that handles authentication.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The jsonOptions.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         public MayanDocumentRepository(
             ILogger<MayanDocumentRepository> logger,
             IHttpClientFactory httpClientFactory,
             IEdmsAuthRepository authRepository,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, configuration, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider)
         {
             _authRepository = authRepository;
         }

source/backend/api/Repositories/Mayan/MayanMetadataRepository.cs

@@ -11,6 +11,7 @@
 using Pims.Api.Models.Mayan;
 using Pims.Api.Models.Mayan.Metadata;
 using Pims.Api.Models.Requests.Http;
+using Polly.Registry;
 
 namespace Pims.Api.Repositories.Mayan
 {
@@ -30,13 +31,15 @@ public class MayanMetadataRepository : MayanBaseRepository, IEdmsMetadataReposit
         /// <param name="authRepository">Injected repository that handles authentication.</param>
         /// <param name="configuration">The injected configuration provider.</param>
         /// <param name="jsonOptions">The json options.</param>
+        /// <param name="pollyPipelineProvider">The polly retry policy.</param>
         public MayanMetadataRepository(
             ILogger<MayanMetadataRepository> logger,
             IHttpClientFactory httpClientFactory,
             IEdmsAuthRepository authRepository,
             IConfiguration configuration,
-            IOptions<JsonSerializerOptions> jsonOptions)
-            : base(logger, httpClientFactory, configuration, jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
+            : base(logger, httpClientFactory, configuration, jsonOptions, pollyPipelineProvider)
         {
             _authRepository = authRepository;
         }

source/backend/api/Startup.cs

@@ -5,6 +5,7 @@
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Reflection;
 using System.Security.Claims;
 using System.Text;
@@ -44,6 +45,7 @@
 using Pims.Core.Api.Exceptions;
 using Pims.Core.Api.Helpers;
 using Pims.Core.Api.Middleware;
+using Pims.Core.Configuration;
 using Pims.Core.Converters;
 using Pims.Core.Http;
 using Pims.Core.Json;
@@ -52,6 +54,7 @@
 using Pims.Dal.Repositories;
 using Pims.Geocoder;
 using Pims.Ltsa;
+using Polly;
 using Prometheus;
 
 namespace Pims.Api
@@ -100,6 +103,7 @@ public void ConfigureServices(IServiceCollection services)
             services.AddSerilogging(this.Configuration);
             var jsonSerializerOptions = this.Configuration.GenerateJsonSerializerOptions();
             var pimsOptions = this.Configuration.GeneratePimsOptions();
+
             services.AddMapster(jsonSerializerOptions, pimsOptions, options =>
             {
                 options.Default.IgnoreNonMapped(true);
@@ -233,6 +237,20 @@ public void ConfigureServices(IServiceCollection services)
                 x.MultipartBodyLengthLimit = maxFileSize; // In case of multipart
             });
 
+            PollyOptions pollyOptions = new();
+            this.Configuration.GetSection("Polly").Bind(pollyOptions);
+
+            services.AddResiliencePipeline<string, HttpResponseMessage>("retry-network-policy", (builder) =>
+            {
+                builder.AddRetry(new()
+                {
+                    BackoffType = DelayBackoffType.Exponential,
+                    Delay = TimeSpan.FromSeconds(pollyOptions.DelayInSeconds),
+                    MaxRetryAttempts = pollyOptions.MaxRetries,
+                    ShouldHandle = new PredicateBuilder<HttpResponseMessage>().Handle<HttpRequestException>().HandleResult(response => (int)response.StatusCode >= 500 && (int)response.StatusCode <= 599),
+                });
+            });
+
             // Export metrics from all HTTP clients registered in services
             services.UseHttpClientMetrics();
 

source/backend/api/appsettings.json

@@ -147,5 +147,9 @@
     "CDogsHost": "[CDOGS_HOST]",
     "ServiceClientId": "[CLIENT_ID]",
     "ServiceClientSecret": "[CLIENT_SECRET]"
+  },
+  "Polly": {
+    "MaxRetries": 3,
+    "DelayInSeconds": 1
   }
 }

source/backend/core.api/Pims.Core.Api.csproj

@@ -14,6 +14,7 @@
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning.ApiExplorer" Version="5.1.0" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
     <PackageReference Include="Microsoft.OpenApi" Version="1.6.22" />
+    <PackageReference Include="Polly.Extensions" Version="8.5.1" />
     <PackageReference Include="Serilog" Version="4.0.2" />
     <PackageReference Include="Serilog.AspNetCore" Version="8.0.3" />
     <PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.1" />

source/backend/core.api/Repositories/RestCommon/BaseRestRepository.cs

@@ -14,6 +14,8 @@
 using Pims.Api.Models;
 using Pims.Api.Models.CodeTypes;
 using Pims.Api.Models.Requests.Http;
+using Polly;
+using Polly.Registry;
 
 namespace Pims.Core.Api.Repositories.Rest
 {
@@ -25,20 +27,25 @@ public abstract class BaseRestRepository : IRestRespository
         protected readonly IHttpClientFactory _httpClientFactory;
         protected readonly ILogger _logger;
         protected readonly IOptions<JsonSerializerOptions> _jsonOptions;
+        private readonly ResiliencePipeline<HttpResponseMessage> _resiliencePipeline;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseRestRepository"/> class.
         /// </summary>
         /// <param name="logger">Injected Logger Provider.</param>
         /// <param name="httpClientFactory">Injected Httpclient factory.</param>
+        /// <param name="jsonOptions"></param>
+        /// <param name="pollyPipelineProvider"></param>
         protected BaseRestRepository(
             ILogger logger,
             IHttpClientFactory httpClientFactory,
-            IOptions<JsonSerializerOptions> jsonOptions)
+            IOptions<JsonSerializerOptions> jsonOptions,
+            ResiliencePipelineProvider<string> pollyPipelineProvider)
         {
             _logger = logger;
             _httpClientFactory = httpClientFactory;
             _jsonOptions = jsonOptions;
+            _resiliencePipeline = pollyPipelineProvider.GetPipeline<HttpResponseMessage>("retry-network-policy");
         }
 
         public abstract void AddAuthentication(HttpClient client, string authenticationToken = null);
@@ -51,7 +58,7 @@ public async Task<ExternalResponse<T>> GetAsync<T>(Uri endpoint, string authenti
             client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json));
             try
             {
-                HttpResponseMessage response = await client.GetAsync(endpoint).ConfigureAwait(true);
+                HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.GetAsync(endpoint, cancellation).ConfigureAwait(true));
                 var result = await ProcessResponse<T>(response);
                 return result;
             }
@@ -74,7 +81,7 @@ public async Task<HttpResponseMessage> GetRawAsync(Uri endpoint, string authenti
             client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json));
             try
             {
-                HttpResponseMessage response = await client.GetAsync(endpoint).ConfigureAwait(true);
+                HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.GetAsync(endpoint, cancellation).ConfigureAwait(true));
                 return response;
             }
             catch (Exception e)
@@ -96,7 +103,7 @@ public async Task<ExternalResponse<T>> PostAsync<T>(Uri endpoint, HttpContent co
 
             try
             {
-                HttpResponseMessage response = await client.PostAsync(endpoint, content).ConfigureAwait(true);
+                HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.PostAsync(endpoint, content, cancellation).ConfigureAwait(true));
                 var result = await ProcessResponse<T>(response);
                 return result;
             }
@@ -120,7 +127,7 @@ public async Task<ExternalResponse<T>> PutAsync<T>(Uri endpoint, HttpContent con
 
             try
             {
-                HttpResponseMessage response = await client.PutAsync(endpoint, content).ConfigureAwait(true);
+                HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.PutAsync(endpoint, content, cancellation).ConfigureAwait(true));
                 var result = await ProcessResponse<T>(response);
                 return result;
             }
@@ -150,7 +157,7 @@ public async Task<ExternalResponse<string>> DeleteAsync(Uri endpoint, string aut
 
             try
             {
-                HttpResponseMessage response = await client.DeleteAsync(endpoint).ConfigureAwait(true);
+                HttpResponseMessage response = await _resiliencePipeline.ExecuteAsync(async cancellation => await client.DeleteAsync(endpoint, cancellation).ConfigureAwait(true));
 
                 _logger.LogTrace("Response: {response}", response);