Skip to content

Commit

Permalink
Update Blob Service Client Registration APIs and Configuration (#1001)
Browse files Browse the repository at this point in the history
- Update blob service client registration for the following reasons:
  1. Integrate blob client and underlying http request pipeline with existing DI service container
  2. Reduce boilerplate
  3. Streamline configuration whose types integrate with existing APIs
- Normalize JSON to 2 spaces for indent
- Bump version of Azurite image such that we can leverage latest blob client API
  • Loading branch information
wsugarman authored Sep 13, 2021
1 parent fb4a116 commit 1dd8218
Show file tree
Hide file tree
Showing 9 changed files with 127 additions and 144 deletions.
2 changes: 1 addition & 1 deletion Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
<FoDicomVersion>4.0.8</FoDicomVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<HealthcareSharedPackageVersion>3.1.23</HealthcareSharedPackageVersion>
<HealthcareSharedPackageVersion>3.2.1</HealthcareSharedPackageVersion>
<HighEntropyVA>true</HighEntropyVA>
<LangVersion>Latest</LangVersion>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
Expand Down
2 changes: 1 addition & 1 deletion docker/docker-compose.dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ services:
- sql
- azurite
azurite:
image: mcr.microsoft.com/azure-storage/azurite:3.13.0@sha256:349f106b240c5fd5ecb146c3aa121c531c8a1080badb762099d886703bc39fa0
image: mcr.microsoft.com/azure-storage/azurite:3.14.2@sha256:443c9b28466219e21005e638a28ac4bcc6d3860509d3fc9ec8753fce64424b29
# # These port bindings [source]:[dest] can be uncommented to connect to the storage emulator via Microsoft Azure Storage Explorer
# # Note that the source ports may need to change if a storage emulator is already running on localhost
# ports:
Expand Down
2 changes: 1 addition & 1 deletion samples/docker/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ services:
- azurite

azurite:
image: mcr.microsoft.com/azure-storage/azurite:3.13.0@sha256:349f106b240c5fd5ecb146c3aa121c531c8a1080badb762099d886703bc39fa0
image: mcr.microsoft.com/azure-storage/azurite:3.14.2@sha256:443c9b28466219e21005e638a28ac4bcc6d3860509d3fc9ec8753fce64424b29
restart: always
# # Uncomment if you want to expose azure storage explorer against localhost
# ports:
Expand Down
17 changes: 5 additions & 12 deletions src/Microsoft.Health.Dicom.Blob/Features/Storage/BlobFileStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Storage;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
Expand All @@ -27,21 +26,21 @@ namespace Microsoft.Health.Dicom.Blob.Features.Storage
public class BlobFileStore : IFileStore
{
private readonly BlobContainerClient _container;
private readonly BlobDataStoreConfiguration _blobDataStoreConfiguration;
private readonly BlobOperationOptions _options;

public BlobFileStore(
BlobServiceClient client,
IOptionsMonitor<BlobContainerConfiguration> namedBlobContainerConfigurationAccessor,
BlobDataStoreConfiguration blobDataStoreConfiguration)
IOptions<BlobOperationOptions> options)
{
EnsureArg.IsNotNull(client, nameof(client));
EnsureArg.IsNotNull(namedBlobContainerConfigurationAccessor, nameof(namedBlobContainerConfigurationAccessor));
EnsureArg.IsNotNull(blobDataStoreConfiguration, nameof(blobDataStoreConfiguration));
EnsureArg.IsNotNull(options?.Value, nameof(options));

BlobContainerConfiguration containerConfiguration = namedBlobContainerConfigurationAccessor.Get(Constants.ContainerConfigurationName);

_container = client.GetBlobContainerClient(containerConfiguration.ContainerName);
_blobDataStoreConfiguration = blobDataStoreConfiguration;
_options = options.Value;
}

/// <inheritdoc />
Expand All @@ -56,13 +55,7 @@ public async Task<Uri> StoreFileAsync(
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
stream.Seek(0, SeekOrigin.Begin);

var blobUploadOptions = new BlobUploadOptions()
{
TransferOptions = new StorageTransferOptions
{
MaximumConcurrency = _blobDataStoreConfiguration.RequestOptions.UploadMaximumConcurrency,
},
};
var blobUploadOptions = new BlobUploadOptions { TransferOptions = _options.Upload };

try
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="$(SdkPackageVersion)" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="$(SdkPackageVersion)" />
<PackageReference Include="Microsoft.Extensions.Options" Version="$(SdkPackageVersion)" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(SdkPackageVersion)" />
<PackageReference Include="Microsoft.Health.Blob" Version="$(HealthcareSharedPackageVersion)" />
<PackageReference Include="Microsoft.Health.Extensions.DependencyInjection" Version="$(HealthcareSharedPackageVersion)" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.1.3" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@

using EnsureThat;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Health.Blob.Configs;
using Microsoft.Health.Blob.Features.Storage;
using Microsoft.Health.Dicom.Blob;
using Microsoft.Health.Dicom.Blob.Features.Health;
using Microsoft.Health.Dicom.Blob.Features.Storage;
Expand All @@ -34,20 +31,27 @@ public static IDicomServerBuilder AddBlobStorageDataStore(this IDicomServerBuild
EnsureArg.IsNotNull(configuration, nameof(configuration));

return serverBuilder
.AddBlobPersistence(configuration)
.AddBlobHealthCheck();
.AddBlobPersistence(configuration)
.AddBlobHealthCheck();
}

private static IDicomServerBuilder AddBlobPersistence(this IDicomServerBuilder serverBuilder, IConfiguration configuration)
{
IServiceCollection services = serverBuilder.Services;

services.AddBlobDataStore();
IConfiguration blobConfig = configuration.GetSection(BlobServiceClientOptions.DefaultSectionName);
services
.AddBlobServiceClient(blobConfig)
.AddBlobContainerInitialization(x => blobConfig
.GetSection(BlobInitializerOptions.DefaultSectionName)
.Bind(x))
.ConfigureContainer(Constants.ContainerConfigurationName, x => configuration
.GetSection(DicomServerBlobConfigurationSectionName)
.Bind(x));

services.Configure<BlobContainerConfiguration>(
Constants.ContainerConfigurationName,
containerConfiguration => configuration.GetSection(DicomServerBlobConfigurationSectionName)
.Bind(containerConfiguration));
services
.AddOptions<BlobOperationOptions>()
.Bind(blobConfig.GetSection(nameof(BlobServiceClientOptions.Operations)));

services.Add<BlobFileStore>()
.Scoped()
Expand All @@ -59,19 +63,6 @@ private static IDicomServerBuilder AddBlobPersistence(this IDicomServerBuilder s
// so we need to register here. Need to some more investigation to see how we might be able to do this.
services.Decorate<IFileStore, LoggingFileStore>();

services.Add(sp =>
{
ILoggerFactory loggerFactory = sp.GetService<ILoggerFactory>();
IOptionsMonitor<BlobContainerConfiguration> namedBlobContainerConfiguration = sp.GetService<IOptionsMonitor<BlobContainerConfiguration>>();
BlobContainerConfiguration blobContainerConfiguration = namedBlobContainerConfiguration.Get(Constants.ContainerConfigurationName);

return new BlobContainerInitializer(
blobContainerConfiguration.ContainerName,
loggerFactory.CreateLogger<BlobContainerInitializer>());
})
.Singleton()
.AsService<IBlobContainerInitializer>();

return serverBuilder;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@

using EnsureThat;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Health.Blob.Configs;
using Microsoft.Health.Blob.Features.Storage;
using Microsoft.Health.Dicom.Core.Features.Common;
using Microsoft.Health.Dicom.Core.Registration;
using Microsoft.Health.Dicom.Metadata;
Expand All @@ -34,33 +31,23 @@ public static IDicomServerBuilder AddMetadataStorageDataStore(this IDicomServerB
EnsureArg.IsNotNull(configuration, nameof(configuration));

return serverBuilder
.AddMetadataPersistence(configuration)
.AddMetadataHealthCheck();
.AddMetadataPersistence(configuration)
.AddMetadataHealthCheck();
}

private static IDicomServerBuilder AddMetadataPersistence(this IDicomServerBuilder serverBuilder, IConfiguration configuration)
{
IServiceCollection services = serverBuilder.Services;

services.AddBlobDataStore();

services.Configure<BlobContainerConfiguration>(
Constants.ContainerConfigurationName,
containerConfiguration => configuration.GetSection(DicomServerBlobConfigurationSectionName)
.Bind(containerConfiguration));

services.Add(sp =>
{
ILoggerFactory loggerFactory = sp.GetService<ILoggerFactory>();
IOptionsMonitor<BlobContainerConfiguration> namedBlobContainerConfiguration = sp.GetService<IOptionsMonitor<BlobContainerConfiguration>>();
BlobContainerConfiguration blobContainerConfiguration = namedBlobContainerConfiguration.Get(Constants.ContainerConfigurationName);

return new BlobContainerInitializer(
blobContainerConfiguration.ContainerName,
loggerFactory.CreateLogger<BlobContainerInitializer>());
})
.Singleton()
.AsService<IBlobContainerInitializer>();
IConfiguration blobConfig = configuration.GetSection(BlobServiceClientOptions.DefaultSectionName);
services
.AddBlobServiceClient(blobConfig)
.AddBlobContainerInitialization(x => blobConfig
.GetSection(BlobInitializerOptions.DefaultSectionName)
.Bind(x))
.ConfigureContainer(Constants.ContainerConfigurationName, x => configuration
.GetSection(DicomServerBlobConfigurationSectionName)
.Bind(x));

services.Add<BlobMetadataStore>()
.Scoped()
Expand Down
173 changes: 92 additions & 81 deletions src/Microsoft.Health.Dicom.Web/appsettings.json
Original file line number Diff line number Diff line change
@@ -1,90 +1,101 @@
{
"AllowedHosts": "*",
"DicomServer": {
"Security": {
"Enabled": false,
"Authentication": {
"Audience": null,
"Authority": null
}
},
"Features": {
"EnableOhifViewer": false,
"EnableFullDicomItemValidation": false,
"EnableExtendedQueryTags": false
},
"Services": {
"DeletedInstanceCleanup": {
"DeleteDelay": "3.00:00:00",
"MaxRetries": 5,
"RetryBackOff": "1.00:00:00",
"PollingInterval": "00:03:00",
"BatchSize": 10
},
"StoreServiceSettings": {
"MaxAllowedDicomFileSize": 2147483647
}
},
"Audit": {
"CustomAuditHeaderPrefix": "X-MS-AZUREDICOM-AUDIT-"
},
"ServerIdentity": {
"UserAssignedAppId": null
},
"Swagger": {
"License": {
"Name": "MIT License",
"Url": "https://github.com/microsoft/dicom-server/blob/main/LICENSE"
}
}
"AllowedHosts": "*",
"DicomServer": {
"Security": {
"Enabled": false,
"Authentication": {
"Audience": null,
"Authority": null
}
},
"Logging": {
"IncludeScopes": false,
"LogLevel": {
"Default": "Warning"
},
"ApplicationInsights": {
"LogLevel": {
"Default": "Information",
"Microsoft.Health": "Information",
"Microsoft": "Warning",
"System": "Warning"
}
}
"Features": {
"EnableOhifViewer": false,
"EnableFullDicomItemValidation": false,
"EnableExtendedQueryTags": false
},
"Services": {
"DeletedInstanceCleanup": {
"DeleteDelay": "3.00:00:00",
"MaxRetries": 5,
"RetryBackOff": "1.00:00:00",
"PollingInterval": "00:03:00",
"BatchSize": 10
},
"StoreServiceSettings": {
"MaxAllowedDicomFileSize": 2147483647
}
},
"Audit": {
"CustomAuditHeaderPrefix": "X-MS-AZUREDICOM-AUDIT-"
},
"ServerIdentity": {
"UserAssignedAppId": null
},
"Swagger": {
"License": {
"Name": "MIT License",
"Url": "https://github.com/microsoft/dicom-server/blob/main/LICENSE"
}
}
},
"Logging": {
"IncludeScopes": false,
"LogLevel": {
"Default": "Warning"
},
"ApplicationInsights": {
"InstrumentationKey": ""
"LogLevel": {
"Default": "Information",
"Microsoft.Health": "Information",
"Microsoft": "Warning",
"System": "Warning"
}
}
},
"ApplicationInsights": {
"InstrumentationKey": ""
},
"DicomWeb": {
"DicomStore": {
"ContainerName": "dicomwebcontainer"
},
"MetadataStore": {
"ContainerName": "metadatacontainer"
}
},
"BlobStore": {
"ConnectionString": null,
"Initialization": {
"RetryDelay": "00:00:15",
"Timeout": "00:06:00"
},
"DicomWeb": {
"DicomStore": {
"ContainerName": "dicomwebcontainer"
},
"MetadataStore": {
"ContainerName": "metadatacontainer"
}
"Operations": {
"Download": {
"MaximumConcurrency": 5
},
"Upload": {
"MaximumConcurrency": 5
}
},
"BlobStore": {
"ConnectionString": null,
"RequestOptions": {
"ExponentialRetryBackoffDeltaInSeconds": 4,
"ExponentialRetryMaxAttempts": 6,
"ServerTimeoutInMinutes": 2,
"DownloadMaximumConcurrency": 5,
"UploadMaximumConcurrency": 5
}
"Retry": {
"Delay": "00:00:04",
"MaxRetries": 6,
"Mode": "Exponential",
"NetworkTimeout": "00:02:00"
}
},
"SqlServer": {
"Initialize": "true",
"AllowDatabaseCreation": "true",
"ConnectionString": "server=(local);Initial Catalog=Dicom;Integrated Security=true",
"TransientFaultRetryPolicy": {
"InitialDelay": "00:00:00.100",
"RetryCount": 3,
"Factor": 2,
"FastFirst": true
},
"SqlServer": {
"Initialize": "true",
"AllowDatabaseCreation": "true",
"ConnectionString": "server=(local);Initial Catalog=Dicom;Integrated Security=true",
"TransientFaultRetryPolicy": {
"InitialDelay": "00:00:00.100",
"RetryCount": 3,
"Factor": 2,
"FastFirst": true
},
"SchemaOptions": {
"AutomaticUpdatesEnabled": true
}
"SchemaOptions": {
"AutomaticUpdatesEnabled": true
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ await blobClientInitializer.InitializeDataStoreAsync(
var jsonSerializer = new JsonSerializer();
jsonSerializer.Converters.Add(new JsonDicomConverter());

FileStore = new BlobFileStore(_blobClient, optionsMonitor, Substitute.For<BlobDataStoreConfiguration>());
FileStore = new BlobFileStore(_blobClient, optionsMonitor, Options.Create(Substitute.For<BlobOperationOptions>()));
MetadataStore = new BlobMetadataStore(_blobClient, jsonSerializer, optionsMonitor, RecyclableMemoryStreamManager);
}

Expand Down

0 comments on commit 1dd8218

Please sign in to comment.