AzureStorageAccountPartnerMetadataStorage implementation (#706)

* Added terraform to create new storage account and metadata container * Added initial implementation for saveMetadata * Added info logging when saving * Added exception handling for azure exceptions * Added needed container access permissions * try changing id to fix issue * Added initial implementation for readMetadata * 672: read and write the metadata in the send order usecase * 672: Use the file metadata storage when running locally * Minor changes for logging * Reverted changes made for testing * Refactored to use lazy initialization for containerClient * Added test to read metadata. Still trying to make it work * Added logs for debugging * Added partial implementation to try to mock blob container client * Add AzureClient interface and implement it in AzureClientImpl The AzureStorageAccountPartnerMetadataStorage class has been refactored to utilize an AzureClient interface for blob operations. The majority of the blob-related logic is now encapsulated within the AzureClientImpl class. This leads to cleaner and more maintainable code by decoupling the storage functionality from the Azure client implementation. * 672: Register the AzureClient implementation into the ApplicationContext * 672: Use the interface * 672: Remove the AzureClient interface to use a direct implementation. * 672: Exception path for reading metadata from Azure bucket * 672: Add more unit tests for AzureStorageAccountPartnerMetadataStorage * 672: Add AzureClient to the JaCoCo code coverage ignore list * Added flag to overwrite uploaded metadata file if it exists. Also did some cleanup * Fixed tests to reflect signature change * 672: Add JavaDocs for AzureClient * Fixed readMetadata implementation (and tests) to return Optional.empty() if blob not found * Added test for path when metadata is not found --------- Co-authored-by: halprin <[email protected]>
CDCgov · Dec 13, 2023 · f883221 · f883221
1 parent ac2f834
commit f883221
Show file tree

Hide file tree

Showing 9 changed files with 263 additions and 6 deletions.
diff --git a/build.gradle b/build.gradle
@@ -94,7 +94,8 @@ ext.jacoco_excludes = [
         '**/hapi/HapiFhirImplementation*',
         '**/jjwt/JjwtEngine*',
         '**/apache/ApacheClient*',
-        '**/azure/AzureSecrets*'
+        '**/azure/AzureSecrets*',
+        '**/azure/AzureClient*'
 ]
 
 tasks.register('allUnitTests') {

diff --git a/etor/build.gradle b/etor/build.gradle
@@ -13,6 +13,9 @@ dependencies {
     implementation project(':shared')
     testImplementation testFixtures(project(':shared'))
 
+    implementation 'com.azure:azure-storage-blob:12.25.0'
+    implementation 'com.azure:azure-identity:1.11.1'
+
     testImplementation 'org.apache.groovy:groovy:4.0.16'
     testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
     testImplementation 'com.openpojo:openpojo:0.9.1'

diff --git a/etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistration.java b/etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistration.java
@@ -19,6 +19,7 @@
 import gov.hhs.cdc.trustedintermediary.etor.orders.OrderSender;
 import gov.hhs.cdc.trustedintermediary.etor.orders.SendOrderUseCase;
 import gov.hhs.cdc.trustedintermediary.etor.orders.UnableToSendOrderException;
+import gov.hhs.cdc.trustedintermediary.external.azure.AzureClient;
 import gov.hhs.cdc.trustedintermediary.external.azure.AzureStorageAccountPartnerMetadataStorage;
 import gov.hhs.cdc.trustedintermediary.external.hapi.HapiOrderConverter;
 import gov.hhs.cdc.trustedintermediary.external.localfile.FilePartnerMetadataStorage;
@@ -75,6 +76,7 @@ public Map<HttpEndpoint, Function<DomainRequest, DomainResponse>> domainRegistra
             ApplicationContext.register(
                     PartnerMetadataStorage.class,
                     AzureStorageAccountPartnerMetadataStorage.getInstance());
+            ApplicationContext.register(AzureClient.class, AzureClient.getInstance());
         }
 
         return endpoints;

diff --git a/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/azure/AzureClient.java b/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/azure/AzureClient.java
@@ -0,0 +1,43 @@
+package gov.hhs.cdc.trustedintermediary.external.azure;
+
+import com.azure.identity.DefaultAzureCredentialBuilder;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import gov.hhs.cdc.trustedintermediary.context.ApplicationContext;
+
+/** This class represents a client for interacting with Azure Blob Storage. */
+public class AzureClient {
+
+    private static final String STORAGE_ACCOUNT_BLOB_ENDPOINT =
+            ApplicationContext.getProperty("STORAGE_ACCOUNT_BLOB_ENDPOINT");
+    private static final String METADATA_CONTAINER_NAME =
+            ApplicationContext.getProperty("METADATA_CONTAINER_NAME");
+
+    private static final AzureClient INSTANCE = new AzureClient();
+
+    private static BlobContainerClient BLOB_CONTAINER_CLIENT;
+
+    private AzureClient() {}
+
+    public static AzureClient getInstance() {
+
+        /*
+        BLOB_CONTAINER_CLIENT is initialized here inside the getInstance method instead of the static context above
+        to ensure that it is not created until it is needed.  This prevents an exception being thrown in the unit
+        test context where `STORAGE_ACCOUNT_BLOB_ENDPOINT` is empty.
+         */
+        BLOB_CONTAINER_CLIENT =
+                new BlobServiceClientBuilder()
+                        .endpoint(STORAGE_ACCOUNT_BLOB_ENDPOINT)
+                        .credential(new DefaultAzureCredentialBuilder().build())
+                        .buildClient()
+                        .getBlobContainerClient(METADATA_CONTAINER_NAME);
+
+        return INSTANCE;
+    }
+
+    public BlobClient getBlobClient(String blobName) {
+        return BLOB_CONTAINER_CLIENT.getBlobClient(blobName);
+    }
+}
diff --git a/...hhs/cdc/trustedintermediary/external/azure/AzureStorageAccountPartnerMetadataStorage.java b/...hhs/cdc/trustedintermediary/external/azure/AzureStorageAccountPartnerMetadataStorage.java
@@ -1,26 +1,71 @@
 package gov.hhs.cdc.trustedintermediary.external.azure;
 
+import com.azure.core.exception.AzureException;
+import com.azure.core.util.BinaryData;
+import com.azure.storage.blob.BlobClient;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadata;
+import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataException;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataStorage;
+import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
+import gov.hhs.cdc.trustedintermediary.wrappers.formatter.Formatter;
+import gov.hhs.cdc.trustedintermediary.wrappers.formatter.FormatterProcessingException;
+import gov.hhs.cdc.trustedintermediary.wrappers.formatter.TypeReference;
 import java.util.Optional;
+import javax.inject.Inject;
 
 /** Implements the {@link PartnerMetadataStorage} using files stored in an Azure Storage Account. */
 public class AzureStorageAccountPartnerMetadataStorage implements PartnerMetadataStorage {
 
     private static final AzureStorageAccountPartnerMetadataStorage INSTANCE =
             new AzureStorageAccountPartnerMetadataStorage();
 
+    @Inject Formatter formatter;
+    @Inject Logger logger;
+    @Inject AzureClient client;
+
     private AzureStorageAccountPartnerMetadataStorage() {}
 
     public static AzureStorageAccountPartnerMetadataStorage getInstance() {
         return INSTANCE;
     }
 
     @Override
-    public Optional<PartnerMetadata> readMetadata(final String uniqueId) {
-        return Optional.empty();
+    public Optional<PartnerMetadata> readMetadata(final String uniqueId)
+            throws PartnerMetadataException {
+        String metadataFileName = getMetadataFileName(uniqueId);
+        try {
+            BlobClient blobClient = client.getBlobClient(metadataFileName);
+            String blobUrl = blobClient.getBlobUrl();
+            logger.logInfo("Reading metadata from " + blobUrl);
+            if (!blobClient.exists()) {
+                logger.logWarning("Metadata blob not found: {}", blobUrl);
+                return Optional.empty();
+            }
+            String content = blobClient.downloadContent().toString();
+            PartnerMetadata metadata =
+                    formatter.convertJsonToObject(content, new TypeReference<>() {});
+            return Optional.ofNullable(metadata);
+        } catch (AzureException | FormatterProcessingException e) {
+            throw new PartnerMetadataException(
+                    "Failed to download metadata file " + metadataFileName, e);
+        }
     }
 
     @Override
-    public void saveMetadata(final PartnerMetadata metadata) {}
+    public void saveMetadata(final PartnerMetadata metadata) throws PartnerMetadataException {
+        String metadataFileName = getMetadataFileName(metadata.uniqueId());
+        try {
+            BlobClient blobClient = client.getBlobClient(metadataFileName);
+            String content = formatter.convertToJsonString(metadata);
+            blobClient.upload(BinaryData.fromString(content), true);
+            logger.logInfo("Saved metadata to " + blobClient.getBlobUrl());
+        } catch (AzureException | FormatterProcessingException e) {
+            throw new PartnerMetadataException(
+                    "Failed to upload metadata file " + metadataFileName, e);
+        }
+    }
+
+    public static String getMetadataFileName(String uniqueId) {
+        return uniqueId + ".json";
+    }
 }
diff --git a/...c/trustedintermediary/external/azure/AzureStorageAccountPartnerMetadataStorageTest.groovy b/...c/trustedintermediary/external/azure/AzureStorageAccountPartnerMetadataStorageTest.groovy
@@ -0,0 +1,141 @@
+package gov.hhs.cdc.trustedintermediary.external.azure
+
+import com.azure.core.exception.AzureException
+import com.azure.core.util.BinaryData
+import com.azure.storage.blob.BlobClient
+import gov.hhs.cdc.trustedintermediary.context.TestApplicationContext
+import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadata
+import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataException
+import gov.hhs.cdc.trustedintermediary.external.jackson.Jackson
+import gov.hhs.cdc.trustedintermediary.wrappers.formatter.Formatter
+import java.time.Instant
+import spock.lang.Specification
+
+class AzureStorageAccountPartnerMetadataStorageTest extends Specification {
+
+    def setup() {
+        TestApplicationContext.reset()
+        TestApplicationContext.init()
+        TestApplicationContext.register(AzureStorageAccountPartnerMetadataStorage, AzureStorageAccountPartnerMetadataStorage.getInstance())
+    }
+
+    def "successfully read metadata"() {
+        given:
+        def expectedUniqueId = "uniqueId"
+        def expectedSender = "sender"
+        def expectedReceiver = "receiver"
+        def expectedTimestamp = Instant.parse("2023-12-04T18:51:48.941875Z")
+        def expectedHash = "abcd"
+
+        PartnerMetadata expectedMetadata = new PartnerMetadata(expectedUniqueId, expectedSender, expectedReceiver, expectedTimestamp, expectedHash)
+
+        String simulatedMetadataJson = """{
+            "uniqueId": "${expectedUniqueId}",
+            "sender": "${expectedSender}",
+            "receiver": "${expectedReceiver}",
+            "timeReceived": "${expectedTimestamp}",
+            "hash": "${expectedHash}"
+        }"""
+
+        def mockBlobClient = Mock(BlobClient)
+        mockBlobClient.exists() >> true
+        mockBlobClient.downloadContent() >> BinaryData.fromString(simulatedMetadataJson)
+
+        def azureClient = Mock(AzureClient)
+        azureClient.getBlobClient(_ as String) >> mockBlobClient
+
+        TestApplicationContext.register(AzureClient, azureClient)
+        TestApplicationContext.register(Formatter, Jackson.getInstance())
+        TestApplicationContext.injectRegisteredImplementations()
+
+        when:
+        def actualMetadata = AzureStorageAccountPartnerMetadataStorage.getInstance().readMetadata(expectedUniqueId)
+
+        then:
+        actualMetadata.get() == expectedMetadata
+    }
+
+    def "readMetadata returns empty when blob does not exist"() {
+        given:
+        def mockBlobClient = Mock(BlobClient)
+        mockBlobClient.exists() >> false
+
+        def azureClient = Mock(AzureClient)
+        azureClient.getBlobClient(_ as String) >> mockBlobClient
+
+        TestApplicationContext.register(AzureClient, azureClient)
+        TestApplicationContext.injectRegisteredImplementations()
+
+        when:
+        def actualMetadata = AzureStorageAccountPartnerMetadataStorage.getInstance().readMetadata("nonexistentId")
+
+        then:
+        actualMetadata.isEmpty()
+    }
+
+    def "exception path while reading metadata"() {
+        given:
+        String expectedUniqueId = "uniqueId"
+        def mockBlobClient = Mock(BlobClient)
+        mockBlobClient.exists() >> true
+        mockBlobClient.downloadContent() >> { throw new AzureException("Download error") }
+
+        def azureClient = Mock(AzureClient)
+        azureClient.getBlobClient(_ as String) >> mockBlobClient
+
+        TestApplicationContext.register(AzureClient, azureClient)
+        TestApplicationContext.injectRegisteredImplementations()
+
+        when:
+        AzureStorageAccountPartnerMetadataStorage.getInstance().readMetadata(expectedUniqueId)
+
+        then:
+        thrown(PartnerMetadataException)
+    }
+
+    def "successfully save metadata"() {
+        given:
+        PartnerMetadata partnerMetadata = new PartnerMetadata("uniqueId", "sender", "receiver", Instant.now(), "abcd")
+
+        def mockBlobClient = Mock(BlobClient)
+        def azureClient = Mock(AzureClient)
+        azureClient.getBlobClient(_ as String) >> mockBlobClient
+
+        def mockFormatter = Mock(Formatter)
+        mockFormatter.convertToJsonString(partnerMetadata) >> "DogCow"
+
+        TestApplicationContext.register(AzureClient, azureClient)
+        TestApplicationContext.register(Formatter, mockFormatter)
+        TestApplicationContext.injectRegisteredImplementations()
+
+        when:
+        AzureStorageAccountPartnerMetadataStorage.getInstance().saveMetadata(partnerMetadata)
+
+        then:
+        1 * mockBlobClient.upload(_ as BinaryData, true)
+    }
+
+    def "failed to save metadata"() {
+        given:
+        PartnerMetadata partnerMetadata = new PartnerMetadata("uniqueId", "sender", "receiver", Instant.now(), "abcd")
+
+        def mockBlobClient = Mock(BlobClient)
+        mockBlobClient.upload(_ as BinaryData, true) >> { throw new AzureException("upload failed") }
+
+        def azureClient = Mock(AzureClient)
+        azureClient.getBlobClient(_ as String) >> mockBlobClient
+
+        def mockFormatter = Mock(Formatter)
+        mockFormatter.convertToJsonString(partnerMetadata) >> "DogCow"
+
+        TestApplicationContext.register(AzureClient, azureClient)
+        TestApplicationContext.register(Formatter, mockFormatter)
+        TestApplicationContext.injectRegisteredImplementations()
+
+        when:
+        AzureStorageAccountPartnerMetadataStorage.getInstance().saveMetadata(partnerMetadata)
+
+        then:
+        thrown(PartnerMetadataException)
+    }
+}
diff --git a/.../gov/hhs/cdc/trustedintermediary/external/localfile/FilePartnerMetadataStorageTest.groovy b/.../gov/hhs/cdc/trustedintermediary/external/localfile/FilePartnerMetadataStorageTest.groovy
@@ -70,6 +70,6 @@ class FilePartnerMetadataStorageTest extends Specification {
         def actualMetadata = FilePartnerMetadataStorage.getInstance().readMetadata("nonexistentId")
 
         then:
-        !actualMetadata.isPresent()
+        actualMetadata.isEmpty()
     }
 }
diff --git a/operations/template/app.tf b/operations/template/app.tf
@@ -34,7 +34,8 @@ resource "azurerm_linux_web_app" "api" {
     ENV                             = var.environment
     REPORT_STREAM_URL_PREFIX        = "https://${local.rs_domain_prefix}prime.cdc.gov"
     KEY_VAULT_NAME                  = azurerm_key_vault.key_storage.name
-
+    STORAGE_ACCOUNT_BLOB_ENDPOINT   = azurerm_storage_account.storage.primary_blob_endpoint
+    METADATA_CONTAINER_NAME         = azurerm_storage_container.metadata.name
   }
 
   identity {

diff --git a/operations/template/storage.tf b/operations/template/storage.tf
@@ -0,0 +1,21 @@
+resource "azurerm_storage_account" "storage" {
+  name                            = "cdcti${var.environment}"
+  resource_group_name             = data.azurerm_resource_group.group.name
+  location                        = data.azurerm_resource_group.group.location
+  account_tier                    = "Standard"
+  account_replication_type        = "GRS"
+  account_kind                    = "StorageV2"
+  allow_nested_items_to_be_public = false
+}
+
+resource "azurerm_storage_container" "metadata" {
+  name                  = "metadata"
+  storage_account_name  = azurerm_storage_account.storage.name
+  container_access_type = "private"
+}
+
+resource "azurerm_role_assignment" "allow_api_read_write" {
+  scope                = azurerm_storage_container.metadata.resource_manager_id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = azurerm_linux_web_app.api.identity.0.principal_id
+}