Story 610 metadata order status (#766)

* Initial commit with fetching and enum added

* Save the delivery status to the database

* Refactored delivery status to avoid null pointer exception

* Updated metadataStorage with delivery status

* Updated PartnerMetadataOchestrator unit tests

* Populating ReST endpoint with status data

* Fixing test

* Defaulting status to PENDING

---------

Co-authored-by: tjohnson7021 <[email protected]>

.github/workflows/terraform-deploy_reusable.yml

@@ -74,7 +74,7 @@ jobs:
       - name: Run Db migration
         run: |
           export PGPASSWORD=$(az account get-access-token --resource-type oss-rdbms --query "[accessToken]" -o tsv)
-          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin;"
+          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "CREATE TYPE message_status AS ENUM ('PENDING', 'DELIVERED', 'FAILED'); CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz, delivery_status message_status); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin;"
 
       - id: export-terraform-output
         name: Export Terraform Output

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistration.java

@@ -26,14 +26,14 @@
 import gov.hhs.cdc.trustedintermediary.external.azure.AzureClient;
 import gov.hhs.cdc.trustedintermediary.external.azure.AzureStorageAccountPartnerMetadataStorage;
 import gov.hhs.cdc.trustedintermediary.external.database.DatabasePartnerMetadataStorage;
+import gov.hhs.cdc.trustedintermediary.external.database.DbDao;
 import gov.hhs.cdc.trustedintermediary.external.database.EtorSqlDriverManager;
 import gov.hhs.cdc.trustedintermediary.external.database.PostgresDao;
 import gov.hhs.cdc.trustedintermediary.external.hapi.HapiOrderConverter;
 import gov.hhs.cdc.trustedintermediary.external.localfile.FilePartnerMetadataStorage;
 import gov.hhs.cdc.trustedintermediary.external.localfile.MockRSEndpointClient;
 import gov.hhs.cdc.trustedintermediary.external.reportstream.ReportStreamEndpointClient;
 import gov.hhs.cdc.trustedintermediary.external.reportstream.ReportStreamOrderSender;
-import gov.hhs.cdc.trustedintermediary.wrappers.DbDao;
 import gov.hhs.cdc.trustedintermediary.wrappers.FhirParseException;
 import gov.hhs.cdc.trustedintermediary.wrappers.HapiFhir;
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
@@ -108,9 +108,9 @@ public Map<HttpEndpoint, Function<DomainRequest, DomainResponse>> domainRegistra
         } else {
             ApplicationContext.register(
                     RSEndpointClient.class, ReportStreamEndpointClient.getInstance());
+
             ApplicationContext.register(AzureClient.class, AzureClient.getInstance());
         }
-
         return endpoints;
     }
 

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/PartnerMetadata.java

@@ -1,5 +1,6 @@
 package gov.hhs.cdc.trustedintermediary.etor.metadata;
 
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus;
 import java.time.Instant;
 
 /**
@@ -11,22 +12,35 @@
  * @param receiver The name of the receiver of the message.
  * @param timeReceived The time the message was received.
  * @param hash The hash of the message.
+ * @param deliveryStatus the status of the message based on an enum
  */
 public record PartnerMetadata(
         String receivedSubmissionId,
         String sentSubmissionId,
         String sender,
         String receiver,
         Instant timeReceived,
-        String hash) {
+        String hash,
+        PartnerMetadataStatus deliveryStatus) {
+
+    // Below is for defaulting status when null
+    public PartnerMetadata {
+        if (deliveryStatus == null) {
+            deliveryStatus = PartnerMetadataStatus.PENDING;
+        }
+    }
 
     public PartnerMetadata(
-            String receivedSubmissionId, String sender, Instant timeReceived, String hash) {
-        this(receivedSubmissionId, null, sender, null, timeReceived, hash);
+            String receivedSubmissionId,
+            String sender,
+            Instant timeReceived,
+            String hash,
+            PartnerMetadataStatus deliveryStatus) {
+        this(receivedSubmissionId, null, sender, null, timeReceived, hash, deliveryStatus);
     }
 
     public PartnerMetadata(String receivedSubmissionId, String hash) {
-        this(receivedSubmissionId, null, null, null, null, hash);
+        this(receivedSubmissionId, null, null, null, null, hash, null);
     }
 
     public PartnerMetadata withSentSubmissionId(String sentSubmissionId) {
@@ -36,7 +50,8 @@ public PartnerMetadata withSentSubmissionId(String sentSubmissionId) {
                 this.sender,
                 this.receiver,
                 this.timeReceived,
-                this.hash);
+                this.hash,
+                this.deliveryStatus);
     }
 
     public PartnerMetadata withReceiver(String receiver) {
@@ -46,6 +61,7 @@ public PartnerMetadata withReceiver(String receiver) {
                 this.sender,
                 receiver,
                 this.timeReceived,
-                this.hash);
+                this.hash,
+                this.deliveryStatus);
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/PartnerMetadataOrchestrator.java

@@ -1,6 +1,7 @@
 package gov.hhs.cdc.trustedintermediary.etor.metadata;
 
 import gov.hhs.cdc.trustedintermediary.etor.RSEndpointClient;
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus;
 import gov.hhs.cdc.trustedintermediary.external.reportstream.ReportStreamEndpointClientException;
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
 import gov.hhs.cdc.trustedintermediary.wrappers.formatter.Formatter;
@@ -71,7 +72,12 @@ public void updateMetadataForReceivedOrder(String receivedSubmissionId, String o
                 sender,
                 timeReceived);
         PartnerMetadata partnerMetadata =
-                new PartnerMetadata(receivedSubmissionId, sender, timeReceived, orderHash);
+                new PartnerMetadata(
+                        receivedSubmissionId,
+                        sender,
+                        timeReceived,
+                        orderHash,
+                        PartnerMetadataStatus.PENDING);
         partnerMetadataStorage.saveMetadata(partnerMetadata);
     }
 

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadataStatus.java

@@ -0,0 +1,11 @@
+package gov.hhs.cdc.trustedintermediary.etor.metadata.partner;
+
+/**
+ * Enum for tracking the status of a message for diagnostic purposes. We store the status in our
+ * database
+ */
+public enum PartnerMetadataStatus {
+    PENDING,
+    DELIVERED,
+    FAILED
+}

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/DatabasePartnerMetadataStorage.java

@@ -3,7 +3,6 @@
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadata;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataException;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataStorage;
-import gov.hhs.cdc.trustedintermediary.wrappers.DbDao;
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
 import java.sql.SQLException;
 import java.util.Optional;
@@ -45,7 +44,8 @@ public void saveMetadata(final PartnerMetadata metadata) throws PartnerMetadataE
                     metadata.sender(),
                     metadata.receiver(),
                     metadata.hash(),
-                    metadata.timeReceived());
+                    metadata.timeReceived(),
+                    metadata.deliveryStatus());
         } catch (SQLException e) {
             throw new PartnerMetadataException("Error saving metadata", e);
         }

shared/src/main/java/gov/hhs/cdc/trustedintermediary/wrappers/DbDao.java → etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/DbDao.java

@@ -1,5 +1,6 @@
-package gov.hhs.cdc.trustedintermediary.wrappers;
+package gov.hhs.cdc.trustedintermediary.external.database;
 
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus;
 import java.sql.SQLException;
 import java.time.Instant;
 
@@ -11,7 +12,8 @@ void upsertMetadata(
             String sender,
             String receiver,
             String hash,
-            Instant timeReceived)
+            Instant timeReceived,
+            PartnerMetadataStatus deliveryStatus)
             throws SQLException;
 
     Object fetchMetadata(String uniqueId) throws SQLException;

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/PostgresDao.java

@@ -2,8 +2,8 @@
 
 import gov.hhs.cdc.trustedintermediary.context.ApplicationContext;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadata;
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus;
 import gov.hhs.cdc.trustedintermediary.external.azure.AzureClient;
-import gov.hhs.cdc.trustedintermediary.wrappers.DbDao;
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
 import gov.hhs.cdc.trustedintermediary.wrappers.SqlDriverManager;
 import java.sql.Connection;
@@ -86,14 +86,15 @@ public synchronized void upsertMetadata(
             String sender,
             String receiver,
             String hash,
-            Instant timeReceived)
+            Instant timeReceived,
+            PartnerMetadataStatus deliveryStatus)
             throws SQLException {
 
         try (Connection conn = connect();
                 PreparedStatement statement =
                         conn.prepareStatement(
                                 """
-                                INSERT INTO metadata VALUES (?, ?, ?, ?, ?, ?)
+                                INSERT INTO metadata VALUES (?, ?, ?, ?, ?, ?, ?)
                                 ON CONFLICT (received_message_id) DO UPDATE SET receiver = EXCLUDED.receiver, sent_message_id = EXCLUDED.sent_message_id
                                 """)) {
 
@@ -109,6 +110,13 @@ ON CONFLICT (received_message_id) DO UPDATE SET receiver = EXCLUDED.receiver, se
             }
             statement.setTimestamp(6, timestamp);
 
+            String deliveryStatusString = null;
+            if (deliveryStatus != null) {
+                deliveryStatusString = deliveryStatus.toString();
+            }
+
+            statement.setString(7, deliveryStatusString);
+
             statement.executeUpdate();
         }
     }
@@ -141,7 +149,8 @@ public synchronized PartnerMetadata fetchMetadata(String receivedSubmissionId)
                     result.getString("sender"),
                     result.getString("receiver"),
                     timeReceived,
-                    result.getString("hash_of_order"));
+                    result.getString("hash_of_order"),
+                    PartnerMetadataStatus.valueOf(result.getString("delivery_status")));
         }
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/hapi/HapiOrderConverter.java

@@ -225,6 +225,12 @@ public FhirMetadata<?> extractPublicMetadataToOperationOutcome(PartnerMetadata m
                 .add(createInformationIssueComponent("order ingestion", orderIngestion));
         operation.getIssue().add(createInformationIssueComponent("payload hash", metadata.hash()));
 
+        operation
+                .getIssue()
+                .add(
+                        createInformationIssueComponent(
+                                "delivery status", metadata.deliveryStatus().toString()));
+
         return new HapiFhirMetadata(operation);
     }
 

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistrationTest.groovy

@@ -14,6 +14,7 @@ import gov.hhs.cdc.trustedintermediary.etor.demographics.PatientDemographicsResp
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadata
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataException
 import gov.hhs.cdc.trustedintermediary.etor.metadata.PartnerMetadataOrchestrator
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus
 import gov.hhs.cdc.trustedintermediary.etor.operationoutcomes.FhirMetadata
 import gov.hhs.cdc.trustedintermediary.etor.orders.Order
 import gov.hhs.cdc.trustedintermediary.etor.orders.OrderController
@@ -324,7 +325,7 @@ class EtorDomainRegistrationTest extends Specification {
         request.setPathParams(["id": "metadataId"])
 
         def mockPartnerMetadataOrchestrator = Mock(PartnerMetadataOrchestrator)
-        mockPartnerMetadataOrchestrator.getMetadata(_ as String) >> Optional.ofNullable(new PartnerMetadata("receivedSubmissionId", "sender", Instant.now(), "hash"))
+        mockPartnerMetadataOrchestrator.getMetadata(_ as String) >> Optional.ofNullable(new PartnerMetadata("receivedSubmissionId", "sender", Instant.now(), "hash", PartnerMetadataStatus.DELIVERED))
         TestApplicationContext.register(PartnerMetadataOrchestrator, mockPartnerMetadataOrchestrator)
 
         def mockResponseHelper = Mock(DomainResponseHelper)

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/metadata/PartnerMetadataOrchestratorTest.groovy

@@ -2,6 +2,7 @@ package gov.hhs.cdc.trustedintermediary.etor.metadata
 
 import gov.hhs.cdc.trustedintermediary.context.TestApplicationContext
 import gov.hhs.cdc.trustedintermediary.etor.RSEndpointClient
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus
 import gov.hhs.cdc.trustedintermediary.etor.orders.OrderConverter
 import gov.hhs.cdc.trustedintermediary.external.hapi.HapiOrderConverter
 import gov.hhs.cdc.trustedintermediary.external.jackson.Jackson
@@ -43,6 +44,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         def hashCode = "123"
         def bearerToken = "token"
         def rsHistoryApiResponse = "{\"sender\": \"${sender}\", \"timestamp\": \"${timestamp}\"}"
+        def deliveryStatus = PartnerMetadataStatus.PENDING
 
         mockFormatter.convertJsonToObject(rsHistoryApiResponse, _ as TypeReference) >> [sender: sender, timestamp: timestamp]
 
@@ -52,7 +54,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         then:
         1 * mockClient.getRsToken() >> bearerToken
         1 * mockClient.requestHistoryEndpoint(receivedSubmissionId, bearerToken) >> rsHistoryApiResponse
-        1 * mockPartnerMetadataStorage.saveMetadata(new PartnerMetadata(receivedSubmissionId, sender, Instant.parse(timestamp), hashCode))
+        1 * mockPartnerMetadataStorage.saveMetadata(new PartnerMetadata(receivedSubmissionId, sender, Instant.parse(timestamp), hashCode, deliveryStatus))
     }
 
     def "updateMetadataForSentOrder test case when sentSubmissionId is null"() {
@@ -147,7 +149,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = "sentSubmissionId"
-        def partnerMetadata = new PartnerMetadata(receivedSubmissionId, "sender", Instant.now(), "hash")
+        def partnerMetadata = new PartnerMetadata(receivedSubmissionId, "sender", Instant.now(), "hash", PartnerMetadataStatus.PENDING)
         def updatedPartnerMetadata = partnerMetadata.withSentSubmissionId(sentSubmissionId)
 
         when:
@@ -174,7 +176,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = "sentSubmissionId"
-        def partnerMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, "sender", null, Instant.now(), "hash")
+        def partnerMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, "sender", null, Instant.now(), "hash", PartnerMetadataStatus.PENDING)
 
         mockPartnerMetadataStorage.readMetadata(receivedSubmissionId) >> Optional.of(partnerMetadata)
         mockClient.getRsToken() >> "token"
@@ -192,7 +194,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = "sentSubmissionId"
         def rsHistoryApiResponse = "{\"destinations\": [{\"organization_id\": \"org\", \"service\": \"service\"}]}"
-        def partnerMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, "sender", null, Instant.now(), "hash")
+        def partnerMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, "sender", null, Instant.now(), "hash", PartnerMetadataStatus.PENDING)
 
         mockPartnerMetadataStorage.readMetadata(receivedSubmissionId) >> Optional.of(partnerMetadata)
         mockClient.getRsToken() >> "token"
@@ -209,7 +211,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
     def "getMetadata retrieves metadata successfully with the sender already filled"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
-        def metadata = new PartnerMetadata(receivedSubmissionId, "sentSubmissionId", "sender", "receiver", Instant.now(), "hash")
+        def metadata = new PartnerMetadata(receivedSubmissionId, "sentSubmissionId", "sender", "receiver", Instant.now(), "hash", PartnerMetadataStatus.DELIVERED)
 
         when:
         def result = PartnerMetadataOrchestrator.getInstance().getMetadata(receivedSubmissionId)
@@ -224,7 +226,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
     def "getMetadata retrieves metadata successfully when receiver is present and sentSubmissionId is missing"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
-        def metadata = new PartnerMetadata(receivedSubmissionId, null, "sender", "receiver", Instant.now(), "hash")
+        def metadata = new PartnerMetadata(receivedSubmissionId, null, "sender", "receiver", Instant.now(), "hash", PartnerMetadataStatus.DELIVERED)
 
         when:
         def result = PartnerMetadataOrchestrator.getInstance().getMetadata(receivedSubmissionId)
@@ -244,8 +246,8 @@ class PartnerMetadataOrchestratorTest extends Specification {
         def hashCode = "123"
         def bearerToken = "token"
         def rsHistoryApiResponse = "{\"destinations\": [{\"organization_id\": \"org\", \"service\": \"service\"}]}"
-        def missingReceiverMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, sender, null, timestamp, hashCode)
-        def expectedMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, sender, "org.service", timestamp, hashCode)
+        def missingReceiverMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, sender, null, timestamp, hashCode, PartnerMetadataStatus.DELIVERED)
+        def expectedMetadata = new PartnerMetadata(receivedSubmissionId, sentSubmissionId, sender, "org.service", timestamp, hashCode, PartnerMetadataStatus.DELIVERED)
 
         mockClient.getRsToken() >> bearerToken
         mockClient.requestHistoryEndpoint(sentSubmissionId, bearerToken) >> rsHistoryApiResponse