Add time of delivery to metadata endpoint (#831)

* Pushing initial branch * Adding time delivered to Operation Outcome API Response object * Adding test coverage * Update time_delivered SQL in PostgresDao.java --------- Co-authored-by: Tiffini Johnson <[email protected]>
CDCgov · Feb 1, 2024 · 0e77fb9 · 0e77fb9
1 parent 6688bee
commit 0e77fb9
Show file tree

Hide file tree

Showing 17 changed files with 194 additions and 62 deletions.
diff --git a/.github/workflows/terraform-deploy_reusable.yml b/.github/workflows/terraform-deploy_reusable.yml
@@ -74,7 +74,7 @@ jobs:
       - name: Run Db migration
         run: |
           export PGPASSWORD=$(az account get-access-token --resource-type oss-rdbms --query "[accessToken]" -o tsv)
-          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "DO \$\$ BEGIN CREATE TYPE message_status AS ENUM ('PENDING', 'DELIVERED', 'FAILED'); EXCEPTION WHEN duplicate_object THEN null; END \$\$; CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz, delivery_status message_status, failure_reason varchar(1000)); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin; ALTER TYPE message_status OWNER TO azure_pg_admin"
+          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "DO \$\$ BEGIN CREATE TYPE message_status AS ENUM ('PENDING', 'DELIVERED', 'FAILED'); EXCEPTION WHEN duplicate_object THEN null; END \$\$; CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz, time_delivered timestamptz, delivery_status message_status, failure_reason varchar(1000)); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin; ALTER TYPE message_status OWNER TO azure_pg_admin"
 
       - id: export-terraform-output
         name: Export Terraform Output

diff --git a/e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/e2e/ResultTest.groovy b/e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/e2e/ResultTest.groovy
@@ -7,7 +7,9 @@ import java.nio.file.Path
 
 class ResultTest extends Specification {
     def resultClient = new EndpointClient("/v1/etor/results")
+
     def labResultJsonFileString = Files.readString(Path.of("../examples/MN/004_MN_ORU_R01_NBS_1_translation_from_initial_hl7_ingestion.fhir"))
+
     def submissionId = "submissionId"
 
     def setup() {

diff --git a/.../src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadata.java b/.../src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadata.java
@@ -10,6 +10,7 @@
  * @param sender The name of the sender of the message.
  * @param receiver The name of the receiver of the message.
  * @param timeReceived The time the message was received.
+ * @param timeDelivered The time the message was delivered.
  * @param hash The hash of the message.
  * @param deliveryStatus the status of the message based on an enum
  */
@@ -19,6 +20,7 @@ public record PartnerMetadata(
         String sender,
         String receiver,
         Instant timeReceived,
+        Instant timeDelivered,
         String hash,
         PartnerMetadataStatus deliveryStatus,
         String failureReason) {
@@ -34,17 +36,27 @@ public PartnerMetadata(
             String receivedSubmissionId,
             String sender,
             Instant timeReceived,
+            Instant timeDelivered,
             String hash,
             PartnerMetadataStatus deliveryStatus) {
-        this(receivedSubmissionId, null, sender, null, timeReceived, hash, deliveryStatus, null);
+        this(
+                receivedSubmissionId,
+                null,
+                sender,
+                null,
+                timeReceived,
+                timeDelivered,
+                hash,
+                deliveryStatus,
+                null);
     }
 
     public PartnerMetadata(String receivedSubmissionId, String hash) {
-        this(receivedSubmissionId, null, null, null, null, hash, null, null);
+        this(receivedSubmissionId, null, null, null, null, null, hash, null, null);
     }
 
     public PartnerMetadata(String receivedSubmissionId, PartnerMetadataStatus deliveryStatus) {
-        this(receivedSubmissionId, null, null, null, null, null, deliveryStatus, null);
+        this(receivedSubmissionId, null, null, null, null, null, null, deliveryStatus, null);
     }
 
     public PartnerMetadata withSentSubmissionId(String sentSubmissionId) {
@@ -54,6 +66,7 @@ public PartnerMetadata withSentSubmissionId(String sentSubmissionId) {
                 this.sender,
                 this.receiver,
                 this.timeReceived,
+                this.timeDelivered,
                 this.hash,
                 this.deliveryStatus,
                 this.failureReason);
@@ -66,6 +79,20 @@ public PartnerMetadata withReceiver(String receiver) {
                 this.sender,
                 receiver,
                 this.timeReceived,
+                this.timeDelivered,
+                this.hash,
+                this.deliveryStatus,
+                this.failureReason);
+    }
+
+    public PartnerMetadata withTimeDelivered(Instant timeDelivered) {
+        return new PartnerMetadata(
+                this.receivedSubmissionId,
+                this.sentSubmissionId,
+                this.sender,
+                this.receiver,
+                this.timeReceived,
+                timeDelivered,
                 this.hash,
                 this.deliveryStatus,
                 this.failureReason);
@@ -78,6 +105,7 @@ public PartnerMetadata withDeliveryStatus(PartnerMetadataStatus deliveryStatus)
                 this.sender,
                 this.receiver,
                 this.timeReceived,
+                this.timeDelivered,
                 this.hash,
                 deliveryStatus,
                 this.failureReason);
@@ -90,6 +118,7 @@ public PartnerMetadata withFailureMessage(String failureMessage) {
                 this.sender,
                 this.receiver,
                 this.timeReceived,
+                this.timeDelivered,
                 this.hash,
                 this.deliveryStatus,
                 failureMessage);

diff --git a/...va/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadataOrchestrator.java b/...va/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadataOrchestrator.java
@@ -44,7 +44,6 @@ public void updateMetadataForReceivedOrder(String receivedSubmissionId, String o
 
         String sender;
         Instant timeReceived;
-
         try {
             String bearerToken = rsclient.getRsToken();
             String responseBody =
@@ -78,6 +77,7 @@ public void updateMetadataForReceivedOrder(String receivedSubmissionId, String o
                         receivedSubmissionId,
                         sender,
                         timeReceived,
+                        null,
                         orderHash,
                         PartnerMetadataStatus.PENDING);
         partnerMetadataStorage.saveMetadata(partnerMetadata);
@@ -128,6 +128,7 @@ public Optional<PartnerMetadata> getMetadata(String receivedSubmissionId)
             String receiver;
             String rsStatus;
             String rsMessage = "";
+            String timeDelivered;
             try {
                 String bearerToken = rsclient.getRsToken();
                 String responseBody =
@@ -136,6 +137,7 @@ public Optional<PartnerMetadata> getMetadata(String receivedSubmissionId)
                 receiver = parsedResponseBody[0];
                 rsStatus = parsedResponseBody[1];
                 rsMessage = parsedResponseBody[2];
+                timeDelivered = parsedResponseBody[3];
             } catch (ReportStreamEndpointClientException | FormatterProcessingException e) {
                 throw new PartnerMetadataException(
                         "Unable to retrieve metadata from RS history API", e);
@@ -148,6 +150,8 @@ public Optional<PartnerMetadata> getMetadata(String receivedSubmissionId)
 
             if (ourStatus == PartnerMetadataStatus.FAILED) {
                 partnerMetadata = partnerMetadata.withFailureMessage(rsMessage);
+            } else if (ourStatus == PartnerMetadataStatus.DELIVERED && timeDelivered != null) {
+                partnerMetadata = partnerMetadata.withTimeDelivered(Instant.parse(timeDelivered));
             }
 
             partnerMetadataStorage.saveMetadata(partnerMetadata);
@@ -214,6 +218,7 @@ String[] getDataFromReportStream(String responseBody) throws FormatterProcessing
         // {
         //    ...
         //    "overallStatus": "Waiting to Deliver",
+        //    "actualCompletionAt": "2023-10-24T19:48:26.921Z"
         //    ...
         //    "destinations" : [ {
         //        ...
@@ -265,7 +270,16 @@ String[] getDataFromReportStream(String responseBody) throws FormatterProcessing
                     "Unable to extract failure reason due to unexpected format", e);
         }
 
-        return new String[] {receiver, overallStatus, errorMessages.toString()};
+        String timeDelivered = null;
+        try {
+
+            timeDelivered = (String) responseObject.get("actualCompletionAt");
+        } catch (Exception e) {
+            throw new FormatterProcessingException(
+                    "Unable to extract timeDelivered due to unexpected format", e);
+        }
+
+        return new String[] {receiver, overallStatus, errorMessages.toString(), timeDelivered};
     }
 
     PartnerMetadataStatus ourStatusFromReportStreamStatus(String rsStatus) {

diff --git a/...ava/gov/hhs/cdc/trustedintermediary/external/database/DatabasePartnerMetadataStorage.java b/...ava/gov/hhs/cdc/trustedintermediary/external/database/DatabasePartnerMetadataStorage.java
@@ -46,6 +46,7 @@ public void saveMetadata(final PartnerMetadata metadata) throws PartnerMetadataE
                     metadata.receiver(),
                     metadata.hash(),
                     metadata.timeReceived(),
+                    metadata.timeDelivered(),
                     metadata.deliveryStatus(),
                     metadata.failureReason());
         } catch (SQLException e) {

diff --git a/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/DbDao.java b/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/DbDao.java
@@ -15,6 +15,7 @@ void upsertMetadata(
             String receiver,
             String hash,
             Instant timeReceived,
+            Instant timeDelivered,
             PartnerMetadataStatus deliveryStatus,
             String failureReason)
             throws SQLException;

diff --git a/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/PostgresDao.java b/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/PostgresDao.java
@@ -78,6 +78,7 @@ public synchronized void upsertMetadata(
             String receiver,
             String hash,
             Instant timeReceived,
+            Instant timeDelivered,
             PartnerMetadataStatus deliveryStatus,
             String failureReason)
             throws SQLException {
@@ -86,8 +87,8 @@ public synchronized void upsertMetadata(
                 PreparedStatement statement =
                         conn.prepareStatement(
                                 """
-                                INSERT INTO metadata VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-                                ON CONFLICT (received_message_id) DO UPDATE SET receiver = EXCLUDED.receiver, sent_message_id = EXCLUDED.sent_message_id, delivery_status = EXCLUDED.delivery_status, failure_reason = EXCLUDED.failure_reason
+                                INSERT INTO metadata VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                                ON CONFLICT (received_message_id) DO UPDATE SET receiver = EXCLUDED.receiver, sent_message_id = EXCLUDED.sent_message_id, time_delivered = EXCLUDED.time_delivered, delivery_status = EXCLUDED.delivery_status, failure_reason = EXCLUDED.failure_reason
                                 """)) {
 
             statement.setString(1, receivedSubmissionId);
@@ -96,20 +97,27 @@ ON CONFLICT (received_message_id) DO UPDATE SET receiver = EXCLUDED.receiver, se
             statement.setString(4, receiver);
             statement.setString(5, hash);
 
-            Timestamp timestamp = null;
+            Timestamp timestampReceived = null;
             if (timeReceived != null) {
-                timestamp = Timestamp.from(timeReceived);
+                timestampReceived = Timestamp.from(timeReceived);
             }
-            statement.setTimestamp(6, timestamp);
+            statement.setTimestamp(6, timestampReceived);
+
+            Timestamp timestampDelivered = null;
+
+            if (timeDelivered != null) {
+                timestampDelivered = Timestamp.from(timeDelivered);
+            }
+            statement.setTimestamp(7, timestampDelivered);
 
             String deliveryStatusString = null;
             if (deliveryStatus != null) {
                 deliveryStatusString = deliveryStatus.toString();
             }
 
-            statement.setObject(7, deliveryStatusString, Types.OTHER);
+            statement.setObject(8, deliveryStatusString, Types.OTHER);
 
-            statement.setString(8, failureReason);
+            statement.setString(9, failureReason);
 
             statement.executeUpdate();
         }
@@ -157,9 +165,15 @@ public synchronized PartnerMetadata fetchMetadata(String submissionId) throws SQ
 
     private PartnerMetadata partnerMetadataFromResultSet(ResultSet resultSet) throws SQLException {
         Instant timeReceived = null;
-        Timestamp timestamp = resultSet.getTimestamp("time_received");
-        if (timestamp != null) {
-            timeReceived = timestamp.toInstant();
+        Instant timeDelivered = null;
+        Timestamp timestampReceived = resultSet.getTimestamp("time_received");
+        Timestamp timestampDelivered = resultSet.getTimestamp("time_delivered");
+        if (timestampReceived != null) {
+            timeReceived = timestampReceived.toInstant();
+        }
+
+        if (timestampDelivered != null) {
+            timeDelivered = timestampDelivered.toInstant();
         }
 
         return new PartnerMetadata(
@@ -168,6 +182,7 @@ private PartnerMetadata partnerMetadataFromResultSet(ResultSet resultSet) throws
                 resultSet.getString("sender"),
                 resultSet.getString("receiver"),
                 timeReceived,
+                timeDelivered,
                 resultSet.getString("hash_of_order"),
                 PartnerMetadataStatus.valueOf(resultSet.getString("delivery_status")),
                 resultSet.getString("failure_reason"));

diff --git a/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/hapi/HapiOrderConverter.java b/etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/hapi/HapiOrderConverter.java
@@ -217,15 +217,22 @@ public FhirMetadata<?> extractPublicMetadataToOperationOutcome(
                 .add(createInformationIssueComponent("receiver name", metadata.receiver()));
 
         String orderIngestion = null;
+        String orderDelivered = null;
         if (metadata.timeReceived() != null) {
             orderIngestion = metadata.timeReceived().toString();
         }
+        if (metadata.timeDelivered() != null) {
+            orderDelivered = metadata.timeDelivered().toString();
+        }
 
         operation
                 .getIssue()
                 .add(createInformationIssueComponent("order ingestion", orderIngestion));
-        operation.getIssue().add(createInformationIssueComponent("payload hash", metadata.hash()));
 
+        operation.getIssue().add(createInformationIssueComponent("payload hash", metadata.hash()));
+        operation
+                .getIssue()
+                .add(createInformationIssueComponent("order delivered", orderDelivered));
         operation
                 .getIssue()
                 .add(

diff --git a/...rc/main/java/gov/hhs/cdc/trustedintermediary/external/localfile/MockRSEndpointClient.java b/...rc/main/java/gov/hhs/cdc/trustedintermediary/external/localfile/MockRSEndpointClient.java
@@ -46,6 +46,7 @@ public String requestHistoryEndpoint(String submissionId, String bearerToken) {
                 {
                     "timestamp" : "2020-01-01T00:00:00.000Z",
                     "sender" : "flexion.simulated-hospital",
+                    "actualCompletionAt" : "2023-10-24T19:48:26.921Z",
                     "overallStatus": "Not Delivering",
                     "destinations": [{
                         "organization_id": "flexion",

diff --git a/etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistrationTest.groovy b/etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistrationTest.groovy
@@ -381,7 +381,7 @@ class EtorDomainRegistrationTest extends Specification {
         request.setPathParams(["id": "metadataId"])
 
         def mockPartnerMetadataOrchestrator = Mock(PartnerMetadataOrchestrator)
-        mockPartnerMetadataOrchestrator.getMetadata(_ as String) >> Optional.ofNullable(new PartnerMetadata("receivedSubmissionId", "sender", Instant.now(), "hash", PartnerMetadataStatus.DELIVERED))
+        mockPartnerMetadataOrchestrator.getMetadata(_ as String) >> Optional.ofNullable(new PartnerMetadata("receivedSubmissionId", "sender", Instant.now(), null, "hash", PartnerMetadataStatus.DELIVERED))
         TestApplicationContext.register(PartnerMetadataOrchestrator, mockPartnerMetadataOrchestrator)
 
         def mockResponseHelper = Mock(DomainResponseHelper)