Merge branch 'main' into story-672-documentation

.github/workflows/terraform-deploy_reusable.yml

@@ -74,7 +74,7 @@ jobs:
       - name: Run Db migration
         run: |
           export PGPASSWORD=$(az account get-access-token --resource-type oss-rdbms --query "[accessToken]" -o tsv)
-          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "CREATE TABLE IF NOT EXISTS metadata (message_id varchar(30) PRIMARY KEY, sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin;"
+          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin;"
 
       - id: export-terraform-output
         name: Export Terraform Output

.secrets.baseline

@@ -77,7 +77,7 @@
     },
     {
       "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".secrets.baseline"
+      "filename": "./.secrets.baseline"
     },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
@@ -142,6 +142,16 @@
         "is_secret": false
       }
     ],
+    "docker-compose.postgres.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "docker-compose.postgres.yml",
+        "hashed_secret": "0f7866a6cab6f2793ea9f68e92935e4d726d58b5",
+        "is_verified": false,
+        "line_number": 11,
+        "is_secret": false
+      }
+    ],
     "mock_credentials/organization-report-stream-private-key-local.pem": [
       {
         "type": "Private Key",
@@ -212,6 +222,16 @@
         "is_secret": false
       }
     ],
+    "operations/template/db.tf": [
+      {
+        "type": "Secret Keyword",
+        "filename": "operations/template/db.tf",
+        "hashed_secret": "7cb6efb98ba5972a9b5090dc2e517fe14d12cb04",
+        "is_verified": false,
+        "line_number": 15,
+        "is_secret": false
+      }
+    ],
     "shared/src/main/java/gov/hhs/cdc/trustedintermediary/external/jjwt/JjwtEngine.java": [
       {
         "type": "Private Key",
@@ -233,5 +253,5 @@
       }
     ]
   },
-  "generated_at": "2023-11-30T22:27:41Z"
+  "generated_at": "2024-01-08T18:22:05Z"
 }

docker-compose.postgres.yml

@@ -1,3 +1,5 @@
+# Find the SQL schema in `/.github/workflows/terraform-deploy_reusable.yml`.
+
 version: "3.7"
 
 services:

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/PartnerMetadata.java

@@ -25,15 +25,6 @@ public PartnerMetadata(
         this(receivedSubmissionId, null, sender, null, timeReceived, hash);
     }
 
-    public PartnerMetadata(
-            String receivedSubmissionId,
-            String sender,
-            String receiver,
-            Instant timeReceived,
-            String hash) {
-        this(receivedSubmissionId, null, sender, receiver, timeReceived, hash);
-    }
-
     public PartnerMetadata(String receivedSubmissionId, String hash) {
         this(receivedSubmissionId, null, null, null, null, hash);
     }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/PartnerMetadataOrchestrator.java

@@ -156,6 +156,9 @@ String getReceiverName(String responseBody) throws FormatterProcessingException
             Map<?, ?> destination = (Map<?, ?>) destinations.get(0);
             organizationId = destination.get("organization_id").toString();
             service = destination.get("service").toString();
+        } catch (IndexOutOfBoundsException e) {
+            // the destinations have not been determined yet by RS
+            return null;
         } catch (Exception e) {
             throw new FormatterProcessingException(
                     "Unable to extract receiver name from response due to unexpected format", e);

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/DatabasePartnerMetadataStorage.java

@@ -41,6 +41,7 @@ public void saveMetadata(final PartnerMetadata metadata) throws PartnerMetadataE
         try {
             dao.upsertMetadata(
                     metadata.receivedSubmissionId(),
+                    metadata.sentSubmissionId(),
                     metadata.sender(),
                     metadata.receiver(),
                     metadata.hash(),

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/database/PostgresDao.java

@@ -82,6 +82,7 @@ public static PostgresDao getInstance() {
     @Override
     public synchronized void upsertMetadata(
             String receivedSubmissionId,
+            String sentSubmissionId,
             String sender,
             String receiver,
             String hash,
@@ -92,15 +93,21 @@ public synchronized void upsertMetadata(
                 PreparedStatement statement =
                         conn.prepareStatement(
                                 """
-                                INSERT INTO metadata VALUES (?, ?, ?, ?, ?)
-                                ON CONFLICT (message_id) DO UPDATE SET receiver = EXCLUDED.receiver
+                                INSERT INTO metadata VALUES (?, ?, ?, ?, ?, ?)
+                                ON CONFLICT (received_message_id) DO UPDATE SET receiver = EXCLUDED.receiver, sent_message_id = EXCLUDED.sent_message_id
                                 """)) {
 
             statement.setString(1, receivedSubmissionId);
-            statement.setString(2, sender);
-            statement.setString(3, receiver);
-            statement.setString(4, hash);
-            statement.setTimestamp(5, Timestamp.from(timeReceived));
+            statement.setString(2, sentSubmissionId);
+            statement.setString(3, sender);
+            statement.setString(4, receiver);
+            statement.setString(5, hash);
+
+            Timestamp timestamp = null;
+            if (timeReceived != null) {
+                timestamp = Timestamp.from(timeReceived);
+            }
+            statement.setTimestamp(6, timestamp);
 
             statement.executeUpdate();
         }
@@ -111,7 +118,8 @@ public synchronized PartnerMetadata fetchMetadata(String receivedSubmissionId)
             throws SQLException {
         try (Connection conn = connect();
                 PreparedStatement statement =
-                        conn.prepareStatement("SELECT * FROM metadata where message_id = ?")) {
+                        conn.prepareStatement(
+                                "SELECT * FROM metadata where received_message_id = ?")) {
 
             statement.setString(1, receivedSubmissionId);
 
@@ -121,11 +129,18 @@ public synchronized PartnerMetadata fetchMetadata(String receivedSubmissionId)
                 return null;
             }
 
+            Instant timeReceived = null;
+            Timestamp timestamp = result.getTimestamp("time_received");
+            if (timestamp != null) {
+                timeReceived = timestamp.toInstant();
+            }
+
             return new PartnerMetadata(
-                    result.getString("message_id"),
+                    result.getString("received_message_id"),
+                    result.getString("sent_message_id"),
                     result.getString("sender"),
                     result.getString("receiver"),
-                    result.getTimestamp("time_received").toInstant(),
+                    timeReceived,
                     result.getString("hash_of_order"));
         }
     }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/hapi/HapiOrderConverter.java

@@ -214,11 +214,15 @@ public FhirMetadata<?> extractPublicMetadataToOperationOutcome(PartnerMetadata m
         operation
                 .getIssue()
                 .add(createInformationIssueComponent("receiver name", metadata.receiver()));
+
+        String orderIngestion = null;
+        if (metadata.timeReceived() != null) {
+            orderIngestion = metadata.timeReceived().toString();
+        }
+
         operation
                 .getIssue()
-                .add(
-                        createInformationIssueComponent(
-                                "order ingestion", metadata.timeReceived().toString()));
+                .add(createInformationIssueComponent("order ingestion", orderIngestion));
         operation.getIssue().add(createInformationIssueComponent("payload hash", metadata.hash()));
 
         return new HapiFhirMetadata(operation);

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/metadata/PartnerMetadataOrchestratorTest.groovy

@@ -2,15 +2,13 @@ package gov.hhs.cdc.trustedintermediary.etor.metadata
 
 import gov.hhs.cdc.trustedintermediary.context.TestApplicationContext
 import gov.hhs.cdc.trustedintermediary.etor.RSEndpointClient
-import gov.hhs.cdc.trustedintermediary.etor.orders.Order
 import gov.hhs.cdc.trustedintermediary.etor.orders.OrderConverter
 import gov.hhs.cdc.trustedintermediary.external.hapi.HapiOrderConverter
 import gov.hhs.cdc.trustedintermediary.external.jackson.Jackson
 import gov.hhs.cdc.trustedintermediary.external.reportstream.ReportStreamEndpointClientException
 import gov.hhs.cdc.trustedintermediary.wrappers.formatter.Formatter
 import gov.hhs.cdc.trustedintermediary.wrappers.formatter.FormatterProcessingException
 import gov.hhs.cdc.trustedintermediary.wrappers.formatter.TypeReference
-
 import java.time.Instant
 import spock.lang.Specification
 
@@ -219,7 +217,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
 
     def "getMetadata retrieves metadata successfully"() {
         given:
-        String receivedSubmissionId = "receivedSubmissionId"
+        def receivedSubmissionId = "receivedSubmissionId"
         def metadata = new PartnerMetadata(receivedSubmissionId, "sentSubmissionId", "sender", "receiver", Instant.now(), "hash")
 
         when:
@@ -231,6 +229,20 @@ class PartnerMetadataOrchestratorTest extends Specification {
         1 * mockPartnerMetadataStorage.readMetadata(receivedSubmissionId) >> Optional.of(metadata)
     }
 
+    def "getMetadata retrieves metadata successfully when receiver is present and sentSubmissionId is missing"() {
+        given:
+        def receivedSubmissionId = "receivedSubmissionId"
+        def metadata = new PartnerMetadata(receivedSubmissionId, null, "sender", "receiver", Instant.now(), "hash")
+
+        when:
+        def result = PartnerMetadataOrchestrator.getInstance().getMetadata(receivedSubmissionId)
+
+        then:
+        result.isPresent()
+        result.get() == metadata
+        1 * mockPartnerMetadataStorage.readMetadata(receivedSubmissionId) >> Optional.of(metadata)
+    }
+
     def "getMetadata gets receiver if missing from metadata"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
@@ -262,60 +274,61 @@ class PartnerMetadataOrchestratorTest extends Specification {
 
     def "getReceiverName returns correct receiver name from valid JSON response"() {
         given:
-        String validJson = "{\"destinations\": [{\"organization_id\": \"org_id\", \"service\": \"service_name\"}]}"
+        def validJson = "{\"destinations\": [{\"organization_id\": \"org_id\", \"service\": \"service_name\"}]}"
 
         TestApplicationContext.register(Formatter, Jackson.getInstance())
         TestApplicationContext.injectRegisteredImplementations()
 
         when:
-        String receiverName = PartnerMetadataOrchestrator.getInstance().getReceiverName(validJson)
+        def receiverName = PartnerMetadataOrchestrator.getInstance().getReceiverName(validJson)
 
         then:
         receiverName == "org_id.service_name"
     }
 
-    def "getReceiverName throws FormatterProcessingException for unexpected format response"() {
+    def "getReceiverName throws FormatterProcessingException or returns null for unexpected format response"() {
         given:
         TestApplicationContext.register(Formatter, Jackson.getInstance())
         TestApplicationContext.injectRegisteredImplementations()
 
         when:
-        String invalidJson = "invalid JSON"
+        def invalidJson = "invalid JSON"
         PartnerMetadataOrchestrator.getInstance().getReceiverName(invalidJson)
 
         then:
         thrown(FormatterProcessingException)
 
         when:
-        String emptyJson = "{}"
+        def emptyJson = "{}"
         PartnerMetadataOrchestrator.getInstance().getReceiverName(emptyJson)
 
         then:
         thrown(FormatterProcessingException)
 
         when:
-        String jsonWithoutDestinations = "{\"someotherkey\": \"value\"}"
+        def jsonWithoutDestinations = "{\"someotherkey\": \"value\"}"
         PartnerMetadataOrchestrator.getInstance().getReceiverName(jsonWithoutDestinations)
 
         then:
         thrown(FormatterProcessingException)
 
         when:
-        String jsonWithEmptyDestinations = "{\"destinations\": []}"
-        PartnerMetadataOrchestrator.getInstance().getReceiverName(jsonWithEmptyDestinations)
+
+        def jsonWithEmptyDestinations = "{\"destinations\": []}"
+        def receiverName = PartnerMetadataOrchestrator.getInstance().getReceiverName(jsonWithEmptyDestinations)
 
         then:
-        thrown(FormatterProcessingException)
+        receiverName == null
 
         when:
-        String jsonWithoutOrgId = "{\"destinations\":[{\"service\":\"service\"}]}"
+        def jsonWithoutOrgId = "{\"destinations\":[{\"service\":\"service\"}]}"
         PartnerMetadataOrchestrator.getInstance().getReceiverName(jsonWithoutOrgId)
 
         then:
         thrown(FormatterProcessingException)
 
         when:
-        String jsonWithoutService = "{\"destinations\":[{\"organization_id\":\"org_id\"}]}"
+        def jsonWithoutService = "{\"destinations\":[{\"organization_id\":\"org_id\"}]}"
         PartnerMetadataOrchestrator.getInstance().getReceiverName(jsonWithoutService)
 
         then:

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/orders/SendOrderUsecaseTest.groovy

@@ -121,4 +121,19 @@ class SendOrderUsecaseTest extends Specification {
         1 * mockLogger.logError(_, _)
         noExceptionThrown()
     }
+
+    def "convertAndSend logs event when submissionId is null"() {
+        given:
+        def mockOrder = Mock(Order)
+        TestApplicationContext.injectRegisteredImplementations()
+
+        mockSender.sendOrder(_) >> Optional.empty()
+
+        when:
+        SendOrderUseCase.getInstance().convertAndSend(mockOrder, "receivedId")
+
+        then:
+        1 * mockLogger.logWarning(_)
+        0 * mockOrchestrator.updateMetadataForSentOrder(_ as String, _ as String)
+    }
 }