Merge branch 'azure-load-tests' of https://github.com/CDCgov/trusted-…

…intermediary into azure-load-tests

app/build.gradle

@@ -23,12 +23,12 @@ dependencies {
     implementation project(':etor')
     testImplementation testFixtures(project(':shared'))
 
-    implementation 'io.javalin:javalin:6.3.0'
+    implementation 'io.javalin:javalin:6.4.0'
 
     testImplementation 'org.apache.groovy:groovy:4.0.24'
     testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
     testImplementation 'com.openpojo:openpojo:0.9.1'
-    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.17.5'
+    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.18'
 }
 
 jacocoTestCoverageVerification {

app/src/main/java/gov/hhs/cdc/trustedintermediary/external/javalin/App.java

@@ -48,7 +48,12 @@ public static void main(String[] args) {
 
         // apply this security header to all responses, but allow it to be overwritten by a specific
         // endpoint by using `before` if needed
-        app.before(ctx -> ctx.header("X-Content-Type-Options", "nosniff"));
+        app.before(
+                ctx -> {
+                    ctx.header("X-Content-Type-Options", "nosniff");
+                    // Fix for https://www.zaproxy.org/docs/alerts/90004
+                    ctx.header("Cross-Origin-Resource-Policy", "cross-origin");
+                });
 
         try {
             app.get(HEALTH_API_ENDPOINT, ctx -> ctx.result("Operational"));

app/src/main/java/gov/hhs/cdc/trustedintermediary/external/javalin/DomainsRegistration.java

@@ -122,7 +122,9 @@ static DomainConnector constructNewDomainConnector(Class<? extends DomainConnect
     static Handler createHandler(
             Function<DomainRequest, DomainResponse> handler, boolean isProtected) {
         return (Context ctx) -> {
-            ApplicationContext.clearThreadRegistrations();
+            ApplicationContext
+                    .clearThreadRegistrations(); // clear this thread's specific registrations from
+            // its previous use
 
             LOGGER.logInfo(ctx.method().name() + " " + ctx.url());
 

e2e/build.gradle

@@ -19,10 +19,10 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2'
 
     //fhir
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:7.6.0'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.6.0'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.6.0'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:7.6.1'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.6.1'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.6.1'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.6.1'
     implementation 'org.fhir:ucum:1.0.9'
 
     testImplementation 'org.apache.groovy:groovy:4.0.24'

etor/build.gradle

@@ -19,7 +19,7 @@ dependencies {
     testImplementation 'org.apache.groovy:groovy:4.0.24'
     testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
     testImplementation 'com.openpojo:openpojo:0.9.1'
-    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.17.5'
+    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.18'
 }
 
 jacocoTestCoverageVerification {

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/EtorDomainRegistration.java

@@ -134,7 +134,6 @@ public Map<HttpEndpoint, Function<DomainRequest, DomainResponse>> domainRegistra
                     MessageLinkStorage.class, FileMessageLinkStorage.getInstance());
         }
 
-        // these are default implementations
         if (ApplicationContext.isPropertyPresent("REPORT_STREAM_URL_PREFIX")) {
             ApplicationContext.register(
                     RSEndpointClient.class, ReportStreamEndpointClient.getInstance());
@@ -174,6 +173,14 @@ DomainResponse handleResults(DomainRequest request) {
     }
 
     DomainResponse handleMetadata(DomainRequest request) {
+        if (Boolean.parseBoolean(request.getHeaders().get("load-test"))
+                && ApplicationContext.isPropertyPresent("REPORT_STREAM_URL_PREFIX")) {
+            // register the mock RS endpoint for this HTTP request because we don't want to call RS
+            // for real when doing a load test.
+            ApplicationContext.registerForThread(
+                    RSEndpointClient.class, MockRSEndpointClient.getInstance());
+        }
+
         try {
             String metadataId = request.getPathParams().get("id");
             Optional<PartnerMetadata> metadataOptional =
@@ -227,8 +234,10 @@ protected DomainResponse handleMessageRequest(
         boolean markMetadataAsFailed = false;
         String errorMessage = "";
 
-        if ("True".equals(request.getHeaders().get("load-test"))
+        if (Boolean.parseBoolean(request.getHeaders().get("load-test"))
                 && ApplicationContext.isPropertyPresent("REPORT_STREAM_URL_PREFIX")) {
+            // register the mock RS endpoint for this HTTP request because we don't want to call RS
+            // for real when doing a load test.
             ApplicationContext.registerForThread(
                     RSEndpointClient.class, MockRSEndpointClient.getInstance());
         }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadataOrchestrator.java

@@ -32,7 +32,6 @@ public class PartnerMetadataOrchestrator {
 
     @Inject PartnerMetadataStorage partnerMetadataStorage;
     @Inject MessageLinkStorage messageLinkStorage;
-    //    @Inject RSEndpointClient rsclient;
     @Inject Formatter formatter;
     @Inject Logger logger;
 
@@ -45,6 +44,7 @@ private PartnerMetadataOrchestrator() {}
     public void updateMetadataForInboundMessage(PartnerMetadata partnerMetadata)
             throws PartnerMetadataException {
 
+        // can't @Inject because the implementation can be different for this specific thread
         RSEndpointClient rsclient = ApplicationContext.getImplementation(RSEndpointClient.class);
 
         logger.logInfo(
@@ -134,6 +134,7 @@ public Optional<PartnerMetadata> getMetadata(String inboundReportId)
         var outboundReportId = partnerMetadata.outboundReportId();
         if (metadataIsStale(partnerMetadata) && outboundReportId != null) {
 
+            // can't @Inject because the implementation can be different for this specific thread
             RSEndpointClient rsclient =
                     ApplicationContext.getImplementation(RSEndpointClient.class);
 

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/CopyOrcOrderProviderToObrOrderProvider.java

@@ -28,16 +28,8 @@ public void transform(HealthData<?> resource, Map<String, Object> args) {
             return;
         }
 
-        // Extract or create the OBR-16 extension from the ServiceRequest
         Extension obrExtension =
                 HapiHelper.ensureExtensionExists(serviceRequest, HapiHelper.EXTENSION_OBR_URL);
-
-        // Extract or create the OBR-16 data type extension
-        Extension obr16Extension =
-                HapiHelper.ensureSubExtensionExists(
-                        obrExtension, HapiHelper.EXTENSION_OBR16_DATA_TYPE.toString());
-
-        // Set the ORC-12 Practitioner in the OBR-16 extension
-        HapiHelper.setOBR16WithPractitioner(obr16Extension, practitionerRole);
+        HapiHelper.setOBR16WithPractitioner(obrExtension, practitionerRole);
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientIdentifiers.java

@@ -15,7 +15,8 @@ public class RemovePatientIdentifiers implements CustomFhirTransformation {
     @Override
     public void transform(HealthData<?> resource, Map<String, Object> args) {
         Bundle bundle = (Bundle) resource.getUnderlyingData();
-        HapiHelper.setPID3_4Value(bundle, ""); // remove PID.3-4
-        HapiHelper.setPID3_5Value(bundle, ""); // remove PID.3-5
+
+        HapiHelper.setPID3_4Value(bundle, null);
+        HapiHelper.removePID3_5Value(bundle);
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientNameTypeCode.java

@@ -12,8 +12,6 @@ public class RemovePatientNameTypeCode implements CustomFhirTransformation {
     @Override
     public void transform(final HealthData<?> resource, final Map<String, Object> args) {
         Bundle bundle = (Bundle) resource.getUnderlyingData();
-        // Need to set the value for extension to empty instead of removing the extension,
-        // otherwise RS will set its own value in its place
-        HapiHelper.setPID5_7ExtensionValue(bundle, null);
+        HapiHelper.removePID5_7Value(bundle);
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/localfile/MockRSEndpointClient.java

@@ -30,7 +30,6 @@ private MockRSEndpointClient() {}
 
     @Override
     public String getRsToken() {
-        logger.logInfo("!!!!!!!!! WE ARE IN THE MOCK IMPLEMENTATION !!!!!!!!!");
         return "token";
     }
 

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/reportstream/ReportStreamEndpointClient.java

@@ -151,7 +151,6 @@ protected String requestToken() throws ReportStreamEndpointClientException {
 
     @Override
     public String getRsToken() throws ReportStreamEndpointClientException {
-        logger.logInfo("#########  {[{[{[{[{[REAL IMPLEMENTATION]}]}]}]}]} #########");
         logger.logInfo("Looking up ReportStream token");
 
         var token = cache.get(RS_TOKEN_CACHE_ID);

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/reportstream/ReportStreamSenderHelper.java

@@ -18,7 +18,6 @@
 public class ReportStreamSenderHelper {
     private static final ReportStreamSenderHelper INSTANCE = new ReportStreamSenderHelper();
 
-    //    @Inject RSEndpointClient rsclient;
     @Inject Formatter formatter;
     @Inject Logger logger;
     @Inject MetricMetadata metadata;
@@ -43,6 +42,7 @@ protected Optional<String> sendToReportStream(
             String body, String fhirResourceId, PartnerMetadataMessageType messageType)
             throws UnableToSendMessageException {
 
+        // can't @Inject because the implementation can be different for this specific thread
         RSEndpointClient rsclient = ApplicationContext.getImplementation(RSEndpointClient.class);
 
         String bearerToken;

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/CopyOrcOrderProviderToObrOrderProviderTest.groovy

@@ -7,8 +7,6 @@ import gov.hhs.cdc.trustedintermediary.external.hapi.HapiHelper
 import gov.hhs.cdc.trustedintermediary.wrappers.MetricMetadata
 import org.hl7.fhir.r4.model.Bundle
 import org.hl7.fhir.r4.model.DiagnosticReport
-import org.hl7.fhir.r4.model.Practitioner
-import org.hl7.fhir.r4.model.Reference
 import org.hl7.fhir.r4.model.ServiceRequest
 import spock.lang.Specification
 
@@ -119,7 +117,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
         evaluateOrc12Values(bundle, EXPECTED_NPI, EXPECTED_FIRST_NAME, EXPECTED_LAST_NAME, EXPECTED_NAME_TYPE_CODE, EXPECTED_IDENTIFIER_TYPE_CODE)
 
         // OBR16 should not exist initially
-        def obr16Practitioner = getObr16ExtensionPractitioner(serviceRequest)
+        def obr16Practitioner = HapiHelper.getObr16ExtensionPractitioner(serviceRequest)
         obr16Practitioner == null
 
         when:
@@ -191,7 +189,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
     }
 
     void evaluateOrc12IsNull(Bundle bundle) {
-        assert getOrc12ExtensionPractitioner(bundle) == null
+        assert HapiHelper.getOrc12ExtensionPractitioner(bundle) == null
     }
 
     void evaluateOrc12Values(
@@ -201,7 +199,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
             String expectedLastName,
             String expectedNameTypeCode,
             String expectedIdentifierTypeCode) {
-        def practitioner = getOrc12ExtensionPractitioner(bundle)
+        def practitioner = HapiHelper.getOrc12ExtensionPractitioner(bundle)
         def xcnExtension = practitioner.getExtensionByUrl(PRACTITIONER_EXTENSION_URL)
 
         assert practitioner.identifier[0]?.value == expectedNpi
@@ -214,7 +212,8 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
     }
 
     void evaluateObr16IsNull(ServiceRequest serviceRequest) {
-        assert getObr16ExtensionPractitioner(serviceRequest) == null
+        assert HapiHelper.getObr16Extension(serviceRequest) == null
+        assert HapiHelper.getObr16ExtensionPractitioner(serviceRequest) == null
     }
 
     void evaluateObr16Values(
@@ -224,7 +223,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
             String expectedLastName,
             String expectedNameTypeCode,
             String expectedIdentifierTypeCode) {
-        def practitioner = getObr16ExtensionPractitioner(serviceRequest)
+        def practitioner = HapiHelper.getObr16ExtensionPractitioner(serviceRequest)
         def xcnExtension = practitioner.getExtensionByUrl(PRACTITIONER_EXTENSION_URL)
 
         assert practitioner.identifier[0]?.value == expectedNpi
@@ -234,40 +233,4 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
         def codingSystem = practitioner.identifier[0]?.type?.coding
         assert codingSystem == null || codingSystem[0]?.code == expectedIdentifierTypeCode
     }
-
-    Practitioner getObr16ExtensionPractitioner (serviceRequest) {
-        def resource
-        try {
-            def extensionByUrl1 =  serviceRequest.getExtensionByUrl(HapiHelper.EXTENSION_OBR_URL)
-            def extensionByUrl2 = extensionByUrl1.getExtensionByUrl(HapiHelper.EXTENSION_OBR16_DATA_TYPE.toString())
-            def value = extensionByUrl2.value
-            resource = value.getResource()
-            return resource
-        } catch(Exception ignored) {
-            resource = null
-            return resource
-        }
-    }
-
-    Practitioner getOrc12ExtensionPractitioner(Bundle bundle) {
-        def diagnosticReport = HapiHelper.getDiagnosticReport(bundle)
-        def serviceRequest = HapiHelper.getServiceRequest(diagnosticReport)
-
-        def orcExtension = serviceRequest.getExtensionByUrl(HapiHelper.EXTENSION_ORC_URL)
-        def orc12Extension = orcExtension.getExtensionByUrl(HapiHelper.EXTENSION_ORC12_URL)
-
-        if (orc12Extension == null) {
-            return null
-        }
-
-        def practitionerReference = (Reference) orc12Extension.getValue()
-        def practitionerUrl = practitionerReference.getReference()
-
-        for (Bundle.BundleEntryComponent entry : bundle.getEntry()) {
-            if (Objects.equals(entry.getFullUrl(), practitionerUrl) && entry.getResource() instanceof Practitioner)
-                return (Practitioner) entry.getResource()
-        }
-
-        return null
-    }
 }

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientIdentifierTest.groovy

@@ -33,10 +33,10 @@ class RemovePatientIdentifierTest extends Specification {
 
         when:
         transformClass.transform(fhirResource, null)
-        def actualPid3_4 = HapiFhirHelper.getPID3_4Value(bundle)
-        def actualPid3_5 = HapiFhirHelper.getPID3_5Value(bundle)
 
         then:
+        def actualPid3_4 = HapiFhirHelper.getPID3_4Value(bundle)
+        def actualPid3_5 = HapiFhirHelper.getPID3_5Value(bundle)
         actualPid3_4 == null || actualPid3_4.isEmpty()
         actualPid3_5 == null || actualPid3_5.isEmpty()
     }

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientNameTypeCodeTest.groovy

@@ -22,18 +22,22 @@ class RemovePatientNameTypeCodeTest extends Specification {
 
     def "remove PID.5-7 from Bundle"() {
         given:
-        def fhirResource = ExamplesHelper.getExampleFhirResource("../CA/002_CA_ORU_R01_initial_translation.fhir")
+        def fhirResource = ExamplesHelper.getExampleFhirResource("../CA/007_CA_ORU_R01_CDPH_produced_UCSD2024-07-11-16-02-17-749_1_hl7_translation.fhir")
         def bundle = fhirResource.getUnderlyingData() as Bundle
-        def pid5_7 = HapiFhirHelper.getPID5_7Value(bundle)
+        def pid5Extension = HapiHelper.getPID5Extension(bundle)
+        def patientName = HapiHelper.getPIDPatient(bundle).getNameFirstRep()
+        def patientNameUse = patientName.getUse()
 
         expect:
-        pid5_7 != null
+        pid5Extension.getExtensionByUrl(HapiHelper.EXTENSION_XPN7_URL) != null
+        patientNameUse.toString() == "OFFICIAL"
 
         when:
         transformClass.transform(fhirResource, null)
 
         then:
-        HapiFhirHelper.getPID5_7Value(bundle) == null
+        pid5Extension.getExtensionByUrl(HapiHelper.EXTENSION_XPN7_URL) == null
+        !patientName.hasUse()
     }
 
     def "don't throw exception if patient resource not present"() {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -86,8 +86,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

operations/locustfile.py

@@ -72,6 +72,7 @@ def post_message_request(self, endpoint, message):
             headers={
                 "Authorization": self.access_token,
                 "RecordId": self.submission_id,
+                "Load-Test": "true",
             },
             data=message.replace("{{placer_order_id}}", poi),
         )
@@ -91,7 +92,10 @@ def get_v1_etor_metadata(self):
         if self.message_api_called:
             self.client.get(
                 f"{METADATA_ENDPOINT}/{self.submission_id}",
-                headers={"Authorization": self.access_token},
+                headers={
+                    "Authorization": self.access_token,
+                    "Load-Test": "true",
+                },
                 name=f"{METADATA_ENDPOINT}/{{id}}",
             )
 

operations/template/logs.tf

@@ -47,14 +47,24 @@ resource "azurerm_log_analytics_query_pack" "application_logs_pack" {
   }
 }
 
-resource "azurerm_log_analytics_query_pack_query" "application_logs" {
-  display_name = "TI's Raw Application Logs"
-  description  = "View all TI's application logs in a structured format"
+resource "azurerm_log_analytics_query_pack_query" "live_application_logs" {
+  display_name = "TI's Live Slot Raw Application Logs"
+  description  = "View all TI's live slot application logs in a structured format"
 
   query_pack_id = azurerm_log_analytics_query_pack.application_logs_pack.id
   categories    = ["applications"]
 
-  body = "AppServiceConsoleLogs | project JsonResult = parse_json(ResultDescription) | evaluate bag_unpack(JsonResult) | project-reorder ['@timestamp'], level, message"
+  body = "AppServiceConsoleLogs | where _ResourceId !contains 'pre-live' | project JsonResult = parse_json(ResultDescription) | evaluate bag_unpack(JsonResult) | project-reorder ['@timestamp'], level, message"
+}
+
+resource "azurerm_log_analytics_query_pack_query" "prelive_application_logs" {
+  display_name = "TI's Pre-Live Slot Raw Application Logs"
+  description  = "View all TI's pre-live slot application logs in a structured format"
+
+  query_pack_id = azurerm_log_analytics_query_pack.application_logs_pack.id
+  categories    = ["applications"]
+
+  body = "AppServiceConsoleLogs | where _ResourceId contains 'pre-live' | project JsonResult = parse_json(ResultDescription) | evaluate bag_unpack(JsonResult) | project-reorder ['@timestamp'], level, message"
 }
 
 resource "azurerm_log_analytics_query_pack_query" "application_error_logs" {