CNDIT-1212 + CNDIT-1519: Cherry pick from NEDSS-DataAccess

investigation-service/src/test/java/gov/cdc/etldatapipeline/investigation/InvestigationDataProcessingTests.java

@@ -63,7 +63,7 @@ void setUp() {
     }
 
     @Test
-    public void testConfirmationMethod() {
+    void testConfirmationMethod() {
         Investigation investigation = new Investigation();
 
         investigation.setPublicHealthCaseUid(investigationUid);
@@ -84,13 +84,13 @@ public void testConfirmationMethod() {
         verify(kafkaTemplate, times(2)).send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture());
         assertEquals(CONFIRMATION_TOPIC, topicCaptor.getValue());
 
-        Function<InvestigationConfirmationMethod, List<String>> cmDetailsFn = (m) -> Arrays.asList(
+        Function<InvestigationConfirmationMethod, List<String>> cmDetailsFn = m -> Arrays.asList(
                 String.valueOf(m.getPublicHealthCaseUid()),
                 m.getConfirmationMethodCd(),
                 m.getConfirmationMethodDescTxt(),
                 m.getConfirmationMethodTime());
 
-        Function<InvestigationConfirmationMethodKey, List<String>> cmKeyFn = (k) -> Arrays.asList(
+        Function<InvestigationConfirmationMethodKey, List<String>> cmKeyFn = k -> Arrays.asList(
                 String.valueOf(k.getPublicHealthCaseUid()),
                 k.getConfirmationMethodCd());
 

investigation-service/src/test/java/gov/cdc/etldatapipeline/investigation/InvestigationModelMappingTests.java

@@ -8,11 +8,11 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
-public class InvestigationModelMappingTests {
+class InvestigationModelMappingTests {
     private final ModelMapper modelMapper = new ModelMapper();
 
     @Test
-    public void testInvestigationReporting() {
+    void testInvestigationReporting() {
         final var investigation = getInvestigation();
 
         InvestigationReporting reporting = modelMapper.map(investigation, InvestigationReporting.class);

organization-service/src/main/java/gov/cdc/etldatapipeline/organization/service/OrganizationService.java

@@ -82,12 +82,12 @@ public void processMessage(String message,
                     String reportingKey = transformer.buildOrganizationKey(org);
                     String reportingData = transformer.processData(org, OrganizationType.ORGANIZATION_REPORTING);
                     kafkaTemplate.send(orgReportingOutputTopic, reportingKey, reportingData);
-                    log.info("Organization Reporting: {}", reportingData.toString());
+                    log.info("Organization Reporting: {}", reportingData);
 
                     String elasticKey = transformer.buildOrganizationKey(org);
                     String elasticData = transformer.processData(org, OrganizationType.ORGANIZATION_ELASTIC_SEARCH);
                     kafkaTemplate.send(orgElasticSearchTopic, elasticKey, elasticData);
-                    log.info("Organization Elastic: {}", elasticData!= null ? elasticData.toString() : "");
+                    log.info("Organization Elastic: {}", elasticData!= null ? elasticData : "");
                 });
             }
             else {

post-processing-service/Dockerfile

@@ -6,6 +6,7 @@ COPY gradlew /usr/src/gradlew
 COPY settings.gradle /usr/src/settings.gradle
 
 #Copy sources
+COPY common-util /usr/src/common-util
 COPY post-processing-service /usr/src/post-processing-service
 
 #cd to root directory

post-processing-service/build.gradle

@@ -56,18 +56,23 @@ dependencies {
     implementation 'net.logstash.logback:logstash-logback-encoder:7.4'
     implementation 'org.springframework.kafka:spring-kafka:3.1.2'
     implementation 'org.apache.kafka:connect-api:3.7.0'
+    implementation 'org.modelmapper:modelmapper:3.2.0'
+    implementation 'commons-io:commons-io:2.15.0'
+    implementation 'com.google.guava:guava:33.1.0-jre'
 
-    testImplementation 'org.springframework.kafka:spring-kafka-test:3.1.0'
+    if (findProject(':common-util')) {
+        implementation project(':common-util')
+    } else {
+        // this mostly needed when testing the app on idea
+        implementation files('libs/common-util-0.0.1-SNAPSHOT.jar')
+    }
 
     testImplementation(platform('org.junit:junit-bom:5.9.1'))
-    testImplementation('org.junit.jupiter:junit-jupiter:5.9.2')
-    testImplementation 'org.testcontainers:testcontainers:1.17.3'
-    testImplementation "org.testcontainers:kafka:1.17.3"
-    testImplementation "org.testcontainers:mssqlserver:1.17.3"
-    testImplementation "org.testcontainers:junit-jupiter:1.17.3"
-    testImplementation 'org.springframework.boot:spring-boot-starter-test:3.2.0'
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
     testImplementation 'org.springframework.security:spring-security-test:6.2.0'
-    testImplementation 'org.apache.camel:camel-test-spring-junit5:4.0.0'
+    testImplementation 'org.testcontainers:testcontainers:1.19.8'
+    testImplementation "org.testcontainers:kafka:1.19.8"
+    testImplementation "org.testcontainers:junit-jupiter:1.19.8"
 }
 
 tasks.named('test') {

post-processing-service/src/main/java/gov/cdc/etldatapipeline/postprocessingservice/config/KafkaProducerConfig.java

@@ -0,0 +1,35 @@
+package gov.cdc.etldatapipeline.postprocessingservice.config;
+
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.core.ProducerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Configuration
+public class KafkaProducerConfig {
+    @Value("${spring.kafka.bootstrap-servers}")
+    private String bootstrapServers = "";
+
+    @Bean
+    public ProducerFactory<String, String> producerFactory() {
+        final Map<String, Object> config = new HashMap<>();
+        config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+        config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+
+        return new DefaultKafkaProducerFactory<>(config);
+    }
+
+    @Bean
+    public KafkaTemplate<String, String> kafkaTemplate() {
+        // set factory for both producer and consumer
+        return new KafkaTemplate<>(producerFactory());
+    }
+}

post-processing-service/src/main/java/gov/cdc/etldatapipeline/postprocessingservice/repository/InvestigationRepository.java

@@ -14,6 +14,14 @@ public interface InvestigationRepository extends JpaRepository<InvestigationResu
     @Query(value = "EXEC sp_nrt_investigation_postprocessing :publicHealthCaseUids", nativeQuery = true)
     List<InvestigationResult> executeStoredProcForPublicHealthCaseIds(@Param("publicHealthCaseUids") String publicHealthCaseUids);
 
+    @Procedure("sp_page_builder_postprocessing")
+    void executeStoredProcForPageBuilder(@Param("phcUid") Long phcUid, @Param("rdbTableNmLst") String rdbTableNmLst);
+
     @Procedure("sp_f_page_case_postprocessing")
     void executeStoredProcForFPageCase(@Param("publicHealthCaseUids") String publicHealthCaseUids);
+
+    @Procedure("sp_hepatitis_datamart_postprocessing")
+    void executeStoredProcForHepDatamart(
+            @Param("publicHealthCaseUids") String publicHealthCaseUids,
+            @Param("patientUids") String patientUids);
 }

post-processing-service/src/main/java/gov/cdc/etldatapipeline/postprocessingservice/repository/model/InvestigationResult.java

@@ -10,8 +10,8 @@
 public class InvestigationResult {
 
     @Id
-    @Column(name = "case_uid")
-    private Long caseUid;
+    @Column(name = "public_health_case_uid")
+    private Long publicHealthCaseUid;
 
     @Column(name = "patient_uid")
     private Long patientUid;

post-processing-service/src/main/java/gov/cdc/etldatapipeline/postprocessingservice/repository/model/dto/Datamart.java

@@ -0,0 +1,28 @@
+package gov.cdc.etldatapipeline.postprocessingservice.repository.model.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+public class Datamart {
+    @JsonProperty("public_health_case_uid")
+    private Long publicHealthCaseUid;
+
+    @JsonProperty("patient_uid")
+    private Long patientUid;
+
+    @JsonProperty("investigation_key")
+    private Long investigationKey;
+
+    @JsonProperty("patient_key")
+    private Long patientKey;
+
+    @JsonProperty("condition_cd")
+    private String conditionCd;
+
+    @JsonProperty("datamart")
+    private String datamart;
+
+    @JsonProperty("stored_procedure")
+    private String storedProcedure;
+}

post-processing-service/src/main/java/gov/cdc/etldatapipeline/postprocessingservice/repository/model/dto/DatamartKey.java

@@ -0,0 +1,14 @@
+package gov.cdc.etldatapipeline.postprocessingservice.repository.model.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NonNull;
+
+@Data
+@Builder
+public class DatamartKey {
+    @NonNull
+    @JsonProperty("public_health_case_uid")
+    private Long publicHealthCaseUid;
+}