Skip to content

Commit

Permalink
CNDIT-1587: Increase logging to post-processing service (#21)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sveselev
sveselev authored Aug 19, 2024
1 parent 0902af0 commit 049a61d
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 26 deletions.
20 changes: 12 additions & 8 deletions ...ain/java/gov/cdc/etldatapipeline/postprocessingservice/service/PostProcessingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class PostProcessingService {
private final ProcessDatamartData datamartProcessor;

static final String PAYLOAD = "payload";
static final String SP_EXECUTION_COMPLETED = "Stored proc execution completed";
static final String SP_EXECUTION_COMPLETED = "Stored proc execution completed: {}";

private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());

Expand Down Expand Up @@ -109,6 +109,7 @@ public void postProcessMessage(
@Header(KafkaHeaders.RECEIVED_KEY) String key,
@Payload String payload) {
Long id = extractIdFromMessage(topic, key, payload);
logger.info("Adding id to cache: {} for topic: {}", id, topic);
if (id != null) {
idCache.computeIfAbsent(topic, k -> new CopyOnWriteArrayList<>()).add(id);
}
Expand Down Expand Up @@ -171,7 +172,10 @@ protected void processCachedIds() {
List<Long> ids = entry.getValue();
idCache.put(keyTopic, new ArrayList<>());

logger.info("Processing {} ids from topic: {}", ids.size(), keyTopic);

Entity entity = getEntityByTopic(keyTopic);
logger.info("{} data is about to be processed...", entity.getName());
switch (entity) {
case ORGANIZATION:
processTopic(keyTopic, entity, ids,
Expand Down Expand Up @@ -231,10 +235,10 @@ protected void processDatamartIds() {
String patients =
dmSet.stream().flatMap(m -> m.values().stream().map(String::valueOf)).collect(Collectors.joining(","));

logger.info("Processing {} message topic. Calling stored proc: {}('{}','{}')", dmType,
logger.info("Processing {} message topic. Calling stored proc: {} '{}','{}'", dmType,
"sp_hepatitis_datamart_postprocessing", cases, patients);
investigationRepository.executeStoredProcForHepDatamart(cases, patients);
completeLog();
completeLog("sp_hepatitis_datamart_postprocessing");
}
} else {
logger.info("No data to process from the datamart topics.");
Expand Down Expand Up @@ -281,22 +285,22 @@ private Entity getEntityByTopic(String topic) {
private void processTopic(String keyTopic, Entity entity, List<Long> ids, Consumer<String> repositoryMethod) {
String idsString = prepareAndLog(keyTopic, entity, ids);
repositoryMethod.accept(idsString);
completeLog();
completeLog(entity.getStoredProcedure());
}

private <T> List<T> processTopic(String keyTopic, Entity entity, List<Long> ids,
Function<String, List<T>> repositoryMethod) {
String idsString = prepareAndLog(keyTopic, entity, ids);
List<T> result = repositoryMethod.apply(idsString);
completeLog();
completeLog(entity.getStoredProcedure());
return result;
}

private void processTopic(String keyTopic, Entity entity, Long id, String vals, BiConsumer<Long, String> repositoryMethod) {
logger.info("Processing {} for topic: {}. Calling stored proc: {} '{}', '{}'", StringUtils.capitalize(entity.getName()), keyTopic,
entity.getStoredProcedure(), id, vals);
repositoryMethod.accept(id, vals);
completeLog();
completeLog(entity.getStoredProcedure());
}

private String prepareAndLog(String keyTopic, Entity entity, List<Long> ids) {
Expand All @@ -306,7 +310,7 @@ private String prepareAndLog(String keyTopic, Entity entity, List<Long> ids) {
return idsString;
}

private void completeLog() {
logger.info(SP_EXECUTION_COMPLETED);
private void completeLog(String sp) {
logger.info(SP_EXECUTION_COMPLETED, sp);
}
}
36 changes: 18 additions & 18 deletions ...java/gov/cdc/etldatapipeline/postprocessingservice/service/PostProcessingServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ void testPostProcessPatientMessage() {
assertTrue(postProcessingServiceMock.idCache.containsKey(topic));

List<ILoggingEvent> logs = listAppender.list;
assertEquals(3, logs.size());
assertTrue(logs.get(2).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(6, logs.size());
assertTrue(logs.get(5).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand All @@ -108,8 +108,8 @@ void testPostProcessProviderMessage() {
assertTrue(postProcessingServiceMock.idCache.containsKey(topic));

List<ILoggingEvent> logs = listAppender.list;
assertEquals(3, logs.size());
assertTrue(logs.get(2).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(6, logs.size());
assertTrue(logs.get(5).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand All @@ -125,8 +125,8 @@ void testPostProcessOrganizationMessage() {
assertTrue(postProcessingServiceMock.idCache.containsKey(topic));

List<ILoggingEvent> logs = listAppender.list;
assertEquals(3, logs.size());
assertTrue(logs.get(2).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(6, logs.size());
assertTrue(logs.get(5).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand All @@ -143,9 +143,9 @@ void testPostProcessInvestigationMessage() {
assertTrue(postProcessingServiceMock.idCache.containsKey(topic));

List<ILoggingEvent> logs = listAppender.list;
assertEquals(5, logs.size());
assertTrue(logs.get(1).getFormattedMessage().contains(PostProcessingService.Entity.INVESTIGATION.getStoredProcedure()));
assertTrue(logs.get(4).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(8, logs.size());
assertTrue(logs.get(4).getFormattedMessage().contains(PostProcessingService.Entity.INVESTIGATION.getStoredProcedure()));
assertTrue(logs.get(7).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand All @@ -161,9 +161,9 @@ void testPostProcessNotificationMessage() {
assertTrue(postProcessingServiceMock.idCache.containsKey(topic));

List<ILoggingEvent> logs = listAppender.list;
assertEquals(3, logs.size());
assertTrue(logs.get(1).getFormattedMessage().contains(PostProcessingService.Entity.NOTIFICATIONS.getStoredProcedure()));
assertTrue(logs.get(2).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(6, logs.size());
assertTrue(logs.get(4).getFormattedMessage().contains(PostProcessingService.Entity.NOTIFICATIONS.getStoredProcedure()));
assertTrue(logs.get(5).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand All @@ -186,8 +186,8 @@ void testPostProcessPageBuilder() {
expectedRdbTableNames);

List<ILoggingEvent> logs = listAppender.list;
assertEquals(7, logs.size());
assertTrue(logs.get(6).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(10, logs.size());
assertTrue(logs.get(9).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand All @@ -203,9 +203,9 @@ void testPostProcessLdfData() {
assertTrue(postProcessingServiceMock.idCache.containsKey(topic));

List<ILoggingEvent> logs = listAppender.list;
assertEquals(3, logs.size());
assertTrue(logs.get(1).getFormattedMessage().contains(PostProcessingService.Entity.LDF_DATA.getStoredProcedure()));
assertTrue(logs.get(2).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
assertEquals(6, logs.size());
assertTrue(logs.get(4).getFormattedMessage().contains(PostProcessingService.Entity.LDF_DATA.getStoredProcedure()));
assertTrue(logs.get(5).getMessage().contains(PostProcessingService.SP_EXECUTION_COMPLETED));
}

@Test
Expand Down Expand Up @@ -338,7 +338,7 @@ void testPostProcessPageBuilderWithNoTableCache() {

verify(investigationRepositoryMock, never()).executeStoredProcForPageBuilder(anyLong(), anyString());
List<ILoggingEvent> logs = listAppender.list;
assertEquals(5, logs.size());
assertEquals(8, logs.size());
}

@Test
Expand Down

0 comments on commit 049a61d

Please sign in to comment.