fix: (2.39) Prioritise payload event data to trigger rule-engine notifications [DHIS2-19156]

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/programrule/DefaultTrackerProgramRuleService.java

@@ -30,6 +30,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -204,22 +205,36 @@ private ProgramInstance getEnrollment(TrackerBundle bundle, String enrollmentUid
 
   private Set<ProgramStageInstance> getEventsFromEnrollment(
       String enrollmentUid, TrackerBundle bundle) {
+
+    // Fetch events linked to the enrollment from the database
     EventQueryParams eventQueryParams = new EventQueryParams();
     eventQueryParams.setProgramInstances(Set.of(enrollmentUid));
-    List<org.hisp.dhis.dxf2.events.event.Event> events =
+    List<org.hisp.dhis.dxf2.events.event.Event> dbEvents =
         eventService.getEvents(eventQueryParams).getEvents();
 
-    Stream<ProgramStageInstance> programStageInstances =
-        events.stream().map(e -> programStageInstanceService.getProgramStageInstance(e.getUid()));
+    // Convert DB events to ProgramStageInstances
+    Map<String, ProgramStageInstance> dbProgramStageInstances =
+        dbEvents.stream()
+            .collect(
+                Collectors.toMap(
+                    org.hisp.dhis.dxf2.events.event.Event::getUid,
+                    e -> programStageInstanceService.getProgramStageInstance(e.getUid())));
 
-    // All events in the payload that are linked to enrollment
-    Stream<ProgramStageInstance> bundleEvents =
+    // Fetch events from the payload for the given enrollment
+    Map<String, ProgramStageInstance> payloadProgramStageInstances =
         bundle.getEvents().stream()
-            .filter(e -> e.getEnrollment().equals(enrollmentUid))
-            .map(
-                event ->
-                    eventTrackerConverterService.fromForRuleEngine(bundle.getPreheat(), event));
-
-    return Stream.concat(programStageInstances, bundleEvents).collect(Collectors.toSet());
+            .filter(event -> event.getEnrollment().equals(enrollmentUid))
+            .collect(
+                Collectors.toMap(
+                    Event::getEvent,
+                    event ->
+                        eventTrackerConverterService.fromForRuleEngine(
+                            bundle.getPreheat(), event)));
+
+    // Merge payload events (prioritized) with DB events
+    dbProgramStageInstances.putAll(payloadProgramStageInstances);
+
+    // Return a Set of unique ProgramStageInstances
+    return new HashSet<>(dbProgramStageInstances.values());
   }
 }

dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/programrule/DefaultTrackerProgramRuleServiceTest.java

@@ -121,6 +121,9 @@ void shouldInvokeSpecificMethodWhenCalculatingRuleEffectsForTrackerEvents() {
     when(trackerBundle.getPreheat()).thenReturn(preheat);
     when(eventTrackerConverterService.fromForRuleEngine(any(TrackerPreheat.class), anyList()))
         .thenReturn(programStageInstances);
+    when(eventTrackerConverterService.fromForRuleEngine(
+            any(TrackerPreheat.class), any(Event.class)))
+        .thenReturn(programStageInstances.get(0));
     when(programRuleEngine.evaluateEnrollmentAndEvents(
             any(ProgramInstance.class), anySet(), anyList()))
         .thenReturn(Collections.emptyList());

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/Assertions.java

@@ -46,10 +46,14 @@
 import org.hisp.dhis.dataelement.DataElement;
 import org.hisp.dhis.trackedentitycomment.TrackedEntityComment;
 import org.hisp.dhis.trackedentitydatavalue.TrackedEntityDataValueAudit;
+import org.hisp.dhis.tracker.job.TrackerSideEffectDataBundle;
 import org.hisp.dhis.tracker.report.TrackerErrorCode;
 import org.hisp.dhis.tracker.report.TrackerImportReport;
 import org.hisp.dhis.tracker.report.TrackerStatus;
+import org.hisp.dhis.tracker.report.TrackerTypeReport;
 import org.hisp.dhis.tracker.report.TrackerValidationReport;
+import org.hisp.dhis.tracker.sideeffect.TrackerRuleEngineSideEffect;
+import org.hisp.dhis.tracker.sideeffect.TrackerSendMessageSideEffect;
 import org.hisp.dhis.util.DateUtils;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
@@ -189,6 +193,52 @@ public static void assertNoErrors(TrackerImportReport report) {
             "Expected import with status OK, instead got:\n", report.getValidationReport()));
   }
 
+  public static void assertHasNoNotificationSideEffects(TrackerImportReport report) {
+    assertNotNull(report, "The ImportReport should not be null.");
+
+    TrackerTypeReport typeReport =
+        report.getBundleReport().getTypeReportMap().get(TrackerType.EVENT);
+
+    assertNotNull(typeReport, "The TrackerTypeReport for EVENT should not be null.");
+    assertFalse(
+        typeReport.getSideEffectDataBundles().isEmpty(),
+        "Expected side effect data bundles but none were found.");
+
+    TrackerSideEffectDataBundle sideEffectDataBundle = typeReport.getSideEffectDataBundles().get(0);
+
+    List<TrackerRuleEngineSideEffect> ruleEngineSideEffects =
+        sideEffectDataBundle.getEventRuleEffects().values().stream()
+            .flatMap(List::stream)
+            .collect(Collectors.toList());
+
+    assertTrue(
+        ruleEngineSideEffects.stream().noneMatch(TrackerSendMessageSideEffect.class::isInstance),
+        "Unexpected notification side effect (TrackerSendMessageSideEffect) found.");
+  }
+
+  public static void assertHasNotificationSideEffects(TrackerImportReport report) {
+    assertNotNull(report, "The ImportReport should not be null.");
+
+    TrackerTypeReport typeReport =
+        report.getBundleReport().getTypeReportMap().get(TrackerType.EVENT);
+
+    assertNotNull(typeReport, "The TrackerTypeReport for EVENT should not be null.");
+    assertFalse(
+        typeReport.getSideEffectDataBundles().isEmpty(),
+        "Expected side effect data bundles but none were found.");
+
+    TrackerSideEffectDataBundle sideEffectDataBundle = typeReport.getSideEffectDataBundles().get(0);
+
+    List<TrackerRuleEngineSideEffect> ruleEngineSideEffects =
+        sideEffectDataBundle.getEventRuleEffects().values().stream()
+            .flatMap(List::stream) // Flatten the list of lists into a single stream
+            .collect(Collectors.toList()); // Collect into a single list
+
+    assertTrue(
+        ruleEngineSideEffects.stream().anyMatch(TrackerSendMessageSideEffect.class::isInstance),
+        "Expected notification side effect (TrackerSendMessageSideEffect) but none were found.");
+  }
+
   public static void assertNoErrors(TrackerValidationReport report) {
     assertNotNull(report);
     assertFalse(

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/programrule/ProgramRuleIntegrationTest.java

@@ -32,6 +32,8 @@
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasProperty;
 import static org.hisp.dhis.programrule.ProgramRuleActionType.SHOWERROR;
+import static org.hisp.dhis.tracker.Assertions.assertHasNoNotificationSideEffects;
+import static org.hisp.dhis.tracker.Assertions.assertHasNotificationSideEffects;
 import static org.hisp.dhis.tracker.Assertions.assertHasOnlyErrors;
 import static org.hisp.dhis.tracker.Assertions.assertNoErrors;
 import static org.hisp.dhis.tracker.report.TrackerErrorCode.E1300;
@@ -68,6 +70,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 
 class ProgramRuleIntegrationTest extends TrackerTest {
+  private static final String TEMPLATE_UID = "D9PbzJY8bNH";
+
   @Autowired private TrackerImportService trackerImportService;
 
   @Autowired private ProgramRuleService programRuleService;
@@ -78,43 +82,57 @@ class ProgramRuleIntegrationTest extends TrackerTest {
 
   @Autowired private ConstantService constantService;
 
-  private Program program;
+  private Program programWithRegistration;
 
   private Program programWithoutRegistration;
 
+  private ProgramStage programStageOnInsert;
+
   private DataElement dataElement1;
 
   private DataElement dataElement2;
 
+  private DataElement dataElement3;
+
   @Override
   public void initTest() throws IOException {
     ObjectBundle bundle = setUpMetadata("tracker/simple_metadata.json");
-    program = bundle.getPreheat().get(PreheatIdentifier.UID, Program.class, "BFcipDERJnf");
+    programWithRegistration =
+        bundle.getPreheat().get(PreheatIdentifier.UID, Program.class, "BFcipDERJnf");
     programWithoutRegistration =
         bundle.getPreheat().get(PreheatIdentifier.UID, Program.class, "BFcipDERJne");
     dataElement1 = bundle.getPreheat().get(PreheatIdentifier.UID, DataElement.class, "DATAEL00001");
     dataElement2 = bundle.getPreheat().get(PreheatIdentifier.UID, DataElement.class, "DATAEL00002");
-    ProgramStage programStage =
+    dataElement3 = bundle.getPreheat().get(PreheatIdentifier.UID, DataElement.class, "DATAEL00006");
+    programStageOnInsert =
         bundle.getPreheat().get(PreheatIdentifier.UID, ProgramStage.class, "NpsdDv6kKSO");
+
     ProgramRuleVariable programRuleVariable =
-        createProgramRuleVariableWithDataElement('A', program, dataElement2);
+        createProgramRuleVariableWithDataElement('A', programWithRegistration, dataElement2);
+
+    ProgramRuleVariable programRuleVariableDE6 =
+        createProgramRuleVariableWithDataElement('D', programWithRegistration, dataElement3);
+    programRuleVariableDE6.setName("integer_prv_de6");
+
     programRuleVariableService.addProgramRuleVariable(programRuleVariable);
-    ProgramRule programRuleA = createProgramRule('A', program);
+    programRuleVariableService.addProgramRuleVariable(programRuleVariableDE6);
+
+    ProgramRule programRuleA = createProgramRule('A', programWithRegistration);
     programRuleA.setUid("ProgramRule");
     programRuleService.addProgramRule(programRuleA);
-    ProgramRule errorProgramRule = createProgramRule('E', program);
+    ProgramRule errorProgramRule = createProgramRule('E', programWithRegistration);
     errorProgramRule.setCondition("ERROR");
     errorProgramRule.setUid("ProgramRulE");
     programRuleService.addProgramRule(errorProgramRule);
-    ProgramRule programRuleC = createProgramRule('C', program);
+    ProgramRule programRuleC = createProgramRule('C', programWithRegistration);
     programRuleC.setUid("ProgramRulC");
     programRuleC.setCondition(
         "d2:daysBetween('2019-01-28', d2:lastEventDate('ProgramRuleVariableA')) < 5");
     programRuleService.addProgramRule(programRuleC);
     ProgramRule programRuleWithoutRegistration = createProgramRule('W', programWithoutRegistration);
     programRuleService.addProgramRule(programRuleWithoutRegistration);
-    ProgramRule programRuleB = createProgramRule('B', program);
-    programRuleB.setProgramStage(programStage);
+    ProgramRule programRuleB = createProgramRule('B', programWithRegistration);
+    programRuleB.setProgramStage(programStageOnInsert);
     programRuleService.addProgramRule(programRuleB);
     ProgramRuleAction programRuleActionShowWarning = createProgramRuleAction('A', programRuleA);
     programRuleActionShowWarning.setProgramRuleActionType(ProgramRuleActionType.SHOWWARNING);
@@ -256,6 +274,28 @@ void testImportEventInProgramStageSuccessWithWarningRaised() throws IOException
                     "Generated by program rule (`ProgramRulC`) - DataElement `DATAEL00001` is being replaced in event `EVENT123456`"))));
   }
 
+  @Test
+  void shouldImportEventWithNotificationActionWhenPayloadEventDataIsPrioritized()
+      throws IOException {
+    storeNotificationProgramRule(
+        'I',
+        programWithRegistration,
+        programStageOnInsert,
+        TEMPLATE_UID,
+        "#{integer_prv_de6} > 10");
+    TrackerImportReport report =
+        trackerImportService.importTracker(
+            fromJson("tracker/programrule/tei_enrollment_with_event_and_no_datavalues.json"));
+    assertHasNoNotificationSideEffects(report);
+
+    TrackerImportParams importParams =
+        fromJson("tracker/programrule/event_updated_datavalues.json");
+    importParams.setImportStrategy(TrackerImportStrategy.UPDATE);
+    report = trackerImportService.importTracker(importParams);
+
+    assertHasNotificationSideEffects(report);
+  }
+
   @Test
   void shouldNotImportProgramEventWhenAnErrorIsTriggeredBasedOnConditionEvaluatingAConstant()
       throws IOException {
@@ -301,4 +341,23 @@ private Constant constant() {
     constant.setShortName("Gravity");
     return constant;
   }
+
+  private void storeNotificationProgramRule(
+      char uniqueCharacter,
+      Program program,
+      ProgramStage programStage,
+      String notification,
+      String condition) {
+    ProgramRule programRule = createProgramRule(uniqueCharacter, program);
+    programRule.setProgramStage(programStage);
+    programRule.setCondition(condition);
+
+    programRuleService.addProgramRule(programRule);
+    ProgramRuleAction sendMessageProgramRuleAction =
+        createProgramRuleAction(programRule, ProgramRuleActionType.SENDMESSAGE, null, null);
+    sendMessageProgramRuleAction.setTemplateUid(notification);
+    programRuleActionService.addProgramRuleAction(sendMessageProgramRuleAction);
+    programRule.getProgramRuleActions().add(sendMessageProgramRuleAction);
+    programRuleService.updateProgramRule(programRule);
+  }
 }

dhis-2/dhis-test-integration/src/test/resources/tracker/programrule/event_updated_datavalues.json

@@ -0,0 +1,86 @@
+{
+  "importMode": "COMMIT",
+  "idSchemes": {
+    "dataElementIdScheme": {
+      "idScheme": "UID"
+    },
+    "orgUnitIdScheme": {
+      "idScheme": "UID"
+    },
+    "programIdScheme": {
+      "idScheme": "UID"
+    },
+    "programStageIdScheme": {
+      "idScheme": "UID"
+    },
+    "idScheme": {
+      "idScheme": "UID"
+    },
+    "categoryOptionComboIdScheme": {
+      "idScheme": "UID"
+    },
+    "categoryOptionIdScheme": {
+      "idScheme": "UID"
+    }
+  },
+  "importStrategy": "CREATE",
+  "atomicMode": "ALL",
+  "flushMode": "AUTO",
+  "validationMode": "FULL",
+  "skipPatternValidation": false,
+  "skipSideEffects": false,
+  "skipRuleEngine": false,
+  "trackedEntities": [],
+  "enrollments": [],
+  "events": [
+    {
+      "event": "D9PbzJY8bJO",
+      "status": "ACTIVE",
+      "program": {
+        "idScheme": "UID",
+        "identifier": "BFcipDERJnf"
+      },
+      "programStage": {
+        "idScheme": "UID",
+        "identifier": "NpsdDv6kKSO"
+      },
+      "enrollment": "TvctPPhpD8u",
+      "orgUnit": {
+        "idScheme": "UID",
+        "identifier": "h4w96yEMlzO"
+      },
+      "relationships": [],
+      "occurredAt": "2019-01-28T00:00:00.000",
+      "scheduledAt": "2019-01-28T12:10:38.100",
+      "storedBy": "admin",
+      "followup": false,
+      "deleted": false,
+      "createdAt": "2019-01-28T12:10:38.108",
+      "createdAtClient": "2019-01-28T11:10:38.108",
+      "updatedAt": "2019-01-28T12:10:38.109",
+      "updatedAtClient": "2019-01-28T10:10:38.109",
+      "attributeOptionCombo": {
+        "idScheme": "UID"
+      },
+      "attributeCategoryOptions": [],
+      "completedBy": "admin",
+      "completedAt": "2019-01-28T00:00:00.000",
+      "dataValues": [
+        {
+          "createdAt": "2019-01-28T12:10:38.113",
+          "updatedAt": "2019-01-28T12:10:38.113",
+          "storedBy": "admin",
+          "providedElsewhere": false,
+          "dataElement": {
+            "idScheme": "UID",
+            "identifier": "DATAEL00006"
+          },
+          "value": 12
+        }
+      ],
+      "notes": []
+    }
+  ],
+  "relationships": [],
+  "username": "system-process"
+}