Adding more test suites for replay pipeline, code cleanup

.github/workflows/auto_execute_target.yml

@@ -52,23 +52,9 @@ jobs:
           echo "# AutoExecutor CI results" >> $GITHUB_STEP_SUMMARY
           echo "**SDK version** : ${{ env.RELEASE_VERSION }}" >> $GITHUB_STEP_SUMMARY
 
-      - name: Read maven-demo's markdown report
-        id: maventest
-        uses: jaywcjlove/github-action-read-file@main
-        with:
-          localfile: target/test-classes/auto-test-resources/AutoExecutorReports/unlogged-spring-maven-demo-summary.md
-        if: always()
-
-      - name: Read webflux-demo's markdown report
-        id: webfluxtest
-        uses: jaywcjlove/github-action-read-file@main
-        with:
-          localfile: target/test-classes/auto-test-resources/AutoExecutorReports/unlogged-spring-webflux-maven-demo-summary.md
-        if: always()
-
       - name: Add report contents
         run: |
-          echo "${{ steps.maventest.outputs.content }}" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.webfluxtest.outputs.content }}" >> $GITHUB_STEP_SUMMARY
+          cat target/test-classes/auto-test-resources/AutoExecutorReports/unlogged-spring-maven-demo-summary.md >> $GITHUB_STEP_SUMMARY
+          cat target/test-classes/auto-test-resources/AutoExecutorReports/unlogged-spring-webflux-maven-demo-summary.md >> $GITHUB_STEP_SUMMARY
         if: always()
 

.github/workflows/compile_target.yml

@@ -54,6 +54,7 @@ jobs:
 
       - name: Write report summary
         run: |
+          echo "# Replay Pipeline Report" >> $GITHUB_STEP_SUMMARY
           echo "${{ steps.compiletest.outputs.content }}" >> $GITHUB_STEP_SUMMARY
         if: always()
 

.github/workflows/replay_target.yml

@@ -54,6 +54,7 @@ jobs:
 
       - name: Write report summary
         run: |
+          echo "# Replay Pipeline Report" >> $GITHUB_STEP_SUMMARY
           echo "${{ steps.replaytest.outputs.content }}" >> $GITHUB_STEP_SUMMARY
         if: always()
 
@@ -64,4 +65,4 @@ jobs:
           name: Replay Pipeline reports
           path: |
             **/replay-pipeline-result.md
-            **/replay_report.xml
+            **/replayReports/

.gitignore

@@ -4,3 +4,4 @@ dependency-tree.txt
 src/test/python/venv/
 src/test/python/.gradle/
 src/test/python/__pycache__/
+.idea/

src/test/java/io/unlogged/autoexecutor/AutoExecutorCITest.java

@@ -5,10 +5,7 @@
 import io.unlogged.autoexecutor.testutils.autoCIUtils.AssertionUtils;
 import io.unlogged.autoexecutor.testutils.autoCIUtils.ParseUtils;
 import io.unlogged.autoexecutor.testutils.autoCIUtils.XlsxUtils;
-import io.unlogged.autoexecutor.testutils.entity.AutoAssertionResult;
-import io.unlogged.autoexecutor.testutils.entity.TestConfig;
-import io.unlogged.autoexecutor.testutils.entity.TestResultSummary;
-import io.unlogged.autoexecutor.testutils.entity.TestUnit;
+import io.unlogged.autoexecutor.testutils.entity.*;
 import io.unlogged.command.AgentCommand;
 import io.unlogged.command.AgentCommandRequest;
 import io.unlogged.command.AgentCommandRequestType;
@@ -111,7 +108,7 @@ public void runTests(TestConfig testconfig) {
             logger.info("\n    " + resultSummary.getMode() + " Mode ->" + "\n"
                     + "    Total number of cases run : " + resultSummary.getNumberOfCases() + "\n"
                     + "    Number of Failing cases : " + resultSummary.getFailingCasesCount() + "\n"
-                    + "    Failing tests (Row numbers) : " + resultSummary.getFailingCases().toString());
+                    + "    Failing tests (Row numbers) : " + resultSummary.getFailingCaseNumbers().toString());
             if (resultSummary.getFailingCasesCount() > 0) {
                 overallStatus = false;
             }
@@ -129,7 +126,7 @@ public TestResultSummary executeMethods(URL pathToUrl, AgentClientLite agentClie
         int count = 0;
         int passing = 0;
         int failing = 0;
-        List<Integer> failingCaseIds = new ArrayList<>();
+        List<AssertionDetails> failingAssertionDetails = new ArrayList<>();
 
         while (rowIterator.hasNext()) {
             if (count == 0) {
@@ -207,7 +204,9 @@ public TestResultSummary executeMethods(URL pathToUrl, AgentClientLite agentClie
                     AutoAssertionResult result = AssertionUtils.assertCase(testUnit);
                     if (!result.isPassing()) {
                         failing++;
-                        failingCaseIds.add(count);
+                        failingAssertionDetails.add(new AssertionDetails(
+                                count, result.getExpected(),
+                                result.getActual(), result.getAssertionType()));
                     } else {
                         passing++;
                     }
@@ -229,7 +228,7 @@ public TestResultSummary executeMethods(URL pathToUrl, AgentClientLite agentClie
                 }
             }
         }
-        return new TestResultSummary(count - 2, passing, failing, failingCaseIds);
+        return new TestResultSummary(count - 2, passing, failing, failingAssertionDetails);
     }
 
     private String limitResponseSize(String input) {

src/test/java/io/unlogged/autoexecutor/report/MarkdownReportGenerator.java

@@ -1,5 +1,6 @@
 package io.unlogged.autoexecutor.report;
 
+import io.unlogged.autoexecutor.testutils.entity.AssertionDetails;
 import io.unlogged.autoexecutor.testutils.entity.TestResultSummary;
 
 import java.io.BufferedWriter;
@@ -30,21 +31,33 @@ public static void generateAndWriteMarkdownReport(String project, String path, L
                     .append("\n\n")
                     .append("<details>\n" +
                             "<summary>Failing cases</summary>\n" +
-                            "\n");
+                            "\n<ul>");
             if (testResultSummary.getFailingCasesCount() == 0) {
-                stringBuilder.append("There are no failing cases.");
+                stringBuilder.append("<li>There are no failing cases.</li>");
             } else {
-                testResultSummary.getFailingCases().forEach(failingCaseId -> {
-                    stringBuilder.append("- " + failingCaseId).append("\n");
+                testResultSummary.getFailingCases().forEach(failingCase -> {
+                    stringBuilder.append(generateFailingCaseSummary(failingCase));
                 });
             }
-            stringBuilder.append("\n").append("</details>").append("\n")
-                    .append(generatePieChart(testResultSummary));
+            stringBuilder.append("\n\n").append("</ul></details>").append("\n")
+                    .append(generatePieChart(testResultSummary))
+                    .append("\n\n");
 
         });
         writeFile(project + "-summary", stringBuilder.toString(), path);
     }
 
+    private static String generateFailingCaseSummary(AssertionDetails assertionDetails) {
+        return new StringBuilder().append("\n")
+                .append("<li><details>\n")
+                .append("<summary> Case ID : ").append(assertionDetails.getCaseId()).append("</summary>\n\n")
+                .append("| Operation Type | ").append(assertionDetails.getAssertionType()).append(" |\n")
+                .append("|----------------|------|\n")
+                .append("| Expected | ").append(capSize(assertionDetails.getExpected())).append("| \n")
+                .append("| Actual | ").append(capSize(assertionDetails.getActual())).append("| \n\n")
+                .append("</details></li>\n\n").toString();
+    }
+
     public static void generateReportForSkippedTest(String project, String path) {
         StringBuilder stringBuilder = new StringBuilder();
         stringBuilder.append("## Project : `" + project + "`")
@@ -70,12 +83,19 @@ private static String generatePieChart(TestResultSummary testResultSummary) {
         pieChartBuilder.append("<details>").append("\n")
                 .append("<summary>Status Chart</summary>").append("\n\n")
                 .append("```mermaid\n" +
-                        "%%{init: {'theme': 'default', 'themeVariables': {'pie1': '#238636', 'pie2': '#da3633'}}}%%\n" +
+                        "%%{init: {'theme': 'base', 'themeVariables': {'pie1': '#238636', 'pie2': '#da3633','primaryTextColor': '#fff', 'primaryBorderColor': '#7C0000'}}}%%\n" +
                         "pie title Status Chart\n" +
                         "    \"Passing\" : " + testResultSummary.getPassingCasesCount() + "\n" +
                         "    \"Failing\" : " + testResultSummary.getFailingCasesCount() + "\n" +
                         "```").append("\n")
                 .append("</details>").append("\n\n");
         return pieChartBuilder.toString();
     }
+
+    private static String capSize(String payload) {
+        if (payload.length() > 10000) {
+            return "Too large to show here, please refer to Logs";
+        }
+        return payload;
+    }
 }

src/test/java/io/unlogged/autoexecutor/testutils/autoCIUtils/AssertionUtils.java

@@ -26,6 +26,9 @@ public static AutoAssertionResult assertCase(TestUnit testUnit) {
             actualResponse = "null - from agent";
         }
 
+        assertionResult.setExpected(refOut);
+        assertionResult.setActual(actualResponse);
+
         boolean result = false;
         assertionResult.setAssertionType(assertionType);
         String message = "";
@@ -67,6 +70,8 @@ public static AutoAssertionResult assertCase(TestUnit testUnit) {
                 } else {
                     message = "Received Exception when it was not expected";
                 }
+                assertionResult.setExpected("NO EXCEPTION");
+                assertionResult.setActual(actualResponse);
                 break;
             case "EQUAL EXCEPTION":
                 //should be an equal exception
@@ -86,6 +91,8 @@ public static AutoAssertionResult assertCase(TestUnit testUnit) {
                 } else {
                     message = "Received a null response";
                 }
+                assertionResult.setExpected("NOT NULL");
+                assertionResult.setActual(actualResponse);
                 break;
             case "NOT EQUAL":
                 //the response shouldn't be equal to reference output
@@ -98,6 +105,7 @@ public static AutoAssertionResult assertCase(TestUnit testUnit) {
         }
         assertionResult.setPassing(result);
         assertionResult.setMessage(message);
+
         return assertionResult;
     }
 

src/test/java/io/unlogged/autoexecutor/testutils/entity/AssertionDetails.java

@@ -0,0 +1,31 @@
+package io.unlogged.autoexecutor.testutils.entity;
+
+public class AssertionDetails {
+    private int caseId;
+    private String expected;
+    private String actual;
+    private String assertionType;
+
+    public AssertionDetails(int caseId, String expected, String actual, String assertionType) {
+        this.caseId = caseId;
+        this.expected = expected;
+        this.actual = actual;
+        this.assertionType = assertionType;
+    }
+
+    public int getCaseId() {
+        return caseId;
+    }
+
+    public String getExpected() {
+        return expected;
+    }
+
+    public String getActual() {
+        return actual;
+    }
+
+    public String getAssertionType() {
+        return assertionType;
+    }
+}

src/test/java/io/unlogged/autoexecutor/testutils/entity/AutoAssertionResult.java

@@ -4,6 +4,8 @@ public class AutoAssertionResult {
     private boolean passing;
     private String message;
     private String assertionType;
+    private String expected;
+    private String actual;
 
     public boolean isPassing() {
         return passing;
@@ -28,4 +30,20 @@ public String getAssertionType() {
     public void setAssertionType(String assertionType) {
         this.assertionType = assertionType;
     }
+
+    public String getExpected() {
+        return expected;
+    }
+
+    public void setExpected(String expected) {
+        this.expected = expected;
+    }
+
+    public String getActual() {
+        return actual;
+    }
+
+    public void setActual(String actual) {
+        this.actual = actual;
+    }
 }

src/test/java/io/unlogged/autoexecutor/testutils/entity/TestResultSummary.java

@@ -1,16 +1,17 @@
 package io.unlogged.autoexecutor.testutils.entity;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 public class TestResultSummary {
     private int numberOfCases;
     private int passingCasesCount;
     private int failingCasesCount;
     private String mode;
-    private List<Integer> failingCases;
+    private List<AssertionDetails> failingCases;
 
     public TestResultSummary(int numberOfCases, int passingCasesCount, int failingCasesCount,
-                             List<Integer> failingCaseIds) {
+                             List<AssertionDetails> failingCaseIds) {
         this.numberOfCases = numberOfCases;
         this.passingCasesCount = passingCasesCount;
         this.failingCasesCount = failingCasesCount;
@@ -37,7 +38,11 @@ public void setMode(String mode) {
         this.mode = mode;
     }
 
-    public List<Integer> getFailingCases() {
+    public List<AssertionDetails> getFailingCases() {
         return failingCases;
     }
+
+    public List<Integer> getFailingCaseNumbers() {
+        return failingCases.stream().map(AssertionDetails::getCaseId).collect(Collectors.toList());
+    }
 }