Release

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ## [Unreleased]
+### Fixed
+- Backgrounds finish with `FAILED` status, by @HardNorth
 
 ## [5.0.2]
 ### Changed

src/main/java/com/epam/reportportal/karate/ReportPortalHook.java

@@ -23,6 +23,7 @@
 import com.epam.reportportal.service.Launch;
 import com.epam.reportportal.service.ReportPortal;
 import com.epam.reportportal.utils.MemoizingSupplier;
+import com.epam.reportportal.utils.StatusEvaluation;
 import com.epam.ta.reportportal.ws.model.FinishExecutionRQ;
 import com.epam.ta.reportportal.ws.model.FinishTestItemRQ;
 import com.epam.ta.reportportal.ws.model.StartTestItemRQ;
@@ -59,6 +60,7 @@ public class ReportPortalHook implements RuntimeHook {
 	private final BlockingConcurrentHashMap<String, Maybe<String>> featureIdMap = new BlockingConcurrentHashMap<>();
 	private final Map<String, Maybe<String>> scenarioIdMap = new ConcurrentHashMap<>();
 	private final Map<String, Maybe<String>> backgroundIdMap = new ConcurrentHashMap<>();
+	private final Map<String, ItemStatus> backgroundStatusMap = new ConcurrentHashMap<>();
 	private final Map<String, Maybe<String>> stepIdMap = new ConcurrentHashMap<>();
 	private final Map<Maybe<String>, Date> stepStartTimeMap = new ConcurrentHashMap<>();
 	private volatile Thread shutDownHook;
@@ -228,6 +230,7 @@ protected StartTestItemRQ buildStartBackgroundRq(@Nonnull Step step, @Nonnull Sc
 	 *
 	 * @param step Karate's Step object instance
 	 * @param sr   Karate's ScenarioRuntime object instance
+	 * @return item ID Future
 	 */
 	public Maybe<String> startBackground(@Nonnull Step step, @Nonnull ScenarioRuntime sr) {
 		return backgroundIdMap.computeIfAbsent(sr.scenario.getUniqueId(), k -> {
@@ -239,27 +242,28 @@ public Maybe<String> startBackground(@Nonnull Step step, @Nonnull ScenarioRuntim
 	/**
 	 * Build ReportPortal request for finish Background event.
 	 *
-	 * @param step Karate's Step object instance
-	 * @param sr   Karate's ScenarioRuntime object instance
+	 * @param stepResult Karate's StepResult class instance
+	 * @param sr         Karate's ScenarioRuntime object instance
 	 * @return request to ReportPortal
 	 */
 	@Nonnull
 	@SuppressWarnings("unused")
-	protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable Step step, @Nonnull ScenarioRuntime sr) {
-		return buildFinishTestItemRq(Calendar.getInstance().getTime(), null);
+	protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable StepResult stepResult, @Nonnull ScenarioRuntime sr) {
+		return buildFinishTestItemRq(Calendar.getInstance().getTime(), backgroundStatusMap.remove(sr.scenario.getUniqueId()));
 
 	}
 
 	/**
 	 * Finish sending Scenario data to ReportPortal.
 	 *
-	 * @param step Karate's Step object instance
-	 * @param sr   Karate's ScenarioRuntime object instance
+	 * @param stepResult Karate's StepResult class instance
+	 * @param sr         Karate's ScenarioRuntime object instance
 	 */
-	public void finishBackground(@Nullable Step step, @Nonnull ScenarioRuntime sr) {
-		Maybe<String> backgroundId = backgroundIdMap.remove(sr.scenario.getUniqueId());
+	public void finishBackground(@Nullable StepResult stepResult, @Nonnull ScenarioRuntime sr) {
+		String uniqueId = sr.scenario.getUniqueId();
+		Maybe<String> backgroundId = backgroundIdMap.remove(uniqueId);
 		if (backgroundId != null) {
-			FinishTestItemRQ finishRq = buildFinishBackgroundRq(step, sr);
+			FinishTestItemRQ finishRq = buildFinishBackgroundRq(stepResult, sr);
 			//noinspection ReactiveStreamsUnusedPublisher
 			launch.get().finishTestItem(backgroundId, finishRq);
 		}
@@ -272,10 +276,10 @@ public void afterScenario(ScenarioRuntime sr) {
 			LOGGER.error("ERROR: Trying to finish unspecified scenario.");
 		}
 
+		finishBackground(null, sr);
 		FinishTestItemRQ rq = buildFinishScenarioRq(sr);
 		//noinspection ReactiveStreamsUnusedPublisher
 		launch.get().finishTestItem(scenarioId, rq);
-		finishBackground(null, sr);
 	}
 
 	/**
@@ -331,8 +335,6 @@ public boolean beforeStep(Step step, ScenarioRuntime sr) {
 		Maybe<String> backgroundId = null;
 		if (background) {
 			backgroundId = startBackground(step, sr);
-		} else {
-			finishBackground(step, sr);
 		}
 		StartTestItemRQ stepRq = buildStartStepRq(step, sr);
 
@@ -383,8 +385,21 @@ protected FinishTestItemRQ buildFinishStepRq(@Nonnull StepResult stepResult, @No
 		return buildFinishTestItemRq(Calendar.getInstance().getTime(), getStepStatus(stepResult.getResult().getStatus()));
 	}
 
+	private void saveBackgroundStatus(@Nonnull StepResult stepResult, @Nonnull ScenarioRuntime sr) {
+		backgroundStatusMap.put(sr.scenario.getUniqueId(),
+				StatusEvaluation.evaluateStatus(backgroundStatusMap.get(sr.scenario.getUniqueId()),
+						getStepStatus(stepResult.getResult().getStatus())
+				)
+		);
+	}
+
 	@Override
 	public void afterStep(StepResult stepResult, ScenarioRuntime sr) {
+		boolean background = stepResult.getStep().isBackground();
+		if (!background) {
+			finishBackground(stepResult, sr);
+		}
+
 		sendStepResults(stepResult, sr);
 		Maybe<String> stepId = stepIdMap.get(sr.scenario.getUniqueId());
 		if (stepId == null) {
@@ -393,6 +408,9 @@ public void afterStep(StepResult stepResult, ScenarioRuntime sr) {
 		}
 
 		FinishTestItemRQ rq = buildFinishStepRq(stepResult, sr);
+		if (background) {
+			saveBackgroundStatus(stepResult, sr);
+		}
 		//noinspection ReactiveStreamsUnusedPublisher
 		launch.get().finishTestItem(stepId, rq);
 	}

src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java

@@ -22,6 +22,7 @@
 import com.epam.reportportal.service.Launch;
 import com.epam.reportportal.service.ReportPortal;
 import com.epam.reportportal.utils.MemoizingSupplier;
+import com.epam.reportportal.utils.StatusEvaluation;
 import com.epam.ta.reportportal.ws.model.FinishExecutionRQ;
 import com.epam.ta.reportportal.ws.model.FinishTestItemRQ;
 import com.epam.ta.reportportal.ws.model.StartTestItemRQ;
@@ -52,6 +53,7 @@ public class ReportPortalPublisher {
 	private final Map<String, Maybe<String>> scenarioIdMap = new HashMap<>();
 	private final Map<Maybe<String>, Long> stepStartTimeMap = new HashMap<>();
 	private Maybe<String> backgroundId;
+	private ItemStatus backgroundStatus;
 	private Maybe<String> stepId;
 	private Thread shutDownHook;
 
@@ -270,7 +272,7 @@ public void startBackground(@Nonnull StepResult stepResult, @Nonnull ScenarioRes
 	@Nonnull
 	@SuppressWarnings("unused")
 	protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable StepResult stepResult, @Nonnull ScenarioResult scenarioResult) {
-		return buildFinishTestItemRq(Calendar.getInstance().getTime(), null);
+		return buildFinishTestItemRq(Calendar.getInstance().getTime(), backgroundStatus);
 
 	}
 
@@ -281,12 +283,13 @@ protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable StepResult stepResu
 	 * @param scenarioResult scenario result
 	 */
 	public void finishBackground(@Nullable StepResult stepResult, @Nonnull ScenarioResult scenarioResult) {
-		ofNullable(backgroundId).ifPresent(id -> {
+		Maybe<String> myBackgroundId = backgroundId;
+		backgroundId = null;
+		ofNullable(myBackgroundId).ifPresent(id -> {
 			FinishTestItemRQ finishRq = buildFinishBackgroundRq(stepResult, scenarioResult);
 			//noinspection ReactiveStreamsUnusedPublisher
 			launch.get().finishTestItem(id, finishRq);
 		});
-		backgroundId = null;
 	}
 
 	/**
@@ -332,14 +335,17 @@ protected StartTestItemRQ buildStartStepRq(@Nonnull StepResult stepResult, @Nonn
 	 */
 	public void startStep(StepResult stepResult, ScenarioResult scenarioResult) {
 		Step step = stepResult.getStep();
-		if (step.isBackground()) {
+		boolean background = step.isBackground();
+		if (background) {
 			startBackground(stepResult, scenarioResult);
-		} else {
-			finishBackground(stepResult, scenarioResult);
 		}
+
 		StartTestItemRQ stepRq = buildStartStepRq(stepResult, scenarioResult);
 		stepId = launch.get()
-				.startTestItem(backgroundId != null ? backgroundId : scenarioIdMap.get(scenarioResult.getScenario().getName()), stepRq);
+				.startTestItem(
+						background && backgroundId != null ? backgroundId : scenarioIdMap.get(scenarioResult.getScenario().getName()),
+						stepRq
+				);
 		stepStartTimeMap.put(stepId, stepRq.getStartTime().getTime());
 		ofNullable(stepRq.getParameters()).filter(params -> !params.isEmpty())
 				.ifPresent(params -> sendLog(stepId, String.format(PARAMETERS_PATTERN, formatParametersAsTable(params)), LogLevel.INFO));
@@ -363,19 +369,33 @@ protected FinishTestItemRQ buildFinishStepRq(@Nonnull StepResult stepResult, @No
 		return buildFinishTestItemRq(Calendar.getInstance().getTime(), getStepStatus(stepResult.getResult().getStatus()));
 	}
 
+	@SuppressWarnings("unused")
+	private void saveBackgroundStatus(@Nonnull StepResult stepResult, @Nonnull ScenarioResult scenarioResult) {
+		backgroundStatus = StatusEvaluation.evaluateStatus(backgroundStatus, getStepStatus(stepResult.getResult().getStatus()));
+	}
+
 	/**
 	 * Finish sending Step data to ReportPortal.
 	 *
 	 * @param stepResult     Karate's StepResult class instance
 	 * @param scenarioResult Karate's ScenarioResult class instance
 	 */
 	public void finishStep(StepResult stepResult, ScenarioResult scenarioResult) {
+		Step step = stepResult.getStep();
+		boolean background = step.isBackground();
+		if (!background) {
+			finishBackground(stepResult, scenarioResult);
+		}
+
 		if (stepId == null) {
 			LOGGER.error("ERROR: Trying to finish unspecified step.");
 			return;
 		}
 
 		FinishTestItemRQ rq = buildFinishStepRq(stepResult, scenarioResult);
+		if (background) {
+			saveBackgroundStatus(stepResult, scenarioResult);
+		}
 		//noinspection ReactiveStreamsUnusedPublisher
 		launch.get().finishTestItem(stepId, rq);
 	}

src/test/java/com/epam/reportportal/karate/status/OneExampleWithBackgroundFailedTest.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2024 EPAM Systems
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.reportportal.karate.status;
+
+import com.epam.reportportal.karate.utils.TestUtils;
+import com.epam.reportportal.listeners.ItemStatus;
+import com.epam.reportportal.service.ReportPortal;
+import com.epam.reportportal.service.ReportPortalClient;
+import com.epam.reportportal.util.test.CommonUtils;
+import com.epam.ta.reportportal.ws.model.FinishTestItemRQ;
+import com.intuit.karate.Results;
+import org.apache.commons.lang3.tuple.Pair;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.epam.reportportal.karate.utils.TestUtils.*;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.*;
+import static org.mockito.Mockito.*;
+
+public class OneExampleWithBackgroundFailedTest {
+	private static final String TEST_FEATURE = "classpath:feature/examples_one_failed_with_background.feature";
+	private final String featureId = CommonUtils.namedId("feature_");
+	private final List<String> exampleIds = Stream.generate(() -> CommonUtils.namedId("example_")).limit(2).collect(Collectors.toList());
+	private final List<Pair<String, List<String>>> stepIds = exampleIds.stream()
+			.map(e -> Pair.of(e, Stream.generate(() -> CommonUtils.namedId("step_")).limit(3).collect(Collectors.toList())))
+			.collect(Collectors.toList());
+	private final List<Pair<String, String>> nestedSteps =
+			Arrays.asList(
+					Pair.of(stepIds.get(0).getValue().get(0), CommonUtils.namedId("nested_step_")),
+					Pair.of(stepIds.get(1).getValue().get(0), CommonUtils.namedId("nested_step_"))
+			);
+
+	private final ReportPortalClient client = mock(ReportPortalClient.class);
+	private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor());
+
+	@BeforeEach
+	public void setupMock() {
+		mockLaunch(client, null, featureId, stepIds);
+		mockNestedSteps(client, nestedSteps);
+		mockBatchLogging(client);
+	}
+
+	private static void verifyStatus(List<FinishTestItemRQ> rqs, ItemStatus... statuses) {
+		for (int i = 0; i < rqs.size(); i++) {
+			ItemStatus statusTest = i >= statuses.length ? statuses[statuses.length - 1] : statuses[i];
+			assertThat(
+					"Failed verifying request number: " + (i + 1),
+					rqs.get(i).getStatus(),
+					allOf(notNullValue(), equalTo(statusTest.name()))
+			);
+		}
+	}
+
+	@ParameterizedTest
+	@ValueSource(booleans = { true, false })
+	public void test_simple_one_step_failed(boolean report) {
+		Results results;
+		if (report) {
+			results = TestUtils.runAsReport(rp, TEST_FEATURE);
+		} else {
+			results = TestUtils.runAsHook(rp, TEST_FEATURE);
+		}
+		assertThat(results.getFailCount(), equalTo(1));
+
+		ArgumentCaptor<FinishTestItemRQ> featureCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(featureId), featureCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> firstExampleCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(exampleIds.get(0)), firstExampleCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> secondExampleCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(exampleIds.get(1)), secondExampleCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> firstExampleFirstStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(stepIds.get(0).getValue().get(0)), firstExampleFirstStepCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> firstExampleSecondStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(stepIds.get(0).getValue().get(1)), firstExampleSecondStepCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> firstExampleThirdStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(stepIds.get(0).getValue().get(2)), firstExampleThirdStepCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> secondExampleFirstStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(stepIds.get(1).getValue().get(0)), secondExampleFirstStepCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> secondExampleSecondStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(stepIds.get(1).getValue().get(1)), secondExampleSecondStepCaptor.capture());
+		ArgumentCaptor<FinishTestItemRQ> secondExampleThirdStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class);
+		verify(client).finishTestItem(same(stepIds.get(1).getValue().get(2)), secondExampleThirdStepCaptor.capture());
+
+		FinishTestItemRQ featureRq = featureCaptor.getValue();
+		assertThat(featureRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.FAILED.name())));
+
+		verifyStatus(Arrays.asList(firstExampleCaptor.getValue(), secondExampleCaptor.getValue()), ItemStatus.PASSED, ItemStatus.FAILED);
+
+		List<FinishTestItemRQ> steps = Arrays.asList(
+				firstExampleFirstStepCaptor.getValue(),
+				firstExampleSecondStepCaptor.getValue(),
+				firstExampleThirdStepCaptor.getValue(),
+				secondExampleFirstStepCaptor.getValue(),
+				secondExampleSecondStepCaptor.getValue(),
+				secondExampleThirdStepCaptor.getValue()
+		);
+
+		verifyStatus(steps, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.FAILED);
+	}
+}

src/test/resources/feature/examples_one_failed_with_background.feature

@@ -0,0 +1,13 @@
+Feature: math tests with examples
+
+  Background: Set varb
+    Given def varb = 2
+
+  Scenario Outline: Verify different maths
+    Given def mathResult = vara + varb
+    Then assert mathResult == result
+
+    Examples:
+      | vara! | result! |
+      | 2     | 4       |
+      | 1     | 4       |