Merge pull request #134 from pitest/task_81-UI_tests_randomly_fail

Task 81 UI tests randomly fail
pitest · Jul 9, 2021 · ff70c30 · ff70c30
2 parents 7a8dd8b + 77b7ee5
commit ff70c30
Show file tree

Hide file tree

Showing 43 changed files with 1,239 additions and 220 deletions.
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -23,6 +23,10 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 11
+    - name: Install WebKit # required for SWT Browser
+      run: |
+        sudo apt update
+        sudo apt install -y libwebkit2gtk-4.0-37
     - name: Cache Maven packages
       uses: actions/cache@v2
       with:
@@ -42,7 +46,13 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Archive Test Results
       uses: actions/upload-artifact@v2
-      if: ${{ failure() }}
+      if: ${{ failure() || cancelled() }}
       with:
         name: test-results-${{ runner.os }}
         path: '**/*.tests/target'
+    - name: Archive UI Tests Screenshots
+      uses: actions/upload-artifact@v2
+      if: ${{ failure() || cancelled() }}
+      with:
+        name: screenshots-${{ runner.os }}
+        path: '**/*.tests/screenshots'
diff --git a/.github/workflows/windows-mac.yml b/.github/workflows/windows-mac.yml
@@ -7,10 +7,6 @@ on:
   push:
     paths-ignore:
       - 'README.md'
-    # Sequence of patterns matched against refs/heads
-    branches:
-      # Push events on the main branch
-      - master
   pull_request:
     paths-ignore:
       - 'README.md'
@@ -39,7 +35,13 @@ jobs:
       run: mvn '-Dtycho.disableP2Mirrors=true' verify
     - name: Archive Test Results
       uses: actions/upload-artifact@v2
-      if: ${{ failure() }}
+      if: ${{ failure() || cancelled() }}
       with:
         name: test-results-${{ runner.os }}
         path: '**/*.tests/target'
+    - name: Archive UI Tests Screenshots
+      uses: actions/upload-artifact@v2
+      if: ${{ failure() || cancelled() }}
+      with:
+        name: screenshots-${{ runner.os }}
+        path: '**/*.tests/screenshots'
diff --git a/...itest.pitclipse.core/src/org/pitest/pitclipse/core/preferences/PreferenceInitializer.java b/...itest.pitclipse.core/src/org/pitest/pitclipse/core/preferences/PreferenceInitializer.java
@@ -19,6 +19,7 @@
 import org.eclipse.core.runtime.preferences.AbstractPreferenceInitializer;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.pitest.pitclipse.core.PitCoreActivator;
+import org.pitest.pitclipse.runner.config.PitConfiguration;
 
 import static org.pitest.pitclipse.core.preferences.PitPreferences.AVOID_CALLS_TO;
 import static org.pitest.pitclipse.core.preferences.PitPreferences.EXCLUDED_CLASSES;
@@ -53,7 +54,7 @@ public void initializeDefaultPreferences() {
         store.setDefault(DEFAULT_MUTATORS, "defaultMutators");
         store.setDefault(TIMEOUT, DEFAULT_TIMEOUT);
         store.setDefault(TIMEOUT_FACTOR, DEFAULT_TIMEOUT_FACTOR.toString());
-        store.setDefault(EXCLUDED_CLASSES, "*Test");
+        store.setDefault(EXCLUDED_CLASSES, PitConfiguration.DEFAULT_EXCLUDED_CLASSES);
     }
 
 }
diff --git a/bundles/org.pitest.pitclipse.launch.ui/META-INF/MANIFEST.MF b/bundles/org.pitest.pitclipse.launch.ui/META-INF/MANIFEST.MF
@@ -18,3 +18,4 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="[3.11.1,4.0.0)",
  org.eclipse.ui.workbench;bundle-version="[3.107.1,4.0.0)",
  org.eclipse.jdt.debug.ui;bundle-version="[3.7.101,4.0.0)"
 Import-Package: com.google.common.collect;version="21.0.0"
+Export-Package: org.pitest.pitclipse.launch.ui
diff --git a/.../org.pitest.pitclipse.launch.ui/src/org/pitest/pitclipse/launch/ui/PitLaunchShortcut.java b/.../org.pitest.pitclipse.launch.ui/src/org/pitest/pitclipse/launch/ui/PitLaunchShortcut.java
@@ -111,7 +111,8 @@ private void launch(Object[] elements, String mode) {
         try {
             if (elements.length == 1) {
                 Optional<IJavaElement> selected = asJavaElement(elements[0]);
-                Optional<IJavaElement> launchElement = selected.map(this::getLaunchElementFor).get();
+                Optional<IJavaElement> launchElement =
+                        selected.flatMap(this::getLaunchElementFor);
 
                 if (launchElement.isPresent()) {
                     performLaunch(launchElement.get(), mode);

diff --git a/bundles/org.pitest.pitclipse.runner/META-INF/MANIFEST.MF b/bundles/org.pitest.pitclipse.runner/META-INF/MANIFEST.MF
@@ -15,8 +15,7 @@ Export-Package: org.pitest.pitclipse.runner,
  org.pitest.pitclipse.runner.results,
  org.pitest.pitclipse.runner.results.mutations,
  org.pitest.pitclipse.runner.results.summary,
- org.pitest.pitclipse.runner.server,
- org.pitest.pitclipse.runner.service
+ org.pitest.pitclipse.runner.server
 Import-Package: com.google.common.annotations;version="21.0.0",
  com.google.common.base;version="21.0.0",
  com.google.common.collect;version="21.0.0",

diff --git a/.../org.pitest.pitclipse.runner/src/org/pitest/pitclipse/runner/config/PitConfiguration.java b/.../org.pitest.pitclipse.runner/src/org/pitest/pitclipse/runner/config/PitConfiguration.java
@@ -23,6 +23,7 @@
 public class PitConfiguration {
     public static final String DEFAULT_AVOID_CALLS_TO_LIST = "java.util.logging, org.apache.log4j, org.slf4j, org.apache.commons.logging, org.apache.logging.log4j";
     public static final String DEFAULT_MUTATORS = "DEFAULTS";
+    public static final String DEFAULT_EXCLUDED_CLASSES = "*Test";
     public static final int DEFAULT_TIMEOUT = 3000;
     public static final BigDecimal DEFAULT_TIMEOUT_FACTOR = BigDecimal.valueOf(1.25);
 
@@ -36,7 +37,7 @@ public class PitConfiguration {
     private final int timeout;
     private final BigDecimal timeoutFactor;
 
-    private PitConfiguration(PitExecutionMode executionMode, boolean parallelExecution, boolean incrementalAnalysis,
+    private PitConfiguration(PitExecutionMode executionMode, boolean parallelExecution, boolean incrementalAnalysis, // NOSONAR this is used by our builder
             String excludedClasses, String excludedMethods, String avoidCallsTo, String mutators, int timeout,
             BigDecimal timeoutFactor) {
         this.executionMode = executionMode;

diff --git a/.../org.pitest.pitclipse.ui/src/org/pitest/pitclipse/ui/view/mutations/PitMutationsView.java b/.../org.pitest.pitclipse.ui/src/org/pitest/pitclipse/ui/view/mutations/PitMutationsView.java
@@ -39,7 +39,6 @@ private void createTreeViewer(Composite parent) {
         viewer.setLabelProvider(new ViewLabelProvider());
         viewer.addDoubleClickListener(ExpandingDoubleClick.LISTENER);
         viewer.addDoubleClickListener(OpenMutationDoubleClick.LISTENER);
-        // viewer.addSelectionChangedListener(listener);
         viewer.setInput(MutationsModel.EMPTY_MODEL);
     }
 

diff --git a/releng/org.pitest.pitclipse.target/org.pitest.pitclipse.target.target b/releng/org.pitest.pitclipse.target/org.pitest.pitclipse.target.target
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?pde version="3.8"?>
-<target name="org.pitest.pitclipse.target" sequenceNumber="1">
+<target name="org.pitest.pitclipse.target" sequenceNumber="2">
 	<locations>
 		<location includeAllPlatforms="false" includeConfigurePhase="true" includeMode="planner" includeSource="true" type="InstallableUnit">
 			<unit id="org.eclipse.e4.core.tools.feature.source.feature.group" version="0.0.0"/>
@@ -10,7 +10,7 @@
 			<unit id="org.eclipse.sdk.feature.group" version="0.0.0"/>
 			<unit id="org.eclipse.swtbot.eclipse.feature.group" version="0.0.0"/>
 			<unit id="org.eclipse.swtbot.feature.group" version="0.0.0"/>
-			<repository location="https://download.eclipse.org/releases/oxygen"/>
+			<repository location="https://download.eclipse.org/releases/photon"/>
 		</location>
 		<location includeAllPlatforms="false" includeConfigurePhase="true" includeMode="planner" includeSource="true" type="InstallableUnit">
 			<unit id="com.google.guava" version="0.0.0"/>

diff --git a/tests/org.pitest.pitclipse.tests.coverage.report/pom.xml b/tests/org.pitest.pitclipse.tests.coverage.report/pom.xml
@@ -88,14 +88,11 @@
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
-		<!-- Hangs in GitHub Actions so it's temporarily disabled -->
-		<!--
 		<dependency>
 			<groupId>org.pitest</groupId>
 			<artifactId>org.pitest.pitclipse.ui.tests</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
-		-->
 	</dependencies>
 </project>
diff --git a/tests/org.pitest.pitclipse.ui.tests/.gitignore b/tests/org.pitest.pitclipse.ui.tests/.gitignore
@@ -0,0 +1 @@
+/screenshots/
diff --git a/tests/org.pitest.pitclipse.ui.tests/META-INF/MANIFEST.MF b/tests/org.pitest.pitclipse.ui.tests/META-INF/MANIFEST.MF
@@ -24,4 +24,5 @@ Require-Bundle: org.eclipse.swtbot.eclipse.finder,
  org.pitest.pitclipse.launch;bundle-version="2.0.0",
  org.pitest.pitclipse.launch.ui;bundle-version="2.0.0",
  org.pitest.pitclipse.preferences.ui;bundle-version="2.0.0",
- io.cucumber;bundle-version="4.3.0"
+ io.cucumber;bundle-version="4.3.0",
+ org.eclipse.debug.core
diff --git a/...tclipse/ui/BundleResourceClassLoader.java → ...d-cucumber/BundleResourceClassLoader.java b/...tclipse/ui/BundleResourceClassLoader.java → ...d-cucumber/BundleResourceClassLoader.java
diff --git a/...test/pitclipse/ui/OsgiCucumberRunner.java → ...ests/old-cucumber/OsgiCucumberRunner.java b/...test/pitclipse/ui/OsgiCucumberRunner.java → ...ests/old-cucumber/OsgiCucumberRunner.java
diff --git a/...t/pitclipse/ui/PitclipseFeaturesTest.java → ...s/old-cucumber/PitclipseFeaturesTest.java b/...t/pitclipse/ui/PitclipseFeaturesTest.java → ...s/old-cucumber/PitclipseFeaturesTest.java
diff --git a/.../pitclipse/ui/j00_default_options.feature → .../old-cucumber/j00_default_options.feature b/.../pitclipse/ui/j00_default_options.feature → .../old-cucumber/j00_default_options.feature
diff --git a/...t/pitclipse/ui/j01_simple_project.feature → ...s/old-cucumber/j01_simple_project.feature b/...t/pitclipse/ui/j01_simple_project.feature → ...s/old-cucumber/j01_simple_project.feature
diff --git a/...i/j02_tests_in_different_packages.feature → ...r/j02_tests_in_different_packages.feature b/...i/j02_tests_in_different_packages.feature → ...r/j02_tests_in_different_packages.feature
diff --git a/...i/j03_refactored_tests_can_be_run.feature → ...r/j03_refactored_tests_can_be_run.feature b/...i/j03_refactored_tests_can_be_run.feature → ...r/j03_refactored_tests_can_be_run.feature
diff --git a/...04_multiple projects_in_workspace.feature → ...04_multiple projects_in_workspace.feature b/...04_multiple projects_in_workspace.feature → ...04_multiple projects_in_workspace.feature
diff --git a/...ces_apply_to_launch_configuration.feature → ...ces_apply_to_launch_configuration.feature b/...ces_apply_to_launch_configuration.feature → ...ces_apply_to_launch_configuration.feature
diff --git a/...t/pitclipse/ui/j06_mutations_view.feature → ...s/old-cucumber/j06_mutations_view.feature b/...t/pitclipse/ui/j06_mutations_view.feature → ...s/old-cucumber/j06_mutations_view.feature
diff --git a/tests/org.pitest.pitclipse.ui.tests/org.pitest.pitclipse.ui.tests.launch b/tests/org.pitest.pitclipse.ui.tests/org.pitest.pitclipse.ui.tests.launch
@@ -28,7 +28,7 @@
     <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-os ${target.os} -ws ${target.ws} -arch ${target.arch} -nl ${target.nl} -consoleLog"/>
     <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="org.pitest.pitclipse.ui.tests"/>
     <stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.pde.ui.workbenchClasspathProvider"/>
-    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>
+    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea -Dorg.eclipse.swtbot.search.timeout=20000"/>
     <stringAttribute key="pde.version" value="3.3"/>
     <stringAttribute key="product" value="org.eclipse.platform.ide"/>
     <booleanAttribute key="show_selected_only" value="false"/>

diff --git a/tests/org.pitest.pitclipse.ui.tests/pom.xml b/tests/org.pitest.pitclipse.ui.tests/pom.xml
@@ -1,49 +1,123 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 	<modelVersion>4.0.0</modelVersion>
-    <groupId>org.pitest</groupId>
-    <artifactId>org.pitest.pitclipse.ui.tests</artifactId>
-    <packaging>eclipse-test-plugin</packaging>
+	<groupId>org.pitest</groupId>
+	<artifactId>org.pitest.pitclipse.ui.tests</artifactId>
+	<packaging>eclipse-test-plugin</packaging>
 
 	<parent>
 		<groupId>org.pitest</groupId>
 		<artifactId>org.pitest.pitclipse.tests</artifactId>
 		<version>2.1.2-SNAPSHOT</version>
 	</parent>
 
+	<properties>
+		<!-- For tests for Java 9 and above -->
+		<moduleProperties></moduleProperties>
+		<!-- OS specific flags -->
+		<os-jvm-flags />
+		<!-- Additional Test arguments -->
+		<additionalTestArgLine>${moduleProperties}</additionalTestArgLine>
+	</properties>
+
+	<profiles>
+		<profile>
+			<id>jdk9-or-newer</id>
+			<activation>
+				<jdk>[9,)</jdk>
+			</activation>
+			<properties>
+				<moduleProperties>--add-modules=ALL-SYSTEM</moduleProperties>
+			</properties>
+		</profile>
+		<profile>
+			<id>macosx-jvm-flags</id>
+			<activation>
+				<os>
+					<family>mac</family>
+				</os>
+			</activation>
+			<properties>
+				<os-jvm-flags>-XstartOnFirstThread</os-jvm-flags>
+			</properties>
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.eclipse.tycho</groupId>
+						<artifactId>target-platform-configuration</artifactId>
+						<configuration>
+							<dependency-resolution>
+								<extraRequirements combine.children="append">
+									<!-- This allows us to take the fragment org.eclipse.jdt.launching.macosx 
+										See https://github.com/LorenzoBettini/jbase/issues/3 https://www.eclipse.org/forums/index.php/t/1073366/
+										Without this in macOS Java projects are not compiled since JRE is not bound. -->
+									<requirement>
+										<type>eclipse-feature</type>
+										<id>org.eclipse.jdt</id>
+										<versionRange>0.0.0</versionRange>
+									</requirement>
+								</extraRequirements>
+							</dependency-resolution>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+		<profile>
+			<id>jacoco</id>
+			<activation>
+				<activeByDefault>false</activeByDefault>
+			</activation>
+			<properties>
+				<additionalTestArgLine>${tycho.testArgLine} ${moduleProperties}</additionalTestArgLine>
+			</properties>
+		</profile>
+	</profiles>
+
 	<build>
-        <plugins>
-			<!-- 
-				Required to run SWTBot 
-			 -->
-            <plugin>
-                <groupId>org.eclipse.tycho</groupId>
-                <artifactId>tycho-surefire-plugin</artifactId>
-                <version>${tycho-version}</version>
-                <configuration>
-                    <useUIHarness>true</useUIHarness>
-                    <useUIThread>false</useUIThread>
-                    <product>org.eclipse.platform.ide</product>
-                    <application>org.eclipse.ui.ide.workbench</application>
-                    <showEclipseLog>true</showEclipseLog>
-                </configuration>
-            </plugin>
-			<!-- 
-				Explicit dependency to the listeners fragment:
-				make Pitclipse's mutation listeners available during test runtime 
-			 -->
+		<plugins>
+			<!-- Required to run SWTBot -->
+			<plugin>
+				<groupId>org.eclipse.tycho</groupId>
+				<artifactId>tycho-surefire-plugin</artifactId>
+				<version>${tycho-version}</version>
+				<configuration>
+					<useUIHarness>true</useUIHarness>
+					<useUIThread>false</useUIThread>
+					<showEclipseLog>true</showEclipseLog>
+					<!-- Increase the timeout for SWTBot especially for the CI -->
+					<argLine>${additionalTestArgLine} ${os-jvm-flags} -Dorg.eclipse.swtbot.search.timeout=60000</argLine>
+				</configuration>
+			</plugin>
+			<!-- Explicit dependency to the listeners fragment: make Pitclipse's mutation 
+				listeners available during test runtime -->
 			<plugin>
 				<groupId>org.eclipse.tycho</groupId>
 				<artifactId>target-platform-configuration</artifactId>
 				<version>${tycho-version}</version>
 				<configuration>
 					<dependency-resolution>
-						<extraRequirements>
+						<extraRequirements combine.children="append">
 							<requirement>
 								<type>eclipse-plugin</type>
 								<id>org.pitest.pitclipse.listeners</id>
 								<versionRange>0.0.0</versionRange>
 							</requirement>
+							<!-- This is required since our SWTBot tests do not
+								depend directly on JUnit 5 and we need JUnit 5 to verify
+								Pitclipse integration with JUnit 5 -->
+							<requirement>
+								<type>eclipse-plugin</type>
+								<id>org.eclipse.jdt.junit5.runtime</id>
+								<versionRange>0.0.0</versionRange>
+							</requirement>
+							<!-- This is required to verify Pitclipse integration with JUnit 5 -->
+							<requirement>
+								<type>eclipse-plugin</type>
+								<id>org.pitest.pitest-junit5-plugin</id>
+								<versionRange>0.0.0</versionRange>
+							</requirement>
 						</extraRequirements>
 					</dependency-resolution>
 				</configuration>

diff --git a/...lipse.ui.tests/src/org/pitest/pitclipse/ui/behaviours/pageobjects/AbstractSyntaxTree.java b/...lipse.ui.tests/src/org/pitest/pitclipse/ui/behaviours/pageobjects/AbstractSyntaxTree.java
@@ -53,7 +53,7 @@ private IJavaProject getJavaProject(ClassContext context) {
         return getJavaProject(context.getProjectName());
     }
 
-    private IJavaProject getJavaProject(String projectName) {
+    public IJavaProject getJavaProject(String projectName) {
         IProject project = ResourcesPlugin.getWorkspace().getRoot().getProject(projectName);
         return JavaCore.create(project);
     }
@@ -80,9 +80,18 @@ public void deleteProject(String projectName) {
     }
 
     public void addJUnitToClassPath(String projectName) {
+        addJUnitToClassPath(projectName, 4);
+    }
+
+    public void addJUnit5ToClassPath(String projectName) {
+        addJUnitToClassPath(projectName, 5);
+    }
+
+    public void addJUnitToClassPath(String projectName, int junitVersion) {
         IJavaProject project = getJavaProject(projectName);
         try {
-            Path junitPath = new Path("org.eclipse.jdt.junit.JUNIT_CONTAINER/4");
+            Path junitPath = new Path(
+                "org.eclipse.jdt.junit.JUNIT_CONTAINER/" + junitVersion);
             IClasspathEntry junitEntry = JavaCore.newContainerEntry(junitPath);
             IClasspathEntry junitClasspath = JavaCore.newContainerEntry(junitEntry.getPath());
 

diff --git a/....pitclipse.ui.tests/src/org/pitest/pitclipse/ui/behaviours/pageobjects/BuildProgress.java b/....pitclipse.ui.tests/src/org/pitest/pitclipse/ui/behaviours/pageobjects/BuildProgress.java
@@ -17,11 +17,12 @@
 package org.pitest.pitclipse.ui.behaviours.pageobjects;
 
 import org.eclipse.swtbot.eclipse.finder.SWTWorkbenchBot;
+import org.eclipse.swtbot.swt.finder.utils.SWTBotPreferences;
 import org.pitest.pitclipse.ui.swtbot.WaitForBuildCondition;
 
 public class BuildProgress {
 
-    private static final long BUILD_TIMEOUT = 20000L;
+    private static final long BUILD_TIMEOUT = SWTBotPreferences.TIMEOUT;
     private final SWTWorkbenchBot bot;
     private WaitForBuildCondition buildCompleteCondition;
 

diff --git a/...tclipse.ui.tests/src/org/pitest/pitclipse/ui/behaviours/pageobjects/NewProjectWizard.java b/...tclipse.ui.tests/src/org/pitest/pitclipse/ui/behaviours/pageobjects/NewProjectWizard.java
@@ -34,8 +34,11 @@ public void createJavaProject(String projectName) {
         bot.tree().expandNode("Java").select("Java Project");
         bot.button("Next >").click();
         bot.textWithLabel("Project name:").setText(projectName);
+        // make sure to set Java 8 otherwise we get another dialog for module-info.java
+        bot.radio("Use an execution environment JRE:").click();
+        bot.comboBox().setSelection("JavaSE-1.8");
         bot.button("Finish").click();
-        
+
         // Ensure the project is fully created before moving on
         bot.waitUntil(Conditions.shellCloses(shell));
     }