Java 11 compatibility (#613)

.github/workflows/ci.yml

@@ -6,14 +6,15 @@ jobs:
     env:
       JDK_OS: linux
       JDK_ARCH: x64
-      JDK_VERSION: 8u192
+      JDK_VERSION: 11u17
+      JDK_DIR: jdk-11.0.17
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 1
         show-progress: false
     - name: Cache JDK
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache-jdk
       with:
         path: ~/.java
@@ -26,11 +27,11 @@ jobs:
         if [ ! -f "$JDK_FILE" ]; then mkdir -p ~/.java && (timeout 400 curl -O -s -S -L --retry 3 "$JDK_URL" || curl -O -L --retry 3 "$JDK_URL") && tar -xzf "$JDK_FILE" -C ~/.java; fi
     - name: Setup JDK
       run: |
-        export JAVA_HOME=$HOME/.java/jdk1.8.0_192
+        export JAVA_HOME=$HOME/.java/$JDK_DIR
         echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV
         echo "$JAVA_HOME/bin" >> $GITHUB_PATH
-    - name: Install diffstat
-      run: sudo apt-get install diffstat
+    - name: Install diffstat and checkbashisms
+      run: sudo apt-get -y install diffstat devscripts
     - name: Install Carton
       uses: perl-actions/install-with-cpanm@v1
       with:
@@ -46,6 +47,9 @@ jobs:
     - run: carton exec ./test_tutorial.sh
       name: test_tutorial.sh
       working-directory: "./framework/test"
+    - run: carton exec ./test_monitor_test.sh
+      name: test_monitor_test.sh
+      working-directory: "./framework/test"
     - run: carton exec ./test_mutation_analysis.sh
       name: test_mutation_analysis.sh
       working-directory: "./framework/test"
@@ -58,6 +62,32 @@ jobs:
     - run: carton exec ./test_bug_mining.sh
       name: test_bug_mining.sh
       working-directory: "./framework/test"
+    - run: carton exec ./test_style.sh
+      name: test_style.sh
+      working-directory: "./framework/test"
+
+    # Verify a few select bugs to detect serious breakages early.
+    - run: carton exec ./test_verify_bugs.sh -p Collections -b 10 -A
+      name: "test_verify_bugs.sh -p Collections -b 10 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Lang -b 24 -A
+      name: "test_verify_bugs.sh -p Lang -b 24 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Lang -b 62 -A
+      name: "test_verify_bugs.sh -p Lang -b 62 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Closure -b 102 -A
+      name: "test_verify_bugs.sh -p Closure -b 102 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Mockito -b 10 -A
+      name: "test_verify_bugs.sh -p Mockito -b 10 -A"
+      working-directory: "./framework/test"
+
+# Verifying the export command for all bugs and verifying reproducibility of all
+# bugs is very time-consuming. The jobs below should be run as part of the
+# release process, but running them on every push and PR is unnecessary.
+#
+#    # Verify that export returns expected values for all projects.
 #    - run: carton exec ./test_export_command.sh -p Chart -p Cli -p Closure
 #      name: "test export 1"
 #      working-directory: "./framework/test"
@@ -70,11 +100,6 @@ jobs:
 #    - run: carton exec ./test_export_command.sh -p Lang -p Math -p Mockito -p Time
 #      name: "test export 4"
 #      working-directory: "./framework/test"
-
-#
-# Verifying all bugs is pretty time-consuming. The jobs below should be run as
-# part of the release process, but running them on every push and PR is
-# unnecessary.
 #
 #    # Verify that all bugs are reproducible (run multiple jobs for projects that
 #    # take a long time to finish).

README.md

@@ -1,4 +1,4 @@
-Defects4J -- version 2.0.1 [![.github/workflows/ci.yml](https://github.com/rjust/defects4j/actions/workflows/ci.yml/badge.svg)](https://github.com/rjust/defects4j/actions/workflows/ci.yml)
+Defects4J -- version 3.0.0 [![.github/workflows/ci.yml](https://github.com/rjust/defects4j/actions/workflows/ci.yml/badge.svg)](https://github.com/rjust/defects4j/actions/workflows/ci.yml)
 ================
 Defects4J is a collection of reproducible bugs and a supporting infrastructure
 with the goal of advancing software engineering research.
@@ -11,29 +11,29 @@ The projects
 
 [comment]: # (Do not edit; generated by framework/util/create_bugs_table.pl .)
 
-Defects4J contains 835 bugs (plus 29 deprecated bugs) from the following open-source projects:
+Defects4J contains 854 bugs (plus 10 deprecated bugs) from the following open-source projects:
 
 | Identifier      | Project name               | Number of active bugs | Active bug ids      | Deprecated bug ids (\*) |
 |-----------------|----------------------------|----------------------:|---------------------|-------------------------| 
 | Chart           | jfreechart                 |           26          | 1-26                | None                    |
 | Cli             | commons-cli                |           39          | 1-5,7-40            | 6                       |
 | Closure         | closure-compiler           |          174          | 1-62,64-92,94-176   | 63,93                   |
 | Codec           | commons-codec              |           18          | 1-18                | None                    |
-| Collections     | commons-collections        |            4          | 25-28               | 1-24                    |
+| Collections     | commons-collections        |           28          | 1-28                | None                    |
 | Compress        | commons-compress           |           47          | 1-47                | None                    |
 | Csv             | commons-csv                |           16          | 1-16                | None                    |
 | Gson            | gson                       |           18          | 1-18                | None                    |
 | JacksonCore     | jackson-core               |           26          | 1-26                | None                    |
-| JacksonDatabind | jackson-databind           |          112          | 1-112               | None                    |
+| JacksonDatabind | jackson-databind           |          110          | 1-64,66-88,90-112   | 65,89                   |
 | JacksonXml      | jackson-dataformat-xml     |            6          | 1-6                 | None                    |
 | Jsoup           | jsoup                      |           93          | 1-93                | None                    |
 | JxPath          | commons-jxpath             |           22          | 1-22                | None                    |
-| Lang            | commons-lang               |           64          | 1,3-65              | 2                       |
+| Lang            | commons-lang               |           61          | 1,3-17,19-24,26-47,49-65 | 2,18,25,48              |
 | Math            | commons-math               |          106          | 1-106               | None                    |
 | Mockito         | mockito                    |           38          | 1-38                | None                    |
 | Time            | joda-time                  |           26          | 1-20,22-27          | 21                      |
 
-\* Due to behavioral changes introduced under Java 8, some bugs are no longer
+\* Due to behavioral changes introduced in newer Java versions, some bugs are no longer
 reproducible. Hence, Defects4J distinguishes between active and deprecated bugs:
 
 - Active bugs can be accessed through `active-bugs.csv`.
@@ -64,31 +64,52 @@ Each bug has the following properties:
 The (b)uggy and (f)ixed program revisions are labelled with `<id>b` and
 `<id>f`, respectively (`<id>` is an integer).
 
+Reproducibility
+================
+
+#### Java version
+All bugs have been reproduced and triggering tests verified, using
+Java 11 (see the [CI configuration](.github/workflows/ci.yml) for specifics).
+Using a different version of Java might result in unexpected failing tests
+and/or non-reproducible bugs.
+
+#### Timezone
+Defects4J generates and executes tests in the timezone `America/Los_Angeles`.
+
+If you are using the bugs _outside_ of the Defects4J framework, set the `TZ`
+environment variable to `America/Los_Angeles` and export it.
+Not setting this option results in unexpected failing tests!
+
+#### Broken and Flaky Tests
+Defects4J excludes broken tests (tests that reliably fail on the fixed and the buggy version)
+and flaky tests (tests that intermittently fail on the fixed or buggy version).
+
+If you are using the bugs _outside_ of the Defects4J framework, make sure to
+verify expected test behavior for your environment.
+
+#### Metadata
+We recommend using Defects4J through the provided command-line interface. All
+reproducibility tests within Defects4J rely on this interface.
+If you are using the bugs _outside_ of the Defects4J framework, make sure to
+use the Defects4J `export` command to obtain relevant metadata such as
+source/test directories, classpath entries, and sets of tests.
+
 
 Setting up Defects4J
 ================
 
 Requirements
 ----------------
- - Java 1.8
+ - Java 11
  - Git >= 1.9
  - Subversion (svn) >= 1.8
  - Perl >= 5.0.12
  - `cpanm`
 
-Defects4J version 1.x and 0.x required Java 1.7.
+Defects4J version 2.x required Java 1.8.
 
+Defects4J version 1.x and 0.x required Java 1.7.
 
-#### Java version
-All bugs have been reproduced and triggering tests verified, using the latest
-version of Java 1.8.
-Using a different version of Java might result in unexpected failing tests on a fixed
-program version. 
-
-#### Timezone
-Defects4J generates and executes tests in the timezone `America/Los_Angeles`.
-If you are using the bugs outside of the Defects4J framework, set the `TZ`
-environment variable to `America/Los_Angeles` and export it.
 
 #### Perl dependencies
 All required Perl modules are listed in [cpanfile](https://github.com/rjust/defects4j/blob/master/cpanfile).

cpanfile

@@ -3,4 +3,6 @@ requires 'DBI',         '>= 1.63';
 requires 'JSON',        '>= 2.97';
 requires 'JSON::Parse', '>= 0.55';
 requires 'List::Util',  '>= 1.33';
+requires 'Perl::Critic';
+requires 'String::Interpolate', '>= 0.33';
 requires 'URI',         '>= 1.72';

developer/README_DEVELOPER.md

@@ -0,0 +1,98 @@
+Defects4J for Developers
+================
+This README provides background and information useful for someone updating
+and/or modifing the Defects4J system.  See [README.md](README.md) in this directory if
+your primary interest is in using Defects4J for research.
+
+How a user's Defects4J is initialized
+----------------
+After cloning defects4j, the user runs the script `init.sh`. This script has
+several steps:
+- set up the project repositories
+- set up the tools for mutation testing
+- set up EvoSuite
+- set up Randoop
+- set up Gradle
+- set up utility programs
+
+The project repositories and the Gradle tools are copied from a protected
+directory on the Defects4J web site:  
+https://defects4j.org/downloads
+
+This directory can be accessed, from a CSE managed machine, at:  
+/cse/web/research/defects4j/downloads
+
+In the details below, the directory containing the user's clone of defects4j
+will be referred to as `BASE` and the defects4j download location will be
+referred to as `HOST_URL`.
+
+### Setting up the project repositories
+The project\_repos directory (`$BASE/project_repos`) is populated by running
+the `get_repos.sh` script in that directory; the repos are not included for
+space reasons.  All project repos are archived in the file:  
+`$HOST_URL/defects4j-repos-v3.zip`
+
+Each repository in that archive (name.git folder) is created by running
+`git clone --bare <URL>;` the README in the archive lists the URLs.
+These name.git folders are essentially the same as the .git folder in a working directory.
+
+### Setting up the tools for mutation testing
+The tools are downloaded from:  
+https://mutation-testing.org/downloads
+
+### Setting up EvoSuite
+EvoSuite is downloaded from:  
+https://github.com/EvoSuite/evosuite/releases/download
+
+### Setting up Randoop
+Randoop is downloaded from:  
+https://github.com/randoop/randoop/releases/download
+
+### Setting up Gradle
+Gradle is downloaded from `HOST_URL`.  As the reproducible bugs in the defect4j
+project repositories are several years old, we must use an older version of gradle
+to build the code defects in the Mockito repository. Version 2.x of Defects4J
+used Gradle version 2.2.1. The current version (3.x) of Defects4J uses Gradle version 4.9.
+
+### Setting up utility programs
+These programs are downloaded from:  
+https://github.com/jose/build-analyzer/releases/download
+
+Notes
+----------------
+#### Testing
+The test\_verfiy\_bugs.sh script is essentially the test oracle -- if it passes,
+the updates/changes were successful.
+
+Note that a requirement before making a release is to run this script with the -A
+option.  See framework/test/README.md for more details about this step and more
+information about all the testing scripts.
+
+#### Project Repos
+* Ideally, we would simply run git pull on each of the name.git repositories to
+update the version control history every once in a while and update the archive
+on the website. Sometimes this does not work as the git history for the repo has
+been rewritten. We synced all repositories for version 3, updating revision ids
+(e.g., in active-bugs.csv) if needed.
+* The D4J website provides an archive with clones of all project repositories (to avoid cloning from multiple sources and to make sure artifacts are reliably available).
+* We expected newer versions of this archive to be a strict superset of a previous version (either more projects or more commits for a given project), and hence did not version the archive file. We started versioning the archive file after version 2.
+* Whenever new bugs are mined for an existing project, the archive is updated (essentially just a pull to update the D4J clone).
+* Whenever new bugs are mined for a new project, a clone for the new project is added to the archive.
+
+Requirements
+------------
+See [README.md](README.md) for system requirements.
+
+Implementation details
+----------------------
+
+Documentation for any script or module is available as
+[HTML documentation][htmldocs].
+
+[htmldocs]: http://defects4j.org/html_doc/index.html
+
+#### Style
+
+Because shell scripts are error-prone, we run a style checker on them.  CI will
+not pass if the style checker issues warnings.  Please address each warning by
+either correcting the problem or suppressing the warning.

framework/bin/d4j/d4j-bids

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-checkout

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-compile

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-coverage

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-env

@@ -1,6 +1,6 @@
 
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-export

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-info

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-monitor.test

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-mutation

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -181,10 +181,11 @@ open(RES, "<$WORK_DIR/summary.csv") or die "Cannot read mutation analysis result
 my $header = <RES>;
 my $line = <RES>;
 close(RES);
-my ($mutGen, $mutCov, $mutKill, $mutLive, $timePreproc, $timeAnalysis) = split(/,/, $line);
+my ($mutGen, $mutRet, $mutCov, $mutKill, $mutLive, $timePreproc, $timeAnalysis) = split(/,/, $line);
 my $killRatioGen = $mutGen == 0 ? 0 : $mutKill/$mutGen*100;
 my $killRatioCov = $mutCov == 0 ? 0 : $mutKill/$mutCov*100;
 printf("%17s: %d\n", "Mutants generated", $mutGen);
+printf("%17s: %d\n", "Mutants retained", $mutRet);
 printf("%17s: %d\n", "Mutants covered", $mutCov);
 printf("%17s: %d\n", "Mutants killed", $mutKill);
 printf("%17s: %.1f%% (%.1f%%)\n", "Mutation score", $killRatioCov, $killRatioGen);

framework/bin/d4j/d4j-pids

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-query

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

framework/bin/d4j/d4j-test

@@ -1,5 +1,5 @@
 #-------------------------------------------------------------------------------
-# Copyright (c) 2014-2019 René Just, Darioush Jalali, and Defects4J contributors.
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal