Merge branch 'java-11-compatibility' into version-3

.github/workflows/ci.yml

@@ -6,14 +6,15 @@ jobs:
     env:
       JDK_OS: linux
       JDK_ARCH: x64
-      JDK_VERSION: 8u192
+      JDK_VERSION: 11u17
+      JDK_DIR: jdk-11.0.17
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 1
         show-progress: false
     - name: Cache JDK
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache-jdk
       with:
         path: ~/.java
@@ -26,7 +27,7 @@ jobs:
         if [ ! -f "$JDK_FILE" ]; then mkdir -p ~/.java && (timeout 400 curl -O -s -S -L --retry 3 "$JDK_URL" || curl -O -L --retry 3 "$JDK_URL") && tar -xzf "$JDK_FILE" -C ~/.java; fi
     - name: Setup JDK
       run: |
-        export JAVA_HOME=$HOME/.java/jdk1.8.0_192
+        export JAVA_HOME=$HOME/.java/$JDK_DIR
         echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV
         echo "$JAVA_HOME/bin" >> $GITHUB_PATH
     - name: Install diffstat
@@ -46,6 +47,9 @@ jobs:
     - run: carton exec ./test_tutorial.sh
       name: test_tutorial.sh
       working-directory: "./framework/test"
+    - run: carton exec ./test_monitor_test.sh
+      name: test_monitor_test.sh
+      working-directory: "./framework/test"
     - run: carton exec ./test_mutation_analysis.sh
       name: test_mutation_analysis.sh
       working-directory: "./framework/test"
@@ -61,6 +65,26 @@ jobs:
     - run: carton exec ./test_style.sh
       name: test_style.sh
       working-directory: "./framework/test"
+
+    # Verify a few select bugs to detect serious breakages early.
+    - run: carton exec ./test_verify_bugs.sh -p Lang -b 24 -A
+      name: "test_verify_bugs.sh -p Lang -b 24 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Lang -b 62 -A
+      name: "test_verify_bugs.sh -p Lang -b 62 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Closure -b 102 -A
+      name: "test_verify_bugs.sh -p Closure -b 102 -A"
+      working-directory: "./framework/test"
+    - run: carton exec ./test_verify_bugs.sh -p Mockito -b 10 -A
+      name: "test_verify_bugs.sh -p Mockito -b 10 -A"
+      working-directory: "./framework/test"
+
+# Verifying the export command for all bugs and verifying reproducibility of all
+# bugs is very time-consuming. The jobs below should be run as part of the
+# release process, but running them on every push and PR is unnecessary.
+#
+#    # Verify that export returns expected values for all projects.
 #    - run: carton exec ./test_export_command.sh -p Chart -p Cli -p Closure
 #      name: "test export 1"
 #      working-directory: "./framework/test"
@@ -73,11 +97,6 @@ jobs:
 #    - run: carton exec ./test_export_command.sh -p Lang -p Math -p Mockito -p Time
 #      name: "test export 4"
 #      working-directory: "./framework/test"
-
-#
-# Verifying all bugs is pretty time-consuming. The jobs below should be run as
-# part of the release process, but running them on every push and PR is
-# unnecessary.
 #
 #    # Verify that all bugs are reproducible (run multiple jobs for projects that
 #    # take a long time to finish).

README.md

@@ -11,7 +11,7 @@ The projects
 
 [comment]: # (Do not edit; generated by framework/util/create_bugs_table.pl .)
 
-Defects4J contains 834 bugs (plus 30 deprecated bugs) from the following open-source projects:
+Defects4J contains 831 bugs (plus 33 deprecated bugs) from the following open-source projects:
 
 | Identifier      | Project name               | Number of active bugs | Active bug ids      | Deprecated bug ids (\*) |
 |-----------------|----------------------------|----------------------:|---------------------|-------------------------|
@@ -24,11 +24,11 @@ Defects4J contains 834 bugs (plus 30 deprecated bugs) from the following open-so
 | Csv             | commons-csv                |           16          | 1-16                | None                    |
 | Gson            | gson                       |           18          | 1-18                | None                    |
 | JacksonCore     | jackson-core               |           26          | 1-26                | None                    |
-| JacksonDatabind | jackson-databind           |          111          | 1-88,90-112         | 89                      |
+| JacksonDatabind | jackson-databind           |          110          | 1-64,66-88,90-112   | 65,89                   |
 | JacksonXml      | jackson-dataformat-xml     |            6          | 1-6                 | None                    |
 | Jsoup           | jsoup                      |           93          | 1-93                | None                    |
 | JxPath          | commons-jxpath             |           22          | 1-22                | None                    |
-| Lang            | commons-lang               |           64          | 1,3-65              | 2                       |
+| Lang            | commons-lang               |           62          | 1,3-24,26-47,49-65  | 2,25,48                 |
 | Math            | commons-math               |          106          | 1-106               | None                    |
 | Mockito         | mockito                    |           38          | 1-38                | None                    |
 | Time            | joda-time                  |           26          | 1-20,22-27          | 21                      |

README_DEVELOPER.md

@@ -0,0 +1,150 @@
+Defects4J for Developers
+================
+This README provides background and information useful for someone updating
+and/or modifing the Defects4J system.  See README.md in this directory if
+your primary interest is in using Defects4J for research.
+
+How a user's Defects4J is initialized
+----------------
+After cloning defects4j, the user runs the script `init.sh`. This script has
+several steps:
+- set up the project repositories
+- set up the tools for mutation testing
+- set up EvoSuite
+- set up Randoop
+- set up Gradle
+- set up utility programs
+
+The project repositories and the Gradle tools are copied from a protected
+directory on the Defects4J web site:  
+https://defects4j.org/downloads
+
+This directory can be accessed, from a CSE managed machine, at:  
+/cse/web/research/defects4j/downloads
+
+In the details below, the directory containing the user's clone of defects4j
+will be referred to as `BASE` and the defects4j download location will be
+referred to as `HOST_URL`.
+
+### Setting up the project repositories
+The project\_repos directory (`$BASE/project_repos`) is populated by running
+the `get_repos.sh` script in that directory; the repos are not included for
+space reasons.  All project repos are archived in the file:  
+`$HOST_URL/defects4j-repos-v3.zip`
+
+Each repository in that archive (name.git folder) is created by running
+`git clone --bare <URL>;` the README in the archive lists the URLs.
+These name.git folders are essentially the same as the .git folder in a working directory.
+
+### Setting up the tools for mutation testing
+The tools are downloaded from:  
+https://mutation-testing.org/downloads
+
+### Setting up EvoSuite
+EvoSuite is downloaded from:  
+https://github.com/EvoSuite/evosuite/releases/download
+
+### Setting up Randoop
+Randoop is downloaded from:  
+https://github.com/randoop/randoop/releases/download
+
+### Setting up Gradle
+Gradle is downloaded from `HOST_URL`.  As the reproducible bugs in the defect4j
+project repositories are several years old, we must use an older version of gradle
+to build the code defects in the Mockito repository. Version 2.x of Defects4J
+used Gradle version 2.2.1. The current version (3.x) of Defects4J uses Gradle version 4.9.
+
+### Setting up utility programs
+These programs are downloaded from:  
+https://github.com/jose/build-analyzer/releases/download
+
+Notes
+----------------
+#### Testing
+The test\_verfiy\_bugs.sh script is essentially the test oracle -- if it passes,
+the updates/changes were successful.
+#### Project Repos
+* Ideally, we would simply run git pull on each of the name.git repositories to
+update the version control history every once in a while and update the archive
+on the website. Sometimes this does not work as the git history for the repo has
+been rewritten. We should be able to update all repositories at some point after we release version 3.
+* The D4J website provides an archive with clones of all project repositories (to avoid cloning from multiple sources and to make sure artifacts are reliably available).
+* We expected newer versions of this archive to be a strict superset of a previous version (either more projects or more commits for a given project), and hence did not version the archive file.
+* Whenever new bugs are mined for an existing project, the archive is updated (essentially just a pull to update the D4J clone).
+* Whenever new bugs are mined for a new project, a clone for the new project is added to the archive.
+
+Requirements (from README.md)
+----------------
+ - Java 11
+ - Git >= 1.9
+ - SVN >= 1.8
+ - Perl >= 5.0.12
+
+Defects4J version 2.x required Java 1.8.
+Defects4J version 1.x and 0.x required Java 1.7.
+
+
+#### Java version
+All bugs have been reproduced and triggering tests verified, using the latest
+version of Java 11.
+Using a different version of Java might result in unexpected failing tests on a fixed
+program version. 
+
+
+Implementation details
+----------------------
+
+Documentation for any script or module is available as
+[HTML documentation][htmldocs].
+
+[htmldocs]: http://defects4j.org/html_doc/index.html
+
+The directory structure of Defects4J is as follows:
+
+    defects4j
+       |
+       |--- project_repos:     The version control repositories of the provided projects.
+       |
+       |--- major:             The Major mutation framework.
+       |
+       |--- framework:         Libraries and executables of the core, test execution,
+           |                   and bug-mining frameworks.
+           |
+           |--- bin:           Command line interface to Defects4J.
+           |
+           |--- bug-mining:    Bug-mining framework.
+           |
+           |--- core:          The modules of the core framework.
+           |
+           |--- lib:           Libraries used in the core framework.
+           |
+           |--- util:          Util scripts used by Defects4J.
+           |
+           |--- projects:      Project-specific resource files.
+           |
+           |--- test:          Scripts to test the framework.
+           
+Versioning information
+----------------------
+Defects4J uses a semantic versioning scheme (`major`.`minor`.`patch`):
+~~~text
+| Change                                  | `major` | `minor` | `patch` |
+|-----------------------------------------|:-------:|:-------:|:-------:|
+| Addition/Deletion of bugs               |    X    |         |         |
+| New/upgraded internal or external tools |         |    X    |         |
+| Fixes and documentation changes         |         |         |    X    |
+~~~
+
+Miscellaneous Notes
+-------------------
+
+#### Build_files
+
+'build_files' associated with the buggy version are not used; both buggy and fixed use the fixed version.
+I believe the buggy versions are are generated and only used during bug-mining.
+They should probably be removed and we should add an issue for not propagating these files from bug-mining to the
+final data set, if this is still the case.
+
+Related, the test patches (<bid>.test.patch) stored in the repo are also not used. I think we
+should remove these as well. Source patches are manually minimized, but test patches are
+not -- these can be easily regenerated by running a diff between the buggy and fixed revisions.

framework/bin/d4j/d4j-mutation

@@ -181,10 +181,11 @@ open(RES, "<$WORK_DIR/summary.csv") or die "Cannot read mutation analysis result
 my $header = <RES>;
 my $line = <RES>;
 close(RES);
-my ($mutGen, $mutCov, $mutKill, $mutLive, $timePreproc, $timeAnalysis) = split(/,/, $line);
+my ($mutGen, $mutRet, $mutCov, $mutKill, $mutLive, $timePreproc, $timeAnalysis) = split(/,/, $line);
 my $killRatioGen = $mutGen == 0 ? 0 : $mutKill/$mutGen*100;
 my $killRatioCov = $mutCov == 0 ? 0 : $mutKill/$mutCov*100;
 printf("%17s: %d\n", "Mutants generated", $mutGen);
+printf("%17s: %d\n", "Mutants retained", $mutRet);
 printf("%17s: %d\n", "Mutants covered", $mutCov);
 printf("%17s: %d\n", "Mutants killed", $mutKill);
 printf("%17s: %.1f%% (%.1f%%)\n", "Mutation score", $killRatioCov, $killRatioGen);

framework/core/Constants.pm

@@ -231,8 +231,8 @@ my $java_version_output = `java -version 2>&1`;
 
 # Extract the imajor version number using regular expressions
 if ($java_version_output =~ 'version "?(?:1\.)?(\K\d+)') {
-    if ($1 != 8) {
-        die ("Java 8 is required!\n\n");
+    if ($1 != 11) {
+        die ("Java 11 is required!\n\n");
     }
 } else {
     die ("Failed to parse Java version! Is Java installed/on the execution path?\n\n");

framework/core/Mutation.pm

@@ -73,7 +73,7 @@ sub create_mml {
     my ($instrument_classes, $out_file, $mut_ops) = @_;
 
     my $OUT_DIR = Utils::get_dir($out_file);
-    my $TEMPLATE = `cat $MAJOR_ROOT/mml/template.mml` or die "Cannot read mml template: $!";
+    my $TEMPLATE = `cat $UTIL_DIR/template.mml` or die "Cannot read mml template: $!";
 
     system("mkdir -p $OUT_DIR");
 
@@ -267,7 +267,7 @@ sub _build_mut_map {
     $_ = <IN>;
     while(<IN>) {
         chomp;
-        /(\d+),(TIME|EXC|FAIL|LIVE)/ or die "Wrong format of kill details file";
+        /(\d+),(TIME|EXC|FAIL|LIVE|UNCOV)/ or die "Wrong format of kill details file";
         $mut_map->{$1}=$2;
     }
     close(IN);
@@ -292,7 +292,7 @@ sub _exclude_mutants {
 
     open(OUT, ">$project->{prog_root}/$EXCL_FILE") or die "Cannot open exclude file!";
     foreach my $mut_id (keys %{$mut_map}) {
-        next if ($mut_map->{$mut_id} eq "LIVE");
+        next if ($mut_map->{$mut_id} =~ "(LIVE|UNCOV)");
         print OUT "$mut_id\n";
     }
 

framework/core/Project.pm

@@ -749,7 +749,9 @@ sub monitor_test {
         #  - A "normal" class named with one or more $ symbols, e.g., com.google.gson.internal.$Gson$Types
         #    from Gson-{14,16,18}.
         #
-        s/\[Loaded (.*) from.*/$1/;
+        # First match corresponds to what a Java-8 JVM outputs; the second match
+        # corresponds to what a Java-11 JVM outputs.
+        s/\[Loaded (.*) from.*/$1/ or s/\S* (.*) source: .*/$1/;
         my $found = 0;
         if (defined $src->{$_}) {
             $found = 1;
@@ -879,7 +881,7 @@ sub mutate {
     -e "$mml_bin" or die "Mml file does not exist: $mml_bin!";
 
     # Set environment variable MML, which is read by Major
-    $ENV{MML} = $mml_bin;
+    $ENV{MML} = "mml:$mml_bin";
 
     # Mutate and compile sources
     my $ret = $self->_call_major("mutate");
@@ -1165,14 +1167,13 @@ sub _ant_call {
 }
 
 #
-# Ensure backward compatibility with Java 7
 # TODO: Remove after Defects4J downloads and initializes its own version of Ant
 #       Currently, we rely on Major's version of Ant to be properly set up.
 #
 sub _ant_call_comp {
     @_ >= 2 or die $ARG_ERROR;
     my ($self, $target, $option_str, $log_file, $ant_cmd) =  @_;
-    $option_str = "-Dbuild.compiler=javac1.7 " . ($option_str // "");
+    $option_str = ($option_str // "");
     $ant_cmd = "$MAJOR_ROOT/bin/ant" unless defined $ant_cmd;
     return $self->_ant_call($target, $option_str, $log_file, $ant_cmd);
 }

framework/core/Project/Cli.pm

@@ -145,6 +145,14 @@ sub _post_checkout {
         print OUT $converted_file;
         close(OUT);
     }
+
+    # Set default Java target to 6.
+    # either these:
+    Utils::sed_cmd("s/source=\\\"1\.[1-5]\\\"/source=\\\"1.6\\\"/", "$work_dir/maven-build.xml");
+    Utils::sed_cmd("s/target=\\\"1\.[1-5]\\\"/target=\\\"1.6\\\"/", "$work_dir/maven-build.xml");
+    # or these:
+    Utils::sed_cmd("s/source=\\\"1\.[1-5]\\\"/source=\\\"1.6\\\"/", "$work_dir/build.xml");
+    Utils::sed_cmd("s/target=\\\"1\.[1-5]\\\"/target=\\\"1.6\\\"/", "$work_dir/build.xml");
 }
 
 #

framework/core/Project/Closure.pm

@@ -68,6 +68,7 @@ sub determine_layout {
 sub _post_checkout {
     @_ == 3 or die $ARG_ERROR;
     my ($self, $rev_id, $work_dir) = @_;
+    my $vid = $self->{_vcs}->lookup_vid($rev_id);
 
     open FH, "$work_dir/build.xml" or die $!;
     my $build_file = do { local $/; <FH> };
@@ -79,6 +80,28 @@ sub _post_checkout {
     open FH, ">$work_dir/build.xml" or die $!;
     print FH $build_file;
     close FH;
+
+    # Fix compilation errors if necessary
+    my $compile_errors = "$PROJECTS_DIR/$self->{pid}/compile-errors/";
+    opendir(DIR, $compile_errors) or die "Could not find compile-errors directory.";
+    my @entries = readdir(DIR);
+    closedir(DIR);
+    foreach my $file (@entries) {
+        if ($file =~ /-(\d+)-(\d+).diff/) {
+            if ($vid >= $1 && $vid <= $2) {
+                $self->apply_patch($work_dir, "$compile_errors/$file")
+                        or confess("Couldn't apply patch ($file): $!");
+            }
+        }
+    }
+
+    # Set default Java target to 6.
+    # either these:
+    Utils::sed_cmd("s/source-level: 1\.[1-5]/source-level 1.6/", "$work_dir/lib/rhino/build.properties");
+    Utils::sed_cmd("s/target-jvm: 1\.[1-5]/target-jvm 1.6/", "$work_dir/lib/rhino/build.properties");
+    # or these:
+    Utils::sed_cmd("s/source-level: 1\.[1-5]/source-level 1.6/", "$work_dir/lib/rhino/src/mozilla/js/rhino/build.properties");
+    Utils::sed_cmd("s/target-jvm: 1\.[1-5]/target-jvm 1.6/", "$work_dir/lib/rhino/src/mozilla/js/rhino/build.properties");
 }
 
 1;