diff --git a/README.md b/README.md index 415748b62..3d3fbf350 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,30 @@ Each bug has the following properties: The (b)uggy and (f)ixed program revisions are labelled with `b` and `f`, respectively (`` is an integer). +Reproducibility +================ + +#### Java version +All bugs have been reproduced and triggering tests verified, using the latest +version of Java 11. +Using a different version of Java might result in unexpected failing tests +and/or non-reproducible bugs. + +#### Locale +Defects4J uses the JDK-8 compatible locale service provider to ensure +reproducibility of bugs. + +If you are using the bugs _outside_ of the Defects4J framework, [set the +JVM's locale service provider to +COMPAT](https://docs.oracle.com/en%2Fjava%2Fjavase%2F11%2Fdocs%2Fapi%2F%2F/java.base/java/util/spi/LocaleServiceProvider.html) +Specifically, use `java -Djava.locale.providers=COMPAT` to ensure +reproducibility. Not setting this option results in unexpected failing tests! + +#### Timezone +Defects4J generates and executes tests in the timezone `America/Los_Angeles`. + +If you are using the bugs _outside_ of the Defects4J framework, set the `TZ` +environment variable to `America/Los_Angeles` and export it. Setting up Defects4J ================ @@ -77,19 +101,9 @@ Requirements - `cpanm` Defects4J version 2.x required Java 1.8. -Defects4J version 1.x and 0.x required Java 1.7. - -#### Java version -All bugs have been reproduced and triggering tests verified, using the latest -version of Java 11. -Using a different version of Java might result in unexpected failing tests on a fixed -program version. +Defects4J version 1.x and 0.x required Java 1.7. -#### Timezone -Defects4J generates and executes tests in the timezone `America/Los_Angeles`. -If you are using the bugs outside of the Defects4J framework, set the `TZ` -environment variable to `America/Los_Angeles` and export it. #### Perl dependencies All required Perl modules are listed in [cpanfile](https://github.com/rjust/defects4j/blob/master/cpanfile). diff --git a/framework/bug-mining/create-project.pl b/framework/bug-mining/create-project.pl index cd54392aa..a9ef019b7 100755 --- a/framework/bug-mining/create-project.pl +++ b/framework/bug-mining/create-project.pl @@ -160,5 +160,6 @@ =head1 DESCRIPTION close(OUT); # Clone the repository -system("mkdir -p $repo_dir && git clone --bare $URL $repo_dir/$NAME.git 2>&1" - . " && echo $NAME: Cloned from $URL >> $repo_dir/README") == 0 or die "Failed to clone repository"; +Utils::exec_cmd("mkdir -p $repo_dir && cp $REPO_DIR/README $repo_dir", "Prepare repository directory"); +Utils::exec_cmd("git clone --bare $URL $repo_dir/$NAME.git 2>&1", "Clone repository from $URL"); +Utils::exec_cmd("echo $NAME: Cloned from $URL >> $repo_dir/README", "Update repository metadata"); diff --git a/framework/core/Constants.pm b/framework/core/Constants.pm index b23215cdb..1292ffb1e 100644 --- a/framework/core/Constants.pm +++ b/framework/core/Constants.pm @@ -245,22 +245,29 @@ if ($java_version_output =~ 'version "?(?:1\.)?(\K\d+)') { # - External libraries (test generation) available? # _repos_available() - or die("Couldn't find project repositories! Did you (re)run 'defects4j/init.sh'?\n\n"); + or die("Couldn't find up-to-date project repositories! Did you (re)run 'defects4j/init.sh'?\n\n"); + -e "$MAJOR_ROOT/bin/ant" or die("Couldn't find Major mutation framework! Did you (re)run 'defects4j/init.sh'?\n\n"); + -d "$TESTGEN_LIB_DIR" or die("Couldn't find test generation tools! Did you (re)run 'defects4j/init.sh'?\n\n"); + -d "$BUILD_SYSTEMS_LIB_DIR" or die("Couldn't find build system tools! Did you (re)run 'defects4j/init.sh'?\n\n"); + -d "$BUILD_SYSTEMS_LIB_DIR/gradle/dists" or die("Couldn't find gradle distributions! Did you (re)run 'defects4j/init.sh'?\n\n"); + -d "$BUILD_SYSTEMS_LIB_DIR/gradle/deps" or die("Couldn't find gradle dependencies! Did you (re)run 'defects4j/init.sh'?\n\n"); sub _repos_available { - opendir(my $dh, "$REPO_DIR") or die "Cannot read $REPO_DIR: $!"; - # The repos directory by default only contains a helper script - return scalar(grep { $_ ne "." && $_ ne ".." } readdir($dh)) > 1; + -e "$REPO_DIR/README" or return 0; + open(IN, "<$REPO_DIR/README") or return 0; + my $line = ; + close(IN); + $line =~ /Defects4J version 3/ or return 0; } # Add script and core directory to @INC diff --git a/init.sh b/init.sh index 616077164..38ae4d647 100755 --- a/init.sh +++ b/init.sh @@ -4,6 +4,8 @@ # immediately set -e # + +# TODO: Major and the coverage tools should be moved to framework/lib ################################################################################ # This script initializes Defects4J. In particular, it downloads and sets up: # - the project's version control repositories @@ -12,34 +14,6 @@ set -e # - the supported code coverage tools (TODO) ################################################################################ -# Print an error message and terminate the script. -# Takes one argument, a custom error message. -# Prints the supplied error message and a script termination notice to stderr. -# Terminates the script with exit code 1. -print_error_and_exit() { - echo -e "${1} \nTerminating initialization... " >&2 - exit 1 -} - -# TODO: Major and the coverage tools should be moved to framework/lib - -# Check whether wget is available on OSX -if [ "$(uname)" = "Darwin" ] ; then - if ! wget --version > /dev/null 2>&1; then - print_error_and_exit "Couldn't find wget to download dependencies. Please install wget and re-run this script." - fi -fi - -# Check whether curl is available -if ! curl --version > /dev/null 2>&1; then - print_error_and_exit "Couldn't find curl to download dependencies. Please install curl and re-run this script." -fi - -# Check whether unzip is available -if ! unzip -v > /dev/null 2>&1; then - print_error_and_exit "Couldn't find unzip to extract dependencies. Please install unzip and re-run this script." -fi -################################################################################ HOST_URL="https://defects4j.org/downloads" # Directories for project repositories and external libraries @@ -48,13 +22,177 @@ DIR_REPOS="$BASE/project_repos" DIR_LIB_GEN="$BASE/framework/lib/test_generation/generation" DIR_LIB_RT="$BASE/framework/lib/test_generation/runtime" DIR_LIB_GRADLE="$BASE/framework/lib/build_systems/gradle" -mkdir -p "$DIR_LIB_GEN" && mkdir -p "$DIR_LIB_RT" && mkdir -p "$DIR_LIB_GRADLE" + +################################################################################ + +main() { + echo "Checking system configuration ... " + # Check whether wget is available on OSX + if [ "$(uname)" = "Darwin" ] ; then + if ! wget --version > /dev/null 2>&1; then + print_error_and_exit "Couldn't find wget to download dependencies. Please install wget and re-run this script." + fi + fi + + # Check whether curl is available + if ! curl --version > /dev/null 2>&1; then + print_error_and_exit "Couldn't find curl to download dependencies. Please install curl and re-run this script." + fi + + # Check whether unzip is available + if ! unzip -v > /dev/null 2>&1; then + print_error_and_exit "Couldn't find unzip to extract dependencies. Please install unzip and re-run this script." + fi + + # Create lib folders if necessary + mkdir -p "$DIR_LIB_GEN" && mkdir -p "$DIR_LIB_RT" && mkdir -p "$DIR_LIB_GRADLE" + + ############################################################################ + # + # Download project repositories if necessary + # + echo "Setting up project repositories ... " + cd "$DIR_REPOS" && ./get_repos.sh + + ############################################################################ + # + # Download Major + # + # Adapt Major's default wrapper scripts: + # - set headless to true to support Chart on machines without X. + # - do not mutate code unless an MML is specified (for historical reasons, + # major v1 was sometimes called without specifying an MML to simply act as + # javac; Major v2+'s default is to generate all mutants as opposed to none). + # + echo + echo "Setting up Major ... " + MAJOR_VERSION="3.0.1" + MAJOR_URL="https://mutation-testing.org/downloads" + MAJOR_ZIP="major-${MAJOR_VERSION}_jre11.zip" + cd "$BASE" && rm -rf major \ + && download_url_and_unzip "$MAJOR_URL/$MAJOR_ZIP" \ + && rm "$MAJOR_ZIP" \ + && perl -pi -e '$_ .= qq( -Djava.awt.headless=true \\\n -Djava.locale.providers=COMPAT \\\n) if /CodeCacheSize/' \ + major/bin/ant \ + && perl -pi -e '$_ .= qq(\nif [ -z "\$MML" ]; then javac \$*; exit \$?; fi\n) if /^REFACTOR=/' \ + major/bin/major \ + && perl -pi -e '$_ = qq(REFACTOR=\${REFACTOR:-"enable.decl.refactor enable.method.refactor"}\n) if /^REFACTOR=/' \ + major/bin/major \ + + ############################################################################ + # + # Download EvoSuite + # + echo + echo "Setting up EvoSuite ... " + EVOSUITE_VERSION="1.1.0" + EVOSUITE_URL="https://github.com/EvoSuite/evosuite/releases/download/v${EVOSUITE_VERSION}" + EVOSUITE_JAR="evosuite-${EVOSUITE_VERSION}.jar" + EVOSUITE_RT_JAR="evosuite-standalone-runtime-${EVOSUITE_VERSION}.jar" + cd "$DIR_LIB_GEN" && download_url "$EVOSUITE_URL/$EVOSUITE_JAR" + cd "$DIR_LIB_RT" && download_url "$EVOSUITE_URL/$EVOSUITE_RT_JAR" + # Set symlinks for the supported version of EvoSuite + (cd "$DIR_LIB_GEN" && ln -sf "$EVOSUITE_JAR" "evosuite-current.jar") + (cd "$DIR_LIB_RT" && ln -sf "$EVOSUITE_RT_JAR" "evosuite-rt.jar") + + ############################################################################ + # + # Download Randoop + # + echo + echo "Setting up Randoop ... " + RANDOOP_VERSION="4.3.3" + RANDOOP_URL="https://github.com/randoop/randoop/releases/download/v${RANDOOP_VERSION}" + RANDOOP_ZIP="randoop-${RANDOOP_VERSION}.zip" + RANDOOP_JAR="randoop-all-${RANDOOP_VERSION}.jar" + REPLACECALL_JAR="replacecall-${RANDOOP_VERSION}.jar" + COVEREDCLASS_JAR="covered-class-${RANDOOP_VERSION}.jar" + (cd "$DIR_LIB_GEN" && download_url_and_unzip "$RANDOOP_URL/$RANDOOP_ZIP") + # Set symlink for the supported version of Randoop + (cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/$RANDOOP_JAR" "randoop-current.jar") + (cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/$REPLACECALL_JAR" "replacecall-current.jar") + (cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/$COVEREDCLASS_JAR" "covered-class-current.jar") + (cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/jacocoagent.jar" "jacocoagent.jar") + + ############################################################################ + # + # Download build system dependencies + # + echo + echo "Setting up Gradle dependencies ... " + + cd "$DIR_LIB_GRADLE" + + GRADLE_DISTS_ZIP=defects4j-gradle-dists-v3.zip + GRADLE_DEPS_ZIP=defects4j-gradle-deps-v3.zip + + old_dists_ts=0 + old_deps_ts=0 + + if [ -e $GRADLE_DISTS_ZIP ]; then + old_dists_ts=$(get_modification_timestamp $GRADLE_DISTS_ZIP) + fi + if [ -e $GRADLE_DEPS_ZIP ]; then + old_deps_ts=$(get_modification_timestamp $GRADLE_DEPS_ZIP) + fi + + # Only download archive if the server has a newer file + download_url $HOST_URL/$GRADLE_DISTS_ZIP + download_url $HOST_URL/$GRADLE_DEPS_ZIP + new_dists_ts=$(get_modification_timestamp $GRADLE_DISTS_ZIP) + new_deps_ts=$(get_modification_timestamp $GRADLE_DEPS_ZIP) + + # Update gradle distributions/dependencies if a newer archive was available + [ "$old_dists_ts" != "$new_dists_ts" ] && mkdir "dists" && unzip -q -u $GRADLE_DISTS_ZIP -d "dists" + [ "$old_deps_ts" != "$new_deps_ts" ] && unzip -q -u $GRADLE_DEPS_ZIP + + cd "$BASE" + + ############################################################################ + # + # Download utility programs + # + echo + echo "Setting up utility programs ... " + + BUILD_ANALYZER_VERSION="0.0.1" + BUILD_ANALYZER_JAR=build-analyzer-$BUILD_ANALYZER_VERSION.jar + BUILD_ANALYZER_URL="https://github.com/jose/build-analyzer/releases/download/v$BUILD_ANALYZER_VERSION/$BUILD_ANALYZER_JAR" + BUILD_ANALYZER_JAR_LOCAL="analyzer.jar" + cd "$BASE/framework/lib" && download_url "$BUILD_ANALYZER_URL" + rm -f "$BUILD_ANALYZER_JAR_LOCAL" + ln -s "$BUILD_ANALYZER_JAR" "$BUILD_ANALYZER_JAR_LOCAL" + + echo + echo "Defects4J successfully initialized." + echo + echo "|------------------------------------------------------------------------|" + echo "| Defects4J version 3 |" + echo "|------------------------------------------------------------------------|" + echo "| PLEASE READ: |" + echo "| https://github.com/rjust/defects4j/?tab=readme-ov-file#reproducibility |" + echo "|------------------------------------------------------------------------|" + echo "| Important changes: |" + echo "| * Java 11 is required |" + echo "| * Randoop v4.3.3 |" + echo "| * Major v3.0.1 |" + echo "|------------------------------------------------------------------------|" +} ################################################################################ # # Utility functions # +# Print an error message and terminate the script. +# Takes one argument, a custom error message. +# Prints the supplied error message and a script termination notice to stderr. +# Terminates the script with exit code 1. +print_error_and_exit() { + echo -e "${1} \nTerminating initialization... " >&2 + exit 1 +} + # MacOS does not install the timeout command by default. if [ "$(uname)" = "Darwin" ] ; then function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } @@ -125,121 +263,4 @@ get_modification_timestamp() { echo "$ts" } -################################################################################ -# -# Download project repositories if necessary -# -echo "Setting up project repositories ... " -cd "$DIR_REPOS" && ./get_repos.sh - -################################################################################ -# -# Download Major -# -# Adapt Major's default wrapper scripts: -# - set headless to true to support Chart on machines without X. -# - do not mutate code unless an MML is specified (for historical reasons, -# major v1 was sometimes called without specifying an MML to simply act as -# javac; Major v2+'s default is to generate all mutants as opposed to none). -# -echo -echo "Setting up Major ... " -MAJOR_VERSION="3.0.1" -MAJOR_URL="https://mutation-testing.org/downloads" -MAJOR_ZIP="major-${MAJOR_VERSION}_jre11.zip" -cd "$BASE" && rm -rf major \ - && download_url_and_unzip "$MAJOR_URL/$MAJOR_ZIP" \ - && rm "$MAJOR_ZIP" \ - && perl -pi -e '$_ .= qq( -Djava.awt.headless=true \\\n -Djava.locale.providers=COMPAT \\\n) if /CodeCacheSize/' \ - major/bin/ant \ - && perl -pi -e '$_ .= qq(\nif [ -z "\$MML" ]; then javac \$*; exit \$?; fi\n) if /^REFACTOR=/' \ - major/bin/major \ - && perl -pi -e '$_ = qq(REFACTOR=\${REFACTOR:-"enable.decl.refactor enable.method.refactor"}\n) if /^REFACTOR=/' \ - major/bin/major \ - -################################################################################ -# -# Download EvoSuite -# -echo -echo "Setting up EvoSuite ... " -EVOSUITE_VERSION="1.1.0" -EVOSUITE_URL="https://github.com/EvoSuite/evosuite/releases/download/v${EVOSUITE_VERSION}" -EVOSUITE_JAR="evosuite-${EVOSUITE_VERSION}.jar" -EVOSUITE_RT_JAR="evosuite-standalone-runtime-${EVOSUITE_VERSION}.jar" -cd "$DIR_LIB_GEN" && download_url "$EVOSUITE_URL/$EVOSUITE_JAR" -cd "$DIR_LIB_RT" && download_url "$EVOSUITE_URL/$EVOSUITE_RT_JAR" -# Set symlinks for the supported version of EvoSuite -(cd "$DIR_LIB_GEN" && ln -sf "$EVOSUITE_JAR" "evosuite-current.jar") -(cd "$DIR_LIB_RT" && ln -sf "$EVOSUITE_RT_JAR" "evosuite-rt.jar") - -################################################################################ -# -# Download Randoop -# -echo -echo "Setting up Randoop ... " -RANDOOP_VERSION="4.3.3" -RANDOOP_URL="https://github.com/randoop/randoop/releases/download/v${RANDOOP_VERSION}" -RANDOOP_ZIP="randoop-${RANDOOP_VERSION}.zip" -RANDOOP_JAR="randoop-all-${RANDOOP_VERSION}.jar" -REPLACECALL_JAR="replacecall-${RANDOOP_VERSION}.jar" -COVEREDCLASS_JAR="covered-class-${RANDOOP_VERSION}.jar" -(cd "$DIR_LIB_GEN" && download_url_and_unzip "$RANDOOP_URL/$RANDOOP_ZIP") -# Set symlink for the supported version of Randoop -(cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/$RANDOOP_JAR" "randoop-current.jar") -(cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/$REPLACECALL_JAR" "replacecall-current.jar") -(cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/$COVEREDCLASS_JAR" "covered-class-current.jar") -(cd "$DIR_LIB_GEN" && ln -sf "randoop-${RANDOOP_VERSION}/jacocoagent.jar" "jacocoagent.jar") - -################################################################################ -# -# Download build system dependencies -# -echo -echo "Setting up Gradle dependencies ... " - -cd "$DIR_LIB_GRADLE" - -GRADLE_DISTS_ZIP=defects4j-gradle-dists-v3.zip -GRADLE_DEPS_ZIP=defects4j-gradle-deps-v3.zip - -old_dists_ts=0 -old_deps_ts=0 - -if [ -e $GRADLE_DISTS_ZIP ]; then - old_dists_ts=$(get_modification_timestamp $GRADLE_DISTS_ZIP) -fi -if [ -e $GRADLE_DEPS_ZIP ]; then - old_deps_ts=$(get_modification_timestamp $GRADLE_DEPS_ZIP) -fi - -# Only download archive if the server has a newer file -download_url $HOST_URL/$GRADLE_DISTS_ZIP -download_url $HOST_URL/$GRADLE_DEPS_ZIP -new_dists_ts=$(get_modification_timestamp $GRADLE_DISTS_ZIP) -new_deps_ts=$(get_modification_timestamp $GRADLE_DEPS_ZIP) - -# Update gradle distributions/dependencies if a newer archive was available -[ "$old_dists_ts" != "$new_dists_ts" ] && mkdir "dists" && unzip -q -u $GRADLE_DISTS_ZIP -d "dists" -[ "$old_deps_ts" != "$new_deps_ts" ] && unzip -q -u $GRADLE_DEPS_ZIP - -cd "$BASE" - -################################################################################ -# -# Download utility programs -# -echo -echo "Setting up utility programs ... " - -BUILD_ANALYZER_VERSION="0.0.1" -BUILD_ANALYZER_JAR=build-analyzer-$BUILD_ANALYZER_VERSION.jar -BUILD_ANALYZER_URL="https://github.com/jose/build-analyzer/releases/download/v$BUILD_ANALYZER_VERSION/$BUILD_ANALYZER_JAR" -BUILD_ANALYZER_JAR_LOCAL="analyzer.jar" -cd "$BASE/framework/lib" && download_url "$BUILD_ANALYZER_URL" -rm -f "$BUILD_ANALYZER_JAR_LOCAL" -ln -s "$BUILD_ANALYZER_JAR" "$BUILD_ANALYZER_JAR_LOCAL" - -echo -echo "Defects4J successfully initialized." +main diff --git a/project_repos/get_repos.sh b/project_repos/get_repos.sh index 776b2ff64..a5c0db768 100755 --- a/project_repos/get_repos.sh +++ b/project_repos/get_repos.sh @@ -5,6 +5,36 @@ set -e # The name of the archive that contains all project repos ARCHIVE=defects4j-repos-v3.zip +main() { + # The BSD version of stat does not support --version or -c + if stat --version &> /dev/null; then + # GNU version + cmd="stat -c %Y $ARCHIVE" + else + # BSD version + cmd="stat -f %m $ARCHIVE" + fi + + if [ -e $ARCHIVE ]; then + old=$($cmd) + else + old=0 + fi + # Only download repos if the server has a newer file + download_url "https://defects4j.org/downloads/$ARCHIVE" + + new=$($cmd) + + # Exit if no newer file is available + [ "$old" == "$new" ] && exit 0 + + # Remove old files + clean + + # Extract new repos + unzip -q -u $ARCHIVE && mv defects4j/project_repos/* . && rm -r defects4j +} + clean() { rm -rf \ closure-compiler.git \ @@ -19,11 +49,15 @@ clean() { gson.git \ jackson-core.git \ jackson-databind.git \ - jackson-dataformat-xml \ + jackson-dataformat-xml.git \ jfreechart \ + jfreechart.git \ joda-time.git \ jsoup.git \ mockito.git \ + defects4j-repos.zip \ + repos.csv \ + sync.sh \ README } @@ -44,30 +78,4 @@ download_url() { fi } -# The BSD version of stat does not support --version or -c -if stat --version &> /dev/null; then - # GNU version - cmd="stat -c %Y $ARCHIVE" -else - # BSD version - cmd="stat -f %m $ARCHIVE" -fi - -if [ -e $ARCHIVE ]; then - old=$($cmd) -else - old=0 -fi -# Only download repos if the server has a newer file -download_url "https://defects4j.org/downloads/$ARCHIVE" - -new=$($cmd) - -# Exit if no newer file is available -[ "$old" == "$new" ] && exit 0 - -# Remove old files -clean - -# Extract new repos -unzip -q -u $ARCHIVE && mv defects4j/project_repos/* . && rm -r defects4j +main