Updating to match core D4J

README.md

@@ -22,15 +22,15 @@ Each bug has the following properties:
 
 - Issue filed in the corresponding issue tracker, and issue tracker identifier
   mentioned in the fixing commit message.
-- Fixed in a single commit -- manually verified to not include irrelevant
-  changes (e.g., refactorings or feature additions).
+- Fixed in a single commit -- the Defects4J maintainers manually pruned out
+  irrelevant changes (e.g., refactorings or feature additions).
 - Fixed by modifying the source code (as opposed to configuration files,
   documentation, or test files).
 - A triggering test exists that failed before the fix and passes after the fix
   -- the test failure is not random or dependent on test execution order.
 
-(B)uggy and (f)ixed program revisions are labelled with `<id>b` and `<id>f`,
-respectively (`<id>` is an integer).
+The (b)uggy and (f)ixed program revisions are labelled with `<id>b` and
+`<id>f`, respectively (`<id>` is an integer).
 
 Requirements
 ----------------
@@ -94,7 +94,7 @@ Publications
 
 Documentation
 --------------------
-A detailed documentation for any script or module is available as
+Detailed documentation for any script or module is available as
 [html documentation][htmldocs].
 
 [htmldocs]: http://people.cs.umass.edu/~rjust/defects4j/html_doc/index.html
@@ -108,7 +108,7 @@ Use `framework/bin/defects4j` to execute any of the following commands:
 | info           | View configuration of a specific project or summary of a specific bug                             |
 | checkout       | Checkout a buggy or a fixed project version                                                       |
 | compile        | Compile sources and developer-written tests of a buggy or a fixed project version                 |
-| test           | Run a single test or a test suite on a buggy or a fixed project version                           |
+| test           | Run a single test method or a test suite on a buggy or a fixed project version                    |
 | mutation       | Run mutation analysis on a buggy or a fixed project version                                       |
 | coverage       | Run code coverage analysis on a buggy or a fixed project version                                  |
 | monitor.test   | Monitor the class loader during the execution of a single test or a test suite                    |
@@ -128,9 +128,9 @@ directory to export a version-specific property:
 | dir.src.classes  | Source directory of classes (relative to working directory)                         |
 | dir.bin.classes  | Target directory of classes (relative to working directory)                         |
 | dir.src.tests    | Source directory of tests (relative to working directory)                           |
-| tests.all        | List of all developer-written tests                                                 |
-| tests.relevant   | List of relevant tests (i.e., tests that touch at least one of the modified classes |
-| tests.trigger    | List of tests that trigger (expose) the bug                                         |
+| tests.all        | List of all developer-written test classes                                          |
+| tests.relevant   | List of relevant tests classes (a test class is relevant if, when executed, the JVM loads at least one of the modified classes) |
+| tests.trigger    | List of test methods that trigger (expose) the bug                                  |
 
 Test execution framework
 --------------------------
@@ -172,6 +172,19 @@ The directory structure is as follows:
            |
            |--- test:          Scripts to test the framework.
 
+Additional resources
+--------------------
+#### Fault localization (FL)
+  - [Scripts and annotations for evaluating FL techniques][FL-eval]
+
+#### Automated program repair (APR)
+  - [Scripts and annotations for evaluating APR techniques][APR-eval]
+  - [Patches generated with the Nopol, jGenProg, and jKali APR systems][APR-patches-spirals]
+
+[fl-eval]: https://bitbucket.org/rjust/fault-localization-data
+[APR-eval]: https://github.com/LASER-UMASS/AutomatedRepairApplicabilityData
+[APR-patches-spirals]: https://github.com/Spirals-Team/defects4j-repair
+
 License
 ---------
 MIT License, see `license.txt` for more information.

framework/bin/d4j/d4j-mutation

@@ -28,7 +28,7 @@ d4j-mutation -- run mutation analysis for a checked-out project version.
 
 =head1 SYNOPSIS
 
-  d4j-mutation [-w work_dir] [-r | [-t single_test] [-s test_suite]] [-i instrument_classes]
+  d4j-mutation [-w work_dir] [-r | [-t single_test] [-s test_suite]] [-i instrument_classes] [-e exclude_file]
 
 =head1 DESCRIPTION
 
@@ -66,6 +66,11 @@ Mutate all classes listed in F<instrument_classes> (optional). By default, only
 classes modified by the bug fix are mutated. The file F<instrument_classes> must contain
 fully-qualified class names -- one class per line.
 
+=item -e F<exclude_file>
+
+The file that contains the list of all mutant ids (one per row) to exclude (optional).
+By default no exclude file is used and therefore no mutant is excluded.
+
 =back
 
 =head1 Test suites
@@ -99,13 +104,14 @@ use Getopt::Std;
 use Pod::Usage;
 
 my %cmd_opts;
-getopts('t:w:rs:i:', \%cmd_opts) or pod2usage( { -verbose => 1, -input => __FILE__} );
+getopts('t:w:rs:i:e:', \%cmd_opts) or pod2usage( { -verbose => 1, -input => __FILE__} );
 
 my $WORK_DIR    = Utils::get_abs_path($cmd_opts{w} // ".");
 my $REL_TESTS   = defined $cmd_opts{r} ? 1 : 0;
 my $SINGLE_TEST = $cmd_opts{t};
 my $TEST_SUITE  = $cmd_opts{s};
 my $INSTRUMENT  = $cmd_opts{i};
+my $EXCL        = $cmd_opts{e};
 if ((defined $SINGLE_TEST || defined $TEST_SUITE) && $REL_TESTS==1) {
     pod2usage( { -verbose => 1, -input => __FILE__} );
 }
@@ -146,20 +152,18 @@ $ENV{MML} = $mml_file;
 $project->mutate() > 0 or die "Cannot mutate project!";
 
 my $log_file = "$WORK_DIR/.mutation.log";
-# By default we do not exclude any mutants
-system("touch $WORK_DIR/exclude.txt") unless -e "$WORK_DIR/exclude.txt";
 # Run the test suite, according to the provided flags
 my $mut_result;
 if (defined $TEST_SUITE) {
     # Run external test suite
     my $test_dir = "$WORK_DIR/.test_suite";
     Utils::extract_test_suite($TEST_SUITE, $test_dir) or die;
     $project->compile_ext_tests($test_dir) or die "Cannot compile extracted test suite!";
-    $mut_result = $project->mutation_analysis_ext($test_dir, "*.java", $log_file, $SINGLE_TEST);
+    $mut_result = $project->mutation_analysis_ext($test_dir, "*.java", $log_file, $EXCL, $SINGLE_TEST);
 } else {
     # Run developer-written tests
     $project->compile_tests() or die "Cannot compile test suite!";
-    $mut_result = $project->mutation_analysis($log_file, $REL_TESTS, $SINGLE_TEST);
+    $mut_result = $project->mutation_analysis($log_file, $REL_TESTS, $EXCL, $SINGLE_TEST);
 }
 
 if (! $mut_result) {

framework/bin/run_mutation.pl

@@ -30,7 +30,7 @@ =head1 NAME
 
 =head1 SYNOPSIS
 
-  run_mutation.pl -p project_id -d suite_dir -o out_dir [-f include_file_pattern] [-v version_id] [-t tmp_dir] [-D] [-A]
+  run_mutation.pl -p project_id -d suite_dir -o out_dir [-f include_file_pattern] [-v version_id] [-t tmp_dir] [-e exclude_file] [-D] [-A]
 
 =head1 OPTIONS
 
@@ -65,6 +65,11 @@ =head1 OPTIONS
 The temporary root directory to be used to check out program versions (optional).
 The default is F</tmp>.
 
+=item -e F<exclude_file>
+
+The file that contains the list of all mutants ids (one per row) to exclude (optional).
+Per default no exclude file is used and therefore no mutant is excluded.
+
 =item -D
 
 Debug: Enable verbose logging and do not delete the temporary check-out directory
@@ -111,7 +116,7 @@ =head1 DESCRIPTION
 # Process arguments and issue usage message if necessary.
 #
 my %cmd_opts;
-getopts('p:d:v:t:o:f:DA', \%cmd_opts) or pod2usage(1);
+getopts('p:d:v:t:o:f:e:DA', \%cmd_opts) or pod2usage(1);
 
 pod2usage(1) unless defined $cmd_opts{p} and defined $cmd_opts{d} and defined $cmd_opts{o};
 
@@ -122,6 +127,7 @@ =head1 DESCRIPTION
 my $SUITE_DIR = abs_path($cmd_opts{d});
 my $VID = $cmd_opts{v} if defined $cmd_opts{v};
 my $INCL = $cmd_opts{f} // "*.java";
+my $EXCL = $cmd_opts{e};
 # Enable debugging if flag is set
 $DEBUG = 1 if defined $cmd_opts{D};
 
@@ -266,6 +272,19 @@ sub _run_mutation {
     $project->{prog_root} = "$root";
     $project->checkout_vid($vid) or die "Checkout failed";
 
+    my $num_excluded_mutants = 0;
+    if (defined $EXCL) {
+      # Count number of excluded mutants: number of lines, excluding:
+      #   - empty lines, i.e., lines with only spaces or tabs, or truly empty lines
+      #   - lines with comments, i.e., lines that start with '#'
+      open(EXCL_FILE, "<$EXCL") or die "Cannot read exclude file";
+      while (<EXCL_FILE>) {
+        next if /^[[:space:]]*(#.*)?$/;
+        $num_excluded_mutants += 1;
+      }
+      close(EXCL_FILE);
+    }
+
     # Create mutation definitions (mml file)
     my $mml_dir = "$TMP_DIR/.mml";
     system("$UTIL_DIR/create_mml.pl -p $PID -c $TARGET_CLASSES_DIR/$bid.src -o $mml_dir -b $bid");
@@ -283,10 +302,10 @@ sub _run_mutation {
     # No need to run the test suite first. Major's preprocessing verifies that
     # all tests in the test suite pass before performing the mutation analysis.
     my $mut_log = "$TMP_DIR/.mutation.log"; `>$mut_log`;
-    my $mut_map = Mutation::mutation_analysis_ext($project, $test_dir, "$INCL", $mut_log);
+    my $mut_map = Mutation::mutation_analysis_ext($project, $test_dir, "$INCL", $mut_log, $EXCL);
     if (defined $mut_map) {
         # Add results to database table
-        Mutation::insert_row($OUT_DIR, $PID, $vid, $suite_src, $test_id, $gen_mutants, $mut_map);
+        Mutation::insert_row($OUT_DIR, $PID, $vid, $suite_src, $test_id, $gen_mutants, $num_excluded_mutants, $mut_map);
     } else {
         # Add incomplete results to database table to indicate a broken test suite
         Mutation::insert_row($OUT_DIR, $PID, $vid, $suite_src, $test_id, "-");

framework/core/DB.pm

@@ -124,6 +124,17 @@ The name of the database table for the results of patch review (I<review>)
 =cut
 our $TAB_REVIEW = ($ENV{TAB_REVIEW} or "review");
 
+=pod
+
+=item C<TAB_FIX>
+
+=back
+
+The name of the database table for the results of fixing automatically generated test cases (I<fix>)
+
+=cut
+our $TAB_FIX = ($ENV{TAB_FIX} or "fix");
+
 # Common columns for all tables
 our $PROJECT       = "project_id";
 our $ID            = "version_id";
@@ -146,7 +157,7 @@ our $FAIL_M_V1     = "num_fail_methods_t2v1";
 our $FAIL_ISO_V1   = "fail_iso_t2v1";
 our $PASS_ISO_V2   = "pass_iso_t2v2";
 
-# Additional columns of TAB_BUG_DETECTION and TAB_MUTATION
+# Additional columns of TAB_BUG_DETECTION, TAB_MUTATION, and TAB_FIX
 our $TEST_SUITE = "test_suite_source";
 our $TEST_ID    = "test_id";
 
@@ -156,6 +167,7 @@ our $NUM_TRIGGER= "num_trigger";
 
 # Additional columns of TAB_MUTATION
 our $MUT_GEN    = "mut_generated";
+our $MUT_EXCL   = "mut_excluded";
 our $MUT_COV    = "mut_covered";
 our $MUT_KILL   = "mut_killed";
 
@@ -169,6 +181,11 @@ our $BRANCHES_COVERED = "branches_covered";
 our $NUM_COMMITS      = "num_commits";
 our $PASSED_COMMITS   = "passed_commits";
 
+# Additional columns of TAB_FIX
+our $NUM_UNCOMPILABLE_TESTS        = "num_uncompilable_tests";
+our $NUM_UNCOMPILABLE_TEST_CLASSES = "num_uncompilable_test_classes";
+our $NUM_FAILING_TESTS             = "num_failing_tests";
+
 # Table definitions
 my %tables = (
 # TAB_REV_PAIRS
@@ -178,11 +195,13 @@ $TAB_TRIGGER => [$PROJECT, $ID, $FAIL_V2, $FAIL_C_V1, $FAIL_M_V1, $PASS_ISO_V2,
 # Table TAB_BUG_DETECTION
 $TAB_BUG_DETECTION => [$PROJECT, $ID, $TEST_SUITE, $TEST_ID, $TEST_CLASS, $NUM_TRIGGER],
 # Table TAB_MUTATION
-$TAB_MUTATION => [$PROJECT, $ID, $TEST_SUITE, $TEST_ID, $MUT_GEN, $MUT_COV, $MUT_KILL],
+$TAB_MUTATION => [$PROJECT, $ID, $TEST_SUITE, $TEST_ID, $MUT_GEN, $MUT_EXCL, $MUT_COV, $MUT_KILL],
 # Table TAB_COVERAGE
 $TAB_COVERAGE => [$PROJECT, $ID, $TEST_SUITE, $TEST_ID, $LINES_TOTAL, $LINES_COVERED, $BRANCHES_TOTAL, $BRANCHES_COVERED],
 # Table TAB_CODE_EVOLUTION
 $TAB_CODE_EVOLUTION => [$PROJECT, $ID, $TEST_SUITE, $TEST_ID, $NUM_COMMITS, $PASSED_COMMITS, $TEST_CLASS, $NUM_TRIGGER],
+# Table TAB_FIX
+$TAB_FIX => [$PROJECT, $ID, $TEST_SUITE, $TEST_ID, $NUM_UNCOMPILABLE_TESTS, $NUM_UNCOMPILABLE_TEST_CLASSES, $NUM_FAILING_TESTS],
 );
 
 
@@ -194,6 +213,7 @@ our %PRIMARY_KEYS = (
     $TAB_MUTATION => 4,
     $TAB_COVERAGE => 4,
     $TAB_CODE_EVOLUTION => 4,
+    $TAB_FIX => 4,
 );
 
 our @EXPORT = qw(
@@ -205,6 +225,7 @@ $TAB_MUTATION
 $TAB_REVIEW
 $TAB_COVERAGE
 $TAB_CODE_EVOLUTION
+$TAB_FIX
 
 $PROJECT
 $ID
@@ -227,6 +248,7 @@ $TEST_ID
 $TEST_CLASS
 $NUM_TRIGGER
 $MUT_GEN
+$MUT_EXCL
 $MUT_COV
 $MUT_KILL
 $LINES_TOTAL
@@ -235,6 +257,9 @@ $BRANCHES_TOTAL
 $BRANCHES_COVERED
 $NUM_COMMITS
 $PASSED_COMMITS
+$NUM_UNCOMPILABLE_TESTS
+$NUM_UNCOMPILABLE_TEST_CLASSES
+$NUM_FAILING_TESTS
 
 %PRIMARY_KEYS
 );