Merge pull request #108 from rjust/fix_mutant_caching

Clean up mutation-related modules and fix issue #87

framework/bin/d4j/d4j-mutation

@@ -133,23 +133,17 @@ if ($type eq "b") {
     exit 1;
 }
 
+# The mutation operators that should be enabled
+my @MUT_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD");
+
 # Instantiate project and set working directory
 my $project = Project::create_project($pid);
 $project->{prog_root} = $WORK_DIR;
 
 # Classes to mutate -- default is all modified classes
 my $classes = $INSTRUMENT // "$SCRIPT_DIR/projects/$pid/modified_classes/$bid.src";
 
-# Create mutation definitions (mml file)
-my $mml_dir = "$WORK_DIR/mml";
-
-system("$UTIL_DIR/create_mml.pl -p $pid -c $classes -o $mml_dir -b $bid");
-my $mml_file = "$mml_dir/${bid}.mml.bin";
--e $mml_file or die "Mml file does not exist: $mml_file!";
-
-# Mutate and compile sources
-$ENV{MML} = $mml_file;
-$project->mutate() > 0 or die "Cannot mutate project!";
+$project->mutate($classes, \@MUT_OPS) > 0 or die "Cannot mutate project!";
 
 my $log_file = "$WORK_DIR/.mutation.log";
 # Run the test suite, according to the provided flags

framework/bin/run_mutation.pl

@@ -131,6 +131,9 @@ =head1 DESCRIPTION
 # Enable debugging if flag is set
 $DEBUG = 1 if defined $cmd_opts{D};
 
+# The mutation operators that should be enabled
+my @MUT_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD");
+
 # Set up project
 my $project = Project::create_project($PID);
 
@@ -285,15 +288,7 @@ sub _run_mutation {
       close(EXCL_FILE);
     }
 
-    # Create mutation definitions (mml file)
-    my $mml_dir = "$TMP_DIR/.mml";
-    system("$UTIL_DIR/create_mml.pl -p $PID -c $TARGET_CLASSES_DIR/$bid.src -o $mml_dir -b $bid");
-    my $mml_file = "$mml_dir/$bid.mml.bin";
-    -e $mml_file or die "Mml file does not exist: $mml_file!";
-
-    # Mutate source code
-    $ENV{MML} = $mml_file;
-    my $gen_mutants = $project->mutate();
+    my $gen_mutants = $project->mutate("$TARGET_CLASSES_DIR/$bid.src", \@MUT_OPS);
     $gen_mutants > 0 or die "No mutants generated for $vid!";
 
     # Compile generated tests

framework/core/Constants.pm

@@ -196,8 +196,10 @@ our $CONFIG_VID = "vid";
 
 # Filename which stores build properties
 our $PROP_FILE       = "defects4j.build.properties";
-our $PROP_EXCLUDE    = "d4j.tests.exclude";
-our $PROP_INSTRUMENT = "d4j.classes.instrument";
+# Keys of stored properties
+our $PROP_EXCLUDE         = "d4j.tests.exclude";
+our $PROP_INSTRUMENT      = "d4j.classes.instrument";
+our $PROP_MUTATE          = "d4j.classes.mutate";
 our $PROP_DIR_SRC_CLASSES = "d4j.dir.src.classes";
 our $PROP_DIR_SRC_TESTS   = "d4j.dir.src.tests";
 our $PROP_CLASSES_MODIFIED= "d4j.classes.modified";
@@ -234,6 +236,7 @@ $CONFIG_VID
 $PROP_FILE
 $PROP_EXCLUDE
 $PROP_INSTRUMENT
+$PROP_MUTATE
 $PROP_DIR_SRC_CLASSES
 $PROP_DIR_SRC_TESTS
 $PROP_CLASSES_MODIFIED

framework/core/Mutation.pm

@@ -54,6 +54,48 @@ my $SUMMARY_FILE = "summary.csv";
 
 =head2 Static subroutines
 
+  Mutation::create_mml(instrument_classes, out_file, mut_ops)
+
+Generates an mml file, enabling all mutation operators defined by the array
+reference C<mut_ops> for all classes listed in F<instrument_classes>. The mml
+(source) file is written to C<out_file>. This subroutine also compiles the mml
+file to F<'out_file'.bin>.
+
+=cut
+sub create_mml {
+    @_ == 3 or die $ARG_ERROR;
+    my ($instrument_classes, $out_file, $mut_ops) = @_;
+
+    my $OUT_DIR = Utils::get_dir($out_file);
+    my $TEMPLATE = `cat $MAJOR_ROOT/mml/template.mml` or die "Cannot read mml template: $!";
+
+    system("mkdir -p $OUT_DIR");
+
+    open(IN, $instrument_classes);
+    my @classes = <IN>;
+    close(IN);
+
+    # Generate mml file by enabling operators for listed classes only
+    open(FILE, ">$out_file") or die "Cannot write mml file ($out_file): $!";
+    # Add operator definitions from template
+    print FILE $TEMPLATE;
+    # Enable operators for all classes
+    foreach my $class (@classes) {
+        chomp $class;
+        print FILE "\n// Enable operators for $class\n";
+        foreach my $op (@{$mut_ops}) {
+            # Skip disabled operators
+            next if $TEMPLATE =~ /-$op<"$class">/;
+            print FILE "$op<\"$class\">;\n";
+        }
+    }
+    close(FILE);
+    Utils::exec_cmd("$MAJOR_ROOT/bin/mmlc $out_file 2>&1", "Compiling mutant definition (mml)")
+            or die "Cannot compile mml file: $out_file!";
+}
+
+=pod
+
   Mutation::mutation_analysis(project_ref, log_file [, exclude_file, base_map, single_test])
 
 Runs mutation analysis for the developer-written test suites of the provided

framework/core/Project.pm

@@ -83,6 +83,10 @@ Commons lang (L<Vcs::Git> backend)
 
 Commons math (L<Vcs::Git> backend)
 
+=item * L<Mockito|Project::Mockito>
+
+Mockito (L<Vcs::Git> backend)
+
 =item * L<Time|Project::Time>
 
 Joda-time (L<Vcs::Git> backend)
@@ -96,6 +100,7 @@ use warnings;
 use strict;
 use Constants;
 use Utils;
+use Mutation;
 use Carp qw(confess);
 
 =pod
@@ -687,14 +692,43 @@ sub coverage_report {
 
 =pod
 
-  $project->mutate()
+  $project->mutate(instrument_classes, mut_ops)
 
-Mutates the checked-out program version.
+Mutates all classes listed in F<instrument_classes>, using all mutation operators
+defined by the array reference C<mut_ops>, in the checked-out program version.
 Returns the number of generated mutants on success, -1 otherwise.
 
 =cut
 sub mutate {
-    my $self = shift;
+    @_ == 3 or die $ARG_ERROR;
+    my ($self, $instrument_classes, $mut_ops)  = @_;
+    my $work_dir = $self->{prog_root};
+
+    # Read all classes that should be mutated
+    -e $instrument_classes or die "Classes file ($instrument_classes) does not exist!";
+    open(IN, "<$instrument_classes") or die "Cannot read $instrument_classes";
+    my @classes = ();
+    while(<IN>) {
+        s/\r?\n//;
+        push(@classes, $_);
+    }
+    close(IN);
+    # Update properties
+    my $list = join(",", @classes);
+    my $config = {$PROP_MUTATE => $list};
+    Utils::write_config_file("$work_dir/$PROP_FILE", $config);
+
+    # Create mutation definitions (mml file)
+    my $mml_src = "$self->{prog_root}/.mml/default.mml";
+    my $mml_bin = "${mml_src}.bin";
+
+    Mutation::create_mml($instrument_classes, $mml_src, $mut_ops);
+    -e "$mml_bin" or die "Mml file does not exist: $mml_bin!";
+
+    # Set environment variable MML, which is read by Major
+    $ENV{MML} = $mml_bin;
+
+    # Mutate and compile sources
     if (! $self->_ant_call("mutate")) {
         return -1;
     }

framework/core/Utils.pm

@@ -37,6 +37,7 @@ use warnings;
 use strict;
 
 use File::Basename;
+use File::Spec;
 use Cwd qw(abs_path);
 use Carp qw(confess);
 
@@ -77,6 +78,20 @@ sub get_abs_path {
     return abs_path($dir);
 }
 
+=pod
+
+  Utils::get_dir(file)
+
+Returns the directory of the absolute path of F<file>.
+
+=cut
+sub get_dir {
+    @_ == 1 or die $ARG_ERROR;
+    my $path = shift;
+    my ($volume,$dir,$file) = File::Spec->splitpath($path);
+    return get_abs_path($dir);
+}
+
 =pod
 
   Utils::get_failing_tests(test_result_file)

framework/test/test_mutation_analysis.sh

@@ -13,7 +13,8 @@ HERE=$(cd `dirname $0` && pwd)
 source "$HERE/test.include" || exit 1
 init
 
-pid="Lang" # any pid-bid should work
+# Any version should work, but the test cases below are specific to this version
+pid="Lang"
 bid="1f"
 pid_bid_dir="$TMP_DIR/$pid-$bid"
 rm -rf "$pid_bid_dir"
@@ -22,61 +23,80 @@ rm -rf "$pid_bid_dir"
 summary_file="$pid_bid_dir/summary.csv"
 mutants_file="$pid_bid_dir/mutants.log"
 
+################################################################################
+#
+# Check whether the mutation analysis results (summary.csv) match the expectations.
+#
+_check_mutation_result() {
+    [ $# -eq 3 ] || die "usage: ${FUNCNAME[0]} \
+            <expected_mutants_generated> \
+            <expected_mutants_covered> \
+            <expected_mutants_killed>"
+    local exp_mut_gen=$1
+    local exp_mut_cov=$2
+    local exp_mut_kill=$3
+
+    # Make sure Major generated the expected data files
+    [ -s "$mutants_file" ] || die "'$mutants_file' doesn't exist or is empty!"
+    [ -s "$summary_file" ] || die "'$summary_file' doesn't exist or is empty!"
+
+    # The last row of 'summary.csv' does not have an end of line character.
+    # Otherwise, using wc would be more intuitive.
+    local num_rows=$(grep -c "^" "$summary_file")
+    [ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!"
+
+    # Columns of summary (csv) file:
+    # MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds
+    local act_mut_gen=$(tail -n1 "$summary_file" | cut -f1 -d',')
+    local act_mut_cov=$(tail -n1 "$summary_file" | cut -f2 -d',')
+    local act_mut_kill=$(tail -n1 "$summary_file" | cut -f3 -d',')
+
+    [ "$act_mut_gen"  -eq "$exp_mut_gen" ] || die "Unexpected number of mutants generated (expected: $exp_mut_gen, actual: $act_mut_gen)!"
+    [ "$act_mut_cov"  -eq "$exp_mut_cov" ] || die "Unexpected number of mutants covered (expected: $exp_mut_cov, actual: $act_mut_cov)!"
+    [ "$act_mut_kill" -eq "$exp_mut_kill" ] || die "Unexpected number of mutants killed (expected: $exp_mut_kill, actual: $act_mut_kill)!"
+
+    # TODO Would be nice to test the number of excluded mutants. In order to do it
+    # Major has to write that number to the '$pid_bid_dir/summary.csv' file.
+}
+################################################################################
+
 # Checkout project-version
 defects4j checkout -p "$pid" -v "$bid" -w "$pid_bid_dir" || die "It was not possible to checkout $pid-$bid to '$pid_bid_dir'!"
 
 ######################################################
 # Test mutation analysis without excluding any mutants
 
-defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!"
-
-# Make sure Major generated the expected data files
-[ -s "$mutants_file" ] || die "'$mutants_file' doesn't exist or is empty!"
-[ -s "$summary_file" ] || die "'$summary_file' doesn't exist or is empty!"
+# Remove the summary file to ensure it is regenerated
+rm -f "$summary_file"
 
-# The last row of 'summary.csv' does not have an end of line character.
-# Otherwise, using wc would be more intuitive.
-num_rows=$(grep -c "^" "$summary_file")
-[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!"
-
-# Columns of summary (csv) file:
-# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds
-num_mutants_covered=$(tail -n1 "$summary_file" | cut -f2 -d',')
-num_mutants_killed=$(tail -n1 "$summary_file" | cut -f3 -d',')
-
-[ "$num_mutants_covered" -gt 0 ] || die "0 mutants covered!"
-[ "$num_mutants_killed" -gt 0 ] || die "0 mutants killed!"
-
-# TODO Would be nice to test the number of excluded mutants. In order to do it
-# Major has to write that number to the '$pid_bid_dir/summary.csv' file.
-
-# Remove the summary file to ensure it is regenerated later
-rm "$summary_file"
+defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!"
+_check_mutation_result 941 913 646
 
 ###################################################
 # Test mutation analysis when excluding all mutants
 
+# Remove the summary file to ensure it is regenerated
+rm -f "$summary_file"
+
 # Exclude all generated mutants
 exclude_file="$pid_bid_dir/exclude_all_mutants.txt"
 cut -f1 -d':' "$mutants_file" > "$exclude_file"
 
 defects4j mutation -w "$pid_bid_dir" -r -e "$exclude_file" || die "Mutation analysis (excluding all mutants) failed!"
+_check_mutation_result 941 0 0
 
-# The last row of 'summary.csv' does not have an end of line character.
-# Otherwise, using wc would be more intuitive.
-num_rows=$(grep -c "^" "$summary_file")
-[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!"
+##########################################################################
+# Test mutation analysis when explicitly providing the class(es) to mutate
 
-# Columns of summary (csv) file:
-# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds
-num_mutants_covered=$(tail -n1 "$summary_file" | cut -f2 -d',')
-num_mutants_killed=$(tail -n1 "$summary_file" | cut -f3 -d',')
+# Remove the summary file to ensure it is regenerated
+rm -f "$summary_file"
 
-[ "$num_mutants_covered" -eq 0 ] || die "$num_mutants_covered mutants have been covered!"
-[ "$num_mutants_killed" -eq 0 ] || die "$num_mutants_killed mutants have been killed!"
+# Mutate an arbitrary, non-modified class
+instrument_classes="$pid_bid_dir/instrument_classes.txt"
+echo "org.apache.commons.lang3.LocaleUtils" > "$instrument_classes"
 
-# TODO Would be nice to test the number of excluded mutants. In order to do it
-# Major has to write that number to the '$pid_bid_dir/summary.csv' file.
+defects4j mutation -w "$pid_bid_dir" -i "$instrument_classes" || die "Mutation analysis (instrument LocaleUtils) failed!"
+_check_mutation_result 184 184 158
 
 # Clean up
 rm -rf "$pid_bid_dir"