diff --git a/framework/bin/d4j/d4j-mutation b/framework/bin/d4j/d4j-mutation index 53616aa1d..149b8b343 100644 --- a/framework/bin/d4j/d4j-mutation +++ b/framework/bin/d4j/d4j-mutation @@ -133,6 +133,9 @@ if ($type eq "b") { exit 1; } +# The mutation operators that should be enabled +my @MUT_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD"); + # Instantiate project and set working directory my $project = Project::create_project($pid); $project->{prog_root} = $WORK_DIR; @@ -140,16 +143,7 @@ $project->{prog_root} = $WORK_DIR; # Classes to mutate -- default is all modified classes my $classes = $INSTRUMENT // "$SCRIPT_DIR/projects/$pid/modified_classes/$bid.src"; -# Create mutation definitions (mml file) -my $mml_dir = "$WORK_DIR/mml"; - -system("$UTIL_DIR/create_mml.pl -p $pid -c $classes -o $mml_dir -b $bid"); -my $mml_file = "$mml_dir/${bid}.mml.bin"; --e $mml_file or die "Mml file does not exist: $mml_file!"; - -# Mutate and compile sources -$ENV{MML} = $mml_file; -$project->mutate() > 0 or die "Cannot mutate project!"; +$project->mutate($classes, \@MUT_OPS) > 0 or die "Cannot mutate project!"; my $log_file = "$WORK_DIR/.mutation.log"; # Run the test suite, according to the provided flags diff --git a/framework/bin/run_mutation.pl b/framework/bin/run_mutation.pl index 12eecc2fb..548ca6db6 100755 --- a/framework/bin/run_mutation.pl +++ b/framework/bin/run_mutation.pl @@ -131,6 +131,9 @@ =head1 DESCRIPTION # Enable debugging if flag is set $DEBUG = 1 if defined $cmd_opts{D}; +# The mutation operators that should be enabled +my @MUT_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD"); + # Set up project my $project = Project::create_project($PID); @@ -285,15 +288,7 @@ sub _run_mutation { close(EXCL_FILE); } - # Create mutation definitions (mml file) - my $mml_dir = "$TMP_DIR/.mml"; - system("$UTIL_DIR/create_mml.pl -p $PID -c $TARGET_CLASSES_DIR/$bid.src -o $mml_dir -b $bid"); - my $mml_file = "$mml_dir/$bid.mml.bin"; - -e $mml_file or die "Mml file does not exist: $mml_file!"; - - # Mutate source code - $ENV{MML} = $mml_file; - my $gen_mutants = $project->mutate(); + my $gen_mutants = $project->mutate("$TARGET_CLASSES_DIR/$bid.src", \@MUT_OPS); $gen_mutants > 0 or die "No mutants generated for $vid!"; # Compile generated tests diff --git a/framework/core/Constants.pm b/framework/core/Constants.pm index e2a7b4ad2..2935ba6d3 100644 --- a/framework/core/Constants.pm +++ b/framework/core/Constants.pm @@ -196,8 +196,10 @@ our $CONFIG_VID = "vid"; # Filename which stores build properties our $PROP_FILE = "defects4j.build.properties"; -our $PROP_EXCLUDE = "d4j.tests.exclude"; -our $PROP_INSTRUMENT = "d4j.classes.instrument"; +# Keys of stored properties +our $PROP_EXCLUDE = "d4j.tests.exclude"; +our $PROP_INSTRUMENT = "d4j.classes.instrument"; +our $PROP_MUTATE = "d4j.classes.mutate"; our $PROP_DIR_SRC_CLASSES = "d4j.dir.src.classes"; our $PROP_DIR_SRC_TESTS = "d4j.dir.src.tests"; our $PROP_CLASSES_MODIFIED= "d4j.classes.modified"; @@ -234,6 +236,7 @@ $CONFIG_VID $PROP_FILE $PROP_EXCLUDE $PROP_INSTRUMENT +$PROP_MUTATE $PROP_DIR_SRC_CLASSES $PROP_DIR_SRC_TESTS $PROP_CLASSES_MODIFIED diff --git a/framework/core/Mutation.pm b/framework/core/Mutation.pm index 3b3d095c2..c77c88026 100644 --- a/framework/core/Mutation.pm +++ b/framework/core/Mutation.pm @@ -54,6 +54,48 @@ my $SUMMARY_FILE = "summary.csv"; =head2 Static subroutines + Mutation::create_mml(instrument_classes, out_file, mut_ops) + +Generates an mml file, enabling all mutation operators defined by the array +reference C for all classes listed in F. The mml +(source) file is written to C. This subroutine also compiles the mml +file to F<'out_file'.bin>. + +=cut +sub create_mml { + @_ == 3 or die $ARG_ERROR; + my ($instrument_classes, $out_file, $mut_ops) = @_; + + my $OUT_DIR = Utils::get_dir($out_file); + my $TEMPLATE = `cat $MAJOR_ROOT/mml/template.mml` or die "Cannot read mml template: $!"; + + system("mkdir -p $OUT_DIR"); + + open(IN, $instrument_classes); + my @classes = ; + close(IN); + + # Generate mml file by enabling operators for listed classes only + open(FILE, ">$out_file") or die "Cannot write mml file ($out_file): $!"; + # Add operator definitions from template + print FILE $TEMPLATE; + # Enable operators for all classes + foreach my $class (@classes) { + chomp $class; + print FILE "\n// Enable operators for $class\n"; + foreach my $op (@{$mut_ops}) { + # Skip disabled operators + next if $TEMPLATE =~ /-$op<"$class">/; + print FILE "$op<\"$class\">;\n"; + } + } + close(FILE); + Utils::exec_cmd("$MAJOR_ROOT/bin/mmlc $out_file 2>&1", "Compiling mutant definition (mml)") + or die "Cannot compile mml file: $out_file!"; +} + +=pod + Mutation::mutation_analysis(project_ref, log_file [, exclude_file, base_map, single_test]) Runs mutation analysis for the developer-written test suites of the provided diff --git a/framework/core/Project.pm b/framework/core/Project.pm index 7384a974c..cc3e6bb22 100644 --- a/framework/core/Project.pm +++ b/framework/core/Project.pm @@ -83,6 +83,10 @@ Commons lang (L backend) Commons math (L backend) +=item * L + +Mockito (L backend) + =item * L Joda-time (L backend) @@ -96,6 +100,7 @@ use warnings; use strict; use Constants; use Utils; +use Mutation; use Carp qw(confess); =pod @@ -687,14 +692,43 @@ sub coverage_report { =pod - $project->mutate() + $project->mutate(instrument_classes, mut_ops) -Mutates the checked-out program version. +Mutates all classes listed in F, using all mutation operators +defined by the array reference C, in the checked-out program version. Returns the number of generated mutants on success, -1 otherwise. =cut sub mutate { - my $self = shift; + @_ == 3 or die $ARG_ERROR; + my ($self, $instrument_classes, $mut_ops) = @_; + my $work_dir = $self->{prog_root}; + + # Read all classes that should be mutated + -e $instrument_classes or die "Classes file ($instrument_classes) does not exist!"; + open(IN, "<$instrument_classes") or die "Cannot read $instrument_classes"; + my @classes = (); + while() { + s/\r?\n//; + push(@classes, $_); + } + close(IN); + # Update properties + my $list = join(",", @classes); + my $config = {$PROP_MUTATE => $list}; + Utils::write_config_file("$work_dir/$PROP_FILE", $config); + + # Create mutation definitions (mml file) + my $mml_src = "$self->{prog_root}/.mml/default.mml"; + my $mml_bin = "${mml_src}.bin"; + + Mutation::create_mml($instrument_classes, $mml_src, $mut_ops); + -e "$mml_bin" or die "Mml file does not exist: $mml_bin!"; + + # Set environment variable MML, which is read by Major + $ENV{MML} = $mml_bin; + + # Mutate and compile sources if (! $self->_ant_call("mutate")) { return -1; } diff --git a/framework/core/Utils.pm b/framework/core/Utils.pm index bea463641..916d54cb7 100644 --- a/framework/core/Utils.pm +++ b/framework/core/Utils.pm @@ -37,6 +37,7 @@ use warnings; use strict; use File::Basename; +use File::Spec; use Cwd qw(abs_path); use Carp qw(confess); @@ -77,6 +78,20 @@ sub get_abs_path { return abs_path($dir); } +=pod + + Utils::get_dir(file) + +Returns the directory of the absolute path of F. + +=cut +sub get_dir { + @_ == 1 or die $ARG_ERROR; + my $path = shift; + my ($volume,$dir,$file) = File::Spec->splitpath($path); + return get_abs_path($dir); +} + =pod Utils::get_failing_tests(test_result_file) diff --git a/framework/test/test_mutation_analysis.sh b/framework/test/test_mutation_analysis.sh index a7eb4ae3c..f88db363a 100755 --- a/framework/test/test_mutation_analysis.sh +++ b/framework/test/test_mutation_analysis.sh @@ -13,7 +13,8 @@ HERE=$(cd `dirname $0` && pwd) source "$HERE/test.include" || exit 1 init -pid="Lang" # any pid-bid should work +# Any version should work, but the test cases below are specific to this version +pid="Lang" bid="1f" pid_bid_dir="$TMP_DIR/$pid-$bid" rm -rf "$pid_bid_dir" @@ -22,61 +23,80 @@ rm -rf "$pid_bid_dir" summary_file="$pid_bid_dir/summary.csv" mutants_file="$pid_bid_dir/mutants.log" +################################################################################ +# +# Check whether the mutation analysis results (summary.csv) match the expectations. +# +_check_mutation_result() { + [ $# -eq 3 ] || die "usage: ${FUNCNAME[0]} \ + \ + \ + " + local exp_mut_gen=$1 + local exp_mut_cov=$2 + local exp_mut_kill=$3 + + # Make sure Major generated the expected data files + [ -s "$mutants_file" ] || die "'$mutants_file' doesn't exist or is empty!" + [ -s "$summary_file" ] || die "'$summary_file' doesn't exist or is empty!" + + # The last row of 'summary.csv' does not have an end of line character. + # Otherwise, using wc would be more intuitive. + local num_rows=$(grep -c "^" "$summary_file") + [ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!" + + # Columns of summary (csv) file: + # MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds + local act_mut_gen=$(tail -n1 "$summary_file" | cut -f1 -d',') + local act_mut_cov=$(tail -n1 "$summary_file" | cut -f2 -d',') + local act_mut_kill=$(tail -n1 "$summary_file" | cut -f3 -d',') + + [ "$act_mut_gen" -eq "$exp_mut_gen" ] || die "Unexpected number of mutants generated (expected: $exp_mut_gen, actual: $act_mut_gen)!" + [ "$act_mut_cov" -eq "$exp_mut_cov" ] || die "Unexpected number of mutants covered (expected: $exp_mut_cov, actual: $act_mut_cov)!" + [ "$act_mut_kill" -eq "$exp_mut_kill" ] || die "Unexpected number of mutants killed (expected: $exp_mut_kill, actual: $act_mut_kill)!" + + # TODO Would be nice to test the number of excluded mutants. In order to do it + # Major has to write that number to the '$pid_bid_dir/summary.csv' file. +} +################################################################################ + # Checkout project-version defects4j checkout -p "$pid" -v "$bid" -w "$pid_bid_dir" || die "It was not possible to checkout $pid-$bid to '$pid_bid_dir'!" ###################################################### # Test mutation analysis without excluding any mutants -defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!" - -# Make sure Major generated the expected data files -[ -s "$mutants_file" ] || die "'$mutants_file' doesn't exist or is empty!" -[ -s "$summary_file" ] || die "'$summary_file' doesn't exist or is empty!" +# Remove the summary file to ensure it is regenerated +rm -f "$summary_file" -# The last row of 'summary.csv' does not have an end of line character. -# Otherwise, using wc would be more intuitive. -num_rows=$(grep -c "^" "$summary_file") -[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!" - -# Columns of summary (csv) file: -# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds -num_mutants_covered=$(tail -n1 "$summary_file" | cut -f2 -d',') -num_mutants_killed=$(tail -n1 "$summary_file" | cut -f3 -d',') - -[ "$num_mutants_covered" -gt 0 ] || die "0 mutants covered!" -[ "$num_mutants_killed" -gt 0 ] || die "0 mutants killed!" - -# TODO Would be nice to test the number of excluded mutants. In order to do it -# Major has to write that number to the '$pid_bid_dir/summary.csv' file. - -# Remove the summary file to ensure it is regenerated later -rm "$summary_file" +defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!" +_check_mutation_result 941 913 646 ################################################### # Test mutation analysis when excluding all mutants +# Remove the summary file to ensure it is regenerated +rm -f "$summary_file" + # Exclude all generated mutants exclude_file="$pid_bid_dir/exclude_all_mutants.txt" cut -f1 -d':' "$mutants_file" > "$exclude_file" defects4j mutation -w "$pid_bid_dir" -r -e "$exclude_file" || die "Mutation analysis (excluding all mutants) failed!" +_check_mutation_result 941 0 0 -# The last row of 'summary.csv' does not have an end of line character. -# Otherwise, using wc would be more intuitive. -num_rows=$(grep -c "^" "$summary_file") -[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!" +########################################################################## +# Test mutation analysis when explicitly providing the class(es) to mutate -# Columns of summary (csv) file: -# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds -num_mutants_covered=$(tail -n1 "$summary_file" | cut -f2 -d',') -num_mutants_killed=$(tail -n1 "$summary_file" | cut -f3 -d',') +# Remove the summary file to ensure it is regenerated +rm -f "$summary_file" -[ "$num_mutants_covered" -eq 0 ] || die "$num_mutants_covered mutants have been covered!" -[ "$num_mutants_killed" -eq 0 ] || die "$num_mutants_killed mutants have been killed!" +# Mutate an arbitrary, non-modified class +instrument_classes="$pid_bid_dir/instrument_classes.txt" +echo "org.apache.commons.lang3.LocaleUtils" > "$instrument_classes" -# TODO Would be nice to test the number of excluded mutants. In order to do it -# Major has to write that number to the '$pid_bid_dir/summary.csv' file. +defects4j mutation -w "$pid_bid_dir" -i "$instrument_classes" || die "Mutation analysis (instrument LocaleUtils) failed!" +_check_mutation_result 184 184 158 # Clean up rm -rf "$pid_bid_dir" diff --git a/framework/util/create_mml.pl b/framework/util/create_mml.pl index b8bdedc50..5b02b7256 100755 --- a/framework/util/create_mml.pl +++ b/framework/util/create_mml.pl @@ -73,6 +73,7 @@ =head1 OPTIONS use lib abs_path("$FindBin::Bin/../core"); use Constants; +use Mutation; use Utils; # @@ -83,6 +84,7 @@ =head1 OPTIONS pod2usage(1) unless defined $cmd_opts{p} and defined $cmd_opts{c} and defined $cmd_opts{o} and defined $cmd_opts{b}; +# TODO: Unused parameter: PID my $PID = $cmd_opts{p}; my $CLASSES = Utils::get_abs_path($cmd_opts{c}); my $OUT_DIR = Utils::get_abs_path($cmd_opts{o}); @@ -91,33 +93,11 @@ =head1 OPTIONS $BID =~ /^(\d+)$/ or die "Wrong bug id format (\\d+): $BID!"; -e $CLASSES or die "File with classes to mutate does not exist: $CLASSES"; -my $TEMPLATE = `cat $MAJOR_ROOT/mml/template.mml` or die "Cannot read mml template: $!"; - -# The mutation operators that should be enabled in the mml file -my @ops =("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD"); - -system("mkdir -p $OUT_DIR"); - -open(IN, $CLASSES); -my @classes = ; -close(IN); - -my $file = "$OUT_DIR/$BID.mml"; - -# Generate mml file by enabling operators for listed classes only -open(FILE, ">$file") or die "Cannot write mml file ($file): $!"; -# Add operator definitions from template -print FILE $TEMPLATE; -# Enable operators for all classes -foreach my $class (@classes) { - chomp $class; - print FILE "\n// Enable operators for $class\n"; - foreach my $op (@ops) { - # Skip disabled operators - next if $TEMPLATE =~ /-$op<"$class">/; - print FILE "$op<\"$class\">;\n"; - } -} -close(FILE); -my $log = `$MAJOR_ROOT/bin/mmlc $file 2>&1`; -$? == 0 or die "Cannot compile mml file: $file\n$log"; +# The mutation operators that should be enabled +my @MU_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD"); + +my $mml_src = "$OUT_DIR/$BID.mml"; +my $mml_bin = "${mml_src}.bin"; + +Mutation::create_mml($CLASSES, $mml_src, \@MUT_OPS); +-e "$mml_bin" or die "Mml file does not exist: $mml_bin!";