Add support for not covered mutants

docs/generated/CLIOptions.rst

@@ -44,6 +44,8 @@
 
 --coverage-info string		Path to the coverage info file (LLVM's profdata)
 
+--include-not-covered		Include (but do not run) not covered mutants. Disabled by default
+
 --include-path regex		File/directory paths to whitelist (supports regex)
 
 --exclude-path regex		File/directory paths to ignore (supports regex)

include/mull/Config/Configuration.h

@@ -16,6 +16,7 @@ struct Configuration {
   bool captureTestOutput;
   bool captureMutantOutput;
   bool skipSanityCheckRun;
+  bool includeNotCovered;
 
   int timeout;
   unsigned linkerTimeout;

include/mull/ExecutionResult.h

@@ -11,7 +11,8 @@ enum ExecutionStatus {
   Crashed = 4,
   AbnormalExit = 5,
   DryRun = 6,
-  FailFast = 7
+  FailFast = 7,
+  NotCovered = 8
 };
 
 static std::string executionStatusAsString(ExecutionStatus status) {
@@ -34,6 +35,8 @@ static std::string executionStatusAsString(ExecutionStatus status) {
     return "DryRun";
   case FailFast:
     return "FailFast";
+  case NotCovered:
+    return "NotCovered";
   }
 }
 

include/mull/ReachableFunction.h → include/mull/FunctionUnderTest.h

@@ -13,15 +13,17 @@ class Bitcode;
 
 class FunctionUnderTest {
 public:
-  explicit FunctionUnderTest(llvm::Function *function, Bitcode *bitcode);
+  FunctionUnderTest(llvm::Function *function, Bitcode *bitcode, bool covered = true);
   llvm::Function *getFunction() const;
   Bitcode *getBitcode() const;
   const std::vector<llvm::Instruction *> &getSelectedInstructions() const;
+  bool isCovered() const;
   void selectInstructions(const std::vector<InstructionFilter *> &filters);
 
 private:
   llvm::Function *function;
   Bitcode *bitcode;
+  bool covered;
   std::vector<llvm::Instruction *> selectedInstructions;
 };
 

include/mull/Mutant.h

@@ -19,6 +19,7 @@ class Mutant {
   const std::string &getDiagnostics() const;
   const std::string &getReplacement() const;
   const std::string &getMutatorIdentifier() const;
+  bool isCovered() const;
 
 private:
   std::string identifier;

include/mull/MutationPoint.h

@@ -64,13 +64,17 @@ class MutationPoint {
   const SourceLocation sourceLocation;
   irm::IRMutation *irMutator;
   std::string userIdentifier;
+  bool covered;
 
 public:
   MutationPoint(Mutator *mutator, irm::IRMutation *irMutator, llvm::Instruction *instruction,
                 std::string replacement, Bitcode *m, std::string diagnostics);
 
   ~MutationPoint() = default;
 
+  void setCovered(bool isCovered);
+  bool isCovered();
+
   Mutator *getMutator();
   Mutator *getMutator() const;
 

include/mull/MutationsFinder.h

@@ -3,8 +3,8 @@
 #include <map>
 #include <vector>
 
+#include "FunctionUnderTest.h"
 #include "MutationPoint.h"
-#include "ReachableFunction.h"
 #include "mull/Mutators/Mutator.h"
 
 namespace mull {

include/mull/Mutators/CXX/TrivialCXXMutator.h

@@ -3,7 +3,7 @@
 #include "mull/Mutators/Mutator.h"
 #include <irm/irm.h>
 #include <memory>
-#include <mull/ReachableFunction.h>
+#include <mull/FunctionUnderTest.h>
 #include <vector>
 
 namespace mull {

include/mull/Mutators/NegateConditionMutator.h

@@ -3,7 +3,7 @@
 #include "Mutator.h"
 #include <irm/irm.h>
 #include <memory>
-#include <mull/ReachableFunction.h>
+#include <mull/FunctionUnderTest.h>
 #include <vector>
 
 namespace llvm {

include/mull/Mutators/RemoveVoidFunctionMutator.h

@@ -3,7 +3,7 @@
 #include "Mutator.h"
 #include <irm/irm.h>
 #include <memory>
-#include <mull/ReachableFunction.h>
+#include <mull/FunctionUnderTest.h>
 #include <vector>
 
 namespace llvm {

include/mull/Mutators/ReplaceCallMutator.h

@@ -3,7 +3,7 @@
 #include "Mutator.h"
 #include <irm/irm.h>
 #include <memory>
-#include <mull/ReachableFunction.h>
+#include <mull/FunctionUnderTest.h>
 #include <vector>
 
 namespace mull {

include/mull/Mutators/ScalarValueMutator.h

@@ -3,7 +3,7 @@
 #include "Mutator.h"
 #include <irm/irm.h>
 #include <memory>
-#include <mull/ReachableFunction.h>
+#include <mull/FunctionUnderTest.h>
 #include <vector>
 
 namespace llvm {

include/mull/Parallelization/Tasks/FunctionFilterTask.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "mull/ReachableFunction.h"
+#include "mull/FunctionUnderTest.h"
 #include <vector>
 
 namespace mull {

include/mull/Parallelization/Tasks/InstructionSelectionTask.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "mull/ReachableFunction.h"
+#include "mull/FunctionUnderTest.h"
 #include <vector>
 
 namespace mull {

include/mull/Parallelization/Tasks/SearchMutationPointsTask.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
 #include "mull/Mutators/Mutator.h"
-#include "mull/ReachableFunction.h"
 
 namespace mull {
 

lib/CMakeLists.txt

@@ -34,7 +34,7 @@ set(mull_sources
 
   Bitcode.cpp
   MutationPoint.cpp
-  ReachableFunction.cpp
+  FunctionUnderTest.cpp
 
   IDEDiagnostics.cpp
 

lib/Config/Configuration.cpp

@@ -17,7 +17,7 @@ unsigned MullDefaultLinkerTimeoutMilliseconds = 30000;
 
 Configuration::Configuration()
     : debugEnabled(false), dryRunEnabled(false), captureTestOutput(true), captureMutantOutput(true),
-      skipSanityCheckRun(false), timeout(MullDefaultTimeoutMilliseconds),
+      skipSanityCheckRun(false), includeNotCovered(false), timeout(MullDefaultTimeoutMilliseconds),
       linkerTimeout(MullDefaultLinkerTimeoutMilliseconds), diagnostics(IDEDiagnosticsKind::None),
       parallelization(singleThreadParallelization()) {}
 

lib/Driver.cpp

@@ -4,12 +4,12 @@
 #include "mull/Diagnostics/Diagnostics.h"
 #include "mull/Filters/Filters.h"
 #include "mull/Filters/FunctionFilter.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/Mutant.h"
 #include "mull/MutationResult.h"
 #include "mull/MutationsFinder.h"
 #include "mull/Parallelization/Parallelization.h"
 #include "mull/Program/Program.h"
-#include "mull/ReachableFunction.h"
 #include "mull/Result.h"
 #include "mull/Toolchain/Runner.h"
 
@@ -334,10 +334,15 @@ std::vector<FunctionUnderTest> Driver::getFunctionsUnderTest() {
           }
           if (covered) {
             functionsUnderTest.emplace_back(&function, bitcode.get());
+          } else if (config.includeNotCovered) {
+            functionsUnderTest.emplace_back(&function, bitcode.get(), false);
           }
         }
       }
     } else {
+      if (config.includeNotCovered) {
+        diagnostics.warning("-include-not-covered is enabled, but there is no coverage info!");
+      }
       for (auto &bitcode : program.bitcode()) {
         for (llvm::Function &function : *bitcode->getModule()) {
           functionsUnderTest.emplace_back(&function, bitcode.get());

lib/ReachableFunction.cpp → lib/FunctionUnderTest.cpp

@@ -1,12 +1,12 @@
-#include "mull/ReachableFunction.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/Filters/InstructionFilter.h"
 
 #include <llvm/IR/InstIterator.h>
 
 using namespace mull;
 
-FunctionUnderTest::FunctionUnderTest(llvm::Function *function, Bitcode *bitcode)
-    : function(function), bitcode(bitcode) {}
+FunctionUnderTest::FunctionUnderTest(llvm::Function *function, Bitcode *bitcode, bool covered)
+    : function(function), bitcode(bitcode), covered(covered) {}
 
 llvm::Function *FunctionUnderTest::getFunction() const {
   return function;
@@ -20,6 +20,10 @@ const std::vector<llvm::Instruction *> &FunctionUnderTest::getSelectedInstructio
   return selectedInstructions;
 }
 
+bool FunctionUnderTest::isCovered() const {
+  return covered;
+}
+
 void FunctionUnderTest::selectInstructions(const std::vector<InstructionFilter *> &filters) {
   for (llvm::Instruction &instruction : llvm::instructions(function)) {
     bool selected = true;

lib/Mutant.cpp

@@ -32,6 +32,16 @@ const std::string &Mutant::getReplacement() const {
   return points.front()->getReplacement();
 }
 
+bool Mutant::isCovered() const {
+  for (MutationPoint *point : points) {
+    /// Consider a mutant covered if at least one of the mutation points is covered
+    if (point->isCovered()) {
+      return true;
+    }
+  }
+  return false;
+}
+
 bool MutantComparator::operator()(std::unique_ptr<Mutant> &lhs, std::unique_ptr<Mutant> &rhs) {
   return operator()(*lhs, *rhs);
 }

lib/MutationPoint.cpp

@@ -80,12 +80,21 @@ MutationPoint::MutationPoint(Mutator *mutator, irm::IRMutation *irMutator,
     : mutator(mutator), address(MutationPointAddress::addressFromInstruction(instruction)),
       bitcode(m), originalFunction(instruction->getFunction()), mutatedFunction(nullptr),
       diagnostics(std::move(diagnostics)), replacement(std::move(replacement)),
-      sourceLocation(SourceLocation::locationFromInstruction(instruction)), irMutator(irMutator) {
+      sourceLocation(SourceLocation::locationFromInstruction(instruction)), irMutator(irMutator),
+      covered(true) {
   userIdentifier = mutator->getUniqueIdentifier() + ':' + sourceLocation.filePath + ':' +
                    std::to_string(sourceLocation.line) + ':' +
                    std::to_string(sourceLocation.column);
 }
 
+void MutationPoint::setCovered(bool isCovered) {
+  covered = isCovered;
+}
+
+bool MutationPoint::isCovered() {
+  return covered;
+}
+
 Mutator *MutationPoint::getMutator() {
   return mutator;
 }

lib/MutationsFinder.cpp

@@ -1,9 +1,9 @@
 #include "mull/MutationsFinder.h"
 
 #include "mull/Config/Configuration.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/Parallelization/Parallelization.h"
 #include "mull/Program/Program.h"
-#include "mull/ReachableFunction.h"
 
 using namespace mull;
 using namespace llvm;

lib/Mutators/CXX/LogicalAndToOr.cpp

@@ -1,7 +1,7 @@
 #include "mull/Mutators/CXX/LogicalAndToOr.h"
 
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include "mull/SourceLocation.h"
 
 #include <iterator>

lib/Mutators/CXX/LogicalOrToAnd.cpp

@@ -1,7 +1,7 @@
 #include "mull/Mutators/CXX/LogicalOrToAnd.h"
 
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include "mull/SourceLocation.h"
 
 #include <iterator>

lib/Mutators/CXX/TrivialCXXMutator.cpp

@@ -1,6 +1,6 @@
 #include "mull/Mutators/CXX/TrivialCXXMutator.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include <irm/irm.h>
 
 using namespace mull;

lib/Mutators/NegateConditionMutator.cpp

@@ -1,6 +1,6 @@
 #include "mull/Mutators/NegateConditionMutator.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include <cassert>
 #include <irm/irm.h>
 #include <sstream>

lib/Mutators/RemoveVoidFunctionMutator.cpp

@@ -1,6 +1,6 @@
 #include "mull/Mutators/RemoveVoidFunctionMutator.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include <irm/irm.h>
 #include <llvm/IR/Instructions.h>
 #include <sstream>

lib/Mutators/ReplaceCallMutator.cpp

@@ -1,6 +1,6 @@
 #include "mull/Mutators/ReplaceCallMutator.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include <cassert>
 #include <irm/irm.h>
 #include <llvm/IR/Instructions.h>

lib/Mutators/ScalarValueMutator.cpp

@@ -1,6 +1,6 @@
 #include "mull/Mutators/ScalarValueMutator.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/MutationPoint.h"
-#include "mull/ReachableFunction.h"
 #include <irm/irm.h>
 
 using namespace llvm;

lib/Parallelization/Tasks/ApplyMutationTask.cpp

@@ -9,6 +9,8 @@ void ApplyMutationTask::operator()(iterator begin, iterator end, Out &storage,
                                    progress_counter &counter) {
   for (auto it = begin; it != end; ++it, counter.increment()) {
     auto point = *it;
-    point->applyMutation();
+    if (point->isCovered()) {
+      point->applyMutation();
+    }
   }
 }

lib/Parallelization/Tasks/InstructionSelectionTask.cpp

@@ -1,6 +1,6 @@
 #include "mull/Parallelization/Tasks/InstructionSelectionTask.h"
+#include "mull/FunctionUnderTest.h"
 #include "mull/Parallelization/Progress.h"
-#include "mull/ReachableFunction.h"
 #include <cassert>
 
 using namespace mull;

lib/Parallelization/Tasks/MutantExecutionTask.cpp

@@ -19,12 +19,16 @@ void MutantExecutionTask::operator()(iterator begin, iterator end, Out &storage,
   Runner runner(diagnostics);
   for (auto it = begin; it != end; ++it, counter.increment()) {
     auto &mutant = *it;
-    ExecutionResult result = runner.runProgram(executable,
-                                               {},
-                                               { mutant->getIdentifier() },
-                                               baseline.runningTime * 10,
-                                               configuration.captureMutantOutput);
-
+    ExecutionResult result;
+    if (mutant->isCovered()) {
+      result = runner.runProgram(executable,
+                                 {},
+                                 { mutant->getIdentifier() },
+                                 baseline.runningTime * 10,
+                                 configuration.captureMutantOutput);
+    } else {
+      result.status = NotCovered;
+    }
     storage.push_back(std::make_unique<MutationResult>(result, mutant.get()));
   }
 }