diff --git a/docs/generated/CLIOptions.rst b/docs/generated/CLIOptions.rst index 02a280417..3286553e2 100644 --- a/docs/generated/CLIOptions.rst +++ b/docs/generated/CLIOptions.rst @@ -44,6 +44,8 @@ --coverage-info string Path to the coverage info file (LLVM's profdata) +--include-not-covered Include (but do not run) not covered mutants. Disabled by default + --include-path regex File/directory paths to whitelist (supports regex) --exclude-path regex File/directory paths to ignore (supports regex) diff --git a/include/mull/Config/Configuration.h b/include/mull/Config/Configuration.h index 0de38c582..6a04055d9 100644 --- a/include/mull/Config/Configuration.h +++ b/include/mull/Config/Configuration.h @@ -16,6 +16,7 @@ struct Configuration { bool captureTestOutput; bool captureMutantOutput; bool skipSanityCheckRun; + bool includeNotCovered; int timeout; unsigned linkerTimeout; diff --git a/include/mull/ExecutionResult.h b/include/mull/ExecutionResult.h index a96828e11..c50767606 100644 --- a/include/mull/ExecutionResult.h +++ b/include/mull/ExecutionResult.h @@ -11,7 +11,8 @@ enum ExecutionStatus { Crashed = 4, AbnormalExit = 5, DryRun = 6, - FailFast = 7 + FailFast = 7, + NotCovered = 8 }; static std::string executionStatusAsString(ExecutionStatus status) { @@ -34,6 +35,8 @@ static std::string executionStatusAsString(ExecutionStatus status) { return "DryRun"; case FailFast: return "FailFast"; + case NotCovered: + return "NotCovered"; } } diff --git a/include/mull/ReachableFunction.h b/include/mull/FunctionUnderTest.h similarity index 81% rename from include/mull/ReachableFunction.h rename to include/mull/FunctionUnderTest.h index 44ef95d99..2aa46aa16 100644 --- a/include/mull/ReachableFunction.h +++ b/include/mull/FunctionUnderTest.h @@ -13,15 +13,17 @@ class Bitcode; class FunctionUnderTest { public: - explicit FunctionUnderTest(llvm::Function *function, Bitcode *bitcode); + FunctionUnderTest(llvm::Function *function, Bitcode *bitcode, bool covered = true); llvm::Function *getFunction() const; Bitcode *getBitcode() const; const std::vector &getSelectedInstructions() const; + bool isCovered() const; void selectInstructions(const std::vector &filters); private: llvm::Function *function; Bitcode *bitcode; + bool covered; std::vector selectedInstructions; }; diff --git a/include/mull/Mutant.h b/include/mull/Mutant.h index bd87fbc24..264270fa6 100644 --- a/include/mull/Mutant.h +++ b/include/mull/Mutant.h @@ -19,6 +19,7 @@ class Mutant { const std::string &getDiagnostics() const; const std::string &getReplacement() const; const std::string &getMutatorIdentifier() const; + bool isCovered() const; private: std::string identifier; diff --git a/include/mull/MutationPoint.h b/include/mull/MutationPoint.h index 3945d7705..ede569b61 100644 --- a/include/mull/MutationPoint.h +++ b/include/mull/MutationPoint.h @@ -64,6 +64,7 @@ class MutationPoint { const SourceLocation sourceLocation; irm::IRMutation *irMutator; std::string userIdentifier; + bool covered; public: MutationPoint(Mutator *mutator, irm::IRMutation *irMutator, llvm::Instruction *instruction, @@ -71,6 +72,9 @@ class MutationPoint { ~MutationPoint() = default; + void setCovered(bool isCovered); + bool isCovered(); + Mutator *getMutator(); Mutator *getMutator() const; diff --git a/include/mull/MutationsFinder.h b/include/mull/MutationsFinder.h index 4591e5a0f..61f0d8d6c 100644 --- a/include/mull/MutationsFinder.h +++ b/include/mull/MutationsFinder.h @@ -3,8 +3,8 @@ #include #include +#include "FunctionUnderTest.h" #include "MutationPoint.h" -#include "ReachableFunction.h" #include "mull/Mutators/Mutator.h" namespace mull { diff --git a/include/mull/Mutators/CXX/TrivialCXXMutator.h b/include/mull/Mutators/CXX/TrivialCXXMutator.h index 9fe7ab5ef..1af9e0d01 100644 --- a/include/mull/Mutators/CXX/TrivialCXXMutator.h +++ b/include/mull/Mutators/CXX/TrivialCXXMutator.h @@ -3,7 +3,7 @@ #include "mull/Mutators/Mutator.h" #include #include -#include +#include #include namespace mull { diff --git a/include/mull/Mutators/NegateConditionMutator.h b/include/mull/Mutators/NegateConditionMutator.h index 72257c15e..791f69d14 100644 --- a/include/mull/Mutators/NegateConditionMutator.h +++ b/include/mull/Mutators/NegateConditionMutator.h @@ -3,7 +3,7 @@ #include "Mutator.h" #include #include -#include +#include #include namespace llvm { diff --git a/include/mull/Mutators/RemoveVoidFunctionMutator.h b/include/mull/Mutators/RemoveVoidFunctionMutator.h index 3cc54f689..96484f23f 100644 --- a/include/mull/Mutators/RemoveVoidFunctionMutator.h +++ b/include/mull/Mutators/RemoveVoidFunctionMutator.h @@ -3,7 +3,7 @@ #include "Mutator.h" #include #include -#include +#include #include namespace llvm { diff --git a/include/mull/Mutators/ReplaceCallMutator.h b/include/mull/Mutators/ReplaceCallMutator.h index eeef62108..1262c1b22 100644 --- a/include/mull/Mutators/ReplaceCallMutator.h +++ b/include/mull/Mutators/ReplaceCallMutator.h @@ -3,7 +3,7 @@ #include "Mutator.h" #include #include -#include +#include #include namespace mull { diff --git a/include/mull/Mutators/ScalarValueMutator.h b/include/mull/Mutators/ScalarValueMutator.h index 6d6527b2f..58590113d 100644 --- a/include/mull/Mutators/ScalarValueMutator.h +++ b/include/mull/Mutators/ScalarValueMutator.h @@ -3,7 +3,7 @@ #include "Mutator.h" #include #include -#include +#include #include namespace llvm { diff --git a/include/mull/Parallelization/Tasks/FunctionFilterTask.h b/include/mull/Parallelization/Tasks/FunctionFilterTask.h index f6de7ea9f..b6a68193c 100644 --- a/include/mull/Parallelization/Tasks/FunctionFilterTask.h +++ b/include/mull/Parallelization/Tasks/FunctionFilterTask.h @@ -1,6 +1,6 @@ #pragma once -#include "mull/ReachableFunction.h" +#include "mull/FunctionUnderTest.h" #include namespace mull { diff --git a/include/mull/Parallelization/Tasks/InstructionSelectionTask.h b/include/mull/Parallelization/Tasks/InstructionSelectionTask.h index f17d041df..dedc509eb 100644 --- a/include/mull/Parallelization/Tasks/InstructionSelectionTask.h +++ b/include/mull/Parallelization/Tasks/InstructionSelectionTask.h @@ -1,6 +1,6 @@ #pragma once -#include "mull/ReachableFunction.h" +#include "mull/FunctionUnderTest.h" #include namespace mull { diff --git a/include/mull/Parallelization/Tasks/SearchMutationPointsTask.h b/include/mull/Parallelization/Tasks/SearchMutationPointsTask.h index a015cf79a..219f54433 100644 --- a/include/mull/Parallelization/Tasks/SearchMutationPointsTask.h +++ b/include/mull/Parallelization/Tasks/SearchMutationPointsTask.h @@ -1,8 +1,8 @@ #pragma once +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" #include "mull/Mutators/Mutator.h" -#include "mull/ReachableFunction.h" namespace mull { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f37332782..a2d0f1533 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -34,7 +34,7 @@ set(mull_sources Bitcode.cpp MutationPoint.cpp - ReachableFunction.cpp + FunctionUnderTest.cpp IDEDiagnostics.cpp diff --git a/lib/Config/Configuration.cpp b/lib/Config/Configuration.cpp index 2be6ebca2..c55c15d5f 100644 --- a/lib/Config/Configuration.cpp +++ b/lib/Config/Configuration.cpp @@ -17,7 +17,7 @@ unsigned MullDefaultLinkerTimeoutMilliseconds = 30000; Configuration::Configuration() : debugEnabled(false), dryRunEnabled(false), captureTestOutput(true), captureMutantOutput(true), - skipSanityCheckRun(false), timeout(MullDefaultTimeoutMilliseconds), + skipSanityCheckRun(false), includeNotCovered(false), timeout(MullDefaultTimeoutMilliseconds), linkerTimeout(MullDefaultLinkerTimeoutMilliseconds), diagnostics(IDEDiagnosticsKind::None), parallelization(singleThreadParallelization()) {} diff --git a/lib/Driver.cpp b/lib/Driver.cpp index f28af5c9e..768daa6b8 100644 --- a/lib/Driver.cpp +++ b/lib/Driver.cpp @@ -4,12 +4,12 @@ #include "mull/Diagnostics/Diagnostics.h" #include "mull/Filters/Filters.h" #include "mull/Filters/FunctionFilter.h" +#include "mull/FunctionUnderTest.h" #include "mull/Mutant.h" #include "mull/MutationResult.h" #include "mull/MutationsFinder.h" #include "mull/Parallelization/Parallelization.h" #include "mull/Program/Program.h" -#include "mull/ReachableFunction.h" #include "mull/Result.h" #include "mull/Toolchain/Runner.h" @@ -334,10 +334,15 @@ std::vector Driver::getFunctionsUnderTest() { } if (covered) { functionsUnderTest.emplace_back(&function, bitcode.get()); + } else if (config.includeNotCovered) { + functionsUnderTest.emplace_back(&function, bitcode.get(), false); } } } } else { + if (config.includeNotCovered) { + diagnostics.warning("-include-not-covered is enabled, but there is no coverage info!"); + } for (auto &bitcode : program.bitcode()) { for (llvm::Function &function : *bitcode->getModule()) { functionsUnderTest.emplace_back(&function, bitcode.get()); diff --git a/lib/ReachableFunction.cpp b/lib/FunctionUnderTest.cpp similarity index 81% rename from lib/ReachableFunction.cpp rename to lib/FunctionUnderTest.cpp index e11560b5e..ebe318b69 100644 --- a/lib/ReachableFunction.cpp +++ b/lib/FunctionUnderTest.cpp @@ -1,12 +1,12 @@ -#include "mull/ReachableFunction.h" +#include "mull/FunctionUnderTest.h" #include "mull/Filters/InstructionFilter.h" #include using namespace mull; -FunctionUnderTest::FunctionUnderTest(llvm::Function *function, Bitcode *bitcode) - : function(function), bitcode(bitcode) {} +FunctionUnderTest::FunctionUnderTest(llvm::Function *function, Bitcode *bitcode, bool covered) + : function(function), bitcode(bitcode), covered(covered) {} llvm::Function *FunctionUnderTest::getFunction() const { return function; @@ -20,6 +20,10 @@ const std::vector &FunctionUnderTest::getSelectedInstructio return selectedInstructions; } +bool FunctionUnderTest::isCovered() const { + return covered; +} + void FunctionUnderTest::selectInstructions(const std::vector &filters) { for (llvm::Instruction &instruction : llvm::instructions(function)) { bool selected = true; diff --git a/lib/Mutant.cpp b/lib/Mutant.cpp index 7b1458c0f..cf5a763e3 100644 --- a/lib/Mutant.cpp +++ b/lib/Mutant.cpp @@ -32,6 +32,16 @@ const std::string &Mutant::getReplacement() const { return points.front()->getReplacement(); } +bool Mutant::isCovered() const { + for (MutationPoint *point : points) { + /// Consider a mutant covered if at least one of the mutation points is covered + if (point->isCovered()) { + return true; + } + } + return false; +} + bool MutantComparator::operator()(std::unique_ptr &lhs, std::unique_ptr &rhs) { return operator()(*lhs, *rhs); } diff --git a/lib/MutationPoint.cpp b/lib/MutationPoint.cpp index a07217b25..3f0a2c711 100644 --- a/lib/MutationPoint.cpp +++ b/lib/MutationPoint.cpp @@ -80,12 +80,21 @@ MutationPoint::MutationPoint(Mutator *mutator, irm::IRMutation *irMutator, : mutator(mutator), address(MutationPointAddress::addressFromInstruction(instruction)), bitcode(m), originalFunction(instruction->getFunction()), mutatedFunction(nullptr), diagnostics(std::move(diagnostics)), replacement(std::move(replacement)), - sourceLocation(SourceLocation::locationFromInstruction(instruction)), irMutator(irMutator) { + sourceLocation(SourceLocation::locationFromInstruction(instruction)), irMutator(irMutator), + covered(true) { userIdentifier = mutator->getUniqueIdentifier() + ':' + sourceLocation.filePath + ':' + std::to_string(sourceLocation.line) + ':' + std::to_string(sourceLocation.column); } +void MutationPoint::setCovered(bool isCovered) { + covered = isCovered; +} + +bool MutationPoint::isCovered() { + return covered; +} + Mutator *MutationPoint::getMutator() { return mutator; } diff --git a/lib/MutationsFinder.cpp b/lib/MutationsFinder.cpp index 82c487792..d2471682e 100644 --- a/lib/MutationsFinder.cpp +++ b/lib/MutationsFinder.cpp @@ -1,9 +1,9 @@ #include "mull/MutationsFinder.h" #include "mull/Config/Configuration.h" +#include "mull/FunctionUnderTest.h" #include "mull/Parallelization/Parallelization.h" #include "mull/Program/Program.h" -#include "mull/ReachableFunction.h" using namespace mull; using namespace llvm; diff --git a/lib/Mutators/CXX/LogicalAndToOr.cpp b/lib/Mutators/CXX/LogicalAndToOr.cpp index d0142606a..8a9bcf2fe 100644 --- a/lib/Mutators/CXX/LogicalAndToOr.cpp +++ b/lib/Mutators/CXX/LogicalAndToOr.cpp @@ -1,7 +1,7 @@ #include "mull/Mutators/CXX/LogicalAndToOr.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include "mull/SourceLocation.h" #include diff --git a/lib/Mutators/CXX/LogicalOrToAnd.cpp b/lib/Mutators/CXX/LogicalOrToAnd.cpp index 51de9f579..022ab6ebd 100644 --- a/lib/Mutators/CXX/LogicalOrToAnd.cpp +++ b/lib/Mutators/CXX/LogicalOrToAnd.cpp @@ -1,7 +1,7 @@ #include "mull/Mutators/CXX/LogicalOrToAnd.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include "mull/SourceLocation.h" #include diff --git a/lib/Mutators/CXX/TrivialCXXMutator.cpp b/lib/Mutators/CXX/TrivialCXXMutator.cpp index 0a63bdd5a..bbdd22ff9 100644 --- a/lib/Mutators/CXX/TrivialCXXMutator.cpp +++ b/lib/Mutators/CXX/TrivialCXXMutator.cpp @@ -1,6 +1,6 @@ #include "mull/Mutators/CXX/TrivialCXXMutator.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include using namespace mull; diff --git a/lib/Mutators/NegateConditionMutator.cpp b/lib/Mutators/NegateConditionMutator.cpp index f4b0c42e8..796955857 100644 --- a/lib/Mutators/NegateConditionMutator.cpp +++ b/lib/Mutators/NegateConditionMutator.cpp @@ -1,6 +1,6 @@ #include "mull/Mutators/NegateConditionMutator.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include #include #include diff --git a/lib/Mutators/RemoveVoidFunctionMutator.cpp b/lib/Mutators/RemoveVoidFunctionMutator.cpp index 69552990a..d57444751 100644 --- a/lib/Mutators/RemoveVoidFunctionMutator.cpp +++ b/lib/Mutators/RemoveVoidFunctionMutator.cpp @@ -1,6 +1,6 @@ #include "mull/Mutators/RemoveVoidFunctionMutator.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include #include #include diff --git a/lib/Mutators/ReplaceCallMutator.cpp b/lib/Mutators/ReplaceCallMutator.cpp index 2dcf72ebe..2188fedb0 100644 --- a/lib/Mutators/ReplaceCallMutator.cpp +++ b/lib/Mutators/ReplaceCallMutator.cpp @@ -1,6 +1,6 @@ #include "mull/Mutators/ReplaceCallMutator.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include #include #include diff --git a/lib/Mutators/ScalarValueMutator.cpp b/lib/Mutators/ScalarValueMutator.cpp index 380d71a0c..a75b0cffd 100644 --- a/lib/Mutators/ScalarValueMutator.cpp +++ b/lib/Mutators/ScalarValueMutator.cpp @@ -1,6 +1,6 @@ #include "mull/Mutators/ScalarValueMutator.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include using namespace llvm; diff --git a/lib/Parallelization/Tasks/ApplyMutationTask.cpp b/lib/Parallelization/Tasks/ApplyMutationTask.cpp index 0c8415bd8..0376c85a7 100644 --- a/lib/Parallelization/Tasks/ApplyMutationTask.cpp +++ b/lib/Parallelization/Tasks/ApplyMutationTask.cpp @@ -9,6 +9,8 @@ void ApplyMutationTask::operator()(iterator begin, iterator end, Out &storage, progress_counter &counter) { for (auto it = begin; it != end; ++it, counter.increment()) { auto point = *it; - point->applyMutation(); + if (point->isCovered()) { + point->applyMutation(); + } } } diff --git a/lib/Parallelization/Tasks/InstructionSelectionTask.cpp b/lib/Parallelization/Tasks/InstructionSelectionTask.cpp index e4dad7421..82fe91d11 100644 --- a/lib/Parallelization/Tasks/InstructionSelectionTask.cpp +++ b/lib/Parallelization/Tasks/InstructionSelectionTask.cpp @@ -1,6 +1,6 @@ #include "mull/Parallelization/Tasks/InstructionSelectionTask.h" +#include "mull/FunctionUnderTest.h" #include "mull/Parallelization/Progress.h" -#include "mull/ReachableFunction.h" #include using namespace mull; diff --git a/lib/Parallelization/Tasks/MutantExecutionTask.cpp b/lib/Parallelization/Tasks/MutantExecutionTask.cpp index 8ee851c07..a0364379f 100644 --- a/lib/Parallelization/Tasks/MutantExecutionTask.cpp +++ b/lib/Parallelization/Tasks/MutantExecutionTask.cpp @@ -19,12 +19,16 @@ void MutantExecutionTask::operator()(iterator begin, iterator end, Out &storage, Runner runner(diagnostics); for (auto it = begin; it != end; ++it, counter.increment()) { auto &mutant = *it; - ExecutionResult result = runner.runProgram(executable, - {}, - { mutant->getIdentifier() }, - baseline.runningTime * 10, - configuration.captureMutantOutput); - + ExecutionResult result; + if (mutant->isCovered()) { + result = runner.runProgram(executable, + {}, + { mutant->getIdentifier() }, + baseline.runningTime * 10, + configuration.captureMutantOutput); + } else { + result.status = NotCovered; + } storage.push_back(std::make_unique(result, mutant.get())); } } diff --git a/lib/Parallelization/Tasks/MutantPreparationTasks.cpp b/lib/Parallelization/Tasks/MutantPreparationTasks.cpp index 7aca2b4d6..45772b7db 100644 --- a/lib/Parallelization/Tasks/MutantPreparationTasks.cpp +++ b/lib/Parallelization/Tasks/MutantPreparationTasks.cpp @@ -21,6 +21,9 @@ void CloneMutatedFunctionsTask::cloneFunctions(Bitcode &bitcode) { for (auto &pair : bitcode.getMutationPointsMap()) { llvm::Function *original = pair.first; for (MutationPoint *point : pair.second) { + if (!point->isCovered()) { + continue; + } llvm::ValueToValueMapTy map; llvm::Function *mutatedFunction = llvm::CloneFunction(original, map); mutatedFunction->setLinkage(llvm::GlobalValue::InternalLinkage); @@ -38,14 +41,18 @@ void DeleteOriginalFunctionsTask::operator()(iterator begin, iterator end, Out & } void DeleteOriginalFunctionsTask::deleteFunctions(Bitcode &bitcode) { - for (auto pair : bitcode.getMutationPointsMap()) { + for (auto &pair : bitcode.getMutationPointsMap()) { auto original = pair.first; - auto anyPoint = pair.second.front(); - - llvm::ValueToValueMapTy map; - auto originalCopy = CloneFunction(original, map); - originalCopy->setName(anyPoint->getOriginalFunctionName()); - original->dropAllReferences(); + /// Remove the original function if at least one mutant is covered + for (MutationPoint *point : pair.second) { + if (point->isCovered()) { + llvm::ValueToValueMapTy map; + auto originalCopy = CloneFunction(original, map); + originalCopy->setName(point->getOriginalFunctionName()); + original->dropAllReferences(); + break; + } + } } } @@ -66,6 +73,16 @@ void InsertMutationTrampolinesTask::insertTrampolines(Bitcode &bitcode) { llvm::Value *getenv = llvm_compat::getOrInsertFunction(module, "getenv", getEnvType); for (auto pair : bitcode.getMutationPointsMap()) { + bool hasCoveredMutants = false; + for (auto point : pair.second) { + if (point->isCovered()) { + hasCoveredMutants = true; + break; + } + } + if (!hasCoveredMutants) { + continue; + } llvm::Function *original = pair.first; llvm::BasicBlock *entry = llvm::BasicBlock::Create(context, "entry", original); @@ -84,6 +101,9 @@ void InsertMutationTrampolinesTask::insertTrampolines(Bitcode &bitcode) { llvm::BasicBlock *head = originalBlock; for (auto &point : pair.second) { + if (!point->isCovered()) { + continue; + } auto name = llvm::ConstantDataArray::getString(context, point->getUserIdentifier()); auto *global = new llvm::GlobalVariable( *module, name->getType(), true, llvm::GlobalValue::PrivateLinkage, name); diff --git a/lib/Parallelization/Tasks/SearchMutationPointsTask.cpp b/lib/Parallelization/Tasks/SearchMutationPointsTask.cpp index 72cfb5631..a427ae89b 100644 --- a/lib/Parallelization/Tasks/SearchMutationPointsTask.cpp +++ b/lib/Parallelization/Tasks/SearchMutationPointsTask.cpp @@ -23,6 +23,7 @@ void SearchMutationPointsTask::operator()(iterator begin, iterator end, Out &sto for (auto &mutator : mutators) { auto mutants = mutator->getMutations(bitcode, functionUnderTest); for (auto mutant : mutants) { + mutant->setCovered(functionUnderTest.isCovered()); storage.push_back(std::unique_ptr(mutant)); } } diff --git a/lib/Reporters/IDEReporter.cpp b/lib/Reporters/IDEReporter.cpp index 0a854527b..a74df95b2 100644 --- a/lib/Reporters/IDEReporter.cpp +++ b/lib/Reporters/IDEReporter.cpp @@ -19,15 +19,19 @@ static bool mutantSurvived(const ExecutionStatus &status) { return status == ExecutionStatus::Passed; } +static bool mutantNotCovered(const ExecutionStatus &status) { + return status == ExecutionStatus::NotCovered; +} + static void printMutant(Diagnostics &diagnostics, SourceCodeReader &sourceCodeReader, - const Mutant &mutant, bool survived) { + const Mutant &mutant, const std::string &status) { auto &sourceLocation = mutant.getSourceLocation(); assert(!sourceLocation.isNull() && "Debug information is missing?"); std::stringstream stringstream; stringstream << sourceLocation.filePath << ":" << sourceLocation.line << ":" - << sourceLocation.column << ": warning: " << (survived ? "Survived" : "Killed") - << ": " << mutant.getDiagnostics() << " [" << mutant.getMutatorIdentifier() << "]" + << sourceLocation.column << ": warning: " << status << ": " + << mutant.getDiagnostics() << " [" << mutant.getMutatorIdentifier() << "]" << "\n"; stringstream << sourceCodeReader.getSourceLineWithCaret(sourceLocation); @@ -36,6 +40,20 @@ static void printMutant(Diagnostics &diagnostics, SourceCodeReader &sourceCodeRe fflush(stdout); } +static void printMutants(Diagnostics &diagnostics, SourceCodeReader &reader, + const std::vector &mutants, size_t totalSize, + const std::string &status) { + if (mutants.empty()) { + return; + } + std::stringstream stringstream; + stringstream << status << " mutants (" << mutants.size() << "/" << totalSize << "):"; + diagnostics.info(stringstream.str()); + for (auto mutant : mutants) { + printMutant(diagnostics, reader, *mutant, status); + } +} + IDEReporter::IDEReporter(Diagnostics &diagnostics, bool showKilled) : diagnostics(diagnostics), showKilled(showKilled), sourceCodeReader() {} @@ -45,46 +63,36 @@ void IDEReporter::reportResults(const Result &result) { return; } - std::set killedMutants; + std::vector killedMutants; + std::vector survivedMutants; + std::vector notCoveredMutants; for (auto &mutationResult : result.getMutationResults()) { auto mutant = mutationResult->getMutant(); auto &executionResult = mutationResult->getExecutionResult(); - if (!mutantSurvived(executionResult.status)) { - killedMutants.insert(mutant); + if (mutantSurvived(executionResult.status)) { + survivedMutants.push_back(mutant); + } else if (mutantNotCovered(executionResult.status)) { + notCoveredMutants.push_back(mutant); + } else { + killedMutants.push_back(mutant); } } - auto survivedMutantsCount = result.getMutants().size() - killedMutants.size(); - auto killedMutantsCount = killedMutants.size(); - - /// It is important that below we iterate over result.getMutationPoints() - /// because we want the output to be stable across multiple executions of Mull. - /// Optimizing this here was a bad idea: https://github.com/mull-project/mull/pull/640. - if (showKilled && killedMutantsCount > 0) { - std::stringstream stringstream; - stringstream << "Killed mutants (" << killedMutantsCount << "/" << result.getMutants().size() - << "):"; - diagnostics.info(stringstream.str()); - for (auto &mutant : result.getMutants()) { - if (killedMutants.find(mutant.get()) != killedMutants.end()) { - printMutant(diagnostics, sourceCodeReader, *mutant, false); - } - } + assert(killedMutants.size() + survivedMutants.size() + notCoveredMutants.size() == + result.getMutationResults().size()); + + if (showKilled) { + printMutants( + diagnostics, sourceCodeReader, killedMutants, result.getMutants().size(), "Killed"); } - if (survivedMutantsCount > 0) { - std::stringstream stringstream; - stringstream << "Survived mutants (" << survivedMutantsCount << "/" - << result.getMutants().size() << "):"; - diagnostics.info(stringstream.str()); + printMutants( + diagnostics, sourceCodeReader, survivedMutants, result.getMutants().size(), "Survived"); + printMutants( + diagnostics, sourceCodeReader, notCoveredMutants, result.getMutants().size(), "Not Covered"); - for (auto &mutant : result.getMutants()) { - if (killedMutants.find(mutant.get()) == killedMutants.end()) { - printMutant(diagnostics, sourceCodeReader, *mutant, true); - } - } - } else { + if (survivedMutants.empty() && notCoveredMutants.empty()) { diagnostics.info("All mutations have been killed"); } diff --git a/lib/Reporters/MutationTestingElementsReporter.cpp b/lib/Reporters/MutationTestingElementsReporter.cpp index 84f478c1c..0a7de77be 100644 --- a/lib/Reporters/MutationTestingElementsReporter.cpp +++ b/lib/Reporters/MutationTestingElementsReporter.cpp @@ -27,6 +27,7 @@ static bool mutantSurvived(const ExecutionStatus &status) { static json11::Json createFiles(Diagnostics &diagnostics, const Result &result, const std::set &killedMutants, + const std::set ¬CoveredMutants, SourceInfoProvider &sourceInfoProvider) { SourceManager sourceManager; @@ -62,7 +63,12 @@ static json11::Json createFiles(Diagnostics &diagnostics, const Result &result, MutationPointSourceInfo sourceInfo = sourceInfoProvider.getSourceInfo(diagnostics, mutant->getMutationPoints().front()); - std::string status = (killedMutants.count(mutant) == 0) ? "Survived" : "Killed"; + std::string status("Survived"); + if (killedMutants.count(mutant) != 0) { + status = "Killed"; + } else if (notCoveredMutants.count(mutant) != 0) { + status = "NoCoverage"; + } Json mpJson = Json::object{ { "id", mutant->getMutatorIdentifier() }, @@ -120,11 +126,14 @@ void MutationTestingElementsReporter::reportResults(const Result &result) { generateHTMLFile(); std::set killedMutants; + std::set notCoveredMutants; for (auto &mutationResult : result.getMutationResults()) { auto mutant = mutationResult->getMutant(); auto &executionResult = mutationResult->getExecutionResult(); - if (!mutantSurvived(executionResult.status)) { + if (executionResult.status == NotCovered) { + notCoveredMutants.insert(mutant); + } else if (!mutantSurvived(executionResult.status)) { killedMutants.insert(mutant); } } @@ -135,7 +144,8 @@ void MutationTestingElementsReporter::reportResults(const Result &result) { Json json = Json::object{ { "mutationScore", (int)score }, { "thresholds", Json::object{ { "high", 80 }, { "low", 60 } } }, - { "files", createFiles(diagnostics, result, killedMutants, sourceInfoProvider) }, + { "files", + createFiles(diagnostics, result, killedMutants, notCoveredMutants, sourceInfoProvider) }, { "schemaVersion", "1.1.1" }, }; std::string json_str = json.dump(); diff --git a/tests-lit/tests/coverage/02/main.c b/tests-lit/tests/coverage/02/main.c new file mode 100644 index 000000000..6fae04a3f --- /dev/null +++ b/tests-lit/tests/coverage/02/main.c @@ -0,0 +1,19 @@ +void noop(int a, int b) { + a + b; +} + +int main() { + return 0; +} + +// clang-format off + +// RUN: cd / && %clang_cc %s -fembed-bitcode -g -fprofile-instr-generate -fcoverage-mapping -o %s.exe +// RUN: cd %CURRENT_DIR +// RUN: env LLVM_PROFILE_FILE=%s.profraw %s.exe +// RUN: %llvm_profdata merge %s.profraw -o %s.profdata + +// RUN: unset TERM; %MULL_EXEC -linker=%clang_cc -coverage-info=%s.profdata -ide-reporter-show-killed -include-not-covered -linker-flags="-fprofile-instr-generate" -mutators=cxx_add_to_sub %s.exe 2>&1 | %FILECHECK_EXEC %s --dump-input=fail --strict-whitespace --match-full-lines +// CHECK:[info] Not Covered mutants (1/1): +// CHECK:{{^.*}}main.c:2:5: warning: Not Covered: Replaced + with - [cxx_add_to_sub] +// CHECK:[info] Mutation score: 0% diff --git a/tests/Helpers/MutationTestBed.cpp b/tests/Helpers/MutationTestBed.cpp index b4fd7b127..37da3123e 100644 --- a/tests/Helpers/MutationTestBed.cpp +++ b/tests/Helpers/MutationTestBed.cpp @@ -3,9 +3,9 @@ #include "mull/AST/ASTVisitor.h" #include "mull/Bitcode.h" #include "mull/Diagnostics/Diagnostics.h" +#include "mull/FunctionUnderTest.h" #include "mull/JunkDetection/CXX/ASTStorage.h" #include "mull/JunkDetection/CXX/CXXJunkDetector.h" -#include "mull/ReachableFunction.h" using namespace mull; using namespace mull_test; diff --git a/tests/JunkDetection/CXXJunkDetectorTests.cpp b/tests/JunkDetection/CXXJunkDetectorTests.cpp index 71988d58d..62f6829d9 100644 --- a/tests/JunkDetection/CXXJunkDetectorTests.cpp +++ b/tests/JunkDetection/CXXJunkDetectorTests.cpp @@ -1,12 +1,12 @@ #include "FixturePaths.h" #include "mull/BitcodeLoader.h" +#include "mull/FunctionUnderTest.h" #include "mull/JunkDetection/CXX/CXXJunkDetector.h" #include "mull/MutationPoint.h" #include "mull/Mutators/NegateConditionMutator.h" #include "mull/Mutators/RemoveVoidFunctionMutator.h" #include "mull/Mutators/ReplaceCallMutator.h" #include "mull/Mutators/ScalarValueMutator.h" -#include "mull/ReachableFunction.h" #include #include #include diff --git a/tests/MutationPointTests.cpp b/tests/MutationPointTests.cpp index a9bfccdec..cb278c939 100644 --- a/tests/MutationPointTests.cpp +++ b/tests/MutationPointTests.cpp @@ -2,11 +2,11 @@ #include "TestModuleFactory.h" #include "mull/BitcodeLoader.h" #include "mull/Config/Configuration.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" #include "mull/Mutators/CXX/LogicalAndToOr.h" #include "mull/Mutators/ReplaceCallMutator.h" #include "mull/Mutators/ScalarValueMutator.h" -#include "mull/ReachableFunction.h" #include #include diff --git a/tests/MutationTestingElementsReporterTest.cpp b/tests/MutationTestingElementsReporterTest.cpp index c39c49eb5..c5e49513d 100644 --- a/tests/MutationTestingElementsReporterTest.cpp +++ b/tests/MutationTestingElementsReporterTest.cpp @@ -6,11 +6,11 @@ #include "mull/Bitcode.h" #include "mull/BitcodeLoader.h" #include "mull/Config/Configuration.h" +#include "mull/FunctionUnderTest.h" #include "mull/JunkDetection/CXX/ASTStorage.h" #include "mull/Metrics/MetricsMeasure.h" #include "mull/MutationsFinder.h" #include "mull/Program/Program.h" -#include "mull/ReachableFunction.h" #include "mull/Reporters/ASTSourceInfoProvider.h" #include "mull/Result.h" #include diff --git a/tests/Mutators/ConditionalsBoundaryMutatorTests.cpp b/tests/Mutators/ConditionalsBoundaryMutatorTests.cpp index 159eb99ea..454a5a170 100644 --- a/tests/Mutators/ConditionalsBoundaryMutatorTests.cpp +++ b/tests/Mutators/ConditionalsBoundaryMutatorTests.cpp @@ -2,8 +2,8 @@ #include #include #include +#include #include -#include using namespace mull; using namespace llvm; diff --git a/tests/Mutators/NegateConditionMutatorTest.cpp b/tests/Mutators/NegateConditionMutatorTest.cpp index a750dafad..8ab7d05dc 100644 --- a/tests/Mutators/NegateConditionMutatorTest.cpp +++ b/tests/Mutators/NegateConditionMutatorTest.cpp @@ -1,9 +1,9 @@ #include "mull/Mutators/NegateConditionMutator.h" #include "FixturePaths.h" #include "TestModuleFactory.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" #include "mull/Program/Program.h" -#include "mull/ReachableFunction.h" #include #include diff --git a/tests/Mutators/ScalarValueMutatorTest.cpp b/tests/Mutators/ScalarValueMutatorTest.cpp index 10cb07b75..22f96aa61 100644 --- a/tests/Mutators/ScalarValueMutatorTest.cpp +++ b/tests/Mutators/ScalarValueMutatorTest.cpp @@ -2,8 +2,8 @@ #include "FixturePaths.h" #include "TestModuleFactory.h" #include "mull/BitcodeLoader.h" +#include "mull/FunctionUnderTest.h" #include "mull/MutationPoint.h" -#include "mull/ReachableFunction.h" #include #include diff --git a/tests/SQLiteReporterTest.cpp b/tests/SQLiteReporterTest.cpp index ccbb242bc..2f35f67db 100644 --- a/tests/SQLiteReporterTest.cpp +++ b/tests/SQLiteReporterTest.cpp @@ -3,11 +3,11 @@ #include "TestModuleFactory.h" #include "mull/BitcodeLoader.h" #include "mull/Config/Configuration.h" +#include "mull/FunctionUnderTest.h" #include "mull/Metrics/MetricsMeasure.h" #include "mull/Mutant.h" #include "mull/MutationsFinder.h" #include "mull/Program/Program.h" -#include "mull/ReachableFunction.h" #include "mull/Result.h" #include diff --git a/tools/mull-cxx/CLIOptions.cpp b/tools/mull-cxx/CLIOptions.cpp index 68dfd8e10..c133676a7 100644 --- a/tools/mull-cxx/CLIOptions.cpp +++ b/tools/mull-cxx/CLIOptions.cpp @@ -139,6 +139,13 @@ opt tool::EnableAST( llvm::cl::desc("Enable \"white\" AST search (disabled by default)"), llvm::cl::cat(MullCXXCategory), llvm::cl::init(false)); +opt tool::IncludeNotCovered( + "include-not-covered", + desc("Include (but do not run) not covered mutants. Disabled by default"), + Optional, + init(false), + cat(MullCXXCategory)); + list tool::ReportersOption( "reporters", desc("Choose reporters:"), @@ -328,6 +335,7 @@ void tool::dumpCLIInterface(Diagnostics &diagnostics) { &LinkerTimeout, &CoverageInfo, + &IncludeNotCovered, &(Option &)IncludePaths, &(Option &)ExcludePaths, diff --git a/tools/mull-cxx/CLIOptions.h b/tools/mull-cxx/CLIOptions.h index 3a4be7634..2997e2f94 100644 --- a/tools/mull-cxx/CLIOptions.h +++ b/tools/mull-cxx/CLIOptions.h @@ -52,6 +52,8 @@ extern opt EnableAST; extern list ExcludePaths; extern list IncludePaths; +extern opt IncludeNotCovered; + class MutatorsCLIOptions { public: MutatorsCLIOptions(Diagnostics &diagnostics, list ¶meter); diff --git a/tools/mull-cxx/mull-cxx.cpp b/tools/mull-cxx/mull-cxx.cpp index 231c50bd3..3b221020e 100644 --- a/tools/mull-cxx/mull-cxx.cpp +++ b/tools/mull-cxx/mull-cxx.cpp @@ -97,6 +97,7 @@ int main(int argc, char **argv) { configuration.executable = tool::InputFile.getValue(); configuration.coverageInfo = tool::CoverageInfo.getValue(); + configuration.includeNotCovered = tool::IncludeNotCovered.getValue(); if (tool::Workers) { mull::ParallelizationConfig parallelizationConfig;