From ee4f882f603f78f06b188c7f9eca5f75b07ff47c Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 8 Jan 2024 19:12:05 -0700 Subject: [PATCH] Create a tool to generate summary results - Extract ToolResults from TestResults and ToolSummary - Scan test case directory - Scan results directory - Compile summary results for all tools - Write summary markdown file --- CMakeLists.txt | 1 + TestCases/CMakeLists.txt | 2 + TestCases/TestCases.cpp | 17 +++ TestCases/TestCases.h | 12 +- TestCases/ToolResults.cpp | 219 ++++++++++++++++++++++++++++++++++++ TestCases/ToolResults.h | 76 +++++++++++++ TestNames/TestNames.cpp | 8 +- TestResults/TestResults.cpp | 217 +++-------------------------------- ToolSummary/CMakeLists.txt | 12 ++ ToolSummary/ToolSummary.cpp | 132 ++++++++++++++++++++++ 10 files changed, 485 insertions(+), 211 deletions(-) create mode 100644 TestCases/ToolResults.cpp create mode 100644 TestCases/ToolResults.h create mode 100644 ToolSummary/CMakeLists.txt create mode 100644 ToolSummary/ToolSummary.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 55ab7d0..ddd3fec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,3 +10,4 @@ add_subdirectory(TestCases) add_subdirectory(TestDiffs) add_subdirectory(TestNames) add_subdirectory(TestResults) +add_subdirectory(ToolSummary) diff --git a/TestCases/CMakeLists.txt b/TestCases/CMakeLists.txt index 5b314d8..64018d2 100644 --- a/TestCases/CMakeLists.txt +++ b/TestCases/CMakeLists.txt @@ -6,6 +6,8 @@ endif() add_library(test-cases STATIC TestCases.h TestCases.cpp + ToolResults.h + ToolResults.cpp ) set_target_properties(test-cases PROPERTIES CXX_STANDARD 17 diff --git a/TestCases/TestCases.cpp b/TestCases/TestCases.cpp index a996813..4570341 100644 --- a/TestCases/TestCases.cpp +++ b/TestCases/TestCases.cpp @@ -147,6 +147,23 @@ void sortTestCases() } } +const std::vector & getTests() +{ + return g_tests; +} + +const std::map> & getTestCases() +{ + return g_testCases; +} + +const std::vector &getTestCaseLabels(const char *prefix) +{ + const auto it = g_testCases.find(prefix); + static std::vector empty; + return it == g_testCases.cend() ? empty : it->second; +} + std::vector scanTestDirectory(std::string_view dir) { scanTestCaseDirectory(dir); diff --git a/TestCases/TestCases.h b/TestCases/TestCases.h index 11ab8c0..42f99b7 100644 --- a/TestCases/TestCases.h +++ b/TestCases/TestCases.h @@ -15,12 +15,16 @@ struct Test bool diffsRequired{true}; }; -extern std::vector g_tests; -extern std::map> g_testCases; - +const std::vector &getTests(); +const std::map> &getTestCases(); +const std::vector &getTestCaseLabels(const char *prefix); +inline std::size_t getNumTestCases(const char *prefix) +{ + return getTestCaseLabels(prefix).size(); +} std::vector scanTestDirectory(std::string_view dir); bool isDeprecatedLabel(const std::string &label); -const std::vector& getDeprecatedLabels(const char *prefix); +const std::vector &getDeprecatedLabels(const char *prefix); const char *getPrefixForTestName(std::string_view name); } // namespace testCases diff --git a/TestCases/ToolResults.cpp b/TestCases/ToolResults.cpp new file mode 100644 index 0000000..07558c4 --- /dev/null +++ b/TestCases/ToolResults.cpp @@ -0,0 +1,219 @@ +#include + +#include + +#include +#include +#include + +namespace testCases +{ + +bool isTestCaseResult(const std::string &line) +{ + if (line.empty()) + { + return false; + } + const auto bar = line.find_first_of('|'); + if (bar == std::string::npos) + { + return false; + } + const auto firstNonSpace = line.find_first_not_of(' '); + if (firstNonSpace == bar) + { + return false; + } + const auto endFirstWord = line.find_last_not_of(' ', bar); + const std::string firstWord = line.substr(firstNonSpace, endFirstWord - firstNonSpace - 1); + if (firstWord.empty() || firstWord == "Case" || firstWord.find_first_not_of('-') == std::string::npos) + { + return false; + } + const auto beginSecondWord = line.find_first_not_of(' ', bar + 1); + if (beginSecondWord == std::string::npos) + { + // Test label with no result reported + return true; + } + const auto lastNonSpace = line.find_last_not_of(' '); + if (lastNonSpace == bar) + { + return false; + } + const std::string secondWord = line.substr(beginSecondWord, lastNonSpace - beginSecondWord + 1); + return secondWord != "Result" && secondWord.find_first_not_of('-') != std::string::npos; +} + +void ToolResults::scanResultsFile(std::filesystem::path path) +{ + if (!is_regular_file(path)) + { + throw std::runtime_error(path.string() + " is not a plain file."); + } + + std::ifstream file(path); + std::string line; + int lineNum{}; + const auto getLine = [&] + { + std::getline(file, line); + ++lineNum; + }; + while (file) + { + getLine(); + if (!file) + { + break; + } + if (line.find("##") == 0) + { + break; + } + m_preamble.push_back(line); + } + while (file && line.find("##") == 0) + { + const std::string title = line.substr(line.find_first_not_of(' ', line.find_first_of(' '))); + const char *prefix = testCases::getPrefixForTestName(title); + if (prefix == nullptr) + { + m_errors.push_back(path.string() + '(' + std::to_string(lineNum) + "): test title '" + std::string{title} + + "' not found."); + while (file) + { + getLine(); + if (!file || line.find("##") == 0) + { + break; + } + } + continue; + } + + m_testReports.push_back(prefix); + const std::string test = prefix; + std::vector &results = m_testResults[test]; + std::vector &labels = m_testResultsLabels[test]; + while (file) + { + getLine(); + if (!file || line.find("##") == 0) + { + break; + } + if (line.substr(0, test.length()) == test) + { + labels.push_back(line.substr(0, line.find_first_of(' '))); + } + if (!isTestCaseResult(line)) + { + continue; + } + const auto bar = line.find('|'); + bool hasResult = line.find_first_not_of(' ', bar) != std::string::npos; + const bool deprecated = line.find("(deprecated)", bar) != std::string::npos; + bool passed{}; + if (!deprecated) + { + if (line.find("Pass", bar) != std::string::npos) + { + passed = true; + } + else if (line.find("Failure", bar) == std::string::npos) + { + hasResult = false; + } + } + results.push_back({line, hasResult, deprecated, passed}); + } + } +} + +bool ToolResults::markedDeprecated(const std::string &label) +{ + const std::string prefix = label.substr(0, label.find_first_of("0123456789")); + const std::vector &results = m_testResults[prefix]; + const auto matchesLabel = [&](const TestResult &result) { return result.line.find(label) != std::string::npos; }; + const auto pos = std::find_if(results.begin(), results.end(), matchesLabel); + return pos != results.end() && pos->deprecated; +} + +void ToolResults::checkResults() +{ + for (const char *testReport : m_testReports) + { + const std::vector &labels = m_testResultsLabels[testReport]; + auto findLabel = [&](const std::string &label) + { return std::find(labels.begin(), labels.end(), label) != labels.end(); }; + for (const std::string &deprecated : testCases::getDeprecatedLabels(testReport)) + { + if (!findLabel(deprecated)) + { + m_errors.push_back("error: No test results for deprecated test " + deprecated); + } + } + for (const std::string &testCase : getTestCaseLabels(testReport)) + { + if (testCases::isDeprecatedLabel(testCase) && !markedDeprecated(testCase)) + { + m_errors.push_back("error: Test results for " + testCase + " not marked deprecated."); + } + if (std::find(labels.begin(), labels.end(), testCase) == labels.end()) + { + m_errors.push_back("error: No test results for " + testCase); + } + } + for (const TestResult &result : m_testResults[testReport]) + { + const std::string label = result.line.substr(0, result.line.find_first_of(' ')); + if (!result.hasResult) + { + m_warnings.push_back("warning: No result for test " + label); + } + else if (result.deprecated && !testCases::isDeprecatedLabel(label)) + { + m_errors.push_back("error: Test result for " + label + + " is marked deprecated, but test case is not deprecated"); + } + } + } +} + +std::vector ToolResults::getSummary() const +{ + std::vector toolSummary; + for (std::string test : m_testReports) + { + const auto &testResults = m_testResults.find(test); + if (testResults == m_testResults.end()) + { + throw std::runtime_error("No test results available for " + test); + } + TestSummary summary{}; + summary.name = test; + for (const TestResult &result : testResults->second) + { + ++summary.numCases; + if (!result.hasResult) + { + continue; + } + ++summary.numCasesReported; + if (result.passed) + { + ++summary.passes; + } + else + { + ++summary.failures; + } + } + toolSummary.push_back(summary); + } + return toolSummary; +} + +} // namespace testCases diff --git a/TestCases/ToolResults.h b/TestCases/ToolResults.h new file mode 100644 index 0000000..6499e27 --- /dev/null +++ b/TestCases/ToolResults.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace testCases +{ + +struct TestResult +{ + std::string line; + bool hasResult; + bool deprecated; + bool passed; +}; +using ToolTestResults = std::map>; +using ToolTestResultsLabels = std::map>; + +inline std::string toolNameFromResultsFile(std::filesystem::path path) +{ + const std::string file = path.filename().string(); + const auto pos = file.find("Results.md"); + if (pos == std::string::npos) + return {}; + return file.substr(0, pos); +} + +struct TestSummary +{ + std::string name; + int passes; + int failures; + int numCases; + int numCasesReported; +}; + +class ToolResults +{ +public: + explicit ToolResults(std::string name) : m_name(std::move(name)) + { + } + + const std::string &getToolName() const + { + return m_name; + } + const std::vector &getWarnings() const + { + return m_warnings; + } + const std::vector &getErrors() const + { + return m_errors; + } + void scanResultsFile(std::filesystem::path path); + bool markedDeprecated(const std::string &label); + void checkResults(); + std::vector getSummary() const; + +private: + std::string m_name; + std::vector m_warnings; + std::vector m_errors; + std::vector m_diffs; + std::vector m_preamble; + std::vector m_testReports; + ToolTestResults m_testResults; + ToolTestResultsLabels m_testResultsLabels; +}; + +} // namespace testCases diff --git a/TestNames/TestNames.cpp b/TestNames/TestNames.cpp index 5e0f60a..9a4e6f3 100644 --- a/TestNames/TestNames.cpp +++ b/TestNames/TestNames.cpp @@ -16,10 +16,10 @@ void checkMissingTestCases() { const auto extractCaseNum = [](const std::string &label) { return std::stoi(label.substr(label.find_first_of("0123456789"))); }; - for (const testCases::Test &test : testCases::g_tests) + for (const testCases::Test &test : testCases::getTests()) { int num{1}; - for (const std::string &testCase : testCases::g_testCases[test.prefix]) + for (const std::string &testCase : testCases::getTestCaseLabels(test.prefix)) { int caseNum = extractCaseNum(testCase); do @@ -53,12 +53,12 @@ void printMarkDown(std::ostream &out) out << "# Tool\n\n"; - for (const testCases::Test &test : testCases::g_tests) + for (const testCases::Test &test : testCases::getTests()) { out << "\n## " << test.name << "\nCase | Result\n" "---- | ------\n"; - for (const std::string &testCase : testCases::g_testCases[test.prefix]) + for (const std::string &testCase : testCases::getTestCaseLabels(test.prefix)) { out << testCase << " | " << (testCases::isDeprecatedLabel(testCase) ? "(deprecated)" : "") << '\n'; } diff --git a/TestResults/TestResults.cpp b/TestResults/TestResults.cpp index f4650b1..88fe168 100644 --- a/TestResults/TestResults.cpp +++ b/TestResults/TestResults.cpp @@ -1,222 +1,32 @@ #include +#include -#include #include -#include #include -#include #include -#include #include -#include #include namespace testResults { -std::vector g_errors; -std::vector g_warnings; -std::vector g_diffs; -std::vector g_preamble; -std::vector g_testReports; -struct TestResult +bool reportResults(const testCases::ToolResults &result, std::ostream &out) { - std::string line; - bool hasResult; - bool deprecated; - bool passed; -}; -std::map> g_testResults; -std::map> g_testResultsLabels; - -bool isTestCaseResult(const std::string &line) -{ - if (line.empty()) - { - return false; - } - const auto bar = line.find_first_of('|'); - if (bar == std::string::npos) - { - return false; - } - const auto firstNonSpace = line.find_first_not_of(' '); - if (firstNonSpace == bar) - { - return false; - } - const auto endFirstWord = line.find_last_not_of(' ', bar); - const std::string firstWord = line.substr(firstNonSpace, endFirstWord - firstNonSpace - 1); - if (firstWord.empty() || firstWord == "Case" || firstWord.find_first_not_of('-') == std::string::npos) - { - return false; - } - const auto beginSecondWord = line.find_first_not_of(' ', bar + 1); - if (beginSecondWord == std::string::npos) + for (const std::string &warning : result.getWarnings()) { - // Test label with no result reported - return true; + out << warning << '\n'; } - const auto lastNonSpace = line.find_last_not_of(' '); - if (lastNonSpace == bar) + const std::vector &errors = result.getErrors(); + if (errors.empty()) { return false; } - const std::string secondWord = line.substr(beginSecondWord, lastNonSpace - beginSecondWord + 1); - return secondWord != "Result" && secondWord.find_first_not_of('-') != std::string::npos; -} - -void scanResultsFile(std::filesystem::path path) -{ - if (!is_regular_file(path)) - { - throw std::runtime_error(path.string() + " is not a plain file."); - } - - std::ifstream file(path); - std::string line; - int lineNum{}; - const auto getLine = [&] - { - std::getline(file, line); - ++lineNum; - }; - while (file) - { - getLine(); - if (!file) - { - break; - } - if (line.find("##") == 0) - { - break; - } - g_preamble.push_back(line); - } - while (file && line.find("##") == 0) - { - const std::string title = line.substr(line.find_first_not_of(' ', line.find_first_of(' '))); - const char *prefix = testCases::getPrefixForTestName(title); - if (prefix == nullptr) - { - g_errors.push_back(path.string() + '(' + std::to_string(lineNum) + "): test title '" + std::string{title} - + "' not found."); - while (file) - { - getLine(); - if (!file || line.find("##") == 0) - { - break; - } - } - continue; - } - - g_testReports.push_back(prefix); - const std::string test = prefix; - std::vector &results = g_testResults[test]; - std::vector &labels = g_testResultsLabels[test]; - while (file) - { - getLine(); - if (!file || line.find("##") == 0) - { - break; - } - if (line.substr(0, test.length()) == test) - { - labels.push_back(line.substr(0, line.find_first_of(' '))); - } - if (!isTestCaseResult(line)) - { - continue; - } - const auto bar = line.find('|'); - bool hasResult = line.find_first_not_of(' ', bar) != std::string::npos; - const bool deprecated = line.find("(deprecated)", bar) != std::string::npos; - bool passed{}; - if (!deprecated) - { - if (line.find("Pass", bar) != std::string::npos) - { - passed = true; - } - else if (line.find("Failure", bar) == std::string::npos) - { - hasResult = false; - } - } - results.push_back({line, hasResult, deprecated, passed}); - } - } -} - -bool markedDeprecated(const std::string &label) -{ - const std::string prefix = label.substr(0, label.find_first_of("0123456789")); - const std::vector &results = g_testResults[prefix]; - const auto matchesLabel = [&](const TestResult &result) { return result.line.find(label) != std::string::npos; }; - const auto pos = std::find_if(results.begin(), results.end(), matchesLabel); - return pos != results.end() && pos->deprecated; -} -void checkResults() -{ - for (const char *testReport : g_testReports) + for (const std::string &error : errors) { - const std::vector &labels = g_testResultsLabels[testReport]; - auto findLabel = [&](const std::string &label) - { return std::find(labels.begin(), labels.end(), label) != labels.end(); }; - for (const std::string &deprecated : testCases::getDeprecatedLabels(testReport)) - { - if (!findLabel(deprecated)) - { - g_errors.push_back("error: No test results for deprecated test " + deprecated); - } - } - for (const std::string &testCase : testCases::g_testCases[testReport]) - { - if (testCases::isDeprecatedLabel(testCase) && !markedDeprecated(testCase)) - { - g_errors.push_back("error: Test results for " + testCase + " not marked deprecated."); - } - if (std::find(labels.begin(), labels.end(), testCase) == labels.end()) - { - g_errors.push_back("error: No test results for " + testCase); - } - } - for (const TestResult &result : g_testResults[testReport]) - { - const std::string label = result.line.substr(0, result.line.find_first_of(' ')); - if (!result.hasResult) - { - g_warnings.push_back("warning: No result for test " + label); - } - else if (result.deprecated && !testCases::isDeprecatedLabel(label)) - { - g_errors.push_back("error: Test result for " + label - + " is marked deprecated, but test case is not deprecated"); - } - } - } -} - -int reportResults(std::ostream &out) -{ - for (const std::string &warning : g_warnings) - { - std::cerr << warning << '\n'; - } - if (!g_errors.empty()) - { - for (const std::string &error : g_errors) - { - std::cerr << error << '\n'; - } - return 1; + out << error << '\n'; } - return 0; + return true; } int main(const std::vector &args) @@ -228,10 +38,11 @@ int main(const std::vector &args) throw std::runtime_error("Missing directory arguments"); } - g_errors = testCases::scanTestDirectory(args[1]); - scanResultsFile(args[2]); - checkResults(); - return reportResults(std::cout); + testCases::scanTestDirectory(args[1]); + testCases::ToolResults results(testCases::toolNameFromResultsFile(args[2])); + results.scanResultsFile(args[2]); + results.checkResults(); + return reportResults(results, std::cout) ? 1 : 0; } catch (const std::exception &bang) { diff --git a/ToolSummary/CMakeLists.txt b/ToolSummary/CMakeLists.txt new file mode 100644 index 0000000..2b25c8a --- /dev/null +++ b/ToolSummary/CMakeLists.txt @@ -0,0 +1,12 @@ +if(NOT ("cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES)) + message(WARNING tool-summary requires C++17 support) + return() +endif() + +add_executable(tool-summary ToolSummary.cpp) +set_target_properties(tool-summary PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + FOLDER Tools +) +target_link_libraries(tool-summary PUBLIC test-cases) diff --git a/ToolSummary/ToolSummary.cpp b/ToolSummary/ToolSummary.cpp new file mode 100644 index 0000000..2a330d7 --- /dev/null +++ b/ToolSummary/ToolSummary.cpp @@ -0,0 +1,132 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace testResults +{ + +std::vector g_toolResults; + +void scanResultsDirectory(std::filesystem::path dir) +{ + for (auto &entry : std::filesystem::directory_iterator(dir)) + { + if (is_directory(entry)) + { + continue; + } + + const std::string toolName = testCases::toolNameFromResultsFile(entry.path()); + if (toolName.empty()) + { + continue; + } + g_toolResults.emplace_back(toolName); + g_toolResults.back().scanResultsFile(entry); + } +} + +struct ToolSummary +{ + std::string name; + std::vector summary; +}; + +inline std::string percent(int numerator, int denominator) +{ + std::ostringstream str; + const double fraction = 100.0 * double(numerator) / double(denominator); + str << std::setprecision(4) << fraction << '%'; + return str.str(); +} + +void reportSummary() +{ + std::vector summary; + for (testCases::ToolResults &toolResult : g_toolResults) + { + toolResult.checkResults(); + summary.push_back({toolResult.getToolName(), toolResult.getSummary()}); + } + + std::cout << "# Summary Results\n" + "\n" + "Refactoring"; + std::string separator(11, '-'); // len("Refactoring") == 11 + for (const ToolSummary &tool : summary) + { + std::cout << " | " << tool.name; + separator += " | " + std::string(tool.name.size(), '-'); + } + std::cout << '\n' << separator << '\n'; + + for (const testCases::Test &test : testCases::getTests()) + { + std::cout << test.name; + for (const ToolSummary &tool : summary) + { + std::cout << " | "; + auto it = std::find_if(tool.summary.begin(), + tool.summary.end(), + [prefix = test.prefix](const testCases::TestSummary &testSummary) + { return testSummary.name == prefix; }); + if (it == tool.summary.end()) + { + std::cout << "(n/a)"; + } + else + { + std::cout << percent(it->passes, it->numCasesReported) << " (" << it->numCasesReported << '/' + << testCases::getNumTestCases(test.prefix) << ')'; + } + } + std::cout << '\n'; + } +} + +int main(const std::vector &args) +{ + try + { + if (args.size() < 3) + { + throw std::runtime_error("Missing directory arguments"); + } + + std::vector testCaseErrors = testCases::scanTestDirectory(args[1]); + scanResultsDirectory(args[2]); + reportSummary(); + return 0; + } + catch (const std::exception &bang) + { + std::cerr << "Unexpected exception: " << bang.what() << '\n'; + return 2; + } + catch (...) + { + std::cerr << "Unknown exception\n"; + return 3; + } +} + +} // namespace testResults + +int main(int argc, char *argv[]) +{ + std::vector args; + for (int i = 0; i < argc; ++i) + { + args.emplace_back(argv[i]); + } + + return testResults::main(args); +}