Create a tool to generate summary results

- Extract ToolResults from TestResults and ToolSummary - Scan test case directory - Scan results directory - Compile summary results for all tools - Write summary markdown file
LegalizeAdulthood · Jan 9, 2024 · ee4f882 · ee4f882
1 parent 8166228
commit ee4f882
Show file tree

Hide file tree

Showing 10 changed files with 485 additions and 211 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -10,3 +10,4 @@ add_subdirectory(TestCases)
 add_subdirectory(TestDiffs)
 add_subdirectory(TestNames)
 add_subdirectory(TestResults)
+add_subdirectory(ToolSummary)
diff --git a/TestCases/CMakeLists.txt b/TestCases/CMakeLists.txt
@@ -6,6 +6,8 @@ endif()
 add_library(test-cases STATIC
     TestCases.h
     TestCases.cpp
+    ToolResults.h
+    ToolResults.cpp
 )
 set_target_properties(test-cases PROPERTIES
     CXX_STANDARD 17

diff --git a/TestCases/TestCases.cpp b/TestCases/TestCases.cpp
@@ -147,6 +147,23 @@ void sortTestCases()
     }
 }
 
+const std::vector<Test> & getTests()
+{
+    return g_tests;
+}
+
+const std::map<const char *, std::vector<std::string>> & getTestCases()
+{
+    return g_testCases;
+}
+
+const std::vector<std::string> &getTestCaseLabels(const char *prefix)
+{
+    const auto it = g_testCases.find(prefix);
+    static std::vector<std::string> empty;
+    return it == g_testCases.cend() ? empty : it->second;
+}
+
 std::vector<std::string> scanTestDirectory(std::string_view dir)
 {
     scanTestCaseDirectory(dir);

diff --git a/TestCases/TestCases.h b/TestCases/TestCases.h
@@ -15,12 +15,16 @@ struct Test
     bool diffsRequired{true};
 };
 
-extern std::vector<Test> g_tests;
-extern std::map<const char *, std::vector<std::string>> g_testCases;
-
+const std::vector<Test> &getTests();
+const std::map<const char *, std::vector<std::string>> &getTestCases();
+const std::vector<std::string> &getTestCaseLabels(const char *prefix);
+inline std::size_t getNumTestCases(const char *prefix)
+{
+    return getTestCaseLabels(prefix).size();
+}
 std::vector<std::string> scanTestDirectory(std::string_view dir);
 bool isDeprecatedLabel(const std::string &label);
-const std::vector<std::string>& getDeprecatedLabels(const char *prefix);
+const std::vector<std::string> &getDeprecatedLabels(const char *prefix);
 const char *getPrefixForTestName(std::string_view name);
 
 }    // namespace testCases
diff --git a/TestCases/ToolResults.cpp b/TestCases/ToolResults.cpp
@@ -0,0 +1,219 @@
+#include <ToolResults.h>
+
+#include <TestCases.h>
+
+#include <algorithm>
+#include <fstream>
+#include <stdexcept>
+
+namespace testCases
+{
+
+bool isTestCaseResult(const std::string &line)
+{
+    if (line.empty())
+    {
+        return false;
+    }
+    const auto bar = line.find_first_of('|');
+    if (bar == std::string::npos)
+    {
+        return false;
+    }
+    const auto firstNonSpace = line.find_first_not_of(' ');
+    if (firstNonSpace == bar)
+    {
+        return false;
+    }
+    const auto endFirstWord = line.find_last_not_of(' ', bar);
+    const std::string firstWord = line.substr(firstNonSpace, endFirstWord - firstNonSpace - 1);
+    if (firstWord.empty() || firstWord == "Case" || firstWord.find_first_not_of('-') == std::string::npos)
+    {
+        return false;
+    }
+    const auto beginSecondWord = line.find_first_not_of(' ', bar + 1);
+    if (beginSecondWord == std::string::npos)
+    {
+        // Test label with no result reported
+        return true;
+    }
+    const auto lastNonSpace = line.find_last_not_of(' ');
+    if (lastNonSpace == bar)
+    {
+        return false;
+    }
+    const std::string secondWord = line.substr(beginSecondWord, lastNonSpace - beginSecondWord + 1);
+    return secondWord != "Result" && secondWord.find_first_not_of('-') != std::string::npos;
+}
+
+void ToolResults::scanResultsFile(std::filesystem::path path)
+{
+    if (!is_regular_file(path))
+    {
+        throw std::runtime_error(path.string() + " is not a plain file.");
+    }
+
+    std::ifstream file(path);
+    std::string line;
+    int lineNum{};
+    const auto getLine = [&]
+    {
+        std::getline(file, line);
+        ++lineNum;
+    };
+    while (file)
+    {
+        getLine();
+        if (!file)
+        {
+            break;
+        }
+        if (line.find("##") == 0)
+        {
+            break;
+        }
+        m_preamble.push_back(line);
+    }
+    while (file && line.find("##") == 0)
+    {
+        const std::string title = line.substr(line.find_first_not_of(' ', line.find_first_of(' ')));
+        const char *prefix = testCases::getPrefixForTestName(title);
+        if (prefix == nullptr)
+        {
+            m_errors.push_back(path.string() + '(' + std::to_string(lineNum) + "): test title '" + std::string{title}
+                               + "' not found.");
+            while (file)
+            {
+                getLine();
+                if (!file || line.find("##") == 0)
+                {
+                    break;
+                }
+            }
+            continue;
+        }
+
+        m_testReports.push_back(prefix);
+        const std::string test = prefix;
+        std::vector<TestResult> &results = m_testResults[test];
+        std::vector<std::string> &labels = m_testResultsLabels[test];
+        while (file)
+        {
+            getLine();
+            if (!file || line.find("##") == 0)
+            {
+                break;
+            }
+            if (line.substr(0, test.length()) == test)
+            {
+                labels.push_back(line.substr(0, line.find_first_of(' ')));
+            }
+            if (!isTestCaseResult(line))
+            {
+                continue;
+            }
+            const auto bar = line.find('|');
+            bool hasResult = line.find_first_not_of(' ', bar) != std::string::npos;
+            const bool deprecated = line.find("(deprecated)", bar) != std::string::npos;
+            bool passed{};
+            if (!deprecated)
+            {
+                if (line.find("Pass", bar) != std::string::npos)
+                {
+                    passed = true;
+                }
+                else if (line.find("Failure", bar) == std::string::npos)
+                {
+                    hasResult = false;
+                }
+            }
+            results.push_back({line, hasResult, deprecated, passed});
+        }
+    }
+}
+
+bool ToolResults::markedDeprecated(const std::string &label)
+{
+    const std::string prefix = label.substr(0, label.find_first_of("0123456789"));
+    const std::vector<TestResult> &results = m_testResults[prefix];
+    const auto matchesLabel = [&](const TestResult &result) { return result.line.find(label) != std::string::npos; };
+    const auto pos = std::find_if(results.begin(), results.end(), matchesLabel);
+    return pos != results.end() && pos->deprecated;
+}
+
+void ToolResults::checkResults()
+{
+    for (const char *testReport : m_testReports)
+    {
+        const std::vector<std::string> &labels = m_testResultsLabels[testReport];
+        auto findLabel = [&](const std::string &label)
+        { return std::find(labels.begin(), labels.end(), label) != labels.end(); };
+        for (const std::string &deprecated : testCases::getDeprecatedLabels(testReport))
+        {
+            if (!findLabel(deprecated))
+            {
+                m_errors.push_back("error: No test results for deprecated test " + deprecated);
+            }
+        }
+        for (const std::string &testCase : getTestCaseLabels(testReport))
+        {
+            if (testCases::isDeprecatedLabel(testCase) && !markedDeprecated(testCase))
+            {
+                m_errors.push_back("error: Test results for " + testCase + " not marked deprecated.");
+            }
+            if (std::find(labels.begin(), labels.end(), testCase) == labels.end())
+            {
+                m_errors.push_back("error: No test results for " + testCase);
+            }
+        }
+        for (const TestResult &result : m_testResults[testReport])
+        {
+            const std::string label = result.line.substr(0, result.line.find_first_of(' '));
+            if (!result.hasResult)
+            {
+                m_warnings.push_back("warning: No result for test " + label);
+            }
+            else if (result.deprecated && !testCases::isDeprecatedLabel(label))
+            {
+                m_errors.push_back("error: Test result for " + label
+                                   + " is marked deprecated, but test case is not deprecated");
+            }
+        }
+    }
+}
+
+std::vector<TestSummary> ToolResults::getSummary() const
+{
+    std::vector<TestSummary> toolSummary;
+    for (std::string test : m_testReports)
+    {
+        const auto &testResults = m_testResults.find(test);
+        if (testResults == m_testResults.end())
+        {
+            throw std::runtime_error("No test results available for " + test);
+        }
+        TestSummary summary{};
+        summary.name = test;
+        for (const TestResult &result : testResults->second)
+        {
+            ++summary.numCases;
+            if (!result.hasResult)
+            {
+                continue;
+            }
+            ++summary.numCasesReported;
+            if (result.passed)
+            {
+                ++summary.passes;
+            }
+            else
+            {
+                ++summary.failures;
+            }
+        }
+        toolSummary.push_back(summary);
+    }
+    return toolSummary;
+}
+
+}    // namespace testCases
diff --git a/TestCases/ToolResults.h b/TestCases/ToolResults.h
@@ -0,0 +1,76 @@
+#pragma once
+
+#include <filesystem>
+#include <map>
+#include <ostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace testCases
+{
+
+struct TestResult
+{
+    std::string line;
+    bool hasResult;
+    bool deprecated;
+    bool passed;
+};
+using ToolTestResults = std::map<std::string, std::vector<TestResult>>;
+using ToolTestResultsLabels = std::map<std::string, std::vector<std::string>>;
+
+inline std::string toolNameFromResultsFile(std::filesystem::path path)
+{
+    const std::string file = path.filename().string();
+    const auto pos = file.find("Results.md");
+    if (pos == std::string::npos)
+        return {};
+    return file.substr(0, pos);
+}
+
+struct TestSummary
+{
+    std::string name;
+    int passes;
+    int failures;
+    int numCases;
+    int numCasesReported;
+};
+
+class ToolResults
+{
+public:
+    explicit ToolResults(std::string name) : m_name(std::move(name))
+    {
+    }
+
+    const std::string &getToolName() const
+    {
+        return m_name;
+    }
+    const std::vector<std::string> &getWarnings() const
+    {
+        return m_warnings;
+    }
+    const std::vector<std::string> &getErrors() const
+    {
+        return m_errors;
+    }
+    void scanResultsFile(std::filesystem::path path);
+    bool markedDeprecated(const std::string &label);
+    void checkResults();
+    std::vector<TestSummary> getSummary() const;
+
+private:
+    std::string m_name;
+    std::vector<std::string> m_warnings;
+    std::vector<std::string> m_errors;
+    std::vector<std::string> m_diffs;
+    std::vector<std::string> m_preamble;
+    std::vector<const char *> m_testReports;
+    ToolTestResults m_testResults;
+    ToolTestResultsLabels m_testResultsLabels;
+};
+
+}    // namespace testCases
diff --git a/TestNames/TestNames.cpp b/TestNames/TestNames.cpp
@@ -16,10 +16,10 @@ void checkMissingTestCases()
 {
     const auto extractCaseNum = [](const std::string &label)
     { return std::stoi(label.substr(label.find_first_of("0123456789"))); };
-    for (const testCases::Test &test : testCases::g_tests)
+    for (const testCases::Test &test : testCases::getTests())
     {
         int num{1};
-        for (const std::string &testCase : testCases::g_testCases[test.prefix])
+        for (const std::string &testCase : testCases::getTestCaseLabels(test.prefix))
         {
             int caseNum = extractCaseNum(testCase);
             do
@@ -53,12 +53,12 @@ void printMarkDown(std::ostream &out)
 
     out << "# Tool\n\n";
 
-    for (const testCases::Test &test : testCases::g_tests)
+    for (const testCases::Test &test : testCases::getTests())
     {
         out << "\n## " << test.name
             << "\nCase | Result\n"
                "---- | ------\n";
-        for (const std::string &testCase : testCases::g_testCases[test.prefix])
+        for (const std::string &testCase : testCases::getTestCaseLabels(test.prefix))
         {
             out << testCase << " | " << (testCases::isDeprecatedLabel(testCase) ? "(deprecated)" : "") << '\n';
         }