Produce CDash URLs to all builds and nonpassing tests for same repo v…

…ersion (#483) This makes it easy to click on links to these other CDash URLs to see what is happening with all of the builds for the same repo version. The main target is for the GitHub Actions drivers for TriBITS testing. This makes it easy to get at those results. As part of this, I also renamed the function tribits_get_build_url_and_write_to_file() to tribits_get_cdash_build_url() and removed the ability to write to a file. (We just don't need that anymore and it is trivial to write a string to a file.)
TriBITSPub · Jun 6, 2022 · 4354ed0 · 4354ed0
1 parent 32cc1ad
commit 4354ed0
Show file tree

Hide file tree

Showing 6 changed files with 422 additions and 29 deletions.
diff --git a/.github/workflows/tribits_testing.yml b/.github/workflows/tribits_testing.yml
@@ -93,6 +93,9 @@ jobs:
       - name: URL to results on CDash
         run: |
           cd ..
-          echo "Result on CDash are posted at:"
+          echo "See results posted on CDash:"
+          echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
           echo
-          cat tribits-build/BUILD/CDashBuildUrl.txt
+          cat tribits-build/BUILD/CDashResults.txt
+          echo
+          echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
diff --git a/test/ctest_driver/CTestDriverUnitTests.cmake b/test/ctest_driver/CTestDriverUnitTests.cmake
@@ -53,6 +53,42 @@ include(TribitsReadTagFile)
 include(TribitsGetCDashUrlsInsideCTestS)
 
 
+function(unittest_tribits_get_cdash_revision_builds_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_builds_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_builds_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_BUILDS_URL_OUT  cdashRevesionBuildsUrlOut
+     )
+  unittest_compare_const(cdashRevesionBuildsUrlOut
+    "somesite.com/my-cdash/index.php?project=goodProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=abc123")
+
+endfunction()
+
+
+function(unittest_tribits_get_cdash_revision_nonpassing_tests_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_nonpassing_tests_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_nonpassing_tests_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_NONPASSING_TESTS_URL_OUT  cdashRevisionNonpassingTestsUrlOut
+     )
+  unittest_compare_const(cdashRevisionNonpassingTestsUrlOut
+    "somesite.com/my-cdash/queryTests.php?project=goodProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=abc123&field2=status&compare2=62&value2=passed")
+
+endfunction()
+
+
 function(unittest_tribits_read_ctest_tag_file)
 
   message("\n***")
@@ -158,6 +194,8 @@ endfunction()
 unittest_initialize_vars()
 
 # Run the unit test functions
+unittest_tribits_get_cdash_revision_builds_url()
+unittest_tribits_get_cdash_revision_nonpassing_tests_url()
 unittest_tribits_read_ctest_tag_file()
 unittest_tribits_get_cdash_site_from_drop_site_and_location()
 unittest_tribits_get_cdash_index_php_from_drop_site_and_location()

diff --git a/test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt b/test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt
@@ -98,6 +98,19 @@ set(TribitsExampleMetaProject_COMMON_CONFIG_ARGS
 ##########################################################################################
 
 
+  set(cdash_build_url_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/index[.]php[?]project=TribitsExampleMetaProject&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=CustomSite&field2=buildname&compare2=61&value2=CTestDriver_TribitsExMetaProj_clone_default_branch_remote&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Continuous" )
+  # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
+  # the number of digits in the buildstarttime.  CMake regex does not support
+  # \d{8}-\d{4} :-(
+
+  set(cdash_revisions_builds_url_expected_regex
+     "https://testing[.]sandia[.]gov/cdash/index.php[?]project=TribitsExampleMetaProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3")
+
+  set(cdash_revisions_nonpassing_tests_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/queryTests.php[?]project=TribitsExampleMetaProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3&field2=status&compare2=62&value2=passed" )
+
+
 tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_remote
   OVERALL_WORKING_DIRECTORY TEST_NAME
   OVERALL_NUM_MPI_PROCS 1
@@ -111,11 +124,19 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
       ${COMMON_ENV_ARGS}
       TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
       CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote
+      CTEST_SITE=CustomSite
       ${CTEST_S_SCRIPT_ARGS}
     PASS_REGULAR_EXPRESSION_ALL
       "CTEST_NOTES_FILES=''"
       "First perform the initial checkout: .*/git. clone -o origin .*github.com.tribits/TribitsExampleMetaProject.git"
       "Perform checkout in directory: .*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote"
+      "Results will be submitted on CDash at the following links:"
+      "Link to this build's results on CDash:"
+      "${cdash_build_url_expected_regex}"
+      "Link to all builds for this repo version on CDash:"
+      "${cdash_revisions_builds_url_expected_regex}"
+      "Link to all nonpassing tests for all builds for this repo version on CDash:"
+      "${cdash_revisions_nonpassing_tests_expected_regex}"
       "Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote/TribitsExampleMetaProject"
       "Old revision of repository is: [a-z0-9]+"
       "New revision of repository is: [a-z0-9]+"
@@ -315,6 +336,11 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
   # * Makes sure some git commit SHA1s are shown in 'Old/New revision of
   #   repository' printouts from ctest_update().
   #
+  # * Checks that CDash URLs that get produced, including for all builds with
+  # * the same repo version.  (This check has to be here since the base
+  # * project repo must have a .git/ directory and we must know the exact SHA1
+  # * of that repo for the test, which we don in this case.)
+  #
   # NOTE: The reason that so many tests are done in a single ctest test is the
   # cost of the initial configure which checks the compilers which is very
   # expensive.  The other runs use

diff --git a/test/ctest_driver/TribitsExampleProject/CMakeLists.txt b/test/ctest_driver/TribitsExampleProject/CMakeLists.txt
@@ -1264,7 +1264,8 @@ function(generate_aao_tests)
     # CONFIGURE_OPTIONS list is checked to make sure that it does not change
     # without being noticed.  Follow-on tests will not check all of that.
 
-  set(cdash_url_expected_regex
+
+  set(cdash_build_url_expected_regex
     "https://cdash[.]site[.]com/cdash/index[.]php[?]project=CustomTribitsExProj&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=My%20Site&field2=buildname&compare2=61&value2=TriBITS_CTestDriver_AAOP_CDASH_URL&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Custom%20CDash%20Group" )
   # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
   # the number of digits in the buildstarttime.  CMake regex does not support
@@ -1285,7 +1286,7 @@ function(generate_aao_tests)
         CTEST_DASHBOARD_ROOT=PWD
         ${AAO_COMMON_ENV_ARGS}
         CTEST_PARALLEL_LEVEL=3
-        CTEST_DO_SUBMIT=OFF  # Never submit this
+        CTEST_DO_SUBMIT=OFF  # Never submit this (see below note)
         CTEST_SITE="My Site"
         CTEST_DROP_SITE="cdash.site.com"
         CTEST_PROJECT_NAME="CustomTribitsExProj"
@@ -1299,17 +1300,19 @@ function(generate_aao_tests)
         "NONE does not exist, skipping extra repositories"
         "Final set of enabled packages:  SimpleCxx 1"
         "Final set of enabled SE packages:  SimpleCxx 1"
-        "Results will be submitted to CDash at:"
+        "Results will be submitted on CDash at the following links:"
+        "Link to this build's results on CDash:"
+        "${cdash_build_url_expected_regex}"
         "File '' does NOT exist so all tests passed"
-        "See results submitted to CDash at:"
-        "${cdash_url_expected_regex}"
+        "See results submitted on CDash at the following links:"
         "TRIBITS_CTEST_DRIVER: OVERALL: ALL PASSED"
       ALWAYS_FAIL_ON_NONZERO_RETURN
     TEST_2
-      MESSAGE "Check that CDashBuildUrl.txt exists and lists the right URL."
-      CMND cat ARGS BUILD/CDashBuildUrl.txt
+      MESSAGE "Check that CDashResults.txt exists and lists the right URL."
+      CMND cat ARGS BUILD/CDashResults.txt
       PASS_REGULAR_EXPRESSION_ALL
-        "${cdash_url_expected_regex}"
+        "Link to this build's results on CDash:"
+        "${cdash_build_url_expected_regex}"
     )
     # NOTE: The above test is never actually submitted to CDash, even when
     # ${PACKAGE_NAME}_CTEST_DRIVER_SUBMIT_TO is set.  This is so we can

diff --git a/tribits/ctest_driver/TribitsCTestDriverCore.cmake b/tribits/ctest_driver/TribitsCTestDriverCore.cmake
@@ -323,6 +323,7 @@ include(TribitsCTestDriverCoreHelpers)
 # * `Determining what testing-related actions are performed (tribits_ctest_driver())`_
 # * `Determining how the results are displayed on CDash (tribits_ctest_driver())`_
 # * `Specifying where the results go to CDash (tribits_ctest_driver())`_
+# * `Links to results on CDash (tribits_ctest_driver())`_
 # * `Determining what TriBITS repositories are included (tribits_ctest_driver())`_
 # * `All-at-once versus package-by-package mode (tribits_ctest_driver())`_
 # * `Multiple ctest -S invocations (tribits_ctest_driver())`_
@@ -1166,6 +1167,63 @@ include(TribitsCTestDriverCoreHelpers)
 # not be performed.  For more details, see `TRIBITS_2ND_CTEST_DROP_SITE`_ and
 # `TRIBITS_2ND_CTEST_DROP_LOCATION`_.
 #
+# .. _Links to results on CDash (tribits_ctest_driver()):
+#
+# **Links to results on CDash (tribits_ctest_driver()):**
+#
+# Links to where the results will be posted on CDash are printed to STDOUT
+# before it performs any actions and at end after all of the actions and
+# submits have been completed.
+#
+# The results are printed to STDOUT in a section that looks like::
+#
+#   Link to this build's results on CDash:
+#
+#     <cdash-build-url>
+#
+#   Link to all builds for this repo version on CDash:
+#
+#     <cdash-revision-builds-url>
+#
+#   Link to all nonpassing tests for all builds for this repo version on CDash:
+#
+#     <cdash-revision-nonpassing-tests-url>
+#
+# The URL ``<cdash-build-url>`` is created from the buildname, site, and
+# buildstartime fields which is known from the TAG file created by CTest.
+# This allows access the results for this particular build on CDash by just
+# clicking that link.
+#
+# The URL ``<cdash-revision-builds-url>`` provides a link to a CDash
+# ``index.php`` query that includes all of the builds with the same base Git
+# repo SHA1.  This allows comparing the results of this build for other builds
+# for this same version of the base Git repository.
+#
+# The URL ``<cdash-revision-nonpassing-tests-url>`` gives a link to a CDash
+# ``queryTests.php`` query for all of the nonpassing tests for all of the
+# builds with this same base project Git repo SHA1.  This allows comparing
+# test failures across all of the builds for the same base project Git repo
+# version.
+#
+# NOTE: The links ``<cdash-revision-builds-url>`` and
+# ``<cdash-revision-nonpassing-tests-url>`` are only provided if the base
+# project Git repo has the ``.git/`` subdirectory and if ``git log``
+# successfully returns the SHA1 for that base Git repo.
+#
+# NOTE: The links ``<cdash-revision-builds-url>`` and
+# ``<cdash-revision-nonpassing-tests-url>`` only consider the Git SHA1 of the
+# base project Git repo.  For multi-repo projects (see `Multi-Repository
+# Support`_), you may get results for builds with different subrepo versions
+# and therefore may be comparing apples and oranges.  (Projects that commit a
+# ``<Project>SubRepoVersion.txt`` file to their base Git repo or use Git
+# Submodules will have unique base project Git repo SHA1s for different
+# versions of the project's repos.)
+#
+# In addition, a text file ``CDashResults.txt`` will be written in the build
+# directory that contains this same CDash link information shown above.  This
+# allows a process to cat the file ``CDashResults.txt`` to get links to the
+# results on CDash.
+#
 # .. _Determining what TriBITS repositories are included (tribits_ctest_driver()):
 #
 # **Determining what TriBITS repositories are included (tribits_ctest_driver()):**
@@ -2093,10 +2151,15 @@ function(tribits_ctest_driver)
   ctest_start(${CTEST_START_ARGS})
 
   tribits_remember_if_configure_attempted()
-  tribits_get_build_url_and_write_to_file(CDASH_BUILD_URL
-    "${CTEST_BINARY_DIRECTORY}/CDashBuildUrl.txt")
-  tribits_print_cdash_url("Results will be submitted to CDash at:"
-    "${CDASH_BUILD_URL}")
+
+  tribits_get_cdash_results_string_and_write_to_file(
+    CDASH_RESULTS_STRING_OUT  CDASH_RESULTS_STRING
+    CDASH_RESULTS_FILE_OUT "${CTEST_BINARY_DIRECTORY}/CDashResults.txt" )
+  message("Results will be submitted on CDash at the following links:\n\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
+    "${CDASH_RESULTS_STRING}\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+    )
 
   message(
     "\n***"
@@ -2370,8 +2433,11 @@ function(tribits_ctest_driver)
 
   report_queued_errors()
 
-  tribits_print_cdash_url("See results submitted to CDash at:"
-    "${CDASH_BUILD_URL}")
+  message("\nSee results submitted on CDash at the following links:\n\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
+    "${CDASH_RESULTS_STRING}\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+    )
 
   if ((NOT UPDATE_FAILED) AND ("${${PROJECT_NAME}_FAILED_PACKAGES}" STREQUAL ""))
     message(