Produce CDash URLs to all builds and nonpassing tests for same repo v…

…ersion (#483) This makes it easy to click on links to these other CDash URLs to see what is happening with all of the builds for the same repo version. The main target is for the GitHub Actions drivers for TriBITS testing. This makes it easy to get at those results. As part of this, I also renamed the function tribits_get_build_url_and_write_to_file() to tribits_get_cdash_build_url() and removed the ability to write to a file. (We just don't need that anymore and it is trivial to write a string to a file.)
TriBITSPub · Jun 5, 2022 · 30f317d · 30f317d
1 parent 32cc1ad
commit 30f317d
Show file tree

Hide file tree

Showing 6 changed files with 364 additions and 29 deletions.
diff --git a/.github/workflows/tribits_testing.yml b/.github/workflows/tribits_testing.yml
@@ -93,6 +93,9 @@ jobs:
       - name: URL to results on CDash
         run: |
           cd ..
-          echo "Result on CDash are posted at:"
+          echo "See results posted on CDash:"
+          echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
           echo
-          cat tribits-build/BUILD/CDashBuildUrl.txt
+          cat tribits-build/BUILD/CDashResults.txt
+          echo
+          echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
diff --git a/test/ctest_driver/CTestDriverUnitTests.cmake b/test/ctest_driver/CTestDriverUnitTests.cmake
@@ -53,6 +53,42 @@ include(TribitsReadTagFile)
 include(TribitsGetCDashUrlsInsideCTestS)
 
 
+function(unittest_tribits_get_cdash_revision_builds_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_builds_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_builds_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_BUILDS_URL_OUT  cdashRevesionBuildsUrlOut
+     )
+  unittest_compare_const(cdashRevesionBuildsUrlOut
+    "somesite.com/my-cdash/index.php?project=goodProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=abc123")
+
+endfunction()
+
+
+function(unittest_tribits_get_cdash_revision_nonpassing_tests_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_nonpassing_tests_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_nonpassing_tests_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_NONPASSING_TESTS_URL_OUT  cdashRevisionNonpassingTestsUrlOut
+     )
+  unittest_compare_const(cdashRevisionNonpassingTestsUrlOut
+    "somesite.com/my-cdash/queryTests.php?project=goodProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=abc123&field2=status&compare2=62&value2=passed")
+
+endfunction()
+
+
 function(unittest_tribits_read_ctest_tag_file)
 
   message("\n***")
@@ -158,6 +194,8 @@ endfunction()
 unittest_initialize_vars()
 
 # Run the unit test functions
+unittest_tribits_get_cdash_revision_builds_url()
+unittest_tribits_get_cdash_revision_nonpassing_tests_url()
 unittest_tribits_read_ctest_tag_file()
 unittest_tribits_get_cdash_site_from_drop_site_and_location()
 unittest_tribits_get_cdash_index_php_from_drop_site_and_location()

diff --git a/test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt b/test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt
@@ -98,6 +98,19 @@ set(TribitsExampleMetaProject_COMMON_CONFIG_ARGS
 ##########################################################################################
 
 
+  set(cdash_build_url_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/index[.]php[?]project=TribitsExampleMetaProject&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=CustomSite&field2=buildname&compare2=61&value2=CTestDriver_TribitsExMetaProj_clone_default_branch_remote&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Continuous" )
+  # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
+  # the number of digits in the buildstarttime.  CMake regex does not support
+  # \d{8}-\d{4} :-(
+
+  set(cdash_revisions_builds_url_expected_regex
+     "https://testing[.]sandia[.]gov/cdash/index.php[?]project=TribitsExampleMetaProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3")
+
+  set(cdash_revisions_nonpassing_tests_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/queryTests.php[?]project=TribitsExampleMetaProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3&field2=status&compare2=62&value2=passed" )
+
+
 tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_remote
   OVERALL_WORKING_DIRECTORY TEST_NAME
   OVERALL_NUM_MPI_PROCS 1
@@ -111,11 +124,19 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
       ${COMMON_ENV_ARGS}
       TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
       CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote
+      CTEST_SITE=CustomSite
       ${CTEST_S_SCRIPT_ARGS}
     PASS_REGULAR_EXPRESSION_ALL
       "CTEST_NOTES_FILES=''"
       "First perform the initial checkout: .*/git. clone -o origin .*github.com.tribits/TribitsExampleMetaProject.git"
       "Perform checkout in directory: .*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote"
+      "Results will be submitted on CDash at the following links:"
+      "Link to this build's results on CDash:"
+      "${cdash_build_url_expected_regex}"
+      "Link all builds for this repo version on CDash:"
+      "${cdash_revisions_builds_url_expected_regex}"
+      "Link to all nonpassing tests for all builds for this repo version on CDash:"
+      "${cdash_revisions_nonpassing_tests_expected_regex}"
       "Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote/TribitsExampleMetaProject"
       "Old revision of repository is: [a-z0-9]+"
       "New revision of repository is: [a-z0-9]+"
@@ -315,6 +336,11 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
   # * Makes sure some git commit SHA1s are shown in 'Old/New revision of
   #   repository' printouts from ctest_update().
   #
+  # * Checks that CDash URLs that get produced, including for all builds with
+  # * the same repo version.  (This check has to be here since the base
+  # * project repo must have a .git/ directory and we must know the exact SHA1
+  # * of that repo for the test, which we don in this case.)
+  #
   # NOTE: The reason that so many tests are done in a single ctest test is the
   # cost of the initial configure which checks the compilers which is very
   # expensive.  The other runs use

diff --git a/test/ctest_driver/TribitsExampleProject/CMakeLists.txt b/test/ctest_driver/TribitsExampleProject/CMakeLists.txt
@@ -1264,7 +1264,8 @@ function(generate_aao_tests)
     # CONFIGURE_OPTIONS list is checked to make sure that it does not change
     # without being noticed.  Follow-on tests will not check all of that.
 
-  set(cdash_url_expected_regex
+
+  set(cdash_build_url_expected_regex
     "https://cdash[.]site[.]com/cdash/index[.]php[?]project=CustomTribitsExProj&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=My%20Site&field2=buildname&compare2=61&value2=TriBITS_CTestDriver_AAOP_CDASH_URL&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Custom%20CDash%20Group" )
   # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
   # the number of digits in the buildstarttime.  CMake regex does not support
@@ -1285,7 +1286,7 @@ function(generate_aao_tests)
         CTEST_DASHBOARD_ROOT=PWD
         ${AAO_COMMON_ENV_ARGS}
         CTEST_PARALLEL_LEVEL=3
-        CTEST_DO_SUBMIT=OFF  # Never submit this
+        CTEST_DO_SUBMIT=OFF  # Never submit this (see below note)
         CTEST_SITE="My Site"
         CTEST_DROP_SITE="cdash.site.com"
         CTEST_PROJECT_NAME="CustomTribitsExProj"
@@ -1299,17 +1300,19 @@ function(generate_aao_tests)
         "NONE does not exist, skipping extra repositories"
         "Final set of enabled packages:  SimpleCxx 1"
         "Final set of enabled SE packages:  SimpleCxx 1"
-        "Results will be submitted to CDash at:"
+        "Results will be submitted on CDash at the following links:"
+        "Link to this build's results on CDash:"
+        "${cdash_build_url_expected_regex}"
         "File '' does NOT exist so all tests passed"
-        "See results submitted to CDash at:"
-        "${cdash_url_expected_regex}"
+        "See results submitted on CDash at the following links:"
         "TRIBITS_CTEST_DRIVER: OVERALL: ALL PASSED"
       ALWAYS_FAIL_ON_NONZERO_RETURN
     TEST_2
-      MESSAGE "Check that CDashBuildUrl.txt exists and lists the right URL."
-      CMND cat ARGS BUILD/CDashBuildUrl.txt
+      MESSAGE "Check that CDashResults.txt exists and lists the right URL."
+      CMND cat ARGS BUILD/CDashResults.txt
       PASS_REGULAR_EXPRESSION_ALL
-        "${cdash_url_expected_regex}"
+        "Link to this build's results on CDash:"
+        "${cdash_build_url_expected_regex}"
     )
     # NOTE: The above test is never actually submitted to CDash, even when
     # ${PACKAGE_NAME}_CTEST_DRIVER_SUBMIT_TO is set.  This is so we can

diff --git a/tribits/ctest_driver/TribitsCTestDriverCore.cmake b/tribits/ctest_driver/TribitsCTestDriverCore.cmake
@@ -2093,10 +2093,15 @@ function(tribits_ctest_driver)
   ctest_start(${CTEST_START_ARGS})
 
   tribits_remember_if_configure_attempted()
-  tribits_get_build_url_and_write_to_file(CDASH_BUILD_URL
-    "${CTEST_BINARY_DIRECTORY}/CDashBuildUrl.txt")
-  tribits_print_cdash_url("Results will be submitted to CDash at:"
-    "${CDASH_BUILD_URL}")
+
+  tribits_get_cdash_results_string_and_write_to_file(
+    CDASH_RESULTS_STRING_OUT  CDASH_RESULTS_STRING
+    CDASH_RESULTS_FILE_OUT "${CTEST_BINARY_DIRECTORY}/CDashResults.txt" )
+  message("Results will be submitted on CDash at the following links:\n\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
+    "${CDASH_RESULTS_STRING}\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+    )
 
   message(
     "\n***"
@@ -2370,8 +2375,11 @@ function(tribits_ctest_driver)
 
   report_queued_errors()
 
-  tribits_print_cdash_url("See results submitted to CDash at:"
-    "${CDASH_BUILD_URL}")
+  message("\nSee results submitted on CDash at the following links:\n\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
+    "${CDASH_RESULTS_STRING}\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+    )
 
   if ((NOT UPDATE_FAILED) AND ("${${PROJECT_NAME}_FAILED_PACKAGES}" STREQUAL ""))
     message(