Smart summarise weekly test results

Move the message building to a python script to make it easier
to customise the message,
and add some features:

- only notify `@soleng` if there were failed tests
- print a quick summary of number of failed tests

Fixes: #6
Fixes: #5

.github/workflows/weekly_tests.yaml

@@ -55,7 +55,7 @@ jobs:
             workflow_file_name: pr.yaml
 
     steps:
-      - name: Running ${{ matrix.workflow_file_name }} tests for ${{ matrix.repo }}
+      - name: Run ${{ matrix.workflow_file_name }} tests for ${{ matrix.repo }}
         uses: convictional/[email protected]
         id: dispatched-tests
         with:
@@ -66,11 +66,11 @@ jobs:
           ref: main
           wait_interval: 60
 
-      - name: Collect result
+      - name: Collect results
         id: collect-result
         if: always()
         run: |
-          result=$(cat <<-END
+          cat <<-END > "result-${{ strategy.job-index }}.json"
               {
                   "job-index": "${{ strategy.job-index }}",
                   "repo": "${{ matrix.repo }}",
@@ -80,8 +80,6 @@ jobs:
                   "workflow_url": "${{ steps.dispatched-tests.outputs.workflow_url }}"
               }
           END
-          )
-          echo $result > "result-${{ strategy.job-index }}.json"
 
       - uses: actions/upload-artifact@v4
         if: always()
@@ -91,29 +89,22 @@ jobs:
 
   notify:
     needs: weekly-tests
-    if: always()  # these need to be run always
+    if: always()  # always send the summary to mattermost
     name: Notify SolEng with results
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v4
+
       - name: Collect results
         uses: actions/download-artifact@v4
         with:
           merge-multiple: true
-      - name: Install jq
-        run: |
-           sudo apt update
-           sudo apt install jq -y
-      - name: Concatenate results
-        run: |
-          {
-           echo 'results<<EOF'
-           jq -s '.[] | ["- [\(.repo) (\(.branch), \(.workflow_file_name))](\(.workflow_url)) \(if .conclusion == "success" then ":gh-success-octicon-checkcirclefillicon:" else ":gh-failure-octicon-xcirclefillicon:" end)"]' result-*.json | jq -r 'join("\n")'
-           echo EOF
-          } >> "$GITHUB_ENV"
+
+      - name: Generate summary message
+        run: python3 ./scripts/weekly-tests-summary.py "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> "$GITHUB_ENV"
+
       - name: Send the Mattermost Message
         uses: mattermost/action-mattermost-notify@master
         with:
           MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }}
-          TEXT: |
-            :robot_face: @soleng Results from [Weekly tests](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}):
-            ${{ env.results }}
+          TEXT: ${{ env.results }}

scripts/weekly-tests-summary.py

@@ -0,0 +1,60 @@
+# Reads all result-*.json files in the current directory
+# (generated by the weekly_tests workflow),
+# and prints a markdown formatted summary
+# suitable for sending to the team mattermost channel.
+
+import sys
+import glob
+import json
+
+
+def load_results() -> list[dict[str, str]]:
+    """Return a list of result data from all result-*.json files"""
+    results = []
+    for path in glob.glob("result-*.json"):
+        with open(path) as f:
+            results.append(json.load(f))
+    return results
+
+
+def format_result(result: dict[str, str]) -> str:
+    """Take a result and return a markdown string"""
+    icon = (
+        ":gh-success-octicon-checkcirclefillicon:"
+        if result["conclusion"] == "success"
+        else ":gh-failure-octicon-xcirclefillicon:"
+    )
+    repo = result["repo"]
+    branch = result["branch"]
+    workflow = result["workflow_file_name"]
+    url = result["workflow_url"]
+    return f"- [{repo} ({branch}, {workflow})]({url}) {icon}"
+
+
+def summarise(results: list[dict[str, str]], action_url: str) -> str:
+    """Take the results and a link to the github action run and return a markdown summary"""
+    failed = len([result for result in results if result["conclusion"] != "success"])
+    total = len(results)
+
+    if failed == 0:
+        return f":robot_face: all [weekly tests]({action_url}) passed! :tada:"
+    else:
+        return f":robot_face: @soleng {failed}/{total} [weekly tests]({action_url}) failed:"
+
+
+def print_results(results: list[dict[str, str]], action_url: str):
+    """Take the results and a link to the github action run and print a markdown formatted message"""
+    print(summarise(results, action_url))
+    for result in results:
+        print(format_result(result))
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        github_action_url = sys.argv[1]
+    else:
+        print("Usage: summarise-results.py <github-action-url>")
+        sys.exit(1)
+
+    results = load_results()
+    print_results(results, github_action_url)