From f2612a0ca821e30c4067a3c7c1072db3e439d36a Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi <113376043+delucchi-cmu@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:44:51 -0500 Subject: [PATCH 1/2] Slack webhook documentation (#368) --- docs/practices/ci_testing.rst | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/docs/practices/ci_testing.rst b/docs/practices/ci_testing.rst index cbd565c..1a17a90 100644 --- a/docs/practices/ci_testing.rst +++ b/docs/practices/ci_testing.rst @@ -52,3 +52,95 @@ that will print out all packages installed through pip and their installed versi 3. Diff those lists 1. e.g. ``diff pass.txt fail.txt`` 2. Or use an online diff tool like https://www.diffchecker.com/ + +Smoke test Slack integration +------------------------------------------------------------------------------- + +The smoke test is only useful if someone notices that the test has failed, and +looks into the nature of the failure. This can be tricky with github, as a +worfklow failure will, by default, only notify the maintainer who added the +workflow file to the repo. + +We have found a Slack Bot useful to notify the whole team and start a discussion +about triaging and debugging the failures. + +Create a Slack App +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You'll need to create a Slack app. It's not as scary as it sounds! +It will have certain permissions to post to particular channels, and will have +an associated webhook URL that we'll use to send messages from GitHub to the app. + +See `Slack's official documentation `_ +for setting up an app. We really only need steps 1 and 5, summarized below: + +- `Step 1 `_: Create an app + from scratch. The ``App Name`` you select here will appear in the slack + notificationw we create later, so use something descriptive enough, like: + + - Slack Bot + - CI Reporter + +- `Step 5 `_: Add a new + webhook and you'll give it permission to post to a specific slack channel. + Copy the webook URL. + +Github workflow step to post to webhook +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you'll need to configure each project repo to send slack messages. + +In your project repo create a new repo secret: + - "Settings" + - "Secrets and variables" > "Actions" + - "Repository Secrets" > "New repository secret" + - Name: ``SLACK_WEBHOOK_URL`` + - Secret: paste the URL that was copied in the previous step + - "Add secret" + +Add a step in each CI workflow that should send failure alerts to the slack +channel. Typically, this would be any nightly job, like ``smoke_test.yml`` +or ``asv-nightly.yml``. + +.. code-block:: + + - name: Send status to Slack app + if: ${{ failure() && github.event_name != 'workflow_dispatch' }} # Only post if the workflow failed and was not manually started. Customize this as necessary./ + id: slack + uses: slackapi/slack-github-action@v1.24.0 + with: + # For posting a rich message using Block Kit + payload: | # The payload defined here can be customized to you liking https://api.slack.com/reference/block-kit/blocks + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "${{ github.repository }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "GitHub Action build result: *${{ job.status }}* :${{ job.status }}:" # Note that we expect the slack workspace to have an emoji called “failed” in this case. + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # Here is where the webhook URL is provided + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + +An example can be found in the `rail project `_ \ No newline at end of file From 361c5641902f69eb488a84779958b8817de32f72 Mon Sep 17 00:00:00 2001 From: Drew Oldag <47493171+drewoldag@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:51:55 -0800 Subject: [PATCH 2/2] Removing hardcoded `tests` path from invocations of pytest. (#367) --- .github/workflows/ci.yml | 2 +- python-project-template/.github/workflows/smoke-test.yml | 2 +- .../.github/workflows/testing-and-coverage.yml.jinja | 2 +- tests/test_package_creation.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f21fd1..ed64cc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,4 +148,4 @@ jobs: if: ${{ !contains(matrix.copier_config.extra_flags, 'create_example_module=no') }} run: | cd ../test/${{ matrix.copier_config.foldername }} - python -m pytest tests --cov=${{ matrix.copier_config.package_name }} --cov-report=xml + python -m pytest --cov=${{ matrix.copier_config.package_name }} --cov-report=xml diff --git a/python-project-template/.github/workflows/smoke-test.yml b/python-project-template/.github/workflows/smoke-test.yml index 2aec838..64a57f3 100644 --- a/python-project-template/.github/workflows/smoke-test.yml +++ b/python-project-template/.github/workflows/smoke-test.yml @@ -40,4 +40,4 @@ jobs: pip list - name: Run unit tests with pytest run: | - python -m pytest tests + python -m pytest diff --git a/python-project-template/.github/workflows/testing-and-coverage.yml.jinja b/python-project-template/.github/workflows/testing-and-coverage.yml.jinja index bbf0e2d..6b0a440 100644 --- a/python-project-template/.github/workflows/testing-and-coverage.yml.jinja +++ b/python-project-template/.github/workflows/testing-and-coverage.yml.jinja @@ -32,6 +32,6 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run unit tests with pytest run: | - python -m pytest tests --cov={{package_name}} --cov-report=xml + python -m pytest --cov={{package_name}} --cov-report=xml - name: Upload coverage report to codecov uses: codecov/codecov-action@v3 diff --git a/tests/test_package_creation.py b/tests/test_package_creation.py index 6c2a7f2..4a67e18 100644 --- a/tests/test_package_creation.py +++ b/tests/test_package_creation.py @@ -45,7 +45,7 @@ def unit_tests_in_project_run_successfully(result, package_name = "example_packa virtual environment for the project. """ pytest_results = subprocess.run( - ["python", "-m", "pytest", (result.project_dir / f"tests/{package_name}")], + ["python", "-m", "pytest"], cwd=result.project_dir )