From 7ec72976f7ae54ed3d39d7ecff8234a630d305f4 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Fri, 2 Aug 2024 15:35:44 +0200 Subject: [PATCH] ci: Add workflow for ci_test bad, use remote fmf plan * Remove plan from role dir, it's too complicated and long to run tests from tmt * Use plan from linux-system-roles/tft-tests * Move getting datetime to testing-farm job where it's used * Add a workflow for running [citest_bad] * Bump sclorg/testing-farm-as-github-action to v3 * Move concurrency group to first job * Dump GitHub context Signed-off-by: Sergei Petrosian --- .github/workflows/tft.yml | 56 ++++++++++++++++------------ .github/workflows/tft_citest_bad.yml | 43 +++++++++++++++++++++ README.md | 2 +- plans/README-plans.md | 30 --------------- plans/general.fmf | 44 ---------------------- 5 files changed, 77 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/tft_citest_bad.yml delete mode 100644 plans/README-plans.md delete mode 100644 plans/general.fmf diff --git a/.github/workflows/tft.yml b/.github/workflows/tft.yml index 81bdecc..1b27c04 100644 --- a/.github/workflows/tft.yml +++ b/.github/workflows/tft.yml @@ -1,3 +1,4 @@ +--- name: Run integration tests in Testing Farm on: issue_comment: @@ -7,26 +8,32 @@ permissions: contents: read # This is required for the ability to create/update the Pull request status statuses: write -# The concurrency key is used to prevent multiple workflows from running at the same time -concurrency: - # group name contains reponame-pr_num to allow simualteneous runs in different PRs - group: testing-farm-${{ github.event.repository.name }}-${{ github.event.issue.number }} - cancel-in-progress: true jobs: prepare_vars: name: Get info from role and PR to determine if and how to test + # The concurrency key is used to prevent multiple workflows from running at the same time + concurrency: + # group name contains reponame-pr_num to allow simualteneous runs in different PRs + group: testing-farm-${{ github.event.repository.name }}-${{ github.event.issue.number }} + cancel-in-progress: true # Let's schedule tests only on user request. NOT automatically. # Only repository owner or member can schedule tests if: | github.event.issue.pull_request - && (contains(github.event.comment.body, '[citest]') || contains(github.event.comment.body, '[citest-all]')) + && contains(github.event.comment.body, '[citest]') + && (contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.comment.author_association) + || contains('systemroller', github.event.comment.user.login)) runs-on: ubuntu-latest outputs: supported_platforms: ${{ steps.supported_platforms.outputs.supported_platforms }} head_sha: ${{ steps.head_sha.outputs.head_sha }} - datetime: ${{ steps.datetime.outputs.datetime }} memory: ${{ steps.memory.outputs.memory }} steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + shell: bash + env: + GITHUB_CONTEXT: ${{ toJson(github) }} - name: Checkout repo uses: actions/checkout@v4 @@ -41,12 +48,6 @@ jobs: PR_NO: ${{ github.event.issue.number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Get cuurent datetime - id: datetime - run: | - printf -v datetime '%(%Y%m%d-%H%M%S)T' -1 - echo "datetime=$datetime" >> $GITHUB_OUTPUT - - name: Get memory id: memory run: | @@ -114,6 +115,19 @@ jobs: ${{ needs.prepare_vars.outputs.datetime }}/artifacts" ARTIFACT_TARGET_DIR: /srv/pub/alt/linuxsystemroles/logs steps: + - name: Set variables with DATETIME and artifact location + id: set_vars + run: | + printf -v DATETIME '%(%Y%m%d-%H%M%S)T' -1 + ARTIFACTS_DIR_NAME="tf_${{ github.event.repository.name }}-${{ github.event.issue.number }}_\ + ${{ matrix.platform }}-${{ matrix.ansible_version }}_$DATETIME/artifacts" + ARTIFACTS_TARGET_DIR=/srv/pub/alt/linuxsystemroles/logs + ARTIFACTS_DIR=$ARTIFACTS_TARGET_DIR/$ARTIFACTS_DIR_NAME + ARTIFACTS_URL=https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs$ARTIFACTS_DIR_NAME + echo "DATETIME=$DATETIME" >> $GITHUB_OUTPUT + echo "ARTIFACTS_DIR=$ARTIFACTS_DIR" >> $GITHUB_OUTPUT + echo "ARTIFACTS_URL=$ARTIFACTS_URL" >> $GITHUB_OUTPUT + - name: Set commit status as pending if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform) uses: myrotvorets/set-commit-status-action@master @@ -137,19 +151,17 @@ jobs: - name: Run test in testing farm uses: sclorg/testing-farm-as-github-action@v3 if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform) - env: - ARTIFACTS_DIR: ${{ env.ARTIFACT_TARGET_DIR }}/${{ env.ARTIFACTS_DIR_NAME }} - ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }} with: - git_url: ${{ github.server_url }}/${{ github.repository }} - git_ref: ${{ needs.prepare_vars.outputs.head_sha }} + git_url: https://github.com/linux-system-roles/tft-tests + git_ref: main pipeline_settings: '{ "type": "tmt-multihost" }' variables: "ANSIBLE_VER=${{ matrix.ansible_version }};\ REPO_NAME=${{ github.event.repository.name }};\ + GITHUB_ORG=${{ github.repository_owner }};\ GITHUB_ORG=linux-system-roles;\ PR_NUM=${{ github.event.issue.number }};\ - ARTIFACTS_DIR=${{ env.ARTIFACTS_DIR }};\ - ARTIFACTS_URL=${{ env.ARTIFACTS_URL }};\ + ARTIFACTS_DIR=${{ steps.set_vars.outputs.ARTIFACTS_DIR }};\ + ARTIFACTS_URL=${{ steps.set_vars.outputs.ARTIFACTS_URL }};\ TEST_LOCAL_CHANGES=false" # Note that LINUXSYSTEMROLES_SSH_KEY must be single-line, TF doesn't read multi-line variables fine. secrets: "LINUXSYSTEMROLES_USER=${{ secrets.LINUXSYSTEMROLES_USER }};\ @@ -167,11 +179,9 @@ jobs: - name: Set final commit status uses: myrotvorets/set-commit-status-action@master if: always() && contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform) - env: - ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }} with: sha: ${{ needs.prepare_vars.outputs.head_sha }} status: ${{ job.status }} context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }} description: Test finished - targetUrl: ${{ env.ARTIFACTS_URL }} + targetUrl: ${{ steps.set_vars.outputs.ARTIFACTS_URL }} diff --git a/.github/workflows/tft_citest_bad.yml b/.github/workflows/tft_citest_bad.yml new file mode 100644 index 0000000..4b65ebb --- /dev/null +++ b/.github/workflows/tft_citest_bad.yml @@ -0,0 +1,43 @@ +--- +name: Re-run failed testing farm tests +on: + issue_comment: + types: + - created +permissions: + contents: read +jobs: + citest_bad_rerun: + if: | + github.event.issue.pull_request + && github.event.comment.body == '[citest_bad]' + && contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.comment.author_association) + permissions: + actions: write # for re-running failed jobs: https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#re-run-a-job-from-a-workflow-run + runs-on: ubuntu-latest + steps: + - name: Wait 10s until tft.yml workflow is created and skipped because new comment don't match [citest] + run: sleep 10s + + - name: Re-run failed jobs for this PR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_TITLE: ${{ github.event.issue.title }} + run: | + PENDING_RUN=$(gh api "repos/$REPO/actions/workflows/tft.yml/runs?event=issue_comment" \ + | jq -r "[.workflow_runs[] | select( .display_title == \"$PR_TITLE\") | \ + select(.status == \"pending\" or .status == \"queued\" or .status == \"in_progress\") | .id][0]") + # if pending run don't exist, take the last run with failure state + if [ "$PENDING_RUN" != "null" ]; then + echo "The workflow $PENDING_RUN is still running, wait for it to finish to re-run" + exit 1 + fi + RUN_ID=$(gh api "repos/$REPO/actions/workflows/tft.yml/runs?event=issue_comment" \ + | jq -r "[.workflow_runs[] | select( .display_title == \"$PR_TITLE\" ) | select( .conclusion == \"failure\" ) | .id][0]") + if [ "$RUN_ID" = "null" ]; then + echo "Failed workflow not found, exitting" + exit 1 + fi + echo "Re-running workflow $RUN_ID" + gh api --method POST repos/$REPO/actions/runs/$RUN_ID/rerun-failed-jobs diff --git a/README.md b/README.md index c674c3c..023d26c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # systemd -[![ansible-lint.yml](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-test.yml) [![codeql.yml](https://github.com/linux-system-roles/systemd/actions/workflows/codeql.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/codeql.yml) [![markdownlint.yml](https://github.com/linux-system-roles/systemd/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/markdownlint.yml) [![python-unit-test.yml](https://github.com/linux-system-roles/systemd/actions/workflows/python-unit-test.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/python-unit-test.yml) [![shellcheck.yml](https://github.com/linux-system-roles/systemd/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/shellcheck.yml) [![tft.yml](https://github.com/linux-system-roles/systemd/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/tft.yml) [![woke.yml](https://github.com/linux-system-roles/systemd/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/woke.yml) +[![ansible-lint.yml](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/ansible-test.yml) [![codeql.yml](https://github.com/linux-system-roles/systemd/actions/workflows/codeql.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/codeql.yml) [![markdownlint.yml](https://github.com/linux-system-roles/systemd/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/markdownlint.yml) [![python-unit-test.yml](https://github.com/linux-system-roles/systemd/actions/workflows/python-unit-test.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/python-unit-test.yml) [![shellcheck.yml](https://github.com/linux-system-roles/systemd/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/shellcheck.yml) [![tft.yml](https://github.com/linux-system-roles/systemd/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/tft.yml) [![tft_citest_bad.yml](https://github.com/linux-system-roles/systemd/actions/workflows/tft_citest_bad.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/tft_citest_bad.yml) [![woke.yml](https://github.com/linux-system-roles/systemd/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/systemd/actions/workflows/woke.yml) ![template](https://github.com/linux-system-roles/systemd/workflows/tox/badge.svg) diff --git a/plans/README-plans.md b/plans/README-plans.md deleted file mode 100644 index b01a4b0..0000000 --- a/plans/README-plans.md +++ /dev/null @@ -1,30 +0,0 @@ -# Introduction CI Testing Plans - -Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tmt.yml](https://github.com/linux-system-roles/systemd/blob/main/.github/workflows/tmt.yml) GitHub workflow. - -The plans/general.fmf plan is a test plan that is general for all roles. It does the following steps: - -1. Provisions two machines, one used as an Ansible control node, and second used as a managed node. -2. Does the required preparation on machines. -3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh). - -The [tmt.yml](https://github.com/linux-system-roles/systemd/blob/main/.github/workflows/tmt.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access. -This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm). - -## Running Tests - -You can run tests locally with the `tmt try` cli. - -### Prerequisites - -* Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html). - -### Running Tests Locally - -To run tests locally, in the role repository, enter `tmt try -p plans/general `. - -This command identifies the plans/general plan and provisions two local VMs, one used as an Ansible control node, and second used as a managed node. - -tmt try is in development and does not identify tests from URL automatically, so after provisioning the machines, you must type `t`, `p`, `t` from the interactive prompt to identify tests, run preparation steps, and run the tests. - -You can modify environment variables in `plans/general.fmf` to, e.g. run only specified test playbooks by overwriting `SYSTEM_ROLES_ONLY_TESTS`. diff --git a/plans/general.fmf b/plans/general.fmf deleted file mode 100644 index da98982..0000000 --- a/plans/general.fmf +++ /dev/null @@ -1,44 +0,0 @@ -summary: A general test for a system role -provision: - - name: control_node - role: control_node - # TF uses `how: artemis`, tmt try uses `how: virtual`. No need to define `how`. - # `connection: system` is for `how: virtual` to make VMs get a real IP to configure ssh easily. - # This setting is ignored on artemis so we can keep it. - connection: system - - name: managed_node1 - role: managed_node - connection: system -environment: - ANSIBLE_VER: 2.17 - REPO_NAME: systemd - PYTHON_VERSION: 3.12 - SYSTEM_ROLES_ONLY_TESTS: "" # e.g. tests_default.yml - PR_NUM: "" - TEST_LOCAL_CHANGES: true -prepare: - - name: Use vault.centos.org repos (CS 7, 8 EOL workaround) - script: | - if grep -q -e 'CentOS Stream release 8' -e 'CentOS Linux release 7.9' /etc/redhat-release; then - sed -i '/^mirror/d;s/#\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo - fi - - name: Enable epel to install beakerlib on all platforms except CS10 and Fedora, there epel not available and not needed - script: | - if ! grep -q -e 'CentOS Stream release 10' -e 'Fedora release' /etc/redhat-release; then - yum install epel-release -y - fi -discover: - - name: Prepare managed node - how: fmf - url: https://github.com/linux-system-roles/tft-tests - ref: main - where: managed_node - filter: tag:prep_managed_node - - name: Run test playbooks from control_node - how: fmf - url: https://github.com/linux-system-roles/tft-tests - ref: main - where: control_node - filter: tag:test_playbooks -execute: - how: tmt