From 28c73d69898c0aa02952567ee2a9acfa8a51b514 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 29 Aug 2024 13:14:13 +0100 Subject: [PATCH 01/21] CIS - don't shut down when out of space --- etc/kayobe/inventory/group_vars/cis-hardening/cis | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/cis-hardening/cis b/etc/kayobe/inventory/group_vars/cis-hardening/cis index 1ec8c3dcc..37d01492b 100644 --- a/etc/kayobe/inventory/group_vars/cis-hardening/cis +++ b/etc/kayobe/inventory/group_vars/cis-hardening/cis @@ -55,7 +55,8 @@ rhel9cis_no_world_write_adjust: false rhel9cis_auditd: space_left_action: syslog action_mail_acct: root - admin_space_left_action: halt + admin_space_left_action: syslog + max_log_file: 10 max_log_file_action: rotate # Max size of audit logs (MB) @@ -156,7 +157,7 @@ ubtu22cis_suid_adjust: false ubtu22cis_auditd: action_mail_acct: root space_left_action: syslog - admin_space_left_action: halt + admin_space_left_action: syslog max_log_file_action: rotate # Max size of audit logs (MB) From acc31e9fba388b73b000243eab2e59a822e5cb13 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 12 Sep 2024 14:54:08 +0100 Subject: [PATCH 02/21] 2023.1: Fix tag for ironic_dnsmasq container The dnsmasq image was not rebuilt at the same time as the ironic images in 3a97322b6b402485ae5ef7102486803e3a3583af. Set the tag for ironic_dnsmasq to what it was before that change. --- etc/kayobe/kolla-image-tags.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 9a6c89aa3..58d173a04 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -17,6 +17,9 @@ kolla_image_tags: ironic: rocky-9: 2023.1-rocky-9-20240906T144646 ubuntu-jammy: 2023.1-ubuntu-jammy-20240906T144646 + ironic_dnsmasq: + rocky-9: 2023.1-rocky-9-20240709T132012 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240621T104542 kolla_toolbox: rocky-9: 2023.1-rocky-9-20240809T102431 letsencrypt: From 130c0eceb76dd7317c50160ae2904bdce1bb9c25 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 13 Sep 2024 08:51:37 +0100 Subject: [PATCH 03/21] CI: Show Rally logs on failure for ci-aio and ci-multinode environments Without this it is difficult to diagnose a failure to run Tempest. --- .github/workflows/stackhpc-all-in-one.yml | 2 +- etc/kayobe/environments/ci-aio/tempest.yml | 3 +++ etc/kayobe/environments/ci-multinode/tempest.yml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/environments/ci-aio/tempest.yml diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 3f5b0fd73..611a56699 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -378,7 +378,7 @@ jobs: -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ $KAYOBE_IMAGE \ - /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack env: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} diff --git a/etc/kayobe/environments/ci-aio/tempest.yml b/etc/kayobe/environments/ci-aio/tempest.yml new file mode 100644 index 000000000..766776417 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/tempest.yml @@ -0,0 +1,3 @@ +--- +# Show Rally output on failure. +rally_no_sensitive_log: false diff --git a/etc/kayobe/environments/ci-multinode/tempest.yml b/etc/kayobe/environments/ci-multinode/tempest.yml index 93a7cdfe2..0657946bb 100644 --- a/etc/kayobe/environments/ci-multinode/tempest.yml +++ b/etc/kayobe/environments/ci-multinode/tempest.yml @@ -1,3 +1,6 @@ --- +# Show Rally output on failure. +rally_no_sensitive_log: false + # Add the Vault CA certificate to the rally container when running tempest. tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" From e5e06761c33fcdf11de32091d90cb1eacd12e10b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 17 Sep 2024 10:54:07 +0100 Subject: [PATCH 04/21] CI: Add Slack alert to automatically triggered workflows Alert on failures for automatically triggered workflows. For manually triggered workflows we rely on GitHub emails to the individual that triggered the workflow. See the "Workflow builder" approach in https://github.com/slackapi/slack-github-action/. --- .../workflows/stackhpc-build-kayobe-image.yml | 22 +++++++++++++++++++ .github/workflows/stackhpc-ci-cleanup.yml | 21 ++++++++++++++++++ .github/workflows/stackhpc-promote.yml | 22 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml index c45e0456b..cbbac9f71 100644 --- a/.github/workflows/stackhpc-build-kayobe-image.yml +++ b/.github/workflows/stackhpc-build-kayobe-image.yml @@ -98,3 +98,25 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Send message to Slack via Workflow Builder + uses: slackapi/slack-github-action@v1.26.0 + with: + payload: | + { + "channel-id": "${{ env.SLACK_CHANNEL_ID }}", + "inputs": "${{ env.INPUTS }}", + "message": "${{ env.MESSAGE }}", + "results-url": "${{ env.RESULTS_URL }}", + "workflow-url": "${{ env.WORKFLOW_URL }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # #release-train-alerts + SLACK_CHANNEL_ID: C03B28HRP53 + INPUTS: >- + branch: ${{ github.ref_name }} + MESSAGE: "SKC Build Kayobe Image workflow failed :sob:" + RESULTS_URL: "N/A" + WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if: failure() && github.event_name == 'push' diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index ed9ec327c..a14b2970c 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -75,3 +75,24 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Send message to Slack via Workflow Builder + uses: slackapi/slack-github-action@v1.26.0 + with: + payload: | + { + "channel-id": "${{ env.SLACK_CHANNEL_ID }}", + "inputs": "${{ env.INPUTS }}", + "message": "${{ env.MESSAGE }}", + "results-url": "${{ env.RESULTS_URL }}", + "workflow-url": "${{ env.WORKFLOW_URL }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # #release-train-alerts + SLACK_CHANNEL_ID: C03B28HRP53 + INPUTS: "N/A" + MESSAGE: "SKC CI Cleanup workflow failed :sob:" + RESULTS_URL: "N/A" + WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if: failure() diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml index 61385c819..386d4d405 100644 --- a/.github/workflows/stackhpc-promote.yml +++ b/.github/workflows/stackhpc-promote.yml @@ -27,3 +27,25 @@ jobs: - name: Display link to package repository promotion workflows run: | echo "::notice Package repository promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-promote.yml" + + - name: Send message to Slack via Workflow Builder + uses: slackapi/slack-github-action@v1.26.0 + with: + payload: | + { + "channel-id": "${{ env.SLACK_CHANNEL_ID }}", + "inputs": "${{ env.INPUTS }}", + "message": "${{ env.MESSAGE }}", + "results-url": "${{ env.RESULTS_URL }}", + "workflow-url": "${{ env.WORKFLOW_URL }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # #release-train-alerts + SLACK_CHANNEL_ID: C03B28HRP53 + INPUTS: >- + branch: ${{ github.ref_name }} + MESSAGE: "SKC promote workflow failed :sob:" + RESULTS_URL: "N/A" + WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if: failure() From 778bfd2b397996928ea7bd2154ce51cdfcce4210 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 17 Sep 2024 10:51:27 +0100 Subject: [PATCH 05/21] docs: Add page on testing, CI and automation --- .../contributor/environments/ci-aio.rst | 5 + .../contributor/environments/ci-builder.rst | 10 + .../contributor/environments/ci-multinode.rst | 5 + doc/source/contributor/index.rst | 1 + .../contributor/testing-ci-automation.rst | 421 ++++++++++++++++++ 5 files changed, 442 insertions(+) create mode 100644 doc/source/contributor/testing-ci-automation.rst diff --git a/doc/source/contributor/environments/ci-aio.rst b/doc/source/contributor/environments/ci-aio.rst index b46f97774..e6d9ef838 100644 --- a/doc/source/contributor/environments/ci-aio.rst +++ b/doc/source/contributor/environments/ci-aio.rst @@ -10,6 +10,11 @@ automates the manual setup steps below, and is recommended for most users. The manual setup steps are provided for reference, and for users who wish to make changes to the setup process. +.. seealso:: + + All-in-one GitHub CI testing using this environment is described :ref:`here + `. + Prerequisites ============= diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index a57ad630a..f0c6aadba 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -11,6 +11,16 @@ In general it is preferable to use the `container image build CI workflow to build container images, but this manual approach may be useful in some cases. +.. seealso:: + + A container image build GitHub Actions workflow using this environment is + described :ref:`here `. + +.. seealso:: + + An overcloud host image build GitHub Actions workflow using this environment + is described :ref:`here `. + Prerequisites ============= diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst index bc60d4365..9f11a8a33 100644 --- a/doc/source/contributor/environments/ci-multinode.rst +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -14,6 +14,11 @@ beyond the defaults. This includes: * Magnum * Wazuh +.. seealso:: + + On-demand and nightly GitHub Actions workflows workflow using this + environment are described :ref:`here `. + Manila ====== The Multinode environment supports Manila with the CephFS native backend, but it diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 988957541..3408bdd56 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -12,3 +12,4 @@ This guide is for contributors of the StackHPC Kayobe configuration project. environments/index package-updates pre-commit + testing-ci-automation diff --git a/doc/source/contributor/testing-ci-automation.rst b/doc/source/contributor/testing-ci-automation.rst new file mode 100644 index 000000000..026829c5b --- /dev/null +++ b/doc/source/contributor/testing-ci-automation.rst @@ -0,0 +1,421 @@ +======================== +Testing, CI & Automation +======================== + +CI hosting clouds +================= + +Several GitHub Actions workflows run on public runners, but some require +private runners. For these, we use a CI hosting cloud. + +Leafcloud +--------- + +`Leafcloud `_ is currently the only supported cloud +for hosting CI workloads. + +Workloads run in the ``stackhpc-ci`` project, and CI workflows authenticate +using a ci+skc@stackhpc.com user account. This is an alias for the +`ci@stackhpc.com Google group +`_, which may be subscribed to +by multiple StackHPC Engineers. Credentials for this account should be shared +amongst a few StackHPC Engineers. + +An autoscaling `Actions Runner Controller (ARC) +`_ +cluster also lives in the ``stackhpc-ci`` project, and runs several jobs that +require access to the cloud or benefit from data-locality with Ark. + +SMS Lab +------- + +SMS lab will soon be added as a supported CI hosting cloud. Several +considerations must be made when porting CI to Leafcloud. + +Many of our CI and other workflows require access to large volumes of data +stored in Ark. This includes package repositories, container images, disk +images, etc. Naively accessing this data from SMS lab will result in high +Internet usage and slow jobs. The previous incarnation of SMS lab hosted a +"Test Pulp" instance that acted as a local mirror of package repositories and +container images. This worked, but required explicit syncing with Ark when +content is updated, and was a bit brittle. + +For SMS lab 2.0, we propose a different approach. Package repository data is +smaller than container images, but we might still benefit from the use of a +Squid caching proxy. For container images we will use a `Docker registry mirror +`_ as a pull-through cache. + +Container and host image build jobs require significant data uploads, and may +still need to run on Leafcloud to avoid long delays while transferring data to +Ark. + +CI for pull requests (PRs) +========================== + +Continuous Integration (CI) is used within StackHPC Kayobe Configuration to +perform various tests against pull requests. The top-level workflow is in +``.github/workflows/stackhpc-pull-request.yml``. It includes the following +jobs: + +``check-changes`` + Determines which other jobs need to run, based on files that are changed in + the PR. The ``.github/path-filters.yml`` file contains the paths. +``Tox pep8 with Python 3.10`` + Runs the Tox ``pep8`` environment. +``Tox releasenotes with Python 3.10`` + Builds the release notes using the Tox ``releasenotes`` environment. The + separate release notes are not really used - rather they are integrated into + the main documentation. +``Tox docs with Python 3.10`` + Builds the documentation using the Tox ``docs`` environment. +``Build Kayobe Image`` + Builds a Kayobe container image for the pull request and pushes it to GHCR. + Uses the ``.github/workflows/stackhpc-build-kayobe-image.yml`` reusable + workflow. +``Check container image tags`` + Checks that: + + - the image to container mapping in ``tools/kolla-images.py`` matches Kolla + Ansible. + - the container tag hierarchy in ``tools/kolla-images.py`` matches Kolla + Ansible. + - the container image tags defined in ``etc/kayobe/kolla-image-tags.yml`` are + present in the ``stackhpc-dev`` namespace in Ark. + + Uses the ``.github/workflows/stackhpc-check-tags.yml`` reusable workflow, + which runs the ``etc/kayobe/ansible/check-tags.yml`` and + ``etc/kayobe/ansible/check-kolla-images-py.yml`` playbooks. +``aio [upgrade] ( )`` + Runs an all-in-one OpenStack deployment test. + Various jobs are run using different parameters. + Uses the ``.github/workflows/stackhpc-all-in-one.yml`` reusable workflow. + See :ref:`below ` for further details. + +.. _testing-ci-aio: + +All in one testing +------------------ + +The ``.github/workflows/stackhpc-all-in-one.yml`` reusable workflow accepts +various parameters, and the following are used to create a test matrix for PRs: + + - Operating System (Rocky 9, Ubuntu Jammy) + - Neutron plugin (OVS, OVN) + - Upgrade or no upgrade + +The workflow runs on an autoscaling `Actions Runner Controller (ARC) +`_ +cluster, and the GitHub runner acts as both a Terraform client and an Ansible +control host. Kayobe is executed using kayobe-automation within another +container, using the Kayobe container image built in the ``Build Kayobe Image`` +job. + +The workflow performs the following high-level steps: + +#. Deploy a VM on an OpenStack cloud using the `aio + `_ + Terraform configuration. +#. Deploy OpenStack in the VM using Kayobe and the :doc:`ci-aio + ` environment. If this is an upgrade job, the previous + OpenStack release is deployed. +#. Register test resources in the cloud under test (images, flavors, networks, + subnets, routers, etc.). +#. If this is an upgrade job, upgrade the cloud under test to the target + release. +#. Run Tempest and `StackHPC OpenStack Tests + `_ to test the cloud. +#. Collect diagnostic information. +#. Upload results as an artifact. +#. Destroy the VM using Terraform. + +In order to create VMs on the cloud hosting the CI, we need a few things: + +- an OpenStack project with sufficient quota to run CI jobs for several PRs + concurrently +- an OpenStack user account +- a ``clouds.yaml`` file +- an application credential to authenticate with the cloud +- a flavor for the VM (minimum 8GiB RAM) +- a set of images for the VM +- a network and subnet for the VM +- SSH connectivity from the GitHub runner to the VM +- access from the VM to the Internet + +This information is provided to GitHub Actions using `secrets +`_ +and `variables +`_. +`GitHub environments +`_ +are used to allow running jobs on different clouds. + +``KAYOBE_VAULT_PASSWORD`` is a repository-scoped GitHub secret containing the +Ansible Vault password for the ``ci-aio`` Kayobe environment. + +The following GitHub secrets are defined in each GitHub environment: + +- ``CLOUDS_YAML`` +- ``OS_APPLICATION_CREDENTIAL_ID`` +- ``OS_APPLICATION_CREDENTIAL_SECRET`` + +The following GitHub variables are defined in each GitHub environment: + +- ``AIO_FLAVOR`` +- ``AIO_NETWORK`` +- ``AIO_SUBNET`` +- ``OS_CLOUD`` + +Glance images for all-in-one VMs are not configured using GitHub variables. +Instead we use the overcloud host images that are built and uploaded to Ark. +These are also uploaded to clouds running CI, with well-known names using the +versions defined in ``etc/kayobe/pulp-host-image-versions.yml``. + +.. _ci-promotion: + +Promotion +========= + +The ``.github/workflows/stackhpc-promote.yml`` workflow runs on a push to any +release branch of StackHPC Kayobe Configuration. It triggers other workflows in +StackHPC Release Train to promote the `package repositories +`_ +and `container images +`_ +referenced in the configuration. + +The standard GitHub API token available in the workflow (``GITHUB_TOKEN``) is +not allowed to trigger a workflow in another repository. To do this, we use a +`fine-grained PAT token +`_ +owned by the ``stackhpc-ci`` GitHub user. This token has read/write permissions +on actions for the ``stackhpc/stackhpc-release-train`` repository. The token is +stored as the ``STACKHPC_RELEASE_TRAIN_TOKEN`` GitHub secret in the StackHPC +Kayobe Configuration repository. The token expires periodically and must be +regenerated, after which the secret must be updated. + +Tag and release +=============== + +The ``.github/workflows/tag-and-release.yml`` workflow runs on a push to any +release branch of StackHPC Kayobe Configuration. It generates a Git tag and +an accompanying GitHub release. See also the `Release Train documentation +`__. + +CI cleanup +========== + +The ``.github/workflows/stackhpc-ci-cleanup.yml`` workflow runs periodically +(currently every 2 hours). It checks for all-in-one CI VMs older than 3 hours +and deletes them, to avoid excess cloud resource consumption. + +.. _testing-container-images: + +Container images +================ + +The ``.github/workflows/stackhpc-container-image-build.yml`` workflow runs on +demand, and is used to build Kolla container images. The process for building +images and updating the configuration to use them is described in the `Release +Train documentation +`__. + +The workflow runs as a matrix, with a job for each supported container OS +distribution. The workflow runs on an autoscaling `Actions Runner Controller +(ARC) +`_ +cluster, and the GitHub runner acts as both the Ansible control host and +container image build host. + +A Pulp authentication proxy container is deployed on the runner that provides +unauthenticated access to the package repositories in Ark. This avoids leaking +Ark credentials into the built container images. + +Once built, images are scanned for vulnerabilities using `Trivy +`_. Any critical vulnerabilities will break the build, +unless the ``push-dirty`` input is true. + +If the ``push`` input is true, images are pushed to Ark, and a `container sync +`_ +workflow is triggered in the StackHPC Release Train repository. See +:ref:`here ` for information on triggering workflows in another repository. + +An artifact containing image build logs is uploaded on completion. + +.. _testing-host-images: + +Overcloud host images +===================== + +The ``.github/workflows/overcloud-host-image-build.yml`` workflow runs on +demand, and is used to build overcloud host images. + +The workflow runs as a single job, building each supported container OS +distribution sequentially. The workflow runs on an autoscaling `Actions Runner +Controller (ARC) +`_ +cluster, and the GitHub runner acts as both a Terraform client and an Ansible +control host. Similarly to the all-in-one CI testing, Terraform is used to +create a VM on a cloud that is then used for building images. + +The following steps are taken for each supported image: + +#. Build an image using Kayobe +#. Upload the image to Ark +#. Upload the image to clouds hosting CI + +At the end of the job, build logs are uploaded as an artifact and the VM is +destroyed. + +In order to create a VM on the cloud hosting the CI, we need a few things: + +- an OpenStack project with sufficient quota to run at least one build VM +- an OpenStack user account +- a ``clouds.yaml`` file +- an application credential to authenticate with the cloud +- a flavor for the VM (minimum 8GiB RAM) +- a Rocky Linux 9 image for the VM +- a network and subnet for the VM +- SSH connectivity from the GitHub runner to the VM +- access from the VM to the Internet + +This information is provided to GitHub Actions using `secrets +`_ +and `variables +`_. +`GitHub environments +`_ +are used to allow running jobs on different clouds. + +``KAYOBE_VAULT_PASSWORD`` is a repository-scoped GitHub secret containing the +Ansible Vault password for the ``ci-builder`` Kayobe environment. + +The following GitHub secrets are defined in each GitHub environment: + +- ``CLOUDS_YAML`` +- ``OS_APPLICATION_CREDENTIAL_ID`` +- ``OS_APPLICATION_CREDENTIAL_SECRET`` + +The following GitHub variables are defined in each GitHub environment: + +- ``HOST_IMAGE_BUILD_FLAVOR`` +- ``HOST_IMAGE_BUILD_IMAGE`` +- ``HOST_IMAGE_BUILD_NETWORK`` +- ``HOST_IMAGE_BUILD_SUBNET`` +- ``OS_CLOUD`` + +The ``.github/workflows/overcloud-host-image-promote.yml`` workflow runs on +demand and is used to promote overcloud host images. Unlike package +repositories and container images, host image promotion is still an manual +step. + +The ``.github/workflows/overcloud-host-image-upload.yml`` workflow runs on +demand and is used to upload images to clouds hosting CI. It is mainly used +when this step failed in a previous host image build job. + +.. _testing-multinode: + +Multinode test clusters +======================= + +The ``.github/workflows/stackhpc-multinode.yml`` workflow runs on demand and is +used to create a multinode test cluster. The +``.github/workflows/stackhpc-multinode-periodic.yml`` workflow runs +periodically (currently nightly) and runs a random test configuration +(generated by ``.github/workflows/multinode-inputs.py``). + +Both workflows use a `reusable workflow +`_ +in the StackHPC OpenStack GitHub Workflows repository. Note that since this +workflow is in a different repository and we reference it with a tag, changes +to the reusable workflow are not picked up until the tag is bumped. + +The workflow runs on an autoscaling `Actions Runner Controller (ARC) +`_ +cluster, and the GitHub runner acts as a Terraform client. Kayobe is executed +on another VM that acts as the Ansible control host. + +The workflow performs the following high-level steps: + +#. Deploy a set of VMs on an OpenStack cloud using the `Terraform Kayobe + Multinode `_ + Terraform configuration. +#. Configure one of the VMs as an Ansible control host for Kayobe. +#. Deploy OpenStack in the other VMs using Kayobe and the :doc:`ci-multinode + ` environment. If this is an upgrade job, the + previous OpenStack release is deployed. +#. Register test resources in the cloud under test (images, flavors, networks, + subnets, routers, etc.). +#. Run Tempest and `StackHPC OpenStack Tests + `__ to test the cloud. +#. If this is an upgrade job, upgrade the cloud under test to the target + release. +#. Run Tempest and `StackHPC OpenStack Tests + `__ to test the cloud. +#. Collect diagnostic information. +#. Upload results as an artifact. +#. Destroy the VMs using Terraform. +#. For nightly jobs, send a Slack alert to ``#release-train-alerts`` on + failure. + +In order to create VMs on the cloud hosting the CI, we need a few things: + +- an OpenStack project with sufficient quota to create several clusters + concurrently +- an OpenStack user account +- a ``clouds.yaml`` file +- an application credential to authenticate with the cloud +- flavors for each type of VM +- a set of images for the VMs +- a network and subnet for the VMs +- a floating IP pool or external network for the Ansible control host (optional) +- SSH connectivity from the GitHub runner to the Ansible control host VM +- access from the VMs to the Internet + +This information is provided to GitHub Actions using `secrets +`_ +and `variables +`_. +`GitHub environments +`_ +are used to allow running jobs on different clouds. + +``KAYOBE_VAULT_PASSWORD_CI_MULTINODE`` is a repository-scoped GitHub secret +containing the Ansible Vault password for the ``ci-multinode`` Kayobe +environment. + +The following GitHub secrets are defined in each GitHub environment: + +- ``CLOUDS_YAML`` +- ``OS_APPLICATION_CREDENTIAL_ID`` +- ``OS_APPLICATION_CREDENTIAL_SECRET`` + +The following GitHub variables are defined in each GitHub environment: + +- ``MULTINODE_ANSIBLE_CONTROL_VM_FLAVOR`` +- ``MULTINODE_FIP_POOL`` +- ``MULTINODE_FLAVOR`` +- ``MULTINODE_INFRA_VM_FLAVOR`` +- ``MULTINODE_NETWORK`` +- ``MULTINODE_SEED_VM_FLAVOR`` +- ``MULTINODE_STORAGE_FLAVOR`` +- ``MULTINODE_SUBNET`` +- ``OS_CLOUD`` + +Glance images for multinode VMs are not configured using GitHub variables. +Instead we use the overcloud host images that are built and uploaded to Ark. +These are also uploaded to clouds running CI, with well-known names using the +versions defined in ``etc/kayobe/pulp-host-image-versions.yml``. + +For multinode clusters created on demand, it is possible to pause the workflow +execution on certain conditions and gain access to the cluster for a limited +period of time. This can be used to interact with the system to investigate +faults, debug, etc. To do this, use the ``break_on`` and ``break_duration`` +workflow inputs. + +Slack alerts +============ + +Slack alerts are sent when certain automatically-triggered workflows fail. See +the `Release Train documentation +`__ for +more details. From 4e023a9b44efc586db4a1544da36839493be9b0c Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 17 Sep 2024 16:26:47 +0100 Subject: [PATCH 06/21] Add note on id of grafana_local_admin account --- doc/source/operations/secret-rotation.rst | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/doc/source/operations/secret-rotation.rst b/doc/source/operations/secret-rotation.rst index a2dad9b80..0b55a9eaf 100644 --- a/doc/source/operations/secret-rotation.rst +++ b/doc/source/operations/secret-rotation.rst @@ -74,7 +74,7 @@ Full method the state of the cloud before any changes are made 2. Edit your Kolla-Ansible checkout to include changes not yet included - upstream. + upstream. .. _kolla-change: @@ -98,7 +98,7 @@ Full method .. code:: bash git fetch https://review.opendev.org/openstack/kolla-ansible refs/changes/78/903178/2 && git cherry-pick FETCH_HEAD - + 3. Re-install Kolla-Ansible from source in your Kolla-Ansible Python environment @@ -129,7 +129,7 @@ Full method ^redis_master_password ^memcache_secret_key _ssh_key - + private_key public_key ^$ @@ -234,6 +234,28 @@ Full method grafana-cli admin reset-admin-password --password-from-stdin + .. note:: + + If you see an error ``Error: ✗ could not read user from database. Error: user not found`` + from Grafana CLI, it means that the ID of ``grafana_local_admin`` + is not 1 (The default value used with password reset command). + You can find the ID of ``grafana_local_admin`` by accessing MariaDB. + + .. code:: sql + + # Enter MariaDB with + docker exec -u 0 -it mariadb bash + mysql grafana -p + # Enter database password when prompted + + SELECT id,login FROM user WHERE login = "grafana_local_admin"; + + Once you get the ID, run the password reset with user-id option + + .. code:: bash + + grafana-cli admin reset-admin-password --user-id --password-from-stdin + 12. Update the MariaDB database password 1. Generate a new secret: From e4cb3685cf66071412a03fb07b4ddb337583b933 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 18 Sep 2024 10:41:45 +0100 Subject: [PATCH 07/21] Update grafana_admin_password section Updating the procedure instead of adding a note --- doc/source/operations/secret-rotation.rst | 35 +++++++++-------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/doc/source/operations/secret-rotation.rst b/doc/source/operations/secret-rotation.rst index 0b55a9eaf..a01f66fa9 100644 --- a/doc/source/operations/secret-rotation.rst +++ b/doc/source/operations/secret-rotation.rst @@ -222,39 +222,32 @@ Full method 2. Update the value of ``grafana_admin_password`` in ``passwords.yml`` - 3. Exec into the Grafana container on a controller + 3. Exec into the MariaDB container on a controller then login to MariaDB .. code:: bash - sudo docker exec -it grafana bash - - 4. Run the password reset command, then enter the new password - - .. code:: bash + sudo docker exec -u 0 -it mariadb bash + (mariadb) mysql grafana -p + # Enter database password when prompted - grafana-cli admin reset-admin-password --password-from-stdin + 4. Query for the ID of ``grafana_local_admin`` - .. note:: + .. code:: sql - If you see an error ``Error: ✗ could not read user from database. Error: user not found`` - from Grafana CLI, it means that the ID of ``grafana_local_admin`` - is not 1 (The default value used with password reset command). - You can find the ID of ``grafana_local_admin`` by accessing MariaDB. + SELECT id,login FROM user WHERE login = "grafana_local_admin"; + # Take a note of this ID - .. code:: sql + 5. Exec into the Grafana container on a controller - # Enter MariaDB with - docker exec -u 0 -it mariadb bash - mysql grafana -p - # Enter database password when prompted + .. code:: bash - SELECT id,login FROM user WHERE login = "grafana_local_admin"; + sudo docker exec -it grafana bash - Once you get the ID, run the password reset with user-id option + 6. Run the password reset command, then enter the new password - .. code:: bash + .. code:: bash - grafana-cli admin reset-admin-password --user-id --password-from-stdin + grafana-cli admin reset-admin-password --user-id --password-from-stdin 12. Update the MariaDB database password From f0edc21c3cc5c682508cc01ee2c62f92a305f927 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 19 Sep 2024 14:58:16 +0100 Subject: [PATCH 08/21] (2023.1) Reintroduce networking-mlnx (#1211) * Reintroduce networking-mlnx * Use stackhpc fork * Use images built from StackHPC branch * Pin ironic neutron agent This wasn't rebuilt * Update kolla-image-tags.yml * Remove WIP references * Adds release note * Update kolla-image-tags.yml * Simplify * Update etc/kayobe/kolla-image-tags.yml --------- Co-authored-by: Tom Clark --- .../environments/ci-builder/stackhpc-ci.yml | 1 + etc/kayobe/kolla-image-tags.yml | 7 ++++--- etc/kayobe/kolla.yml | 20 +++++++++++++++++++ etc/kayobe/pulp.yml | 1 + ...adds-networking-mlnx-05fdc10ea6132145.yaml | 6 ++++++ 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index ae77f5d90..5394598b0 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -22,6 +22,7 @@ kolla_enable_magnum: true kolla_enable_manila: true kolla_enable_mariabackup: true kolla_enable_neutron_sriov: true +kolla_enable_neutron_mlnx: true kolla_enable_octavia: true kolla_enable_opensearch: true kolla_enable_prometheus: true diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 58d173a04..7e6ac35e0 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -30,10 +30,11 @@ kolla_image_tags: manila: rocky-9: 2023.1-rocky-9-20240809T102431 neutron: - rocky-9: 2023.1-rocky-9-20240809T102431 + rocky-9: 2023.1-rocky-9-20240916T114629 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240916T114629 nova: - rocky-9: 2023.1-rocky-9-20240723T140948 - ubuntu-jammy: 2023.1-ubuntu-jammy-20240723T140948 + rocky-9: 2023.1-rocky-9-20240916T114629 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240916T114629 octavia: rocky-9: 2023.1-rocky-9-20240730T090421 ubuntu-jammy: 2023.1-ubuntu-jammy-20240730T090421 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index e518d4065..e2fbad5cc 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -147,6 +147,14 @@ kolla_sources: type: git location: https://github.com/stackhpc/networking-generic-switch.git reference: stackhpc/{{ openstack_release }} + neutron-server-plugin-networking-mlnx: + type: git + location: https://github.com/stackhpc/networking-mlnx + reference: stackhpc/{{ openstack_release }} + nova-compute-plugin-networking-mlnx: + type: git + location: https://github.com/stackhpc/networking-mlnx + reference: stackhpc/{{ openstack_release }} ############################################################################### # Kolla image build configuration. @@ -339,6 +347,12 @@ kolla_build_blocks: {% set magnum_capi_packages = ['magnum-capi-helm==1.1.0'] %} RUN {{ macros.install_pip(magnum_capi_packages | customizable("pip_packages")) }} {% endraw %} + nova_compute_header: + # See: https://review.opendev.org/c/openstack/kolla/+/878663/1/docker/neutron/neutron-mlnx-agent/Dockerfile.j2 + RUN sed -e "s/^pyzmq===20.0.0/pyzmq==21.*/g" -i /requirements/upper-constraints.txt + neutron_server_header: + # See: https://review.opendev.org/c/openstack/kolla/+/878663/1/docker/neutron/neutron-mlnx-agent/Dockerfile.j2 + RUN sed -e "s/^pyzmq===20.0.0/pyzmq==21.*/g" -i /requirements/upper-constraints.txt prometheus_alertmanager_repository_version: | # 2023.1 kolla has 0.24.0 {% raw %} ARG prometheus_alertmanager_version=0.26.0 @@ -431,6 +445,12 @@ kolla_build_customizations_common: - /additions/* ironic_inspector_pip_packages_append: - /additions/* + neutron_server_packages_append: + - python3-libvirt + - python3-ethtool + nova_compute_packages_append: + - python3-libvirt + - python3-ethtool kolla_build_customizations_rocky: kolla_toolbox_packages_remove: diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 7b7dd6721..619a62427 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -491,6 +491,7 @@ stackhpc_pulp_images_kolla: - neutron-dhcp-agent - neutron-l3-agent - neutron-metadata-agent + - neutron-mlnx-agent - neutron-openvswitch-agent - neutron-server - neutron-sriov-agent diff --git a/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml b/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml new file mode 100644 index 000000000..5f9e3a15b --- /dev/null +++ b/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds the ``networking-mlnx`` mechanism driver to the Neutron Server + container and ``ebrctl``utility to the Nova Compute container. This allows + you to use the ``kolla_enable_neutron_mlnx`` feature flag. From 51a4f7cc068fb8f07c74f8af39e2953350fed68c Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 24 Sep 2024 10:20:40 +0200 Subject: [PATCH 09/21] Fix release note markup --- releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml b/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml index 5f9e3a15b..695ab035c 100644 --- a/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml +++ b/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml @@ -2,5 +2,5 @@ features: - | Adds the ``networking-mlnx`` mechanism driver to the Neutron Server - container and ``ebrctl``utility to the Nova Compute container. This allows + container and ``ebrctl`` utility to the Nova Compute container. This allows you to use the ``kolla_enable_neutron_mlnx`` feature flag. From def9bdab8560f1b177e7644dbf523082d6f807f7 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 24 Sep 2024 11:13:53 +0200 Subject: [PATCH 10/21] Bump Neutron image tags to fix bug 2068644 This bug could prevent associating floating IPs with OVN-based load balancers. --- etc/kayobe/kolla-image-tags.yml | 7 +++++-- .../fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 7e6ac35e0..9dddef8cb 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -20,6 +20,9 @@ kolla_image_tags: ironic_dnsmasq: rocky-9: 2023.1-rocky-9-20240709T132012 ubuntu-jammy: 2023.1-ubuntu-jammy-20240621T104542 + ironic_neutron_agent: + rocky-9: 2023.1-rocky-9-20240916T114629 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240916T114629 kolla_toolbox: rocky-9: 2023.1-rocky-9-20240809T102431 letsencrypt: @@ -30,8 +33,8 @@ kolla_image_tags: manila: rocky-9: 2023.1-rocky-9-20240809T102431 neutron: - rocky-9: 2023.1-rocky-9-20240916T114629 - ubuntu-jammy: 2023.1-ubuntu-jammy-20240916T114629 + rocky-9: 2023.1-rocky-9-20240923T162134 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240923T162134 nova: rocky-9: 2023.1-rocky-9-20240916T114629 ubuntu-jammy: 2023.1-ubuntu-jammy-20240916T114629 diff --git a/releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml b/releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml new file mode 100644 index 000000000..ff8c55bc6 --- /dev/null +++ b/releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Bumps Neutron container image tags to fix `bug 2068644 + `__ which could prevent + associating floating IPs with OVN-based load balancers. From 20ffb9c1116407a4a6db1e986ec430459a5c4baa Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 25 Sep 2024 17:09:10 +0100 Subject: [PATCH 11/21] Add Ceph package repo for Ubuntu Jammy --- etc/kayobe/apt.yml | 8 ++++++++ etc/kayobe/environments/ci-aio/stackhpc-ci.yml | 1 + etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 1 + etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 1 + etc/kayobe/pulp-repo-versions.yml | 1 + etc/kayobe/pulp.yml | 8 ++++++++ etc/kayobe/stackhpc.yml | 4 ++++ 7 files changed, 24 insertions(+) diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 08db21910..cb24d7484 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -33,6 +33,8 @@ apt_config: apt_keys: - url: "https://download.docker.com/linux/ubuntu/gpg" filename: docker.asc + - url: "https://download.ceph.com/keys/release.asc" + filename: ceph.asc # A list of Apt repositories. Each item is a dict with the following keys: # * types: whitespace-separated list of repository types, e.g. deb or deb-src @@ -70,6 +72,12 @@ stackhpc_apt_repositories: signed_by: docker.asc architecture: amd64 required: true + - url: "{{ stackhpc_repo_ceph_reef_debian_url }}" + suites: "{{ ansible_facts.distribution_release }}" + components: main + signed_by: ceph.asc + architecture: amd64 + required: true # Do not replace apt configuration for non-overcloud hosts. This can result in # errors if apt reconfiguration is performed before local repository mirrors diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index 3e4ca8fee..50af5d160 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -30,6 +30,7 @@ stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "" stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}" stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}" stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}" diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index e7f755d59..338b7ef66 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -55,6 +55,7 @@ stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "" stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}" stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}" stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}" diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index e75b7589b..6e10b63ea 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -30,6 +30,7 @@ stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "" stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}" stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}" stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}" diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 6867b40ee..6584b99d3 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -42,3 +42,4 @@ stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20240718T001130 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20240911T041957 stackhpc_pulp_repo_ubuntu_jammy_security_version: 20240911T063424 stackhpc_pulp_repo_ubuntu_jammy_version: 20240911T063424 +stackhpc_pulp_repo_ceph_reef_debian_version: 20240925T152022 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index f05a36a8f..0789644aa 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -122,6 +122,14 @@ stackhpc_pulp_deb_repos: components: "stable" required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + - name: "Docker CE for Ubuntu Jammy" + url: "{{ stackhpc_release_pulp_content_url }}/ceph/debian-reef/{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" + distribution_name: "docker-ce-for-ubuntu-jammy-" + base_path: "docker-ce/ubuntu-jammy/" + distributions: "jammy" + components: "stable" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + # Publication format is a subset of distribution. stackhpc_pulp_publication_deb_development: "{{ stackhpc_pulp_distribution_deb_development }}" diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index c8ce513d5..5f7113117 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -49,6 +49,10 @@ stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_repo_distribution }}" stackhpc_repo_docker_ce_ubuntu_jammy_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/docker-ce/ubuntu-jammy/{{ stackhpc_repo_docker_ce_ubuntu_jammy_version }}" stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_repo_distribution }}" +# Ceph Reef for Debian +stackhpc_repo_ceph_reef_debian_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ceph/debian-reef/{{ stackhpc_repo_ceph_reef_debian_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_repo_distribution }}" + ############################################################################### # RPMs From 56f0cebfcdbf393f3524bb07693e5507c11005b6 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Thu, 26 Sep 2024 12:52:14 +0200 Subject: [PATCH 12/21] Fix rabbitmq builds after move to Cloudsmith mirror (#1301) * Add sed for multiline baseurl in yum repos * use new Rabbitmq Cloudsmith repos * use rabbitmq built from Cloudsmith repo --- etc/kayobe/kolla-image-tags.yml | 2 +- etc/kayobe/kolla.yml | 1 + etc/kayobe/pulp-repo-versions.yml | 4 ++-- etc/kayobe/stackhpc.yml | 4 ++-- .../notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml | 7 +++++++ 5 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 9dddef8cb..f17e60910 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -48,5 +48,5 @@ kolla_image_tags: ovn: rocky-9: 2023.1-rocky-9-20240809T102431 rabbitmq: - rocky-9: 2023.1-rocky-9-20240823T101942 + rocky-9: 2023.1-rocky-9-20240925T133913 ubuntu-jammy: 2023.1-ubuntu-jammy-20240823T101942 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index e2fbad5cc..b6cc34475 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -289,6 +289,7 @@ kolla_build_blocks: {% for repo in base_centos_repo_overrides_post_yum_list %} sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[ \t]*\(https.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \ {% endfor %} diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 50985a30a..bd766b98c 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -11,8 +11,8 @@ stackhpc_pulp_repo_epel_9_version: 20240708T235303 stackhpc_pulp_repo_grafana_version: 20240708T235303 stackhpc_pulp_repo_opensearch_2_x_version: 20240626T000533 stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20240626T000533 -stackhpc_pulp_repo_rabbitmq_erlang_version: 20240506T000343 -stackhpc_pulp_repo_rabbitmq_server_version: 20240704T001154 +stackhpc_pulp_repo_rabbitmq_erlang_version: 20240925T093206 +stackhpc_pulp_repo_rabbitmq_server_version: 20240925T111913 stackhpc_pulp_repo_rhel_9_influxdb_version: 20240702T000233 stackhpc_pulp_repo_rhel_9_mariadb_10_6_version: 20240517T012522 stackhpc_pulp_repo_rhel_9_treasuredata_4_version: 20231130T012511 diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 9606634a9..db9e3bda5 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -57,11 +57,11 @@ stackhpc_repo_grafana_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/grafana/ stackhpc_repo_grafana_version: "{{ stackhpc_repo_distribution }}" # RabbitMQ - Erlang for Redhat family, version 8/9 agnostic -stackhpc_repo_rabbitmq_erlang_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/erlang/el/8/x86_64/{{ stackhpc_repo_rabbitmq_erlang_version }}" +stackhpc_repo_rabbitmq_erlang_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/erlang/el/9/x86_64/{{ stackhpc_repo_rabbitmq_erlang_version }}" stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_repo_distribution }}" # RabbitMQ for Redhat family, version 8/9 agnostic -stackhpc_repo_rabbitmq_server_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/rabbitmq-server/el/8/x86_64/{{ stackhpc_repo_rabbitmq_server_version }}" +stackhpc_repo_rabbitmq_server_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/rabbitmq-server/el/9/noarch/{{ stackhpc_repo_rabbitmq_server_version }}" stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_repo_distribution }}" # CentOS Stream 9 - NFV OpenvSwitch diff --git a/releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml b/releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml new file mode 100644 index 000000000..b1a66e16f --- /dev/null +++ b/releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + RabbitMQ and Erlang packages are now all installed from the Cloudsmith + ``rabbitmq.com`` mirrors since the RabbitMQ ``packagecloud.io`` is getting + shut down August 18st, 2024: + https://www.rabbitmq.com/blog/2024/08/11/package-repository-updates#packagecloud-will-be-discontinued-on-aug-18th-2024 From da02f4de19db7b4e79deec7c70e993063c3ec3ec Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 27 Sep 2024 10:12:08 +0100 Subject: [PATCH 13/21] Add note on Tempest concurrency, increase defaults --- .automation.conf/config.sh | 13 +++++++++---- .../notes/tempest-concurrency-1c86c0ef90e104f6.yaml | 5 +++++ 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml diff --git a/.automation.conf/config.sh b/.automation.conf/config.sh index 84b11dd07..76f2a59a0 100644 --- a/.automation.conf/config.sh +++ b/.automation.conf/config.sh @@ -3,7 +3,14 @@ # See: https://github.com/stackhpc/docker-rally/blob/master/bin/rally-verify-wrapper.sh for a full list of tempest parameters that can be overriden. # You can override tempest parameters like so: -export TEMPEST_CONCURRENCY=2 + +# The Tempest concurrency determines how many tests can be running at once. +# Higher values run tests faster but risk running out of resources and failing tests +# On production systems, Tempest concurrency can usually be set to a high number e.g. 16-64. It is often limited by the number of available floating IPs. +# On virtualised test environments, compute and networking speeds often limit the concurrency to 1-16 before tests begin to fail due to timeouts. +export TEMPEST_CONCURRENCY=16 + + # Specify single test whilst experimenting #export TEMPEST_PATTERN="${TEMPEST_PATTERN:-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name}" @@ -21,9 +28,7 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then fi if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then - # SMSLab is currently running with 1G switches. This causes tests using volumes and images to fail if - # the concurrency is set too high. - export TEMPEST_CONCURRENCY=1 + export TEMPEST_CONCURRENCY=4 # Uncomment this to perform a full tempest test # export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full # export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode-tempest-full diff --git a/releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml b/releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml new file mode 100644 index 000000000..db9658694 --- /dev/null +++ b/releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + The default Tempest concurrency has been increased from 2 to 16. + This is often easily achievable in production systems. From f5ce9416655d505e6da5e3bb58963aaab8cabc17 Mon Sep 17 00:00:00 2001 From: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:37:30 +0100 Subject: [PATCH 14/21] Firewall config (#1016) Add a default firewall configuration --------- Co-authored-by: Mark Goddard Co-authored-by: Will Szumski Co-authored-by: Will Szumski Co-authored-by: Jack Hodgkiss --- .github/workflows/stackhpc-all-in-one.yml | 1 + doc/source/configuration/cephadm.rst | 2 + doc/source/configuration/firewall.rst | 462 ++++++++++++++++++ doc/source/configuration/index.rst | 7 +- etc/kayobe/ansible/firewalld-watchdog.yml | 69 +++ .../environments/ci-aio/controllers.yml | 8 + .../post.d/10-debug-firewall.yml | 9 + .../environments/ci-aio/kolla/globals.yml | 3 + etc/kayobe/environments/ci-aio/networks.yml | 1 + .../environments/ci-multinode/compute.yml | 23 + .../environments/ci-multinode/controllers.yml | 24 + .../environments/ci-multinode/infra-vms.yml | 23 + .../inventory/group_vars/all/firewall.yml | 7 + .../ci-multinode/kolla/globals.yml | 4 + .../environments/ci-multinode/monitoring.yml | 23 + .../environments/ci-multinode/networks.yml | 10 + .../ci-multinode/seed-hypervisor.yml | 23 + etc/kayobe/environments/ci-multinode/seed.yml | 23 + .../environments/ci-multinode/storage.yml | 23 + etc/kayobe/inventory/group_vars/all/firewall | 330 +++++++++++++ etc/kayobe/networks.yml | 6 + .../automated-firewalld-a95e7322fd457259.yaml | 7 + 22 files changed, 1085 insertions(+), 3 deletions(-) create mode 100644 doc/source/configuration/firewall.rst create mode 100644 etc/kayobe/ansible/firewalld-watchdog.yml create mode 100644 etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml create mode 100644 etc/kayobe/environments/ci-multinode/monitoring.yml create mode 100644 etc/kayobe/environments/ci-multinode/seed-hypervisor.yml create mode 100644 etc/kayobe/inventory/group_vars/all/firewall create mode 100644 releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 611a56699..c5aa3e368 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -223,6 +223,7 @@ jobs: admin_bootproto: dhcp admin_ips: controller0: "{{ access_ip_v4.value }}" + admin_zone: admin EOF - name: Write Terraform network interface config diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst index 9c839b42e..bcb13cd6c 100644 --- a/doc/source/configuration/cephadm.rst +++ b/doc/source/configuration/cephadm.rst @@ -344,6 +344,8 @@ should be used in the Kolla Manila configuration e.g.: manila_cephfs_filesystem_name: manila-cephfs +.. _RGWs-With-Ceph: + RADOS Gateways -------------- diff --git a/doc/source/configuration/firewall.rst b/doc/source/configuration/firewall.rst new file mode 100644 index 000000000..2a14d8d6c --- /dev/null +++ b/doc/source/configuration/firewall.rst @@ -0,0 +1,462 @@ +.. _firewall: + +======== +Firewall +======== + +StackHPC Kayobe configuration provides a standardised firewalld configuration. +The configuration uses the :kayobe-doc:`firewall +` host configuration +functionality of Kayobe. + +The firewall configuration is provided in +``etc/kayobe/inventory/group_vars/all/firewall``. + +Enabling StackHPC firewalld rules +================================= + +The standardised firewalld configuration is not enabled by default and must be +actively opted into. To do so, make the following changes in +``etc/kayobe/.yml`` (or +``etc/kayobe/environments//.yml`` if environments are being +used). + +Controller firewalld Configuration +---------------------------------- + +.. code-block:: yaml + :caption: ``controllers.yml`` + + ############################################################################### + # Controller node firewalld configuration. + + # Whether to install and enable firewalld. + controller_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + controller_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Compute firewalld Configuration +------------------------------- + +.. code-block:: yaml + :caption: ``compute.yml`` + + ############################################################################### + # Compute node firewalld configuration. + + # Whether to install and enable firewalld. + compute_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + compute_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Storage firewalld Configuration +------------------------------- + +.. code-block:: yaml + :caption: ``storage.yml`` + + ############################################################################### + # storage node firewalld configuration. + + # Whether to install and enable firewalld. + storage_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + storage_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + storage_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + storage_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +If using RADOS Gateway, you can customise ``stackhpc_ceph_firewalld_radosgw_port`` to match +the ``rgw_frontend_port`` as documented in :ref:`RGWs-with-Ceph`. + +Monitoring firewalld Configuration +---------------------------------- + +.. code-block:: yaml + :caption: ``monitoring.yml`` + + ############################################################################### + # monitoring node firewalld configuration. + + # Whether to install and enable firewalld. + monitoring_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + monitoring_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + monitoring_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + monitoring_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Infrastructure VM firewalld Configuration +----------------------------------------- + +The standard firewalld configuration has rules for wazuh-manager and Ansible +control host Infrastructure VMs. + +.. code-block:: yaml + :caption: ``infra-vms.yml`` + + ############################################################################### + # Infrastructure VM node firewalld configuration + + # Whether to install and enable firewalld. + infra_vm_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + infra_vm_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + infra_vm_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + infra_vm_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +When configuring wazuh-manager, remember to set ``wazuh_dashboard_net_name`` if you have customised +the network where the Wazuh dashboard is exposed. + +Seed firewalld Configuration +---------------------------- + +.. code-block:: yaml + :caption: ``seed.yml`` + + ############################################################################### + # seed node firewalld configuration. + + # Whether to install and enable firewalld. + seed_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + seed_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + seed_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + seed_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Seed Hypervisor firewalld Configuration +--------------------------------------- + +.. code-block:: yaml + :caption: ``seed_hypervisor.yml`` + + ############################################################################### + # seed_hypervisor node firewalld configuration. + + # Whether to install and enable firewalld. + seed_hypervisor_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + seed_hypervisor_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + seed_hypervisor_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + seed_hypervisor_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +The following workaround is needed to prevent VM network traffic from being blocked: + +.. code-block:: yaml + :caption: ``seed_hypervisor.yml`` + + seed_hypervisor_sysctl_parameters: + # By default this is 1, which causes layer 2 traffic flowing through Linux + # bridges to pass through iptables. This blocks traffic from VMs (seed, wazuh) to + # the Internet. + net.bridge.bridge-nf-call-iptables: 0 + +The hope is that in the future this can be replaced by some additional firewalld configuration. + +Kolla-Ansible configuration +--------------------------- + +Ensure Kolla Ansible opens up ports in firewalld for services on the public +API network: + +.. code-block:: yaml + :caption: ``etc/kayobe/kolla/globals.yml`` + + enable_external_api_firewalld: true + external_api_firewalld_zone: "{{ public_net_name | net_zone }}" + +Network configuration +--------------------- + +Ensure every network in ``networks.yml`` has a zone defined. The standard +configuration is to set the internal network zone to ``trusted`` and every +other zone to the name of the network. See +``etc/kayobe/environments/ci-multinode/networks.yml`` for a practical example. + +Custom rules +------------ + +Custom firewalld rules can be added to ``stackhpc_firewalld_rules_extra`` + +The variable is a list of firewall rules to apply. Each item is a dictionary +containing arguments to pass to the firewalld module. The variable can be +defined as a group var or host var in the kayobe inventory. + +The structure of custom rules is different from the default rules. Custom rules +use the firewalld Ansible module format. Arguments are omitted if not provided, +with the following exceptions: + +* ``offline: true`` +* ``permanent: true`` +* ``state: enabled`` + +The main differences are that the ``zone`` argument is mandatory, and the +``network`` argument is not. + +The example below would enable SSH in the ``provision_oc`` zone, and disable +UDP port 1000 in the ``admin_oc`` zone for the Wazuh manager Infrastructure +VM: + +.. code-block:: yaml + :caption: ``etc/kayobe/inventory/group_vars/wazuh_manager/firewall`` + + stackhpc_firewalld_rules_extra: + - service: ssh + zone: "{{ provision_oc_net_name | net_zone }}" + state: enabled + - port: 1000/udp + zone: "{{ admin_oc_net_name | net_zone }}" + state: disabled + +Extra rules have higher precedence than the default rules but are not +validated before being applied. Use with caution. If you need to add a custom +rule, consider adding it to the default rule list with an appropriate boolean +condition, and where possible merge your changes back into upstream SKC. + +Validation +---------- + +The ``kayobe configuration dump`` command can be used to view all the rules +that will be applied to a host. + +.. code-block:: bash + + kayobe configuration dump --var-name stackhpc_firewalld_rules --limit + +A shorter version, ``stackhpc_firewalld_rules_debug`` prints the rules in a +simplified format: + +.. code-block:: bash + + kayobe configuration dump --var-name stackhpc_firewalld_rules_debug --limit + +If the commands above print a template, rather than a list of rules, the +configuration may be invalid. The ``kayobe configuration dump`` command can be +used on other variables such as ``stackhpc_firewalld_rules_default`` or +``stackhpc_*_firewalld_rules_template`` to debug the configuration. See the +`How it works`_ section for more details. + +It can be useful to print the active ports on each type of host, to create +rules for running services. The internal network is currently left open. The +below command will print all other open ports: + +.. code-block:: bash + + ss -lntpu | grep --invert-match '' + +It is strongly recommended that you dry-run the changes using ``--diff`` and +``--check`` before applying to a production system: + +.. code-block:: bash + :caption: ``Overcloud diff example`` + + kayobe overcloud host configure -t firewall --diff --check + +Baseline checks +^^^^^^^^^^^^^^^ + +Before applying, it is a good idea to take note of any actively firing alerts +and run Tempest to gather a baseline. See the :doc:`Tempest +` page for more details. + +Applying changes +---------------- + +Before applying these changes, you should be completely sure you are not going +to lock yourself out of any hosts. If you are deploying these changes to a test +environment, consider setting a password on the stack user so that you can +access the host through a BMC or other virtual console. + +The following Kayobe command can be used to set a password on all overcloud +hosts: + +.. code-block:: bash + + kayobe overcloud host command run --command "echo 'stack:super-secret-password' | sudo chpasswd" --show-output + +The ``firewalld-watchdog.yml`` playbook can be used to set up a timer that +disables the firewalld service after a period of time (default 600s). It should +be used as follows: + +.. code-block:: bash + + # Enable the watchdog BEFORE applying the firewall configuration + kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l + + # Disable the watchdog after applying the firewall configuration + kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l -e firewalld_watchdog_state=absent + +If the firewall rules block connectivity, the second playbook run (disabling +the watchdog) will fail. You will still be able to get in after the watchdog +triggers. Remember to disable the watchdog when you are finished, otherwise the +firewall will be disabled! + +Changes should be applied to controllers one at a time to ensure connectivity +is not lost. + +Once you are sure you know what you are doing, use the ``kayobe * host +configure`` commands to apply the firewall changes: + +.. code-block:: bash + + # For Seed Hypervisor hosts + kayobe seed hypervisor host configure -t network,firewall + # For Seed hosts + kayobe seed host configure -t network,firewall + # For Infrastructure VM hosts + kayobe infra vm host configure -t network,firewall + # For the First Controller + kayobe overcloud host configure -t network,firewall -l controllers[0] + # For the Second Controller + kayobe overcloud host configure -t network,firewall -l controllers[1] + # For the Third Controller + kayobe overcloud host configure -t network,firewall -l controllers[2] + # For the rest of the Overcloud hosts + kayobe overcloud host configure -t network,firewall + +Debugging +--------- + +To test the changes, first check for any firing alerts, then try simple smoke +tests (create a VM, list OpenStack endpoints etc.), then run Tempest. + +If the firewall configuration is causing errors, it is often useful to log +blocked packets. + +.. code-block:: bash + + sudo sed -i s/LogDenied=off/LogDenied=all/g /etc/firewalld/firewalld.conf + sudo systemctl restart firewalld + +Dropped packets will be logged to ``dmesg``. + +How it works +============ + +The standard firewall rule configuration is stored in +``etc/kayobe/inventory/group_vars/all/firewall``. + +The file contains sections for different host groups. There are sections for: + +* Common (all hosts) +* Controllers +* Compute +* Storage +* Monitoring +* Wazuh Manager Infrastructure VM +* Ansible Control host Infrastructure VM +* Seed +* Seed Hypervisor + +Each of these sections contains a template. The template is made of sets of +rules. The rules can then be enabled and disabled in sets, based on properties +of the cloud. For example, if ``kolla_enable_designate`` is true, a set of +rules will be enabled in ``stackhpc_controller_firewalld_rules_template``. + +The templates are combined into a single list, +``stackhpc_firewalld_rules_template``. Templates are selected according to the +host's group membership, as well as a set of common rules, which is enabled for +all hosts. + +The rules are then formatted into a single list of the enabled default rules: +``stackhpc_firewalld_rules_default``. The Rules are manipulated to reduce +duplication. When no zone is specified in a rule template, it is inferred from +the network. They are also validated. Conflicting rules will result in an +error. Non-applicable rules are dropped. + +The default rules are combined with any extra rules defined for the deployment. +The complete set of controller firewalld rules is +``stackhpc_firewalld_rules``. diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index f8be7891a..2a42bd473 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -8,16 +8,17 @@ the various features provided. .. toctree:: :maxdepth: 1 - walled-garden release-train host-images lvm swap cephadm monitoring - wazuh vault + wazuh + walled-garden + security-hardening + firewall magnum-capi ci-cd - security-hardening cloudkitty diff --git a/etc/kayobe/ansible/firewalld-watchdog.yml b/etc/kayobe/ansible/firewalld-watchdog.yml new file mode 100644 index 000000000..874992df7 --- /dev/null +++ b/etc/kayobe/ansible/firewalld-watchdog.yml @@ -0,0 +1,69 @@ +--- +# This playbook can be applied in advance of rolling out a firewall +# configuration. It sets up a timer that disables the firewalld service after a +# period of time (default 600s). It should be used as follows: +# 1. Enable firewalld-watchdog +# kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l +# 2. Apply firewall config +# kayobe host configure -l -t network,firewall +# 3. Disable watchdog +# kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l -e firewalld_watchdog_state=absent +# If the firewall changes result in being locked out of the system, the +# watchdog will disable the firewall after the timeout. +# Remember to disable the watchdog, otherwise the firewall will be disabled! + +- name: Create a systemd timer to stop firewalld + hosts: seed:seed-hypervisor:overcloud:infra-vms + tags: + - firewalld-watchdog + vars: + # Watchdog state: present or absent. + firewalld_watchdog_state: present + # Watchdog timeout in seconds. + firewalld_watchdog_timeout_s: 600 + become: true + tasks: + - when: firewalld_watchdog_state == 'present' + block: + - name: Create firewalld-watchdog service unit file + ansible.builtin.copy: + dest: /etc/systemd/system/firewalld-watchdog.service + content: | + [Unit] + Description=Firewalld watchdog service + + [Service] + Type=oneshot + ExecStart=/usr/bin/systemctl stop firewalld + register: service_result + + - name: Create firewalld-watchdog timer unit file + ansible.builtin.copy: + dest: /etc/systemd/system/firewalld-watchdog.timer + content: | + [Unit] + Description=Firewalld watchdog timer + + [Timer] + OnActiveSec={{ firewalld_watchdog_timeout_s }} + Unit=firewalld-watchdog.service + + [Install] + WantedBy=timers.target + register: timer_result + + - name: Enable or disable firewalld-watchdog timer + ansible.builtin.systemd_service: + name: firewalld-watchdog.timer + daemon_reload: "{{ service_result is changed or timer_result is changed }}" + enabled: false + state: "{{ 'started' if firewalld_watchdog_state == 'present' else 'stopped' }}" + + - name: Remove firewalld-watchdog unit files + ansible.builtin.file: + path: "/etc/systemd/system/{{ item }}" + state: absent + loop: + - firewalld-watchdog.service + - firewalld-watchdog.timer + when: firewalld_watchdog_state == 'absent' diff --git a/etc/kayobe/environments/ci-aio/controllers.yml b/etc/kayobe/environments/ci-aio/controllers.yml index b34536705..8972187df 100644 --- a/etc/kayobe/environments/ci-aio/controllers.yml +++ b/etc/kayobe/environments/ci-aio/controllers.yml @@ -10,3 +10,11 @@ controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' el # for the exact configuration. controller_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" + +# Controller firewalld configuration. See inventory/group_vars/all/firewall for +# the exact configuration. +controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" +controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" +# FIXME(wszumski): Firewall disabled in OVS temporarily until someone has a change to fix it. +controller_firewalld_enabled: "{{ kolla_enable_ovn | bool }}" +controller_firewalld_default_zone: "drop" diff --git a/etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml b/etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml new file mode 100644 index 000000000..f0fa29978 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml @@ -0,0 +1,9 @@ +--- + +- hosts: overcloud + gather_facts: false + tasks: + - name: Configure firewalld to log blocked traffic + command: firewall-cmd --set-log-denied=all + become: true + when: firewalld_enabled | bool diff --git a/etc/kayobe/environments/ci-aio/kolla/globals.yml b/etc/kayobe/environments/ci-aio/kolla/globals.yml index 9f058ed19..2244b5bf5 100644 --- a/etc/kayobe/environments/ci-aio/kolla/globals.yml +++ b/etc/kayobe/environments/ci-aio/kolla/globals.yml @@ -15,6 +15,9 @@ opensearch_heap_size: 200m # Increase Grafana timeout grafana_start_first_node_retries: 20 +# Open up ports in firewalld for services on the public API network. +enable_external_api_firewalld: "{{ controller_firewalld_enabled | bool }}" + # Ensure Rabbit is deployed with HA rather than quorum queues (to test migrations) om_enable_rabbitmq_high_availability: true om_enable_rabbitmq_quorum_queues: false diff --git a/etc/kayobe/environments/ci-aio/networks.yml b/etc/kayobe/environments/ci-aio/networks.yml index 216696eaa..e3cc4d43d 100644 --- a/etc/kayobe/environments/ci-aio/networks.yml +++ b/etc/kayobe/environments/ci-aio/networks.yml @@ -89,6 +89,7 @@ aio_neutron_allocation_pool_end: 192.168.33.127 aio_inspection_allocation_pool_start: 192.168.33.128 aio_inspection_allocation_pool_end: 192.168.33.254 aio_vip_address: 192.168.33.2 +aio_zone: aio ############################################################################### # Network virtual patch link configuration. diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml index 7e7701cf0..a00207b41 100644 --- a/etc/kayobe/environments/ci-multinode/compute.yml +++ b/etc/kayobe/environments/ci-multinode/compute.yml @@ -4,3 +4,26 @@ compute_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else # format. compute_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" + +############################################################################### +# Compute node firewalld configuration. + +# Whether to install and enable firewalld. +compute_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +compute_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml index 73c31c27f..173bcc371 100644 --- a/etc/kayobe/environments/ci-multinode/controllers.yml +++ b/etc/kayobe/environments/ci-multinode/controllers.yml @@ -4,3 +4,27 @@ controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' el # format. controller_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" + + +############################################################################### +# Controller node firewalld configuration. + +# Whether to install and enable firewalld. +controller_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +controller_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/infra-vms.yml b/etc/kayobe/environments/ci-multinode/infra-vms.yml index c388a3c73..33bb0b91e 100644 --- a/etc/kayobe/environments/ci-multinode/infra-vms.yml +++ b/etc/kayobe/environments/ci-multinode/infra-vms.yml @@ -4,3 +4,26 @@ infra_vm_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else # format. infra_vm_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" + +############################################################################### +# Infrastructure VM node firewalld configuration + +# Whether to install and enable firewalld. +infra_vm_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +infra_vm_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +infra_vm_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +infra_vm_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml new file mode 100644 index 000000000..ca1f9b475 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml @@ -0,0 +1,7 @@ +--- + +stackhpc_firewalld_rules_extra: + - port: "{{ vxlan_dstport }}/udp" + network: "{{ admin_oc_net_name }}" + zone: "{{ admin_oc_net_name | net_zone }}" + state: enabled diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 599618500..354a8c310 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -62,6 +62,10 @@ designate_backend: "bind9" designate_recursion: "yes" designate_forwarders_addresses: "1.1.1.1; 8.8.8.8" +# Open up ports in firewalld for services on the public API network. +enable_external_api_firewalld: true +external_api_firewalld_zone: "{{ public_net_name | net_zone }}" + ############################################################################ # RabbitMQ diff --git a/etc/kayobe/environments/ci-multinode/monitoring.yml b/etc/kayobe/environments/ci-multinode/monitoring.yml new file mode 100644 index 000000000..fa9bbf0be --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/monitoring.yml @@ -0,0 +1,23 @@ +--- +############################################################################### +# monitoring node firewalld configuration. + +# Whether to install and enable firewalld. +monitoring_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +monitoring_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +monitoring_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +monitoring_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml index c0a7ff69e..59d3760b9 100644 --- a/etc/kayobe/environments/ci-multinode/networks.yml +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -71,12 +71,16 @@ storage_mgmt_net_name: storage_mgmt ############################################################################### # Network definitions. +# Admin overcloud network +admin_oc_zone: "admin_oc" + # Internal network internal_cidr: 192.168.37.0/24 internal_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" internal_allocation_pool_start: 192.168.37.3 internal_allocation_pool_end: 192.168.37.254 internal_vlan: 101 +internal_zone: "trusted" # External network external_cidr: 192.168.38.0/24 @@ -84,6 +88,7 @@ external_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" external_allocation_pool_start: 192.168.38.3 external_allocation_pool_end: 192.168.38.128 external_vlan: 102 +external_zone: "external" # Public network public_cidr: 192.168.39.0/24 @@ -91,6 +96,7 @@ public_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" public_allocation_pool_start: 192.168.39.3 public_allocation_pool_end: 192.168.39.254 public_vlan: 103 +public_zone: "public" # Tunnel network tunnel_cidr: 192.168.40.0/24 @@ -98,6 +104,7 @@ tunnel_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" tunnel_allocation_pool_start: 192.168.40.3 tunnel_allocation_pool_end: 192.168.40.254 tunnel_vlan: 104 +tunnel_zone: "tunnel" # Storage network storage_cidr: 192.168.41.0/24 @@ -105,6 +112,7 @@ storage_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" storage_allocation_pool_start: 192.168.41.3 storage_allocation_pool_end: 192.168.41.254 storage_vlan: 105 +storage_zone: "storage" # Storage management network # NOTE: Skipping the .42 subnet to avoid a collision with a popular number. @@ -113,6 +121,7 @@ storage_mgmt_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" storage_mgmt_allocation_pool_start: 192.168.43.3 storage_mgmt_allocation_pool_end: 192.168.43.254 storage_mgmt_vlan: 106 +storage_mgmt_zone: "storage_mgmt" # Provision overcloud network provision_oc_cidr: 192.168.33.0/24 @@ -120,6 +129,7 @@ provision_oc_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" provision_oc_allocation_pool_start: 192.168.33.128 provision_oc_allocation_pool_end: 192.168.33.254 provision_oc_vlan: 107 +provision_oc_zone: "provision_oc" ############################################################################### # Network virtual patch link configuration. diff --git a/etc/kayobe/environments/ci-multinode/seed-hypervisor.yml b/etc/kayobe/environments/ci-multinode/seed-hypervisor.yml new file mode 100644 index 000000000..d64b776fc --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/seed-hypervisor.yml @@ -0,0 +1,23 @@ +--- +############################################################################### +# seed_hypervisor node firewalld configuration. + +# Whether to install and enable firewalld. +seed_hypervisor_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +seed_hypervisor_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +seed_hypervisor_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +seed_hypervisor_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index bb9e3c6bf..e732258c1 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -27,3 +27,26 @@ snat_rules_manila: source_ip: "{{ ansible_facts[storage_interface].ipv4.address | default }}" # Only add the storage snat rule if we are using manila-cephfs. snat_rules: "{{ snat_rules_default + snat_rules_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else snat_rules_default }}" + +############################################################################### +# seed node firewalld configuration. + +# Whether to install and enable firewalld. +seed_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +seed_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +seed_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +seed_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/storage.yml b/etc/kayobe/environments/ci-multinode/storage.yml index b152af472..11f7cf71e 100644 --- a/etc/kayobe/environments/ci-multinode/storage.yml +++ b/etc/kayobe/environments/ci-multinode/storage.yml @@ -4,3 +4,26 @@ storage_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else # format. storage_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" + +############################################################################### +# storage node firewalld configuration. + +# Whether to install and enable firewalld. +storage_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +storage_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +storage_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +storage_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/inventory/group_vars/all/firewall b/etc/kayobe/inventory/group_vars/all/firewall new file mode 100644 index 000000000..488e95b65 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/firewall @@ -0,0 +1,330 @@ +--- +######################################### +# StackHPC default firewall configuration +######################################### +# This file contains the reference StackHPC firewalld rule implementation. It +# is designed to cover as many services and use cases as possible, however may +# not be compatible with all deployments. For more information, see the +# firewall page in the configuration section of the SKC docs. + +# A templated list of firewalld zones, according to the host's network +# interfaces. +# This variable can be used to set any *_firewalld_zones kayobe variable e.g. +# compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" in compute.yml +stackhpc_firewalld_zones: | + {% set network_zones = [] %} + {% for network in network_interfaces %} + {% if network | net_zone %} + {% set _ = network_zones.append({'zone': network | net_zone }) %} + {% endif %} + {% endfor %} + {{ network_zones | unique | list }} + +# A templated list of firewalld rules, according to the enabled services, +# host's group membership, and host's network configuration. +# This variable can be used to set any *_firewalld_rules kayobe variable e.g. +# compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" in compute.yml +stackhpc_firewalld_rules: | + {% set stackhpc_firewalld_rules_verified = stackhpc_firewalld_rules_extra %} + {% for rule in stackhpc_firewalld_rules_default | unique %} + {% if rule | ansible.utils.remove_keys('state') not in stackhpc_firewalld_rules_verified | map('ansible.utils.remove_keys', 'state') | default([]) %} + {% if rule.network is not defined %} + {% set _ = stackhpc_firewalld_rules_verified.append(rule) %} + {% elif rule.network in network_interfaces and rule.network | net_zone %} + {% set _ = stackhpc_firewalld_rules_verified.append(rule) %} + {% endif %} + {% endif %} + {% endfor %} + {{ stackhpc_firewalld_rules_verified | list }} + +# A single list of default firewall rules, combining the enabled rules from +# stackhpc_firewalld_rules_template. This variable is designed to fail to +# template if any conflicting rules are found. +stackhpc_firewalld_rules_default: | + {% set stackhpc_firewalld_rules_formatted = [] %} + {% for ruleset in stackhpc_firewalld_rules_template %} + {% if ruleset.enabled | bool %} + {% for rule in ruleset.rules %} + {% if rule.zone is not defined %} + {% set rule = rule | combine({'zone': rule.network | net_zone }) %} + {% endif %} + {% if rule not in stackhpc_firewalld_rules_formatted %} + {% if rule | ansible.utils.remove_keys('state') in stackhpc_firewalld_rules_formatted | map('ansible.utils.remove_keys', 'state') %} + {% set _ = stackhpc_firewalld_rules_formatted.append({'state':'failure'}) %} + {% elif rule.network is not defined %} + {% set _ = stackhpc_firewalld_rules_formatted.append(rule) %} + {% elif rule.network in network_interfaces and rule.network | net_zone %} + {% set _ = stackhpc_firewalld_rules_formatted.append(rule) %} + {% endif %} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {{ undef(hint='ERROR: Conflicting firewall rules found') if ({'state':'failure'} in stackhpc_firewalld_rules_formatted) else stackhpc_firewalld_rules_formatted }} + +stackhpc_firewalld_rules_template: | + {{ stackhpc_common_firewalld_rules_template + + (stackhpc_controller_firewalld_rules_template if 'controllers' in group_names else []) + + (stackhpc_compute_firewalld_rules_template if 'compute' in group_names else []) + + (stackhpc_ceph_firewalld_rules_template if 'ceph' in group_names else []) + + (stackhpc_monitoring_firewalld_rules_template if 'monitoring' in group_names else []) + + (stackhpc_seed_firewalld_rules_template if 'seed' in group_names else []) + + (stackhpc_seed_hypervisor_firewalld_rules_template if 'seed-hypervisor' in group_names else []) + + (stackhpc_wazuh_manager_infra_vm_firewalld_rules_template if 'wazuh-manager' in group_names else []) + + (stackhpc_ansible_control_infra_vm_firewalld_rules_template if inventory_hostname == 'localhost' else []) }} + +############################################################################### +# Debug Vars + +# This variable is not applied anywhere. It exists for debugging purpouses +# only. Print it with: +# kayobe configuration dump --var-name stackhpc_firewalld_rules_debug +stackhpc_firewalld_rules_debug: | + {% set stackhpc_firewalld_services_debug = [] %} + {% for rule in stackhpc_firewalld_rules %} + {% if rule.service is defined %} + {% set _ = stackhpc_firewalld_services_debug.append(rule.service + ' ' + rule.state + ' ' + rule.zone | default()) %} + {% else %} + {% set _ = stackhpc_firewalld_services_debug.append(rule.port + ' ' + rule.state + ' ' + rule.zone | default()) %} + {% endif %} + {% endfor %} + {{ stackhpc_firewalld_services_debug | list }} + +############################################################################### +# Extra firewalld rules + +# Extra firewalld rules. Each item is a dict containing arguments to pass to +# the firewalld module. +# These rules have higher precidence than the +# default rules and are not validated before being applied. Use with caution. +# NOTE: The format of this variable is NOT the same as the +# stackhpc_*_firewalld_rules_template variables found elsewhere in this file. +stackhpc_firewalld_rules_extra: [] + +############################################################################### +# Common firewalld rules + +stackhpc_common_firewalld_rules_template: + # Common + - rules: + - service: ssh + network: "{{ admin_oc_net_name }}" + state: enabled + enabled: true + # Cockpit, dhcpv6-client, and SSH are enabled by default in firewalld + - rules: + - service: cockpit + zone: public + state: disabled + - service: dhcpv6-client + zone: public + state: disabled + - service: ssh + zone: public + state: disabled + enabled: "{{ admin_oc_net_name | net_zone != 'public' }}" + +############################################################################### +# Controller firewalld rules + +stackhpc_controller_firewalld_rules_template: + # Overcloud Ironic + - rules: + # Ironic inspector API + - port: 5050/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + # Ironic API + - port: 6385/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 8089/tcp + network: "{{ provision_wl_net_name }}" + state: enabled + - service: dhcp + network: "{{ provision_wl_net_name }}" + state: enabled + - service: ntp + network: "{{ provision_wl_net_name }}" + state: enabled + - service: tftp + network: "{{ provision_wl_net_name }}" + state: enabled + enabled: "{{ kolla_enable_ironic | bool }}" + # Designate + - rules: + - port: 53/tcp + network: "{{ public_net_name }}" + state: enabled + - port: 53/udp + network: "{{ public_net_name }}" + state: enabled + - port: 5354/tcp + network: "{{ public_net_name }}" + state: enabled + - port: 5354/udp + network: "{{ public_net_name }}" + state: enabled + enabled: "{{ kolla_enable_designate | bool }}" + # GENEVE + - rules: + - port: 6081/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ 'geneve' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types) and 'network' in group_names }}" + # VXLAN + - rules: + - port: 4789/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ 'vxlan' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types) and 'network' in group_names }}" + # Octavia + - rules: + - port: 5555/udp + network: "{{ octavia_net_name | default() }}" + state: enabled + enabled: "{{ kolla_enable_octavia | bool and octavia_net_name is not none }}" + # DHCP, Was required to create VMs at a customer site, need to work out condition + - rules: + - port: 67/udp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: true + +############################################################################### +# Compute firewalld rules + +stackhpc_compute_firewalld_rules_template: + # GENEVE + - rules: + - port: 6081/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ ('geneve' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types)) | bool }}" + # VXLAN + - rules: + - port: 4789/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ ('vxlan' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types)) | bool }}" + +############################################################################### +# Ceph firewalld rules + +# Port on which radosgw is exposed. +# See: https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-2024.1/configuration/cephadm.html#rados-gateways +stackhpc_ceph_firewalld_radosgw_port: 8100 + +stackhpc_ceph_firewalld_rules_template: + # Ceph Prometheus exporter + - rules: + - port: 9283/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ kolla_enable_prometheus_ceph_mgr_exporter | default(false) | bool and 'mgrs' in group_names }}" + # Ceph + - rules: + - service: ceph + network: "{{ storage_net_name }}" + state: enabled + - service: ceph + network: "{{ storage_mgmt_net_name }}" + state: enabled + - service: ceph-mon + network: "{{ storage_net_name }}" + state: "{{ 'enabled' if 'mons' in group_names else 'disabled' }}" + - port: "{{ stackhpc_ceph_firewalld_radosgw_port }}/tcp" + network: "{{ storage_net_name }}" + state: "{{ 'enabled' if 'rgws' in group_names else 'disabled' }}" + enabled: "{{ 'ceph' in group_names }}" + +############################################################################### +# Monitoring firewalld rules + +stackhpc_monitoring_firewalld_rules_template: [] + +############################################################################### +# Infra VM firewalld rules (Wazuh Manager) + +stackhpc_wazuh_manager_infra_vm_firewalld_rules_template: + - rules: + - port: 1514/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 1514/udp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 1515/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 443/tcp + network: "{{ wazuh_dashboard_net_name | default(provision_oc_net_name) }}" + state: enabled + - port: 9200/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 9300-9400/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 55000/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: true + +############################################################################### +# Infra VM firewalld rules (Ansible Control) + +stackhpc_ansible_control_infra_vm_firewalld_rules_template: [] + +############################################################################### +# Seed firewalld rules + +stackhpc_seed_firewalld_rules_template: + # Pulp server + - rules: + - port: "{{ pulp_port }}/tcp" + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ seed_pulp_container_enabled | bool }}" + # Squid proxy + - rules: + - service: squid + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ seed_squid_container_enabled | bool }}" + # Ironic + - rules: + # nginx + - port: 8080/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + # Ironic inspector API + - port: 5050/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + # Ironic API + - port: 6385/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - service: dhcp + network: "{{ provision_oc_net_name }}" + state: enabled + - service: tftp + network: "{{ provision_oc_net_name }}" + state: enabled + - service: ntp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: true #FIXME: Make rules conditional on Bifrost deployment + # Redfish exporter + - rules: + - port: 9610/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ stackhpc_enable_redfish_exporter | default(false) }}" + # TODO: Rules if SNAT enabled on seed + +############################################################################### +# Seed Hypervisor firewalld rules + +stackhpc_seed_hypervisor_firewalld_rules_template: [] diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index e58ab44cf..10feb408c 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -59,6 +59,12 @@ # hosts #cleaning_net_name: +# Name of the network used to manage octavia loadbalancers +#octavia_net_name: + +# Name of the network where Wazuh manager is exposed +#wazuh_dashboard_net_name: + ############################################################################### # Network definitions. diff --git a/releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml b/releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml new file mode 100644 index 000000000..4a06e6705 --- /dev/null +++ b/releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + A default firewall configuration is now included on an opt-in basis. The + rules are defined under ``etc/kayobe/inventory/group_vars/all/firewall``. + More information can be found `here + `__ From 30d670c1afe65355947ebb65c10736292f2573fd Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 4 Oct 2024 08:51:37 +0100 Subject: [PATCH 15/21] Fix file descriptor leaks in networking-mlnx (#1306) * Test patches for resource leaks * Update kolla-image-tags.yml * Switch back to stackhpc branch * Adds a release note --- etc/kayobe/kolla-image-tags.yml | 8 ++++---- .../fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index f17e60910..2d7a5bd65 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -33,11 +33,11 @@ kolla_image_tags: manila: rocky-9: 2023.1-rocky-9-20240809T102431 neutron: - rocky-9: 2023.1-rocky-9-20240923T162134 - ubuntu-jammy: 2023.1-ubuntu-jammy-20240923T162134 + rocky-9: 2023.1-rocky-9-20240926T151818 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240926T151818 nova: - rocky-9: 2023.1-rocky-9-20240916T114629 - ubuntu-jammy: 2023.1-ubuntu-jammy-20240916T114629 + rocky-9: 2023.1-rocky-9-20240926T151818 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240926T151818 octavia: rocky-9: 2023.1-rocky-9-20240730T090421 ubuntu-jammy: 2023.1-ubuntu-jammy-20240730T090421 diff --git a/releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml b/releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml new file mode 100644 index 000000000..415b4d76d --- /dev/null +++ b/releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes a file descriptor leak in networking-mlnx which prevented VMs using + Infiniband virtual functions from provisioning after a period of time. + - | + Fixes ``KeyError: ip_version`` in networking-mlnx when used in conjuction + with OVN mechanism driver. From f918a1a534ff8b189ab4c8510017207d2cc2bed4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 8 Oct 2024 11:09:09 +0100 Subject: [PATCH 16/21] feat: bump `multinode` workflow --- .github/workflows/stackhpc-multinode-periodic.yml | 2 +- .github/workflows/stackhpc-multinode.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stackhpc-multinode-periodic.yml b/.github/workflows/stackhpc-multinode-periodic.yml index cb94705bc..7debe8fa3 100644 --- a/.github/workflows/stackhpc-multinode-periodic.yml +++ b/.github/workflows/stackhpc-multinode-periodic.yml @@ -35,7 +35,7 @@ jobs: name: Multinode periodic needs: - generate-inputs - uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.1.0 + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0 with: multinode_name: mn-prdc-${{ github.run_id }} os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }} diff --git a/.github/workflows/stackhpc-multinode.yml b/.github/workflows/stackhpc-multinode.yml index 5c8b67d1e..3ec055bb4 100644 --- a/.github/workflows/stackhpc-multinode.yml +++ b/.github/workflows/stackhpc-multinode.yml @@ -52,7 +52,7 @@ name: Multinode jobs: multinode: name: Multinode - uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.1.0 + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0 with: multinode_name: ${{ inputs.multinode_name }} os_distribution: ${{ inputs.os_distribution }} From 0770dd67221a9c52cad90b57c7a10daab166458d Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 8 Oct 2024 21:53:38 +0200 Subject: [PATCH 17/21] Install openssh-client in AIO workflow Fixes `ssh-keygen: command not found` in pull requests workflows. --- .github/workflows/stackhpc-all-in-one.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 4526e1359..1909129ac 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -80,7 +80,7 @@ jobs: - name: Install Package uses: ConorMacBride/install-package@main with: - apt: git unzip nodejs + apt: git unzip nodejs openssh-client - uses: actions/checkout@v4 with: From 74eee068db465bffbe48223791dc7ce562a6fcf0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 8 Oct 2024 11:09:09 +0100 Subject: [PATCH 18/21] feat: bump `multinode` workflow --- .github/workflows/stackhpc-multinode-periodic.yml | 2 +- .github/workflows/stackhpc-multinode.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stackhpc-multinode-periodic.yml b/.github/workflows/stackhpc-multinode-periodic.yml index cb94705bc..7debe8fa3 100644 --- a/.github/workflows/stackhpc-multinode-periodic.yml +++ b/.github/workflows/stackhpc-multinode-periodic.yml @@ -35,7 +35,7 @@ jobs: name: Multinode periodic needs: - generate-inputs - uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.1.0 + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0 with: multinode_name: mn-prdc-${{ github.run_id }} os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }} diff --git a/.github/workflows/stackhpc-multinode.yml b/.github/workflows/stackhpc-multinode.yml index 7d77c8742..86a79b11d 100644 --- a/.github/workflows/stackhpc-multinode.yml +++ b/.github/workflows/stackhpc-multinode.yml @@ -52,7 +52,7 @@ name: Multinode jobs: multinode: name: Multinode - uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.1.0 + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0 with: multinode_name: ${{ inputs.multinode_name }} os_distribution: ${{ inputs.os_distribution }} From f9b654475e9c2631024fcbb15a6d1041ea59a7f3 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 9 Oct 2024 16:12:45 +0100 Subject: [PATCH 19/21] Disable Seed firewalld when using OVS VMs can't have outbound connection with default SKC firewalld setup when a multinode uses OVS. Make default Seed firewalld state conditional to avoid having breaking change. --- etc/kayobe/environments/ci-multinode/seed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index e732258c1..630f9a849 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -32,7 +32,7 @@ snat_rules: "{{ snat_rules_default + snat_rules_manila if (kolla_enable_manila | # seed node firewalld configuration. # Whether to install and enable firewalld. -seed_firewalld_enabled: true +seed_firewalld_enabled: "{{ kolla_enable_ovn | bool }}" # A list of zones to create. Each item is a dict containing a 'zone' item. seed_firewalld_zones: "{{ stackhpc_firewalld_zones }}" From ee6381cc006389911e44fefffbde54af7ac08741 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 11 Oct 2024 09:40:23 +0100 Subject: [PATCH 20/21] 2024.1 upgrade doc additions (#1325) * doc: add note about `redis_connection_string` * doc: add `--exclude` to `configuration save` --- doc/source/operations/upgrading-openstack.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index c4e1bb6f0..0bcba45e5 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -113,6 +113,14 @@ The domain set should be something that is not use anywhere else such as The Neuron DNS integration can be disabled by setting ``neutron_dns_integration: false`` in ``kolla/globals.yml`` +Redis Default User +------------------ + +The ``redis_connection_string`` has changed the username used from ``admin`` +to ``default``. Whilst this does not have any negative impact on services +that utilise Redis it will feature prominently in any preview of the overcloud +configuration. + Known issues ============ @@ -486,7 +494,7 @@ Save the old configuration locally. .. code-block:: console - kayobe overcloud service configuration save --node-config-dir /etc/kolla --output-dir ~/kolla-diff/old --limit controllers[0],compute[0],storage[0] + kayobe overcloud service configuration save --node-config-dir /etc/kolla --output-dir ~/kolla-diff/old --limit controllers[0],compute[0],storage[0] --exclude ironic-agent.initramfs,ironic-agent.kernel Generate the new configuration to a tmpdir. @@ -498,7 +506,7 @@ Save the new configuration locally. .. code-block:: console - kayobe overcloud service configuration save --node-config-dir /tmp/kolla --output-dir ~/kolla-diff/new --limit controllers[0],compute[0],storage[0] + kayobe overcloud service configuration save --node-config-dir /tmp/kolla --output-dir ~/kolla-diff/new --limit controllers[0],compute[0],storage[0] --exclude ironic-agent.initramfs,ironic-agent.kernel The old and new configuration will be saved to ``~/kolla-diff/old`` and ``~/kolla-diff/new`` respectively on the Ansible control host. From 38876d7cef3803083507b74de3c43f2dcdd7505a Mon Sep 17 00:00:00 2001 From: Seunghun Lee <45145778+seunghun1ee@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:42:04 +0100 Subject: [PATCH 21/21] Set selinux_do_reboot to true for ci-multinode (#1324) ``selinux_do_reboot`` is false by default in Kayobe. This causes multinode automation to fail in host configure because selinux state is updated but reboot is not allowed. This allows reboot so, host configure can run without failure. --- etc/kayobe/environments/ci-multinode/globals.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml index 9fe415ecd..de497cb84 100644 --- a/etc/kayobe/environments/ci-multinode/globals.yml +++ b/etc/kayobe/environments/ci-multinode/globals.yml @@ -60,6 +60,9 @@ os_release: >- stackhpc_write_barbican_role_id_to_file: true stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id" +# Enable rebooting to update SELinux state +selinux_do_reboot: true + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes