Merge branch 'cross-arch-builds-2024.1' of github.com:stackhpc/stackh…

…pc-kayobe-config into cross-arch-builds-2024.1

.automation.conf/run-books/pulp-sync-content.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euE
+set -o pipefail
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KAYOBE_AUTOMATION_DIR="$(realpath "${PARENT}/../../.automation")"
+
+function main {
+	if [ "${PULP_DO_CONTAINER_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_CONTAINER_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_REPO_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml'
+	fi
+	if [ "${PULP_DO_REPO_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml'
+	fi
+	if [ "${PULP_DO_REPO_PROMOTE:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml'
+	fi
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+	main
+fi

.github/workflows/overcloud-host-image-build.yml

@@ -32,6 +32,13 @@ jobs:
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
       - name: Install Package
         uses: ConorMacBride/install-package@main
         with:

.github/workflows/overcloud-host-image-promote.yml

@@ -23,6 +23,13 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-22.04
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
       - uses: actions/checkout@v4
         with:
           path: src/kayobe-config

.github/workflows/overcloud-host-image-upload.yml

@@ -34,6 +34,13 @@ jobs:
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
       - name: Install package dependencies
         run: |
           sudo apt update

.github/workflows/stackhpc-container-image-build.yml

@@ -9,12 +9,12 @@ on:
         required: false
         default: ""
       overcloud:
-        description: Build overcloud images?
+        description: Build container images for overcloud services?
         type: boolean
         required: false
         default: true
       seed:
-        description: Build seed images?
+        description: Build container images for seed services?
         type: boolean
         required: false
         default: false
@@ -52,6 +52,17 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky-linux-9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+          if [[ ${{ inputs.overcloud }} == 'false' && ${{ inputs.seed }} == 'false' ]]; then
+            echo "At least one of overcloud or seed must be selected"
+            exit 1
+          fi
+
       - name: Checkout
         uses: actions/checkout@v4
 

doc/source/contributor/environments/ci-multinode.rst

@@ -35,8 +35,15 @@ is not enabled by default. To enable it, set the following in
       kolla_enable_manila: true
       kolla_enable_manila_backend_cephfs_native: true
 
-And re-run ``kayobe overcloud service deploy`` if you are working on an existing
-deployment.
+If you are working on an existing deployment, you need to do the following first.
+
+1. Create CephFS pools: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-pools.yml``
+2. Create cephx key for Manila: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-keys.yml``
+3. Run Manila related Ceph commands: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-commands-post.yml``
+4. Gather Ceph configuration and keyring for Manila: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml``
+5. Configure Storage network on Seed node: ``kayobe seed host configure -t network,ip-allocation,snat``
+
+Then, run ``kayobe overcloud service deploy`` to deploy Manila.
 
 To test it, you will need two virtual machines. Cirros does not support the Ceph
 kernel client, so you will need to use a different image. Any regular Linux
@@ -108,35 +115,35 @@ Then create a share type and share:
 
 .. code-block:: bash
 
-      manila type-create cephfs-type false --is_public true
-      manila type-key cephfs-type set vendor_name=Ceph storage_protocol=CEPHFS
-      manila create --name test-share --share-type cephfs-type CephFS 2
+      openstack share type create cephfs-type false --public true
+      openstack share type set cephfs-type --extra-specs vendor_name=Ceph, storage_protocol=CEPHFS
+      openstack share create --name test-share --share-type cephfs-type --public true CephFS 2
 
 Wait until the share is available:
 
 .. code-block:: bash
 
-      manila list
+      openstack share list
 
 Then allow access to the shares to two users:
 
 .. code-block:: bash
 
-      manila access-allow test-share cephx alice
-      manila access-allow test-share cephx bob
+      openstack share access create test-share cephx alice
+      openstack share access create test-share cephx bob
 
 Show the access list to make sure the state of both entries is ``active`` and
 take note of the access keys:
 
 .. code-block:: bash
 
-      manila access-list test-share
+      openstack share access list test-share
 
 And take note of the path to the share:
 
 .. code-block:: bash
 
-      manila share-export-location-list test-share
+      openstack share export location list test-share
 
 SSH into the first instance, create a directory for the share, and mount it:
 

doc/source/operations/upgrading-openstack.rst

@@ -124,16 +124,11 @@ configuration.
 Known issues
 ============
 
-* OVN breaks on Rocky 9 deployments where hostnames are FQDNs.
-  Before upgrading, you must make sure no compute or controller nodes have any
-  ``.`` characters in their hostnames. Run the command below to check:
-
-  .. code-block:: bash
-
-     kayobe overcloud host command run --command "grep -v \'\.\' /etc/hostname" --show-output
-
-  There is currently no known fix for this issue aside from reprovisioning. A
-  patch will be developed soon.
+* Due to an incorrect default value NGS will attempt to use v3alpha for the api
+  path when communicating with etcd3. This isn't possible as in Caracal etcd is
+  running a newer version that has dropped support for v3alpha. You can work
+  around this in custom config, see the SMS PR for an example:
+  https://github.com/stackhpc/smslab-kayobe-config/pull/354
 
 Security baseline
 =================

etc/kayobe/ansible/requirements.yml

@@ -11,7 +11,7 @@ collections:
   - name: stackhpc.hashicorp
     version: 2.5.1
   - name: stackhpc.kayobe_workflows
-    version: 1.0.3
+    version: 1.1.0
 roles:
   - src: stackhpc.vxlan
   - name: ansible-lockdown.ubuntu22_cis

etc/kayobe/kolla-image-tags.yml

@@ -28,3 +28,12 @@ kolla_image_tags:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20240910T072617
   rabbitmq:
     rocky-9: 2024.1-rocky-9-20240927T152945
+  ironic:
+    rocky-9: 2024.1-rocky-9-20241022T090648
+    ubuntu-jammy: 2024.1-ubuntu-jammy-20241022T090648
+  ironic_dnsmasq:
+    rocky-9: 2024.1-rocky-9-20241022T090648
+    ubuntu-jammy: 2024.1-ubuntu-jammy-20241022T090648
+  ironic_neutron_agent:
+    rocky-9: 2024.1-rocky-9-20241022T090648
+    ubuntu-jammy: 2024.1-ubuntu-jammy-20241022T090648

etc/kayobe/pulp.yml

@@ -122,12 +122,12 @@ stackhpc_pulp_deb_repos:
     components: "stable"
     required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}"
 
-  - name: "Docker CE for Ubuntu Jammy"
+  - name: "Ceph Reef for Debian"
     url: "{{ stackhpc_release_pulp_content_url }}/ceph/debian-reef/{{ stackhpc_pulp_repo_ceph_reef_debian_version }}"
-    distribution_name: "docker-ce-for-ubuntu-jammy-"
-    base_path: "docker-ce/ubuntu-jammy/"
+    distribution_name: "ceph-reef-debian-"
+    base_path: "ceph/debian-reef/"
     distributions: "jammy"
-    components: "stable"
+    components: "main"
     required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}"
 
 # Publication format is a subset of distribution.
@@ -216,40 +216,29 @@ stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}"
 stackhpc_pulp_repo_rocky_9_minor_version: 4
 # Rocky 9 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
+
 # Define the architecture suffix
 arch_suffix: "{{ '_aarch64' if kolla_base_arch == 'aarch64' else '' }}"
 
-# Lists of repositories
-rocky_minor_version_repos:
-  - baseos
-  - appstream
-  - extras
-  - crb
-  - highavailability
-
-rocky_multiarch_repos:
-  - stackhpc_pulp_repo_rhel9_rabbitmq_erlang
-  - stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch
-  - stackhpc_pulp_repo_centos_stream_9_openstack_caracal
-  - stackhpc_pulp_repo_centos_stream_9_opstools
-  - stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef
-  - stackhpc_pulp_repo_centos_stream_9_docker
-  - stackhpc_pulp_repo_rhel_9_treasuredata_5
-  - stackhpc_pulp_repo_rhel_9_mariadb_10_11
-  - stackhpc_pulp_repo_rhel_9_influxdb
-  - stackhpc_pulp_repo_epel_9
-  - stackhpc_pulp_repo_rocky_9_sig_security_common
-
-# Set Rocky repository versions
-{% for repo in rocky_minor_version_repos %}
-stackhpc_pulp_repo_rocky_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_{{ repo }}_version: >-
-  {{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_' ~ repo ~ arch_suffix ~ '_version') }}
-{% endfor %}
-
-# Set Rocky multiarch repository versions
-{% for repo in rocky_multiarch_repos %}
-{{ repo }}_version: "{{ lookup('vars', repo ~ arch_suffix ~ '_version') }}"
-{% endfor %}
+# Rocky 9 Snapshot versions
+stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_baseos' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_appstream' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_extras' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_crb' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_highavailability' ~ arch_suffix ~ '_version') }}"
+
+# Rocky 9 Multiarch repositories
+stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel9_rabbitmq_erlang' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_openstack_caracal' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_opstools' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_docker' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_treasuredata_5' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_mariadb_10_11' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_influxdb' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_epel_9_version: "{{ lookup('vars', 'stackhpc_pulp_repo_epel_9' ~ arch_suffix ~ '_version') }}"
+stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_sig_security_common' ~ arch_suffix ~ '_version') }}"
 
 # Whether to sync packages common to all RHEL 9 derivatives.
 stackhpc_pulp_sync_el_9: "{{ stackhpc_pulp_sync_rocky_9 | bool }}"

releasenotes/notes/fix-octavia-tls-terminated-pkcs12-4f7e32a6f5ca0143.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes creation and failover of Octavia TLS-terminated load balancers when
+    storing the certificate and key as a PKCS12 bundle in Barbican.

releasenotes/notes/fix-ossa-2024-004-f732e58c12e26785.yaml

@@ -0,0 +1,6 @@
+---
+security:
+  - |
+    Fixes `OSSA-2024-004
+    <https://security.openstack.org/ossa/OSSA-2024-004.html>`_ with updated
+    container images for Ironic.

releasenotes/notes/update-kayobe-automation-55fc9c5c380d819c.yaml

@@ -0,0 +1,23 @@
+---
+features:
+  - |
+    Upgrades kayobe-automation submodule to ``7676aa8``.
+    
+    Upgrades kayobe-workflows collection to ``v1.1.0``.
+
+    Kayobe-automation config-diff now runs in parallel and generates both
+    the old and new configuration at the same time. This should improve
+    config-diff wait times.
+
+    Add support for the `pulp-sync-content` run book.
+deprecations:
+  - |
+    Kayobe-automation will now automatically detect vaulted files for the
+    purpose of config-diff therefore, ``KAYOBE_CONFIG_SECRET_PATHS_EXTRA`` and
+    ``KAYOBE_CONFIG_VAULTED_FILES_PATHS_EXTRA`` are no longer used
+security:
+  - |
+    The upgraded kayobe-workflows collection increases the version of various
+    Actions and containers used within GitHub based workflows, including increasing
+    Docker in Docker to version ``27.3.1`` thus removing the vunerabilities present
+    in ``24.0-git``.