Add osbuild-ci images based on c8s and c9s (COMPOSER-2125)

docker-bake.hcl

@@ -215,16 +215,29 @@ target "nfsd-latest" {
  * osbuild-ci - OSBuild CI Images
  *
  * The following groups and targets build the CI images used by osbuild. They
- * build on the official fedora images.
+ * build on the official fedora and cXs images.
+ *
+ * The `osbuild-ci-cXs-latest` images are missing some packages, compared to
+ * the `osbuild-ci-latest` image, because they are not available in
+ * the cXs repositories. The intention is to use them only to run unit
+ * tests in osbuild upstream. The Fedora image should be used for all the
+ * other tests, such as linters and running unit tests on multiple Python
+ * versions.
+ *
+ * NB: Docker bake HCL does not support definig arrays as a variable or calling
+ * functions in variable definitions, so we need to duplicate the package list
+ * in Fedora and cXs targets.
  */
 
 group "all-osbuild-ci" {
         targets = [
                 "osbuild-ci-latest",
+                "osbuild-ci-c8s-latest",
+                "osbuild-ci-c9s-latest",
         ]
 }
 
-target "virtual-osbuild-ci" {
+target "virtual-osbuild-ci-base" {
         args = {
                 OSB_DNF_PACKAGES = join(",", [
                         "bash",
@@ -288,7 +301,7 @@ target "virtual-osbuild-ci" {
                         "util-linux",
                 ]),
                 OSB_DNF_GROUPS = join(",", [
-                        "development-tools",
+                        "development tools",
                         "rpm-development-tools",
                 ]),
         }
@@ -301,13 +314,121 @@ target "virtual-osbuild-ci" {
 
 target "osbuild-ci-latest" {
         args = {
-                OSB_FROM = "docker.io/library/fedora:latest",
+                OSB_FROM = "registry.fedoraproject.org/fedora:latest",
+        }
+        inherits = [
+                "virtual-osbuild-ci-base",
+        ]
+        tags = concat(
+                mirror("osbuild-ci-fedora", "latest", "", OSB_UNIQUEID),
+        )
+}
+
+target "virtual-osbuild-ci-cXs" {
+        args = {
+                OSB_DNF_PACKAGES = join(",", [
+                        "bash",
+                        //"btrfs-progs",         // not available in cXs
+                        "bubblewrap",
+                        "coreutils",
+                        "cryptsetup",
+                        "curl",
+                        "dnf",
+                        "dnf-plugins-core",
+                        "dosfstools",
+                        "e2fsprogs",
+                        "findutils",
+                        "git",
+                        "glibc",
+                        "iproute",
+                        "lvm2",
+                        "make",
+                        //"nbd",                 // not available in cXs
+                        //"nbd-cli",             // not available in cXs
+                        "ostree",
+                        //"pacman",              // not available in cXs
+                        "policycoreutils",
+                        //"pylint",              // not available in cXs
+                        "python-rpm-macros",
+                        "python3",               // install just the default version
+                        //"python3.6",
+                        //"python3.7",
+                        //"python3.8",
+                        //"python3.9",
+                        //"python3.10",
+                        //"python3.12",
+                        //"python3-autopep8",    // not available in cXs
+                        //"python3-boto3",       // not available in cXs
+                        //"python3-botocore",    // not available in cXs
+                        //"python3-docutils",    // not available in cXs
+                        "python3-devel",
+                        "python3-iniparse",
+                        //"python3-isort",       // not available in cXs
+                        "python3-jsonschema",
+                        "python3-librepo",
+                        "python3-mako",
+                        //"python3-mypy",        // not available in cXs
+                        "python3-pip",
+                        //"python3-pylint",      // not available in cXs
+                        //"python3-pytest",      // too old in cXs
+                        //"python3-pytest-cov",  // not available in cXs
+                        "python3-pyyaml",
+                        "python3-rpm-generators",
+                        "python3-rpm-macros",
+                        "qemu-img",
+                        //"qemu-system-x86",     // not available in cXs
+                        "rpm",
+                        "rpm-build",
+                        "rpm-ostree",
+                        "rpmdevtools",
+                        "skopeo",
+                        "systemd",
+                        "systemd-container",
+                        "tar",
+                        //"tox",                 // not available in cXs
+                        "util-linux",
+                ]),
+                OSB_PIP_PACKAGES = join(",", [
+                        "autopep8",
+                        "boto3",
+                        "botocore",
+                        "docutils",
+                        "isort",
+                        "mypy",
+                        "pylint",
+                        "pytest",
+                        "pytest-cov",
+                        "tox",
+                ]),
+                OSB_DNF_ALLOW_ERASING = 1,
+        }
+        dockerfile = "src/images/osbuild-ci-cstream.Dockerfile"
+        inherits = [
+                "virtual-osbuild-ci-base",
+        ]
+}
+
+target "osbuild-ci-c8s-latest" {
+        args = {
+                OSB_FROM = "quay.io/centos/centos:stream8",
+        }
+        inherits = [
+                "virtual-osbuild-ci-cXs",
+        ]
+        tags = concat(
+                mirror("osbuild-ci-c8s", "latest", "", OSB_UNIQUEID),
+        )
+}
+
+target "osbuild-ci-c9s-latest" {
+        args = {
+                OSB_FROM = "quay.io/centos/centos:stream9",
         }
         inherits = [
-                "virtual-osbuild-ci",
+                "virtual-osbuild-ci-cXs",
         ]
         tags = concat(
-                mirror("osbuild-ci", "latest", "", OSB_UNIQUEID),
+                mirror("osbuild-ci-c9s", "latest", "", OSB_UNIQUEID),
         )
 }
 

src/config/google-cloud-sdk.repo

@@ -3,6 +3,6 @@ name=Google Cloud SDK
 baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64
 enabled=1
 gpgcheck=1
-repo_gpgcheck=1
+repo_gpgcheck=0
 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
        https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

src/images/osbuild-ci-cstream.Dockerfile

@@ -0,0 +1,63 @@
+#
+# osbuild-ci - OSBuild CI Images
+#
+# This image provides the OS environment for the osbuild continuous integration
+# on GitHub Actions. It is based on CentOS Stream and includes all the required
+# packages and utilities for running unit-tests.
+#
+# Arguments:
+#
+#   * OSB_FROM="quay.io/centos/centos:stream9"
+#       This controls the host container used as base for the CI image.
+#
+#   * OSB_DNF_PACKAGES=""
+#       Specify the packages to install into the container. Separate packages
+#       by comma. By default, no package is pulled in.
+#
+#   * OSB_DNF_GROUPS=""
+#       Specify the package groups to install into the container. Separate
+#       groups by comma. By default, no group is pulled in.
+#
+#   * OSB_PIP_PACKAGES=""
+#       Specify the packages to install into the container using pip. Separate
+#       packages by comma. By default, no packages are installed.
+#
+
+ARG             OSB_FROM="quay.io/centos/centos:stream9"
+FROM            "${OSB_FROM}" AS target
+
+#
+# Import our build sources and prepare the target environment. When finished,
+# we drop the build sources again, to keep the target image small.
+#
+
+WORKDIR         /osb
+COPY            src src
+
+ARG             OSB_DNF_PACKAGES=""
+ARG             OSB_DNF_GROUPS=""
+ARG             OSB_PIP_PACKAGES=""
+ARG             OSB_DNF_ALLOW_ERASING=""
+RUN             ./src/scripts/dnf.sh "${OSB_DNF_PACKAGES}" "${OSB_DNF_GROUPS}" ${OSB_DNF_ALLOW_ERASING}
+RUN             ./src/scripts/pip.sh "${OSB_PIP_PACKAGES}"
+COPY            src/scripts/osbuild-ci.sh .
+
+RUN             rm -rf /osb/src
+
+#
+# Allow cross-UID git access. Git users must be careful not to invoke git from
+# within untrusted directory-paths.
+#
+
+RUN             git config --global --add safe.directory '*'
+
+#
+# Rebuild from scratch to drop all intermediate layers and keep the final image
+# as small as possible. Then setup the entrypoint.
+#
+
+FROM            scratch
+COPY            --from=target . .
+
+WORKDIR         /osb/workdir
+ENTRYPOINT      ["/osb/osbuild-ci.sh"]

src/scripts/dnf.sh

@@ -14,6 +14,8 @@ OSB_IFS=$IFS
 # Parse command-line arguments into local variables. We accept:
 #   @1: Comma-separated list of packages to install.
 #   @2: Comma-separated list of comp-groups to install.
+#   @3: 0 or 1 to enable or disable --allowerasing when installing packages.
+#       Disabled by default. (optional)
 #
 
 if (( $# > 0 )) ; then
@@ -25,10 +27,19 @@ if (( $# > 1 )) ; then
         IFS=$OSB_IFS
 fi
 if (( $# > 2 )) ; then
+        if [[ ! $3 =~ ^[01]$ ]] ; then
+                echo >&2 "ERROR: invalid value for the third argument '$3'"
+                echo >&2 "       only 0 or 1 are allowed"
+                exit 1
+        fi
+fi
+if (( $# > 3 )) ; then
         echo >&2 "ERROR: invalid number of arguments"
         exit 1
 fi
 
+ALLOW_ERASING=${3:-0}
+
 #
 # Clean all caches so we force a metadata refresh. Then make sure to update
 # the system to avoid unsynchronized installs. Note that we force a metadata
@@ -45,11 +56,18 @@ dnf -y upgrade
 # step to keep the number of duplicate installs low.
 #
 
+EXTRA_ARGS=""
+
+if [[ "$ALLOW_ERASING" == 1 ]] ; then
+        EXTRA_ARGS+=" --allowerasing"
+fi
+
 if (( ${#OSB_PACKAGES[@]} )) ; then
         dnf -y \
                 --nodocs \
                 --setopt=fastestmirror=True \
                 --setopt=install_weak_deps=False \
+                $EXTRA_ARGS \
                 install \
                 -- \
                         "${OSB_PACKAGES[@]}"
@@ -60,6 +78,7 @@ if (( ${#OSB_GROUPS[@]} )) ; then
                 --nodocs \
                 --setopt=fastestmirror=True \
                 --setopt=install_weak_deps=False \
+                $EXTRA_ARGS \
                 group install \
                 -- \
                         "${OSB_GROUPS[@]}"

src/scripts/install-oci-cli.sh

@@ -2,4 +2,18 @@
 
 set -eox pipefail
 
-bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
+# temporary filename for the script
+INSTALL_SCRIPT_PATH="mktemp"
+
+curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh --output "${INSTALL_SCRIPT_PATH}"
+
+# install python3.11 if the python3 version is 3.12 or newer
+# https://github.com/oracle/oci-cli/issues/742
+if [[ $(python3 -c "import sys; too_new = sys.version_info >= (3, 12, 0); print(1 if too_new else 0)") -eq 1 ]]; then
+    sudo dnf install -y python3.11
+    sudo dnf clean all
+    sed -i 's/for try_python_exe in /&python3.11 /' "${INSTALL_SCRIPT_PATH}"
+fi
+
+
+bash "${INSTALL_SCRIPT_PATH}" --accept-all-defaults

src/scripts/pip.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#
+# This script is a pip package install helper for container images. It takes
+# packages as argument and then installs them via `pip3`.
+#
+
+set -eox pipefail
+
+OSB_IFS=$IFS
+
+#
+# Parse command-line arguments into local variables. We accept:
+#   @1: Comma-separated list of packages to install.
+#
+
+if (( $# > 0 )) ; then
+        IFS=',' read -r -a PIP_PACKAGES <<< "$1"
+        IFS=$OSB_IFS
+fi
+if (( $# > 1 )) ; then
+        echo >&2 "ERROR: invalid number of arguments"
+        exit 1
+fi
+
+#
+# Install the specified packages.
+#
+
+if (( ${#PIP_PACKAGES[@]} )) ; then
+        pip3 install --upgrade "${PIP_PACKAGES[@]}"
+fi