From 19a977ebe86e0ff7bdc10e63640d8985a611cf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Wed, 17 Jan 2024 09:05:01 +0100 Subject: [PATCH] Add osbuild-ci images based on c8s and c9s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two new variants of the osbuild-ci image based on c8s and c9s. CentOS Stream is lacking some packages. Install all of the unavailable python packages using pip and also pytest on cXs images. The reason is for not installing pytest from cXs repositories is to get the latest version of it to not have to deal with lack of features on older releases. In addition, the cXs base images contain minimal variants of some packages, which we want to install into them. To resolve the conflicts, allow erasing of packages during package installation. The intention is that these cXs images will be used mostly to run the unit tests. The Fedora image should be used for all the additional testing (such as pylint, mypy, isort, ...), including unit tests. Signed-off-by: Tomáš Hozza --- docker-bake.hcl | 133 ++++++++++++++++++++++- src/images/osbuild-ci-cstream.Dockerfile | 63 +++++++++++ src/scripts/pip.sh | 32 ++++++ 3 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 src/images/osbuild-ci-cstream.Dockerfile create mode 100755 src/scripts/pip.sh diff --git a/docker-bake.hcl b/docker-bake.hcl index 4513f3a..295850a 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -215,16 +215,29 @@ target "nfsd-latest" { * osbuild-ci - OSBuild CI Images * * The following groups and targets build the CI images used by osbuild. They - * build on the official fedora images. + * build on the official fedora and cXs images. + * + * The `osbuild-ci-cXs-latest` images are missing some packages, compared to + * the `osbuild-ci-latest` images, because they are not available in + * the cXs repositories. Their main purpose is to use them only to run unit + * tests in osbuild upstream. The Fedora image should be used for all the + * other tests, such as linters and running unit tests on multiple Python + * versions. + * + * NB: Docker bake HCL does not support definig arrays as a variable or calling + * functions in variable definitions, so we need to duplicate the package list + * in Fedora and cXs targets. */ group "all-osbuild-ci" { targets = [ "osbuild-ci-latest", + "osbuild-ci-c8s-latest", + "osbuild-ci-c9s-latest", ] } -target "virtual-osbuild-ci" { +target "virtual-osbuild-ci-base" { args = { OSB_DNF_PACKAGES = join(",", [ "bash", @@ -288,7 +301,7 @@ target "virtual-osbuild-ci" { "util-linux", ]), OSB_DNF_GROUPS = join(",", [ - "development-tools", + "development tools", "rpm-development-tools", ]), } @@ -301,13 +314,121 @@ target "virtual-osbuild-ci" { target "osbuild-ci-latest" { args = { - OSB_FROM = "docker.io/library/fedora:latest", + OSB_FROM = "registry.fedoraproject.org/fedora:latest", + } + inherits = [ + "virtual-osbuild-ci-base", + ] + tags = concat( + mirror("osbuild-ci-fedora", "latest", "", OSB_UNIQUEID), + ) +} + +target "virtual-osbuild-ci-cXs" { + args = { + OSB_DNF_PACKAGES = join(",", [ + "bash", + //"btrfs-progs", // not available in cXs + "bubblewrap", + "coreutils", + "cryptsetup", + "curl", + "dnf", + "dnf-plugins-core", + "dosfstools", + "e2fsprogs", + "findutils", + "git", + "glibc", + "iproute", + "lvm2", + "make", + //"nbd", // not available in cXs + //"nbd-cli", // not available in cXs + "ostree", + //"pacman", // not available in cXs + "policycoreutils", + //"pylint", // not available in cXs + "python-rpm-macros", + "python3", // install just the default version + //"python3.6", + //"python3.7", + //"python3.8", + //"python3.9", + //"python3.10", + //"python3.12", + //"python3-autopep8", // not available in cXs + //"python3-boto3", // not available in cXs + //"python3-botocore", // not available in cXs + //"python3-docutils", // not available in cXs + "python3-devel", + "python3-iniparse", + //"python3-isort", // not available in cXs + "python3-jsonschema", + "python3-librepo", + "python3-mako", + //"python3-mypy", // not available in cXs + "python3-pip", + //"python3-pylint", // not available in cXs + //"python3-pytest", // too old in cXs + //"python3-pytest-cov", // not available in cXs + "python3-pyyaml", + "python3-rpm-generators", + "python3-rpm-macros", + "qemu-img", + //"qemu-system-x86", // not available in cXs + "rpm", + "rpm-build", + "rpm-ostree", + "rpmdevtools", + "skopeo", + "systemd", + "systemd-container", + "tar", + //"tox", // not available in cXs + "util-linux", + ]), + OSB_PIP_PACKAGES = join(",", [ + "autopep8", + "boto3", + "botocore", + "docutils", + "isort", + "mypy", + "pylint", + "pytest", + "pytest-cov", + "tox", + ]), + OSB_DNF_ALLOW_ERASING = 1, + } + dockerfile = "src/images/osbuild-ci-cstream.Dockerfile" + inherits = [ + "virtual-osbuild-ci-base", + ] +} + +target "osbuild-ci-c8s-latest" { + args = { + OSB_FROM = "quay.io/centos/centos:stream8", + } + inherits = [ + "virtual-osbuild-ci-cXs", + ] + tags = concat( + mirror("osbuild-ci-c8s", "latest", "", OSB_UNIQUEID), + ) +} + +target "osbuild-ci-c9s-latest" { + args = { + OSB_FROM = "quay.io/centos/centos:stream9", } inherits = [ - "virtual-osbuild-ci", + "virtual-osbuild-ci-cXs", ] tags = concat( - mirror("osbuild-ci", "latest", "", OSB_UNIQUEID), + mirror("osbuild-ci-c9s", "latest", "", OSB_UNIQUEID), ) } diff --git a/src/images/osbuild-ci-cstream.Dockerfile b/src/images/osbuild-ci-cstream.Dockerfile new file mode 100644 index 0000000..0033356 --- /dev/null +++ b/src/images/osbuild-ci-cstream.Dockerfile @@ -0,0 +1,63 @@ +# +# osbuild-ci - OSBuild CI Images +# +# This image provides the OS environment for the osbuild continuous integration +# on GitHub Actions. It is based on CesnOS Stream and includes all the required +# packages and utilities for running unit-tests. +# +# Arguments: +# +# * OSB_FROM="quay.io/centos/centos:stream9" +# This controls the host container used as base for the CI image. +# +# * OSB_DNF_PACKAGES="" +# Specify the packages to install into the container. Separate packages +# by comma. By default, no package is pulled in. +# +# * OSB_DNF_GROUPS="" +# Specify the package groups to install into the container. Separate +# groups by comma. By default, no group is pulled in. +# +# * OSB_PIP_PACKAGES="" +# Specify the packages to install into the container using pip. Separate +# packages by comma. By default, no packages are installed. +# + +ARG OSB_FROM="quay.io/centos/centos:stream9" +FROM "${OSB_FROM}" AS target + +# +# Import our build sources and prepare the target environment. When finished, +# we drop the build sources again, to keep the target image small. +# + +WORKDIR /osb +COPY src src + +ARG OSB_DNF_PACKAGES="" +ARG OSB_DNF_GROUPS="" +ARG OSB_PIP_PACKAGES="" +ARG OSB_DNF_ALLOW_ERASING="" +RUN ./src/scripts/dnf.sh "${OSB_DNF_PACKAGES}" "${OSB_DNF_GROUPS}" ${OSB_DNF_ALLOW_ERASING} +RUN ./src/scripts/pip.sh "${OSB_PIP_PACKAGES}" +COPY src/scripts/osbuild-ci.sh . + +RUN rm -rf /osb/src + +# +# Allow cross-UID git access. Git users must be careful not to invoke git from +# within untrusted directory-paths. +# + +RUN git config --global --add safe.directory '*' + +# +# Rebuild from scratch to drop all intermediate layers and keep the final image +# as small as possible. Then setup the entrypoint. +# + +FROM scratch +COPY --from=target . . + +WORKDIR /osb/workdir +ENTRYPOINT ["/osb/osbuild-ci.sh"] diff --git a/src/scripts/pip.sh b/src/scripts/pip.sh new file mode 100755 index 0000000..2ab8b8f --- /dev/null +++ b/src/scripts/pip.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# +# This script is a pip package install helper for container images. It takes +# packages as argument and then installs them via `pip3`. +# + +set -eox pipefail + +OSB_IFS=$IFS + +# +# Parse command-line arguments into local variables. We accept: +# @1: Comma-separated list of packages to install. +# + +if (( $# > 0 )) ; then + IFS=',' read -r -a PIP_PACKAGES <<< "$1" + IFS=$OSB_IFS +fi +if (( $# > 1 )) ; then + echo >&2 "ERROR: invalid number of arguments" + exit 1 +fi + +# +# Install the specified packages. +# + +if (( ${#PIP_PACKAGES[@]} )) ; then + pip3 install --upgrade "${PIP_PACKAGES[@]}" +fi