From 3b432ebca8c1d7b4282db44ef19dd0492e886577 Mon Sep 17 00:00:00 2001 From: Jason Frey Date: Tue, 12 Mar 2024 17:31:40 -0400 Subject: [PATCH 1/3] Collapse building of base image into the build loop for other images --- bin/build | 58 ++++++++++++++++++++++++------------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/bin/build b/bin/build index aaaa32af..042980e2 100755 --- a/bin/build +++ b/bin/build @@ -52,45 +52,37 @@ if [ -n "$REBUILD_RPM" ]; then fi pushd $IMAGE_DIR - cmd="$CONTAINER_COMMAND build" - - # --format docker is needed for podman to ensure the builds are built with docker v2 manifests - if [ "$CONTAINER_COMMAND" == "podman" ]; then - cmd+=" --format docker" - fi - - cmd+=" --tag $REPO/manageiq-base:$TAG \ - --pull \ - --build-arg RPM_PREFIX=$RPM_PREFIX" - - if [ -n "$NO_CACHE" ]; then - cmd+=" --no-cache" - fi - - if [ -n "$LOCAL_RPM" ] || [ -n "$REBUILD_RPM" ]; then - cmd+=" --build-arg LOCAL_RPM=true" - fi - - if [ -n "$RELEASE_BUILD" ]; then - cmd+=" --build-arg RELEASE_BUILD=true" - fi - - echo "Building manageiq-base: $cmd" - $cmd manageiq-base - - build_images="manageiq-base-worker manageiq-orchestrator manageiq-webserver-worker manageiq-ui-worker" + build_images="manageiq-base manageiq-base-worker manageiq-orchestrator manageiq-webserver-worker manageiq-ui-worker" for image in $build_images; do - cmd="$CONTAINER_COMMAND build" + cmd="$CONTAINER_COMMAND build --tag $REPO/$image:$TAG" + + if [ -n "$VERBOSE" ]; then + cmd+=" --progress plain" + fi + # --format docker is needed for podman to ensure the builds are built with docker v2 manifests if [ "$CONTAINER_COMMAND" == "podman" ]; then cmd+=" --format docker" fi - cmd+=" --tag $REPO/$image:$TAG \ - --build-arg FROM_REPO=$REPO \ - --build-arg FROM_TAG=$TAG \ - --build-arg RPM_PREFIX=$RPM_PREFIX \ - $image" + if [ -n "$NO_CACHE" ]; then + cmd+=" --no-cache" + fi + + if [ "$image" == "manageiq-base" ]; then + cmd+=" --pull \ + --build-arg RPM_PREFIX=$RPM_PREFIX" + + if [ -n "$RELEASE_BUILD" ]; then + cmd+=" --build-arg RELEASE_BUILD=true" + fi + else + cmd+=" --build-arg FROM_REPO=$REPO \ + --build-arg FROM_TAG=$TAG \ + --build-arg RPM_PREFIX=$RPM_PREFIX" + fi + + cmd+=" $image" echo "Building $image: $cmd" $cmd From dc5a853d1f0f0b0315a00787433ae31d630eef04 Mon Sep 17 00:00:00 2001 From: Jason Frey Date: Tue, 12 Mar 2024 21:45:55 -0400 Subject: [PATCH 2/3] Move RPM copying into a separate image This allows the RUN command that need then to mount that image, without actually including the RPM artifacts in the final image itself. --- .gitignore | 2 +- bin/build | 35 +++++++------------ images/manageiq-base/Dockerfile | 22 ++++++++---- ..._yum_repo.sh => prepare_local_yum_repo.sh} | 7 +--- images/manageiq-base/rpms/.gitkeep | 0 images/manageiq-rpms/Dockerfile | 22 ++++++++++++ images/manageiq-webserver-worker/Dockerfile | 11 ++++-- 7 files changed, 59 insertions(+), 40 deletions(-) rename images/manageiq-base/container-assets/{create_local_yum_repo.sh => prepare_local_yum_repo.sh} (64%) delete mode 100644 images/manageiq-base/rpms/.gitkeep create mode 100644 images/manageiq-rpms/Dockerfile diff --git a/.gitignore b/.gitignore index 87567eb1..f2ebdea5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ Gemfile.lock build.log images/manageiq-base-worker/container-assets/VMware-* -images/manageiq-base/rpms +images/manageiq-rpms/rpms manageiq-operator/build/_output/* manageiq-operator/vendor/* tools/certs/*.crt diff --git a/bin/build b/bin/build index 042980e2..a1b4e48d 100755 --- a/bin/build +++ b/bin/build @@ -6,27 +6,21 @@ RPM_BUILD_OPTIONS=${RPM_BUILD_OPTIONS:-""} RPM_BUILD_IMAGE=${RPM_BUILD_IMAGE:-"manageiq/rpm_build:$TAG"} RPM_PREFIX=${RPM_PREFIX:-"manageiq"} -while getopts "t:c:d:r:hblnops" opt; do +while getopts "t:c:d:r:hbnops" opt; do case $opt in b) REBUILD_RPM="true" ;; c) CONTAINER_PREFIX=$OPTARG ;; d) BASE_DIR=$OPTARG ;; - l) LOCAL_RPM="true" ;; n) NO_CACHE="true" ;; o) NO_OPERATOR="true" ;; p) PUSH="true" ;; r) REPO=$OPTARG ;; s) RELEASE_BUILD="true" ;; t) TAG=$OPTARG ;; - h) echo "Usage: $0 [-hblnops] [-c CONTAINER_PREFIX] [-d BASE_DIR] [-r IMAGE_REPOSITORY] [-t IMAGE_TAG]"; exit 1 + h) echo "Usage: $0 [-hbnops] [-c CONTAINER_PREFIX] [-d BASE_DIR] [-r IMAGE_REPOSITORY] [-t IMAGE_TAG]"; exit 1 esac done -if [ -n "$LOCAL_RPM" ] && [ -n "$REBUILD_RPM" ]; then - echo "Local rpm (-l) and rebuild rpm (-b) options can't be used together" - exit 1 -fi - BASE_DIR=${BASE_DIR:-$PWD} CONTAINER_PREFIX=${CONTAINER_PREFIX:-manageiq} IMAGE_DIR="$BASE_DIR/images" @@ -38,7 +32,7 @@ CONTAINER_COMMAND="$(which podman &>/dev/null && echo "podman" || echo "docker") set -e if [ -n "$REBUILD_RPM" ]; then - rpm_dir=$IMAGE_DIR/manageiq-base/rpms + rpm_dir=$IMAGE_DIR/manageiq-rpms/rpms rm -rf $rpm_dir/* options="-v $PWD/$rpm_dir:/root/BUILD/rpms" if [ -n "$RPM_BUILD_OPTIONS" ]; then @@ -52,9 +46,12 @@ if [ -n "$REBUILD_RPM" ]; then fi pushd $IMAGE_DIR - build_images="manageiq-base manageiq-base-worker manageiq-orchestrator manageiq-webserver-worker manageiq-ui-worker" + # Pull the base image first + $CONTAINER_COMMAND pull registry.access.redhat.com/ubi9/ubi + + build_images="manageiq-rpms manageiq-base manageiq-base-worker manageiq-orchestrator manageiq-webserver-worker manageiq-ui-worker" for image in $build_images; do - cmd="$CONTAINER_COMMAND build --tag $REPO/$image:$TAG" + cmd="$CONTAINER_COMMAND build --build-arg FROM_REPO=$REPO --build-arg FROM_TAG=$TAG --build-arg RPM_PREFIX=$RPM_PREFIX --tag $REPO/$image:$TAG" if [ -n "$VERBOSE" ]; then cmd+=" --progress plain" @@ -69,17 +66,8 @@ pushd $IMAGE_DIR cmd+=" --no-cache" fi - if [ "$image" == "manageiq-base" ]; then - cmd+=" --pull \ - --build-arg RPM_PREFIX=$RPM_PREFIX" - - if [ -n "$RELEASE_BUILD" ]; then - cmd+=" --build-arg RELEASE_BUILD=true" - fi - else - cmd+=" --build-arg FROM_REPO=$REPO \ - --build-arg FROM_TAG=$TAG \ - --build-arg RPM_PREFIX=$RPM_PREFIX" + if [ -n "$RELEASE_BUILD" -a "$image" == "manageiq-base" ]; then + cmd+=" --build-arg RELEASE_BUILD=true" fi cmd+=" $image" @@ -106,11 +94,12 @@ if [ -n "$PUSH" ]; then for image in $push_images; do cmd="$CONTAINER_COMMAND push $REPO/$image:$TAG" + # --format docker is needed for podman to ensure the builds are built with docker v2 manifests if [ "$CONTAINER_COMMAND" == "podman" ]; then cmd+=" --format docker" fi - echo "Pushing: $cmd" + echo "Pushing $image: $cmd" $cmd done fi diff --git a/images/manageiq-base/Dockerfile b/images/manageiq-base/Dockerfile index e886764e..c01b5b54 100644 --- a/images/manageiq-base/Dockerfile +++ b/images/manageiq-base/Dockerfile @@ -1,7 +1,14 @@ -FROM registry.access.redhat.com/ubi9/ubi -MAINTAINER ManageIQ https://manageiq.org +ARG FROM_REPO=docker.io/manageiq +ARG FROM_TAG=latest + +# alias stage for rpms +FROM ${FROM_REPO}/manageiq-rpms:${FROM_TAG} as rpms + +################################################################################ -ARG LOCAL_RPM +FROM registry.access.redhat.com/ubi9/ubi +ARG FROM_REPO +ARG FROM_TAG ARG RELEASE_BUILD ARG RPM_PREFIX=manageiq @@ -9,6 +16,7 @@ ENV TERM=xterm \ CONTAINER=true \ APP_ROOT=/var/www/miq/vmdb +MAINTAINER ManageIQ https://manageiq.org LABEL name="manageiq-base" \ vendor="ManageIQ" \ url="https://manageiq.org/" \ @@ -29,11 +37,11 @@ RUN ARCH=$(uname -m) && \ curl -L -o /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} && \ chmod +x /usr/bin/dumb-init -COPY rpms/* /tmp/rpms/ -COPY container-assets/create_local_yum_repo.sh / +COPY container-assets/prepare_local_yum_repo.sh /usr/local/bin COPY container-assets/clean_dnf_rpm /usr/local/bin/ -RUN ARCH=$(uname -m) && \ +RUN --mount=type=bind,from=rpms,source=/tmp/rpms,target=/tmp/rpms \ + ARCH=$(uname -m) && \ dnf config-manager --setopt=tsflags=nodocs --setopt=install_weak_deps=False --save && \ dnf -y --setopt=protected_packages= remove redhat-release && \ dnf -y remove *subscription-manager* && \ @@ -46,7 +54,7 @@ RUN ARCH=$(uname -m) && \ https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ https://rpm.manageiq.org/release/19-spassky/el9/noarch/manageiq-release-19.0-1.el9.noarch.rpm && \ if [[ "$RELEASE_BUILD" != "true" ]]; then dnf config-manager --enable manageiq-19-spassky-nightly; fi && \ - if [[ "$LOCAL_RPM" = "true" ]]; then /create_local_yum_repo.sh; fi && \ + if [[ -n "$(ls /tmp/rpms)" ]]; then /usr/local/bin/prepare_local_yum_repo.sh; fi && \ dnf -y module enable ruby:3.1 && \ dnf -y install \ httpd \ diff --git a/images/manageiq-base/container-assets/create_local_yum_repo.sh b/images/manageiq-base/container-assets/prepare_local_yum_repo.sh similarity index 64% rename from images/manageiq-base/container-assets/create_local_yum_repo.sh rename to images/manageiq-base/container-assets/prepare_local_yum_repo.sh index 72474e8a..21371c62 100755 --- a/images/manageiq-base/container-assets/create_local_yum_repo.sh +++ b/images/manageiq-base/container-assets/prepare_local_yum_repo.sh @@ -1,11 +1,6 @@ #!/bin/bash -yum -y install createrepo_c -rm -rf /tmp/rpms/repodata -createrepo /tmp/rpms -yum -y remove createrepo_c - -ls -al /tmp/rpms +set -e cat > /etc/yum.repos.d/local_rpm.repo << EOF [local-rpm] diff --git a/images/manageiq-base/rpms/.gitkeep b/images/manageiq-base/rpms/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/images/manageiq-rpms/Dockerfile b/images/manageiq-rpms/Dockerfile new file mode 100644 index 00000000..f85c072c --- /dev/null +++ b/images/manageiq-rpms/Dockerfile @@ -0,0 +1,22 @@ +FROM registry.access.redhat.com/ubi9/ubi + +COPY rpms/* /tmp/rpms/ +RUN rm -f /tmp/rpms/.gitkeep + +RUN if [[ -n "$(ls /tmp/rpms)" ]]; then \ + ARCH=$(uname -m) && \ + dnf config-manager --setopt=tsflags=nodocs --setopt=install_weak_deps=False --save && \ + dnf -y --setopt=protected_packages= remove redhat-release && \ + dnf -y remove *subscription-manager* && \ + dnf -y install \ + http://mirror.stream.centos.org/9-stream/BaseOS/${ARCH}/os/Packages/centos-stream-release-9.0-24.el9.noarch.rpm \ + http://mirror.stream.centos.org/9-stream/BaseOS/${ARCH}/os/Packages/centos-stream-repos-9.0-24.el9.noarch.rpm \ + http://mirror.stream.centos.org/9-stream/BaseOS/${ARCH}/os/Packages/centos-gpg-keys-9.0-24.el9.noarch.rpm ; \ + fi + +RUN if [[ -n "$(ls /tmp/rpms)" ]]; then \ + ls -al /tmp/rpms && \ + yum -y install createrepo_c && \ + rm -rf /tmp/rpms/repodata && \ + createrepo /tmp/rpms; \ + fi diff --git a/images/manageiq-webserver-worker/Dockerfile b/images/manageiq-webserver-worker/Dockerfile index 19fcf03c..4c1fe958 100644 --- a/images/manageiq-webserver-worker/Dockerfile +++ b/images/manageiq-webserver-worker/Dockerfile @@ -1,17 +1,22 @@ ARG FROM_REPO=docker.io/manageiq ARG FROM_TAG=latest -FROM ${FROM_REPO}/manageiq-base-worker:${FROM_TAG} -MAINTAINER ManageIQ https://manageiq.org +# alias stage for rpms +FROM ${FROM_REPO}/manageiq-rpms:${FROM_TAG} as rpms + +############################################################################### +FROM ${FROM_REPO}/manageiq-base-worker:${FROM_TAG} ARG RPM_PREFIX=manageiq +MAINTAINER ManageIQ https://manageiq.org LABEL name="manageiq-webserver-worker" \ summary="ManageIQ web server worker image" COPY container-assets/service-worker-entrypoint /usr/local/bin -RUN dnf -y install \ +RUN --mount=type=bind,from=rpms,source=/tmp/rpms,target=/tmp/rpms \ + dnf -y install \ ${RPM_PREFIX}-ui && \ clean_dnf_rpm && \ # Remove httpd default settings From 6c57b61e3434f2533fab0387a4571e6fb3105b41 Mon Sep 17 00:00:00 2001 From: Brandon Dunne Date: Thu, 14 Mar 2024 11:51:11 -0400 Subject: [PATCH 3/3] Ensure rpms directory exists, otherwise there's an error STEP 2/5: COPY rpms/* /tmp/rpms/ Error: building at STEP "COPY rpms/* /tmp/rpms/": checking on sources under "/home/bdunne/projects/manageiq/manageiq-pods/images/manageiq-rpms": Rel: can't make relative to /home/bdunne/projects/manageiq/manageiq-pods/images/manageiq-rpms; copier: stat: ["/rpms/*"]: no such file or directory --- images/manageiq-rpms/rpms/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 images/manageiq-rpms/rpms/.gitkeep diff --git a/images/manageiq-rpms/rpms/.gitkeep b/images/manageiq-rpms/rpms/.gitkeep new file mode 100644 index 00000000..e69de29b