Merge pull request #64 from enp0s3/hack-refactor

Adjust the building process of the project artifacts
openshift-virtualization · Sep 26, 2024 · 26e43e2 · 26e43e2
2 parents d4cbeac + 0878b7f
commit 26e43e2
Show file tree

Hide file tree

Showing 12 changed files with 116 additions and 195 deletions.
diff --git a/Makefile b/Makefile
@@ -26,28 +26,17 @@ all: build
 
 build:  wasp manifest-generator
 
-DOCKER?=1
-ifeq (${DOCKER}, 1)
-	# use entrypoint.sh (default) as your entrypoint into the container
-	DO=./hack/build/in-docker.sh
-	# use entrypoint-bazel.sh as your entrypoint into the container.
-	DO_BAZ=./hack/build/bazel-docker.sh
-else
-	DO=eval
-	DO_BAZ=eval
-endif
-
 ifeq ($(origin KUBEVIRT_RELEASE), undefined)
 	KUBEVIRT_RELEASE="latest_nightly"
 endif
 
 all: manifests build-images
 
 manifests:
-	${DO_BAZ} "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} CR_NAME=${CR_NAME} WASP_NAMESPACE=${WASP_NAMESPACE} MAX_AVERAGE_SWAPIN_PAGES_PER_SECOND=${MAX_AVERAGE_SWAPIN_PAGES_PER_SECOND} MAX_AVERAGE_SWAPOUT_PAGES_PER_SECOND=${MAX_AVERAGE_SWAPOUT_PAGES_PER_SECOND} SWAP_UTILIZATION_THRESHOLD_FACTOR=${SWAP_UTILIZATION_THRESHOLD_FACTOR} AVERAGE_WINDOW_SIZE_SECONDS=${AVERAGE_WINDOW_SIZE_SECONDS}  DEPLOY_PROMETHEUS_RULE=${DEPLOY_PROMETHEUS_RULE} ./hack/build/build-manifests.sh"
+	hack/build/bazel-docker.sh "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} CR_NAME=${CR_NAME} WASP_NAMESPACE=${WASP_NAMESPACE} MAX_AVERAGE_SWAPIN_PAGES_PER_SECOND=${MAX_AVERAGE_SWAPIN_PAGES_PER_SECOND} MAX_AVERAGE_SWAPOUT_PAGES_PER_SECOND=${MAX_AVERAGE_SWAPOUT_PAGES_PER_SECOND} SWAP_UTILIZATION_THRESHOLD_FACTOR=${SWAP_UTILIZATION_THRESHOLD_FACTOR} AVERAGE_WINDOW_SIZE_SECONDS=${AVERAGE_WINDOW_SIZE_SECONDS}  DEPLOY_PROMETHEUS_RULE=${DEPLOY_PROMETHEUS_RULE} ./hack/build/build-manifests.sh"
 
 builder-push:
-	./hack/build/bazel-build-builder.sh
+	./hack/build/build-builder.sh
 
 cluster-up:
 	eval "KUBEVIRT_RELEASE=${KUBEVIRT_RELEASE} KUBEVIRT_SWAP_ON=true ./cluster-up/up.sh"
@@ -71,17 +60,17 @@ cluster-sync: cluster-clean-wasp
 
 test: WHAT = ./pkg/... ./cmd/...
 test: bootstrap-ginkgo
-	${DO_BAZ} "ACK_GINKGO_DEPRECATIONS=${ACK_GINKGO_DEPRECATIONS} ./hack/build/run-unit-tests.sh ${WHAT}"
+	hack/build/bazel-docker.sh "ACK_GINKGO_DEPRECATIONS=${ACK_GINKGO_DEPRECATIONS} ./hack/build/run-unit-tests.sh ${WHAT}"
 
 build-functest:
-	${DO_BAZ} ./hack/build/build-functest.sh
+	hack/build/bazel-docker.sh ./hack/build/build-functest.sh
 
 functest:  WHAT = ./tests/...
 functest: build-functest
 	./hack/build/run-functional-tests.sh ${WHAT} "${TEST_ARGS}"
 
 bootstrap-ginkgo:
-	${DO_BAZ} ./hack/build/bootstrap-ginkgo.sh
+	hack/build/bazel-docker.sh ./hack/build/bootstrap-ginkgo.sh
 
 manifest-generator:
 	GO111MODULE=${GO111MODULE:-off} go build -o manifest-generator -v tools/manifest-generator/*.go

diff --git a/hack/build/bazel-build-builder.sh b/hack/build/bazel-build-builder.sh
diff --git a/hack/build/bazel-docker.sh b/hack/build/bazel-docker.sh
@@ -21,15 +21,17 @@ source "${script_dir}"/config.sh
 
 mkdir -p "${WASP_DIR}/_out"
 
-BUILDER_VOLUME="kubevirt-wasp-volume"
-BAZEL_BUILDER_SERVER="${BUILDER_VOLUME}-bazel-server"
+# update this whenever new builder tag is created
+BUILDER_IMAGE=${BUILDER_IMAGE:-quay.io/openshift-virtualization/wasp-agent-builder:2409050858-9e0a2aa}
+
+BUILDER_VOLUME="wasp-volume"
 DOCKER_CA_CERT_FILE="${DOCKER_CA_CERT_FILE:-}"
 DOCKERIZED_CUSTOM_CA_PATH="/etc/pki/ca-trust/source/anchors/custom-ca.crt"
 
 DISABLE_SECCOMP=${DISABLE_SECCOMP:-}
 
 SYNC_OUT=${SYNC_OUT:-true}
-SYNC_VENDOR=${SYNC_VENDOR:-true}
+SYNC_VENDOR=${SYNC_VENDOR:-false}
 
 # Create the persistent docker volume
 if [ -z "$(${WASP_CRI} volume list | grep ${BUILDER_VOLUME})" ]; then
@@ -38,12 +40,12 @@ fi
 
 # Make sure that the output directory exists
 echo "Making sure output directory exists..."
-${WASP_CRI} run -v "${BUILDER_VOLUME}:/root:rw,z" --security-opt label=disable $DISABLE_SECCOMP --rm --entrypoint "/entrypoint-bazel.sh" ${BUILDER_IMAGE} mkdir -p /root/go/src/kubevirt.io/wasp-agent/_out
+${WASP_CRI} run -v "${BUILDER_VOLUME}:/root:rw,z" --security-opt label=disable $DISABLE_SECCOMP --rm --entrypoint "/entrypoint.sh" ${BUILDER_IMAGE} mkdir -p /root/go/src/github.com/openshift-virtualization/wasp-agent/_out
 
-${WASP_CRI} run -v "${BUILDER_VOLUME}:/root:rw,z" --security-opt label=disable $DISABLE_SECCOMP --rm --entrypoint "/entrypoint-bazel.sh" ${BUILDER_IMAGE} git config --global --add safe.directory /root/go/src/kubevirt.io/wasp-agent
+${WASP_CRI} run -v "${BUILDER_VOLUME}:/root:rw,z" --security-opt label=disable $DISABLE_SECCOMP --rm --entrypoint "/entrypoint.sh" ${BUILDER_IMAGE} git config --global --add safe.directory /root/go/src/github.com/openshift-virtualization/wasp-agent
 echo "Starting rsyncd"
 # Start an rsyncd instance and make sure it gets stopped after the script exits
-RSYNC_CID_WASP=$(${WASP_CRI} run -d -v "${BUILDER_VOLUME}:/root:rw,z" --security-opt label=disable $DISABLE_SECCOMP --cap-add SYS_CHROOT --expose 873 -P --entrypoint "/entrypoint-bazel.sh" ${BUILDER_IMAGE} /usr/bin/rsync --no-detach --daemon --verbose)
+RSYNC_CID_WASP=$(${WASP_CRI} run -d -v "${BUILDER_VOLUME}:/root:rw,z" --security-opt label=disable $DISABLE_SECCOMP --cap-add SYS_CHROOT --expose 873 -P --entrypoint "/entrypoint.sh" ${BUILDER_IMAGE} /usr/bin/rsync --no-detach --daemon --verbose)
 
 function finish() {
     ${WASP_CRI} stop --time 1 ${RSYNC_CID_WASP} >/dev/null 2>&1
@@ -82,54 +84,25 @@ echo "Rsyncing ${WASP_DIR} to container"
 # Copy wasp into the persistent docker volume
 _rsync \
     --delete \
-    --exclude 'bazel-bin' \
-    --exclude 'bazel-genfiles' \
-    --exclude 'bazel-wasp' \
-    --exclude 'bazel-out' \
-    --exclude 'bazel-testlogs' \
     --exclude 'cluster-up/cluster/**/.kubectl' \
     --exclude 'cluster-up/cluster/**/.oc' \
     --exclude 'cluster-up/cluster/**/.kubeconfig' \
     --exclude ".vagrant" \
     ${WASP_DIR}/ \
     "rsync://[email protected]:${RSYNCD_PORT}/build"
 
-volumes="-v ${BUILDER_VOLUME}:/root:rw,z"
-# append .docker directory as volume
-mkdir -p "${HOME}/.docker"
-volumes="$volumes -v ${HOME}/.docker:/root/.docker:ro,z"
-
-if [[ WASP_CRI = podman* ]] && [[ -f "${XDG_RUNTIME_DIR-}/containers/auth.json" ]]; then
-    volumes="$volumes --mount type=bind,source=${XDG_RUNTIME_DIR-}/containers/auth.json,target=/root/.docker/config.json,readonly"
-elif [[ -f "${HOME}/.docker/config.json" ]]; then
-    volumes="$volumes --mount type=bind,source=${HOME}/.docker/config.json,target=/root/.docker/config.json,readonly"
-fi
-
-if [ "${CI}" = "true" ]; then
-    mkdir -p "$HOME/containers"
-    volumes="$volumes -v ${HOME}/containers:/root/containers:ro,z"
-fi
-
-if [ -n "$DOCKER_CA_CERT_FILE" ]; then
-    volumes="$volumes -v ${DOCKER_CA_CERT_FILE}:${DOCKERIZED_CUSTOM_CA_PATH}:ro,z"
-fi
-
-if [ -z "$(${WASP_CRI} ps --format '{{.Names}}' | grep ${BAZEL_BUILDER_SERVER})" ]; then
-   ${WASP_CRI} run --ulimit nofile=10000:10000 $DISABLE_SECCOMP --network host -d ${volumes} --security-opt label=disable --rm --name ${BAZEL_BUILDER_SERVER} -e "GOPATH=/root/go" -w "/root/go/src/kubevirt.io/wasp-agent"  ${BUILDER_IMAGE} hack/build/bazel-server.sh
-fi
-
-echo "Starting bazel server"
 # Run the command
 test -t 1 && USE_TTY="-it"
-${WASP_CRI} exec ${USE_TTY} ${BAZEL_BUILDER_SERVER} /entrypoint-bazel.sh "$@"
+if ! $WASP_CRI exec -w /root/go/src/github.com/openshift-virtualization/wasp-agent ${USE_TTY} ${RSYNC_CID_WASP} /entrypoint.sh "$@"; then
+    # Copy the build output out of the container, make sure that _out exactly matches the build result
+    if [ "$SYNC_OUT" = "true" ]; then
+        _rsync --delete "rsync://[email protected]:${RSYNCD_PORT}/out" ${OUT_DIR}
+    fi
+    exit 1
+fi
 
 # Copy the whole wasp data out to get generated sources and formatting changes
 _rsync \
-    --exclude 'bazel-bin' \
-    --exclude 'bazel-genfiles' \
-    --exclude 'bazel-wasp' \
-    --exclude 'bazel-out' \
-    --exclude 'bazel-testlogs' \
     --exclude 'cluster-up/cluster/**/.kubectl' \
     --exclude 'cluster-up/cluster/**/.oc' \
     --exclude 'cluster-up/cluster/**/.kubeconfig' \
@@ -149,5 +122,4 @@ fi
 # Copy the build output out of the container, make sure that _out exactly matches the build result
 if [ "$SYNC_OUT" = "true" ]; then
     _rsync --delete "rsync://[email protected]:${RSYNCD_PORT}/out" ${OUT_DIR}
-fi
-${WASP_CRI} rm --force kubevirt-wasp-volume-bazel-server
+fi
diff --git a/hack/build/bazel-server.sh b/hack/build/bazel-server.sh
diff --git a/hack/build/build-builder.sh b/hack/build/build-builder.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+#Copyright 2023 The WASP Authors.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+script_dir="$(cd "$(dirname "$0")" && pwd -P)"
+source "${script_dir}"/common.sh
+
+FORCE_PUSH=${FORCE_PUSH:-false}
+
+function build_and_push() {
+  if [ "${WASP_CRI}" = "docker" ]; then
+     WASP_CONTAINER_BUILDCMD=${WASP_CONTAINER_BUILDCMD:-docker}
+  else
+     WASP_CONTAINER_BUILDCMD=${WASP_CONTAINER_BUILDCMD:-buildah}
+  fi
+
+  if [ "${WASP_CONTAINER_BUILDCMD}" = "buildah" ]; then
+      if [ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]; then
+          BUILDAH_PLATFORM_FLAG="--platform linux/amd64,linux/arm64"
+      else
+          echo "No qemu-user-static on host machine, building only native container"
+          BUILDAH_PLATFORM_FLAG=""
+      fi
+  fi
+
+  BUILDER_SPEC="${BUILD_DIR}/docker/builder"
+  UNTAGGED_BUILDER_IMAGE=quay.io/openshift-virtualization/wasp-agent-builder
+  BUILDER_TAG=$(date +"%y%m%d%H%M")-$(git rev-parse --short HEAD)
+  BUILDER_MANIFEST=${UNTAGGED_BUILDER_IMAGE}:${BUILDER_TAG}
+  echo "$DOCKER_PREFIX:$DOCKER_TAG"
+
+  #Build the encapsulated compile and test container
+  if [ "${WASP_CONTAINER_BUILDCMD}" = "buildah" ]; then
+      (cd ${BUILDER_SPEC} && buildah build ${BUILDAH_PLATFORM_FLAG} --manifest ${BUILDER_MANIFEST} .)
+      buildah manifest push --all ${BUILDER_MANIFEST} docker://${BUILDER_MANIFEST}
+  else
+      (cd ${BUILDER_SPEC} && docker build --tag ${BUILDER_MANIFEST} .)
+      docker push ${BUILDER_MANIFEST}
+  fi
+
+  DIGEST=$(${WASP_CRI} images --digests | grep ${UNTAGGED_BUILDER_IMAGE} | grep ${BUILDER_TAG} | awk '{ print $4 }')
+  echo "Image: ${BUILDER_MANIFEST}"
+  echo "Digest: ${DIGEST}"
+}
+
+if ! git diff-index --quiet HEAD~1 "${script_dir}"/docker || \
+    [ "$FORCE_PUSH" == "true" ] ; then
+    #Since this only runs during the post-submit job, the PR will have squashed into a single
+    #commit and we can use HEAD~1 to compare.
+  build_and_push
+fi
diff --git a/hack/build/build-functest.sh b/hack/build/build-functest.sh
@@ -20,6 +20,7 @@ source "${script_dir}"/common.sh
 mkdir -p ${TESTS_OUT_DIR}/
 # use vendor
 export GO111MODULE=${GO111MODULE:-off}
+go build -C vendor/github.com/onsi/ginkgo/v2/ginkgo -o /usr/bin/ginkgo
 ginkgo build ${WASP_DIR}/tests/
 mv ${WASP_DIR}/tests/tests.test ${TESTS_OUT_DIR}/
-cp -f /go/bin/ginkgo ${TESTS_OUT_DIR}/
+cp -f /usr/bin/ginkgo ${TESTS_OUT_DIR}/
diff --git a/hack/build/common.sh b/hack/build/common.sh
@@ -44,11 +44,5 @@ VENDOR_DIR=${WASP_DIR}/vendor
 ARCHITECTURE="${BUILD_ARCH:-$(uname -m)}"
 HOST_ARCHITECTURE="$(uname -m)"
 WASP_CRI="$(determine_wasp_bin)"
-if [ "${WASP_CRI}" = "docker" ]; then
-   WASP_CONTAINER_BUILDCMD=${WASP_CONTAINER_BUILDCMD:-docker}
-else
-   WASP_CONTAINER_BUILDCMD=${WASP_CONTAINER_BUILDCMD:-buildah}
-fi
-echo "WASP_CRI: ${WASP_CRI}, WASP_CONTAINER_BUILDCMD: ${WASP_CONTAINER_BUILDCMD}"
 
 
diff --git a/hack/build/config.sh b/hack/build/config.sh
@@ -25,9 +25,6 @@ SWAP_UTILIZATION_THRESHOLD_FACTOR=${SWAP_UTILIZATION_THRESHOLD_FACTOR:-0.8}
 DEPLOY_PROMETHEUS_RULE=${DEPLOY_PROMETHEUS_RULE:-false}
 CR_NAME=${CR_NAME:-wasp}
 
-# update this whenever new builder tag is created
-BUILDER_IMAGE=${BUILDER_IMAGE:-quay.io/bmordeha/kubevirt-wasp-bazel-builder:2407031059-d673c1a}
-
 function parseTestOpts() {
     pkgs=""
     test_args=""

diff --git a/hack/build/docker/builder/Dockerfile b/hack/build/docker/builder/Dockerfile
@@ -1,6 +1,13 @@
 FROM quay.io/centos/centos:stream9
-LABEL maintainer="The KubeVirt Project <[email protected]>"
 
+ENV GIMME_GO_VERSION=1.22.6
+ENV OPERATOR_COURIER_VERSION=2.1.11
+ENV GOLANGCI_LINT_VERSION=v1.60.1
+ENV GOPATH="/go"
+ENV GOBIN="/usr/bin"
+ENV GO111MODULE="on"
+
+# Install packages
 RUN 	dnf -y install dnf-plugins-core && \
 	dnf config-manager --set-enable crb && dnf update -y && \
 	dnf install -y \
@@ -24,24 +31,17 @@ RUN 	dnf -y install dnf-plugins-core && \
 	nbdkit-devel \
 	unzip \
     protobuf-compiler \
-	java-11-openjdk-devel \
 	&& dnf clean all
 
-RUN pip3 install --upgrade j2cli operator-courier==2.1.11 && \
-	curl -sL https://services.gradle.org/distributions/gradle-6.6-bin.zip -o gradle-6.6-bin.zip && \
-	mkdir /opt/gradle && \
-	unzip -d /opt/gradle gradle-6.6-bin.zip && \
-	ln -s /opt/gradle/gradle-6.6/bin/gradle /usr/local/bin/gradle && \
-	rm gradle-6.6-bin.zip
-
-ENV GIMME_GO_VERSION=1.20.5 GOPATH="/go" GO111MODULE="on"
-
-RUN mkdir -p /gimme && curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | HOME=/gimme bash >> /etc/profile.d/gimme.sh
+# reference to master is for an external repo and can't yet be changed
+RUN mkdir -p /gimme && curl -sL \
+    https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | \
+    HOME=/gimme bash >> /etc/profile.d/gimme.sh
 
+# Install persistent go packages
 RUN \
 	source /etc/profile.d/gimme.sh && \
 	eval $(go env) && \
-	go install github.com/onsi/ginkgo/[email protected] && \
 	go install golang.org/x/tools/cmd/goimports@latest && \
 	go install mvdan.cc/sh/cmd/shfmt@latest && \
 	go install github.com/mattn/goveralls@latest && \
@@ -52,20 +52,17 @@ RUN \
 	go install sigs.k8s.io/controller-tools/cmd/[email protected] && \
     rm -rf "${GOPATH}/pkg"
 
-ENV BAZEL_VERSION 5.3.1
+RUN set -x && \
+    source /etc/profile.d/gimme.sh && \
+    go install -v mvdan.cc/[email protected] && \
+    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOROOT)"/bin $GOLANGCI_LINT_VERSION
 
-COPY output-bazel-arch.sh /output-bazel-arch.sh
-
-RUN curl -L -o /usr/bin/bazel https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-$(sh /output-bazel-arch.sh) && chmod u+x /usr/bin/bazel
-
-# Until we use a version including the fix for this Bazel issue:
-# https://github.com/bazelbuild/bazel/issues/11554
-RUN ln -s /usr/bin/python3 /usr/bin/python
 
 ADD entrypoint.sh /entrypoint.sh
 
-ADD entrypoint-bazel.sh /entrypoint-bazel.sh
-
-COPY rsyncd.conf /etc/rsyncd.conf
+ADD rsyncd.conf /etc/rsyncd.conf
+# Add /root/go/src/github.com/openshift-virtualization/wasp-agent and mark it as a safe directory for git
+RUN mkdir -p /root/go/src/github.com/openshift-virtualization/wasp-agent && \
+    git config --global --add safe.directory /root/go/src/github.com/openshift-virtualization/wasp-agent
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT [ "/entrypoint.sh" ]