From c22dc61eb4441e56d7e496f69cd7393cb6fd075b Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 19 Oct 2023 18:58:29 -0400 Subject: [PATCH] Rework custom-bundle.Dockerfile to use staged data This reworks custom-bundle.Dockerfile so that it involves less stages and can support 2 distinct steps: sync time and build time. This will more cleanly align to our downstream build needs At sync time a bundle_extra_data directory is populated with ENV variables from all operators (required for webhooks). Additionally dataplane-operator CRDs are cached in this repo. At build time the custom-bundle.Dockerfile pulls from bundle_extra_data directly. The csv-merger is used at 'sync time' to extract ENV variables and again at 'build time' to merge those ENV variables along with the dataplane-operators into the combined openstack operator CSV. Jira: OSP-29916 --- .../workflows/build-openstack-operator.yaml | 2 +- .gitignore | 1 + .prow_ci.env | 1 + Makefile | 14 ++- README.md | 2 +- custom-bundle.Dockerfile | 108 ++---------------- hack/bundle-cache-data.sh | 71 ++++++++++++ 7 files changed, 95 insertions(+), 104 deletions(-) create mode 100755 hack/bundle-cache-data.sh diff --git a/.github/workflows/build-openstack-operator.yaml b/.github/workflows/build-openstack-operator.yaml index 98a1ef0d3..51dffcbee 100644 --- a/.github/workflows/build-openstack-operator.yaml +++ b/.github/workflows/build-openstack-operator.yaml @@ -17,7 +17,7 @@ jobs: operator_name: openstack go_version: 1.19.x operator_sdk_version: 1.31.0 - bundle_dockerfile: ./custom-bundle.Dockerfile.pinned + bundle_dockerfile: ./custom-bundle.Dockerfile catalog_extra_bundles_script: ./hack/pin-bundle-images.sh secrets: IMAGENAMESPACE: ${{ secrets.IMAGENAMESPACE }} diff --git a/.gitignore b/.gitignore index 47fb80724..c81d32a07 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ testbin/* bundle/* bundle.Dockerfile +bundle_extra_data custom-bundle.Dockerfile.pinned storage-bundle.Dockerfile.pinned diff --git a/.prow_ci.env b/.prow_ci.env index 2606e6b65..0655cbc07 100644 --- a/.prow_ci.env +++ b/.prow_ci.env @@ -1 +1,2 @@ export USE_IMAGE_DIGESTS=true +export BUNDLE_DOCKERFILE=custom-bundle.Dockerfile diff --git a/Makefile b/Makefile index 39096b9b6..580111a30 100644 --- a/Makefile +++ b/Makefile @@ -234,18 +234,16 @@ $(GINKGO): $(LOCALBIN) test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo .PHONY: bundle -bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files. +bundle: build manifests kustomize bundle-cache-extra-data ## Generate bundle manifests and metadata, then validate generated files. operator-sdk generate kustomize manifests -q cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle $(BUNDLE_GEN_FLAGS) cp dependencies.yaml ./bundle/metadata operator-sdk bundle validate ./bundle - DOCKERFILE=custom-bundle.Dockerfile /bin/bash hack/pin-bundle-images.sh - .PHONY: bundle-build bundle-build: bundle ## Build the bundle image. - podman build -f custom-bundle.Dockerfile.pinned -t $(BUNDLE_IMG) . + podman build -f custom-bundle.Dockerfile -t $(BUNDLE_IMG) . .PHONY: bundle-push bundle-push: ## Push the bundle image. @@ -364,3 +362,11 @@ run-with-webhook: export HEALTH_PORT?=8081 run-with-webhook: manifests generate fmt vet ## Run a controller from your host. /bin/bash hack/configure_local_webhook.sh go run ./main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)" + +# refresh the bundle extra data based on go.mod entries +# bundle extra data includes: +# - extracted ENV vars from all operators (required for webhooks) +# - dataplane-operator bundle (this gets merged into openstack-operator's bundle) +.PHONY: bundle-cache-extra-data +bundle-cache-extra-data: build + bash hack/bundle-cache-data.sh diff --git a/README.md b/README.md index 2f5d65521..2be7b0932 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ bundle. make dep-bundle-build-push ``` -3. Run bundle-build. This will execute podman to build the custom-bundle.Dockerfile.pinned. +3. Run bundle-build. This will execute podman to build the custom-bundle.Dockerfile. ```sh make bundle-build diff --git a/custom-bundle.Dockerfile b/custom-bundle.Dockerfile index 398e42f30..d54c9906e 100644 --- a/custom-bundle.Dockerfile +++ b/custom-bundle.Dockerfile @@ -1,27 +1,5 @@ ARG GOLANG_CTX=golang:1.19 -ARG INFRA_BUNDLE=quay.io/openstack-k8s-operators/infra-operator-bundle:latest -ARG KEYSTONE_BUNDLE=quay.io/openstack-k8s-operators/keystone-operator-bundle:latest -ARG MARIADB_BUNDLE=quay.io/openstack-k8s-operators/mariadb-operator-bundle:latest -ARG PLACEMENT_BUNDLE=quay.io/openstack-k8s-operators/placement-operator-bundle:latest -ARG OVN_BUNDLE=quay.io/openstack-k8s-operators/ovn-operator-bundle:latest -ARG NEUTRON_BUNDLE=quay.io/openstack-k8s-operators/neutron-operator-bundle:latest -ARG ANSIBLEEE_BUNDLE=quay.io/openstack-k8s-operators/openstack-ansibleee-operator-bundle:latest -ARG DATAPLANE_BUNDLE=quay.io/openstack-k8s-operators/dataplane-operator-bundle:latest -ARG NOVA_BUNDLE=quay.io/openstack-k8s-operators/nova-operator-bundle:latest -ARG HEAT_BUNDLE=quay.io/openstack-k8s-operators/heat-operator-bundle:latest -ARG IRONIC_BUNDLE=quay.io/openstack-k8s-operators/ironic-operator-bundle:latest -ARG BAREMETAL_BUNDLE=quay.io/openstack-k8s-operators/openstack-baremetal-operator-bundle:latest -ARG TELEMETRY_BUNDLE=quay.io/openstack-k8s-operators/telemetry-operator-bundle:latest -ARG HORIZON_BUNDLE=quay.io/openstack-k8s-operators/horizon-operator-bundle:latest -ARG GLANCE_BUNDLE=quay.io/openstack-k8s-operators/glance-operator-bundle:latest -ARG CINDER_BUNDLE=quay.io/openstack-k8s-operators/cinder-operator-bundle:latest -ARG MANILA_BUNDLE=quay.io/openstack-k8s-operators/manila-operator-bundle:latest -ARG SWIFT_BUNDLE=quay.io/openstack-k8s-operators/swift-operator-bundle:latest -ARG OCTAVIA_BUNDLE=quay.io/openstack-k8s-operators/octavia-operator-bundle:latest -ARG DESIGNATE_BUNDLE=quay.io/openstack-k8s-operators/designate-operator-bundle:latest -ARG BARBICAN_BUNDLE=quay.io/openstack-k8s-operators/barbican-operator-bundle:latest - -# Build the manager binary + FROM $GOLANG_CTX as builder WORKDIR /workspace @@ -39,90 +17,24 @@ RUN go mod download COPY cmd/csv-merger/csv-merger.go csv-merger.go COPY pkg/ pkg/ -# Build +# Build the csv-merger RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o csv-merger csv-merger.go -FROM $INFRA_BUNDLE as infra-bundle -FROM $KEYSTONE_BUNDLE as keystone-bundle -FROM $MARIADB_BUNDLE as mariadb-bundle -FROM $PLACEMENT_BUNDLE as placement-bundle -FROM $OVN_BUNDLE as ovn-bundle -FROM $NEUTRON_BUNDLE as neutron-bundle -FROM $ANSIBLEEE_BUNDLE as openstack-ansibleee-bundle -FROM $DATAPLANE_BUNDLE as dataplane-bundle -FROM $NOVA_BUNDLE as nova-bundle -FROM $HEAT_BUNDLE as heat-bundle -FROM $IRONIC_BUNDLE as ironic-bundle -FROM $BAREMETAL_BUNDLE as baremetal-bundle -FROM $TELEMETRY_BUNDLE as telemetry-bundle -FROM $HORIZON_BUNDLE as horizon-bundle -FROM $GLANCE_BUNDLE as glance-bundle -FROM $CINDER_BUNDLE as cinder-bundle -FROM $MANILA_BUNDLE as manila-bundle -FROM $SWIFT_BUNDLE as swift-bundle -FROM $OCTAVIA_BUNDLE as octavia-bundle -FROM $DESIGNATE_BUNDLE as designate-bundle -FROM $BARBICAN_BUNDLE as barbican-bundle - FROM $GOLANG_CTX as merger WORKDIR /workspace COPY --from=builder /workspace/csv-merger . # local operator manifests COPY bundle/manifests /manifests/ +COPY bundle_extra_data /bundle_extra_data +RUN cp -a /bundle_extra_data/manifests/* /manifests/ -# Custom Manifests -COPY --from=keystone-bundle /manifests/* /manifests/ -COPY --from=mariadb-bundle /manifests/* /manifests/ -COPY --from=infra-bundle /manifests/* /manifests/ -COPY --from=placement-bundle /manifests/* /manifests/ -COPY --from=ovn-bundle /manifests/* /manifests/ -COPY --from=neutron-bundle /manifests/* /manifests/ -COPY --from=openstack-ansibleee-bundle /manifests/* /manifests/ -COPY --from=dataplane-bundle /manifests/* /manifests/ -COPY --from=nova-bundle /manifests/* /manifests/ -COPY --from=heat-bundle /manifests/* /manifests/ -COPY --from=ironic-bundle /manifests/* /manifests/ -COPY --from=baremetal-bundle /manifests/* /manifests/ -COPY --from=telemetry-bundle /manifests/* /manifests/ -COPY --from=horizon-bundle /manifests/* /manifests/ -COPY --from=glance-bundle /manifests/* /manifests/ -COPY --from=cinder-bundle /manifests/* /manifests/ -COPY --from=manila-bundle /manifests/* /manifests/ -COPY --from=swift-bundle /manifests/* /manifests/ -COPY --from=octavia-bundle /manifests/* /manifests/ -COPY --from=designate-bundle /manifests/* /manifests/ -COPY --from=barbican-bundle /manifests/* /manifests/ - -# extract all the env vars (NOTE/FIXME: base-csv is unused below to be refactored) -RUN /workspace/csv-merger \ - --export-env-file=/env-vars.yaml \ - --mariadb-csv=/manifests/mariadb-operator.clusterserviceversion.yaml \ - --infra-csv=/manifests/infra-operator.clusterserviceversion.yaml \ - --keystone-csv=/manifests/keystone-operator.clusterserviceversion.yaml \ - --placement-csv=/manifests/placement-operator.clusterserviceversion.yaml \ - --ovn-csv=/manifests/ovn-operator.clusterserviceversion.yaml \ - --neutron-csv=/manifests/neutron-operator.clusterserviceversion.yaml \ - --ansibleee-csv=/manifests/openstack-ansibleee-operator.clusterserviceversion.yaml \ - --nova-csv=/manifests/nova-operator.clusterserviceversion.yaml \ - --heat-csv=/manifests/heat-operator.clusterserviceversion.yaml \ - --ironic-csv=/manifests/ironic-operator.clusterserviceversion.yaml \ - --baremetal-csv=/manifests/openstack-baremetal-operator.clusterserviceversion.yaml \ - --horizon-csv=/manifests/horizon-operator.clusterserviceversion.yaml \ - --telemetry-csv=/manifests/telemetry-operator.clusterserviceversion.yaml \ - --glance-csv=/manifests/glance-operator.clusterserviceversion.yaml \ - --cinder-csv=/manifests/cinder-operator.clusterserviceversion.yaml \ - --manila-csv=/manifests/manila-operator.clusterserviceversion.yaml \ - --swift-csv=/manifests/swift-operator.clusterserviceversion.yaml \ - --octavia-csv=/manifests/octavia-operator.clusterserviceversion.yaml \ - --designate-csv=/manifests/designate-operator.clusterserviceversion.yaml \ - --barbican-csv=/manifests/barbican-operator.clusterserviceversion.yaml \ - --base-csv=/manifests/openstack-operator.clusterserviceversion.yaml | tee /fixme-required-for-now-but-will-can-made-optional.yaml - -# apply all the ENV vars to the actual base-csv +# Merge things into our openstack-operator CSV: +# -dataplane-operator CSV +# -ENV vars from all operators (for webhooks) RUN /workspace/csv-merger \ - --import-env-files=/env-vars.yaml \ - --dataplane-csv=/manifests/dataplane-operator.clusterserviceversion.yaml \ + --import-env-files=/bundle_extra_data/env-vars.yaml \ + --dataplane-csv=/bundle_extra_data/manifests/dataplane-operator.clusterserviceversion.yaml \ --base-csv=/manifests/openstack-operator.clusterserviceversion.yaml | tee /openstack-operator.clusterserviceversion.yaml.new # remove all individual operator CSV's @@ -151,7 +63,7 @@ COPY bundle/tests/scorecard /tests/scorecard/ # copy in manifests from operators COPY bundle/manifests /manifests/ -COPY --from=merger /manifests/dataplane.openstack.* /manifests/ +COPY --from=merger /manifests/* /manifests/ # overwrite with the final merged CSV COPY --from=merger /openstack-operator.clusterserviceversion.yaml.new /manifests/openstack-operator.clusterserviceversion.yaml diff --git a/hack/bundle-cache-data.sh b/hack/bundle-cache-data.sh new file mode 100755 index 000000000..5781295e9 --- /dev/null +++ b/hack/bundle-cache-data.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# extract select data from bundles: +# -env vars from all service operators +# -dataplane-operator bundle is cached (in order to merge at build time) +set -ex + +function extract_bundle { + local IN_DIR=$1 + local OUT_DIR=$2 + for X in $(file ${IN_DIR}/* | grep gzip | cut -f 1 -d ':'); do + tar xvf $X -C ${OUT_DIR}/; + done +} + +function extract_csv { + local IN_DIR=$1 + local OUT_DIR=$2 + for X in $(file ${IN_DIR}/* | grep gzip | cut -f 1 -d ':'); do + tar xvf $X -C ${OUT_DIR} --wildcards --no-anchor '**/*clusterserviceversion.yaml' + done +} + +OUT_BUNDLE=bundle_extra_data +EXTRACT_DIR=tmp/bundle_extract + +mkdir -p "$EXTRACT_DIR" +mkdir -p "$EXTRACT_DIR/csvs" +mkdir -p "$OUT_BUNDLE" + +for BUNDLE in $(hack/pin-bundle-images.sh | tr "," " "); do + skopeo copy "docker://$BUNDLE" dir:${EXTRACT_DIR}/tmp; + if echo $BUNDLE | grep dataplane-operator &> /dev/null; then + extract_bundle "${EXTRACT_DIR}/tmp" "${OUT_BUNDLE}/" + else + extract_csv "${EXTRACT_DIR}/tmp" "${EXTRACT_DIR}/csvs" + fi +done + +# Extract the ENV vars from all the CSVs +CSV_DIR="${EXTRACT_DIR}/csvs/manifests" +bin/csv-merger \ + --export-env-file=$OUT_BUNDLE/env-vars.yaml \ + --mariadb-csv=$CSV_DIR/mariadb-operator.clusterserviceversion.yaml \ + --infra-csv=$CSV_DIR/infra-operator.clusterserviceversion.yaml \ + --keystone-csv=$CSV_DIR/keystone-operator.clusterserviceversion.yaml \ + --placement-csv=$CSV_DIR/placement-operator.clusterserviceversion.yaml \ + --ovn-csv=$CSV_DIR/ovn-operator.clusterserviceversion.yaml \ + --neutron-csv=$CSV_DIR/neutron-operator.clusterserviceversion.yaml \ + --ansibleee-csv=$CSV_DIR/openstack-ansibleee-operator.clusterserviceversion.yaml \ + --nova-csv=$CSV_DIR/nova-operator.clusterserviceversion.yaml \ + --heat-csv=$CSV_DIR/heat-operator.clusterserviceversion.yaml \ + --ironic-csv=$CSV_DIR/ironic-operator.clusterserviceversion.yaml \ + --baremetal-csv=$CSV_DIR/openstack-baremetal-operator.clusterserviceversion.yaml \ + --horizon-csv=$CSV_DIR/horizon-operator.clusterserviceversion.yaml \ + --telemetry-csv=$CSV_DIR/telemetry-operator.clusterserviceversion.yaml \ + --glance-csv=$CSV_DIR/glance-operator.clusterserviceversion.yaml \ + --cinder-csv=$CSV_DIR/cinder-operator.clusterserviceversion.yaml \ + --manila-csv=$CSV_DIR/manila-operator.clusterserviceversion.yaml \ + --swift-csv=$CSV_DIR/swift-operator.clusterserviceversion.yaml \ + --octavia-csv=$CSV_DIR/octavia-operator.clusterserviceversion.yaml \ + --designate-csv=$CSV_DIR/designate-operator.clusterserviceversion.yaml \ + --barbican-csv=$CSV_DIR/barbican-operator.clusterserviceversion.yaml \ + --base-csv=config/manifests/bases/openstack-operator.clusterserviceversion.yaml | tee "$EXTRACT_DIR/out.yaml" + +# cleanup our temporary dir used for extraction +rm -Rf "$EXTRACT_DIR" + +# we only keep manifests from extracted (merged) bundles +rm -Rf "$OUT_BUNDLE/metadata" +rm -Rf "$OUT_BUNDLE/tests"