Add default buildspec definition to batch buildspecs

aws · Oct 18, 2023 · 339ea59 · 339ea59
1 parent f5e1420
commit 339ea59
Show file tree

Hide file tree

Showing 11 changed files with 98 additions and 168 deletions.
diff --git a/Makefile b/Makefile
@@ -91,14 +91,14 @@ generate-project-list: | ensure-locale
 .PHONY: generate-staging-buildspec
 generate-staging-buildspec: export BINARY_PLATFORMS=linux/amd64 linux/arm64
 generate-staging-buildspec: | ensure-locale
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/staging-build.yml"
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/checksums-build.yml" true EXCLUDE_FROM_CHECKSUMS_BUILDSPEC CHECKSUMS_BUILDSPECS false buildspecs/checksums-pr-buildspec.yml
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_bottlerocket-bootstrap" "$(BASE_DIRECTORY)/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml" true
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "kubernetes_cloud-provider-vsphere" "$(BASE_DIRECTORY)/projects/kubernetes/cloud-provider-vsphere/buildspecs/batch-build.yml" true
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "kubernetes-sigs_kind" "$(BASE_DIRECTORY)/projects/kubernetes-sigs/kind/buildspecs/batch-build.yml" true
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "containerd_containerd" "$(BASE_DIRECTORY)/projects/containerd/containerd/buildspecs/batch-build.yml" true
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "opencontainers_runc" "$(BASE_DIRECTORY)/projects/opencontainers/runc/buildspecs/batch-build.yml" true
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "torvalds_linux" "$(BASE_DIRECTORY)/projects/torvalds/linux/buildspecs/batch-build.yml" true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/staging-build.yml" "$(BASE_DIRECTORY)/buildspec.yml"
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/checksums-build.yml" "$(BASE_DIRECTORY)/buildspecs/checksums-buildspec.yml" true EXCLUDE_FROM_CHECKSUMS_BUILDSPEC CHECKSUMS_BUILDSPECS false buildspecs/checksums-pr-buildspec.yml
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_bottlerocket-bootstrap" "$(BASE_DIRECTORY)/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "kubernetes_cloud-provider-vsphere" "$(BASE_DIRECTORY)/projects/kubernetes/cloud-provider-vsphere/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "kubernetes-sigs_kind" "$(BASE_DIRECTORY)/projects/kubernetes-sigs/kind/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspecs/images.yml" true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "containerd_containerd" "$(BASE_DIRECTORY)/projects/containerd/containerd/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "opencontainers_runc" "$(BASE_DIRECTORY)/projects/opencontainers/runc/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "torvalds_linux" "$(BASE_DIRECTORY)/projects/torvalds/linux/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
 
 .PHONY: generate
 generate: generate-project-list generate-staging-buildspec

diff --git a/build/lib/generate_staging_buildspec.sh b/build/lib/generate_staging_buildspec.sh
@@ -20,11 +20,12 @@ set -o pipefail
 BASE_DIRECTORY="${1?Specify first argument - Base directory of build-tooling repo}"
 ALL_PROJECTS="${2?Specify second argument - All projects in repo}"
 STAGING_BUILDSPEC_FILE="${3}"
-SKIP_DEPEND_ON="${4:-false}"
-EXCLUDE_VAR="${5:-EXCLUDE_FROM_STAGING_BUILDSPEC}"
-BUILDSPECS_VAR="${6:-BUILDSPECS}"
-FAST_FAIL="${7:-true}"
-ADD_FINAL_STAGE="${8:-}"
+DEFAULT_BUILDSPEC_FILE="${4}"
+SKIP_DEPEND_ON="${5:-false}"
+EXCLUDE_VAR="${6:-EXCLUDE_FROM_STAGING_BUILDSPEC}"
+BUILDSPECS_VAR="${7:-BUILDSPECS}"
+FAST_FAIL="${8:-true}"
+ADD_FINAL_STAGE="${9:-}"
 
 SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 source "${SCRIPT_ROOT}/common.sh"
@@ -45,7 +46,7 @@ MAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 cd $MAKE_ROOT
 
 mkdir -p $(dirname $STAGING_BUILDSPEC_FILE)
-yq eval --null-input ".batch={\"fast-fail\":$FAST_FAIL,\"build-graph\":[]}" > $STAGING_BUILDSPEC_FILE # Creates an empty YAML array
+yq eval --null-input ".batch={\"fast-fail\":$FAST_FAIL,\"build-graph\":[]} | . *= load(\"$DEFAULT_BUILDSPEC_FILE\")" > $STAGING_BUILDSPEC_FILE # Creates an empty YAML array
 
 # TODO: refactor use of release_branch to get git_tag and golang_version in makefile, we should be able to push this to common.mk and avoid needing to pass it here
 RELEASE_BRANCH=$(build::eksd_releases::get_release_branch)
@@ -78,7 +79,11 @@ for project in "${PROJECTS[@]}"; do
         IDENTIFIER="${org//-/_}_${repo//-/_}"
 
         buildspec=${SPECS[$i]}
-
+        buildspec_field="\"buildspec\":\"$buildspec\","
+        if [[ $(realpath $buildspec) == $(realpath $DEFAULT_BUILDSPEC_FILE) ]]; then
+            buildspec_field=""
+        fi
+
         DEPEND_ON=""
         # something other than empty string since some overrides are empty strings
         PROJECT_DEPENDENCIES="false"
@@ -189,7 +194,7 @@ for project in "${PROJECTS[@]}"; do
 
                     ALL_PROJECT_IDS+="\"$IDENTIFIER\","
                     yq eval -i -P \
-                        ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",\"buildspec\":\"$buildspec\",$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\"}}}]" \
+                        ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\"}}}]" \
                         $STAGING_BUILDSPEC_FILE
                 done
             else
@@ -204,15 +209,15 @@ for project in "${PROJECTS[@]}"; do
                         IDENTIFIER=${org//-/_}_${repo//-/_}_${val1//-/_}_${val2//-/_}_${BUILDSPEC_NAME//-/_}
                         ALL_PROJECT_IDS+="\"$IDENTIFIER\","
                         yq eval -i -P \
-                            ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",\"buildspec\":\"$buildspec\",$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\",\"${KEYS[1]}\":\"$val2\"}}}]" \
+                            ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\",\"${KEYS[1]}\":\"$val2\"}}}]" \
                             $STAGING_BUILDSPEC_FILE 
                     done
                 done
             fi
         else
             ALL_PROJECT_IDS+="\"$IDENTIFIER\","
             yq eval -i -P \
-                ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",\"buildspec\":\"$buildspec\",$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL}}}]" \
+                ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL}}}]" \
                 $STAGING_BUILDSPEC_FILE 
         fi
 

diff --git a/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml b/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml
@@ -18,37 +18,40 @@ batch:
   fast-fail: true
   build-graph:
     - identifier: aws_bottlerocket_bootstrap_1_24
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-24
     - identifier: aws_bottlerocket_bootstrap_1_25
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-25
     - identifier: aws_bottlerocket_bootstrap_1_26
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-26
     - identifier: aws_bottlerocket_bootstrap_1_27
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-27
     - identifier: aws_bottlerocket_bootstrap_1_28
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-28
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      - ./build/lib/setup.sh
+  build:
+    commands:
+      - if $(make check-project-path-exists) && make check-for-release-branch-skip -C $PROJECT_PATH; then make release -C $PROJECT_PATH; fi
diff --git a/projects/aws/rolesanywhere-credential-helper/Makefile b/projects/aws/rolesanywhere-credential-helper/Makefile
@@ -11,7 +11,8 @@ EXTRA_GO_LDFLAGS=-X k8s.io/component-base/version.gitVersion=$(GIT_TAG)
 HAS_S3_ARTIFACTS=true
 
 IMAGE_NAMES=
-
+EXCLUDE_FROM_STAGING_BUILDSPEC=true
+SKIP_ON_RELEASE_BRANCH=true
 
 include $(BASE_DIRECTORY)/Common.mk
 

diff --git a/projects/containerd/containerd/buildspecs/batch-build.yml b/projects/containerd/containerd/buildspecs/batch-build.yml
@@ -18,16 +18,22 @@ batch:
   fast-fail: true
   build-graph:
     - identifier: containerd_containerd_linux_amd64
-      buildspec: buildspec.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           BINARY_PLATFORMS: linux/amd64
     - identifier: containerd_containerd_linux_arm64
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           BINARY_PLATFORMS: linux/arm64
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      - ./build/lib/setup.sh
+  build:
+    commands:
+      - if $(make check-project-path-exists) && make check-for-release-branch-skip -C $PROJECT_PATH; then make release -C $PROJECT_PATH; fi
diff --git a/projects/kubernetes-sigs/kind/buildspecs/batch-build.yml b/projects/kubernetes-sigs/kind/buildspecs/batch-build.yml
@@ -23,37 +23,43 @@ batch:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
     - identifier: kubernetes_sigs_kind_1_24
-      buildspec: buildspecs/images.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           RELEASE_BRANCH: 1-24
     - identifier: kubernetes_sigs_kind_1_25
-      buildspec: buildspecs/images.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           RELEASE_BRANCH: 1-25
     - identifier: kubernetes_sigs_kind_1_26
-      buildspec: buildspecs/images.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           RELEASE_BRANCH: 1-26
     - identifier: kubernetes_sigs_kind_1_27
-      buildspec: buildspecs/images.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           RELEASE_BRANCH: 1-27
     - identifier: kubernetes_sigs_kind_1_28
-      buildspec: buildspecs/images.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           RELEASE_BRANCH: 1-28
+version: 0.2
+env:
+  variables:
+    HAS_S3_ARTIFACTS: ""
+phases:
+  pre_build:
+    commands:
+      - ./build/lib/setup.sh
+  build:
+    commands:
+      - if $(make check-project-path-exists); then make release -C $PROJECT_PATH; fi
diff --git a/projects/kubernetes/cloud-provider-vsphere/buildspecs/batch-build.yml b/projects/kubernetes/cloud-provider-vsphere/buildspecs/batch-build.yml
@@ -18,37 +18,40 @@ batch:
   fast-fail: true
   build-graph:
     - identifier: kubernetes_cloud_provider_vsphere_1_24
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-24
     - identifier: kubernetes_cloud_provider_vsphere_1_25
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-25
     - identifier: kubernetes_cloud_provider_vsphere_1_26
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-26
     - identifier: kubernetes_cloud_provider_vsphere_1_27
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-27
     - identifier: kubernetes_cloud_provider_vsphere_1_28
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           RELEASE_BRANCH: 1-28
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      - ./build/lib/setup.sh
+  build:
+    commands:
+      - if $(make check-project-path-exists) && make check-for-release-branch-skip -C $PROJECT_PATH; then make release -C $PROJECT_PATH; fi
diff --git a/projects/opencontainers/runc/buildspecs/batch-build.yml b/projects/opencontainers/runc/buildspecs/batch-build.yml
@@ -18,16 +18,22 @@ batch:
   fast-fail: true
   build-graph:
     - identifier: opencontainers_runc_linux_amd64
-      buildspec: buildspec.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           BINARY_PLATFORMS: linux/amd64
     - identifier: opencontainers_runc_linux_arm64
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           BINARY_PLATFORMS: linux/arm64
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      - ./build/lib/setup.sh
+  build:
+    commands:
+      - if $(make check-project-path-exists) && make check-for-release-branch-skip -C $PROJECT_PATH; then make release -C $PROJECT_PATH; fi
diff --git a/projects/torvalds/linux/buildspecs/batch-build.yml b/projects/torvalds/linux/buildspecs/batch-build.yml
@@ -18,16 +18,22 @@ batch:
   fast-fail: true
   build-graph:
     - identifier: torvalds_linux_linux_amd64
-      buildspec: buildspec.yml
       env:
         type: LINUX_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           BINARY_PLATFORMS: linux/amd64
     - identifier: torvalds_linux_linux_arm64
-      buildspec: buildspec.yml
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
         variables:
           BINARY_PLATFORMS: linux/arm64
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      - ./build/lib/setup.sh
+  build:
+    commands:
+      - if $(make check-project-path-exists) && make check-for-release-branch-skip -C $PROJECT_PATH; then make release -C $PROJECT_PATH; fi