CI/CD - use separate instance to build arm64 images (#2286)

# Conflicts: # .circleci/config.yml # build/build-image.sh # build/push-image.sh
cortexlabs · Jun 23, 2021 · 9622653 · 9622653
1 parent 27fca9f
commit 9622653
Show file tree

Hide file tree

Showing 8 changed files with 219 additions and 31 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -77,7 +77,8 @@ commands:
 jobs:
   test:
     machine:
-      image: ubuntu-1604:202104-01  # machine executor necessary to run go integration tests
+      image: ubuntu-1604:202104-01
+    resource_class: medium
     steps:
       - checkout
       - install-go
@@ -87,6 +88,17 @@ jobs:
             go get -u -v golang.org/x/lint/golint
             go get -u -v github.com/kyoh86/looppointer/cmd/looppointer
             pip3 install black aiohttp
+      - run:
+          name: Lint
+          command: make lint
+
+  test:
+    machine:
+      image: ubuntu-1604:202104-01  # machine executor necessary to run go integration tests
+    resource_class: medium
+    steps:
+      - checkout
+      - install-go
       - run:
           name: Initialize Credentials
           command: |
@@ -112,30 +124,67 @@ jobs:
           name: Go Tests
           command: make test
 
-  build-and-deploy:
+  build-and-upload-cli:
     docker:
       - image: cimg/python:3.6
-    environment:
-      DOCKER_CLI_EXPERIMENTAL: enabled
+    resource_class: medium
     steps:
       - setup_remote_docker
       - checkout
       - install-go
-      - run:
-          name: Configure Docker Buildx
-          command: |
-            docker context create custom-context --default-stack-orchestrator swarm --docker "host=$DOCKER_HOST,ca=$DOCKER_CERT_PATH/ca.pem,cert=$DOCKER_CERT_PATH/cert.pem,key=$DOCKER_CERT_PATH/key.pem"
-            docker buildx create --name builder --platform linux/amd64,linux/arm64 --use custom-context && docker buildx inspect --bootstrap
       - run: pip install awscli
       - run: make ci-build-cli
       - return-if-not-deployed-branch
       - run: make ci-build-and-upload-cli
+
+  build-and-push-images-amd64:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: medium
+    steps:
+      - checkout
+      - return-if-not-deployed-branch
+      - run:
+          name: Build CI Images (amd64)
+          command: make ci-build-images-amd64
+          no_output_timeout: 40m
       - quay-login
       - run:
-          name: Build and Push CI Images
-          command: |
-            make ci-build-images
-            make ci-push-images
+          name: Push CI Images (amd64)
+          command: make ci-push-images-amd64
+          no_output_timeout: 40m
+
+  build-and-push-images-arm64:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
+    steps:
+      - checkout
+      - return-if-not-deployed-branch
+      - run:
+          name: Build CI Images (arm64)
+          command: make ci-build-images-arm64
+          no_output_timeout: 40m
+      - quay-login
+      - run:
+          name: Push CI Images (arm64)
+          command: make ci-push-images-arm64
+          no_output_timeout: 40m
+
+  amend-images:
+    docker:
+      - image: cimg/python:3.6
+    environment:
+      DOCKER_CLI_EXPERIMENTAL: enabled
+    resource_class: medium
+    steps:
+      - setup_remote_docker
+      - checkout
+      - return-if-not-deployed-branch
+      - quay-login
+      - run:
+          name: Amend CI Images
+          command: make ci-amend-images
           no_output_timeout: 40m
 
   e2e-tests:
@@ -206,7 +255,7 @@ workflows:
             branches:
               only:
                 - /^[0-9]+\.[0-9]+$/
-      - build-and-deploy:
+      - build-and-upload-cli:
           requires:
             - test
             - build-and-deploy-approval
@@ -215,6 +264,34 @@ workflows:
               only:
                 - master
                 - /^[0-9]+\.[0-9]+$/
+      - build-and-push-images-amd64:
+          requires:
+            - build-and-upload-cli
+            - build-and-deploy-approval
+          filters:
+            branches:
+              only:
+                - master
+                - /^[0-9]+\.[0-9]+$/
+      - build-and-push-images-arm64:
+          requires:
+            - build-and-upload-cli
+            - build-and-deploy-approval
+          filters:
+            branches:
+              only:
+                - master
+                - /^[0-9]+\.[0-9]+$/
+      - amend-images:
+          requires:
+            - build-and-push-images-amd64
+            - build-and-push-images-arm64
+            - build-and-deploy-approval
+          filters:
+            branches:
+              only:
+                - master
+                - /^[0-9]+\.[0-9]+$/
   nightly:
     triggers:
       - schedule:

diff --git a/Makefile b/Makefile
@@ -200,11 +200,20 @@ lint-docs:
 # CI Commands #
 ###############
 
-ci-build-images:
-	@./build/build-images.sh
+ci-build-images-amd64:
+	@./build/build-images.sh amd64 quay.io docker.io
 
-ci-push-images:
-	@./build/push-images.sh quay.io docker.io
+ci-build-images-arm64:
+	@./build/build-images.sh arm64 quay.io docker.io
+
+ci-push-images-amd64:
+	@./build/push-images.sh amd64 quay.io docker.io
+
+ci-push-images-arm64:
+	@./build/push-images.sh arm64 quay.io docker.io
+
+ci-amend-images:
+	@./build/amend-images.sh quay.io docker.io
 
 ci-build-cli:
 	@./build/cli.sh

diff --git a/build/amend-image.sh b/build/amend-image.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2021 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -euo pipefail
+
+CORTEX_VERSION=0.37.0
+
+host_primary=$1
+host_backup=$2
+image=$3
+
+hosts=(
+    "$host_primary"
+    "$host_backup"
+)
+echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+
+for host in "${hosts[@]}"; do
+    docker manifest create $host/cortexlabs/${image}:${CORTEX_VERSION} \
+        -a $host/cortexlabs/${image}:manifest-${CORTEX_VERSION}-amd64 \
+        -a $host/cortexlabs/${image}:manifest-${CORTEX_VERSION}-arm64
+    docker manifest push $host/cortexlabs/${image}:${CORTEX_VERSION}
+done
diff --git a/build/amend-images.sh b/build/amend-images.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2021 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
+
+source $ROOT/build/images.sh
+source $ROOT/dev/util.sh
+
+host_primary=$1
+host_backup=$2
+
+for image in "${all_images[@]}"; do
+    if in_array $image "multi_arch_images"; then
+        $ROOT/build/amend-image.sh $host_primary $host_backup $image
+    fi
+done
diff --git a/build/build-image.sh b/build/build-image.sh
@@ -21,10 +21,25 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 
 CORTEX_VERSION=0.37.0
 
-image=$1
-platforms=$2
+host_primary=$1
+host_backup=$2
+image=$3
+is_multi_arch=$4
+arch=$5
 
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
 fi
-docker buildx build $ROOT --progress plain -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} --platform $platforms
+
+if [ "$is_multi_arch" = "true" ]; then
+  tag="manifest-${CORTEX_VERSION}-$arch"
+else
+  tag="${CORTEX_VERSION}"
+fi
+
+docker build $ROOT \
+  --build-arg TARGETOS=linux \
+  --build-arg TARGETARCH=$arch \
+  -f $ROOT/images/$image/Dockerfile \
+  -t $host_primary/cortexlabs/${image}:${tag} \
+  -t $host_backup/cortexlabs/${image}:${tag}
diff --git a/build/build-images.sh b/build/build-images.sh
@@ -22,10 +22,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
+arch=$1
+host_primary=$2
+host_backup=$3
+
 for image in "${all_images[@]}"; do
-  platforms="linux/amd64"
+  is_multi_arch="false"
   if in_array $image "multi_arch_images"; then
-    platforms+=",linux/arm64"
+    is_multi_arch="true"
+    $ROOT/build/build-image.sh $host_primary $host_backup $image $is_multi_arch $arch
+  elif [ "$arch" = "amd64" ]; then
+    $ROOT/build/build-image.sh $host_primary $host_backup $image $is_multi_arch $arch
   fi
-  $ROOT/build/build-image.sh $image $platforms
 done
diff --git a/build/push-image.sh b/build/push-image.sh
@@ -24,7 +24,16 @@ CORTEX_VERSION=0.37.0
 host_primary=$1
 host_backup=$2
 image=$3
-platforms=$4
+is_multi_arch=$4
+arch=$5
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-docker buildx build $ROOT --progress plain -f $ROOT/images/$image/Dockerfile -t $host_primary/cortexlabs/${image}:${CORTEX_VERSION} -t $host_backup/cortexlabs/${image}:${CORTEX_VERSION} --platform $platforms --push
+
+if [ "$is_multi_arch" = "true" ]; then
+  tag="manifest-${CORTEX_VERSION}-$arch"
+else
+  tag="${CORTEX_VERSION}"
+fi
+
+docker push $host_primary/cortexlabs/${image}:${tag}
+docker push $host_backup/cortexlabs/${image}:${tag}
diff --git a/build/push-images.sh b/build/push-images.sh
@@ -22,13 +22,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
-host_primary=$1
-host_backup=$2
+arch=$1
+host_primary=$2
+host_backup=$3
 
 for image in "${all_images[@]}"; do
-  platforms="linux/amd64"
+  is_multi_arch="false"
   if in_array $image "multi_arch_images"; then
-    platforms+=",linux/arm64"
+    is_multi_arch="true"
+    $ROOT/build/push-image.sh $host_primary $host_backup $image $is_multi_arch $arch
+  elif [ "$arch" = "amd64" ]; then
+    $ROOT/build/push-image.sh $host_primary $host_backup $image $is_multi_arch $arch
   fi
-  $ROOT/build/push-image.sh $host_primary $host_backup $image $platforms
 done