From b88be99214e1627df46e5dbb862c1208e86a4f7f Mon Sep 17 00:00:00 2001 From: dineshkumar-j-genea Date: Fri, 4 Oct 2024 17:58:42 +0530 Subject: [PATCH 1/2] feat: add support to build image from docker file This commit introduces the ability to build images directly from a Dockerfile, enhancing the build process for containerized applications. Changelog: Added support for building images from Dockerfiles. Ticket: MEN-7539 Signed-off-by: dineshkumar-j-genea --- .gitlab-ci.yml | 3 +++ gen/app-gen | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f9bec1..c3cc6dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,6 +108,9 @@ test:acceptancetests: - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl - chmod +x ./kubectl - mv ./kubectl /usr/bin/kubectl + - wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O yq + - chmod +x yq + - mv yq /usr/local/bin script: - ./acceptance-tests/run.sh diff --git a/gen/app-gen b/gen/app-gen index 122f352..542fd16 100755 --- a/gen/app-gen +++ b/gen/app-gen @@ -489,9 +489,35 @@ prepare_images() { local url_current local url_new local output_dir + local services + + # Extract services from the Docker Compose file + services=$(docker compose --project-directory "$manifests_dir" config --services) + for service in $services; do + # Convert Docker Compose config to JSON + config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json) + # Check if the service has a build context + build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty") + if [ -n "$build_context" ]; then + echo "Building Images from Project" + # Build the image locally using Docker Compose + docker compose --project-directory "$manifests_dir" build "$service" + image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") + images+=("$image_name") + images+=("$image_name") + else + echo "Pulling Images from Directory" + # Get the image name from the Docker Compose file + image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") + images+=("$image_name") + images+=("$image_name") + # Pull the image from the registry + pull_image "$image_name" + fi + # images_shas+=($(get_image_sha "$image_name")) + done for ((i = 0; i < ${#images[@]}; i++)); do - pull_image "${images[${i}]}" images_shas+=($(get_image_sha "${images[${i}]}")) done declare -p images_shas @@ -565,7 +591,13 @@ generate_metadata() { prepare_images "${temp_dir}/images" ( cd "${temp_dir}" && tar czvf images.tar.gz images ) -cp -a "${manifests_dir}" "${temp_dir}/manifests" +mkdir "${temp_dir}/manifests" +for file in "${manifests_dir}/docker-compose.{yaml,yml}"; do + if [ -e "$file" ]; then + cp -a "$file" "${temp_dir}/manifests/docker-compose.yaml" + break # Exit the loop after copying the first found file + fi +done ( cd "${temp_dir}" && tar czvf "manifests.tar.gz" manifests ) generate_metadata "${temp_dir}/metadata.json" From a9b3515ddca1d71e681143c72f162b38b74e24a9 Mon Sep 17 00:00:00 2001 From: Dineshkumar J Date: Fri, 4 Oct 2024 18:55:13 +0530 Subject: [PATCH 2/2] ci: added yaml to json converter package This will modify the - pull images passed as args - image_not_present test case updated - ci config to install the yq package before testing - update the test script to use debian version instead of the alpine version due to the platform conflict. Signed-off-by: Dineshkumar J --- .gitlab-ci.yml | 3 + .../03.basic.failed.images.not.present.sh | 20 +++-- .../06.rollback.on.images.not.present.sh | 25 ++++-- gen/app-gen | 76 +++++++++++-------- .../data/docker/manifests/docker-compose.yml | 2 + 5 files changed, 80 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3cc6dd..5f36163 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,6 +73,9 @@ test:smoketests: alias: docker before_script: - apk --update --no-cache add bash gawk curl aws-cli tree xz jq wget xdelta3 openssl1.1-compat gcompat + - wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O yq + - chmod +x yq + - mv yq /usr/local/bin script: - ./tests/app/run.sh - ./tests/gen/run.sh diff --git a/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh b/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh index 1ac2001..053ae12 100644 --- a/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh +++ b/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh @@ -51,12 +51,20 @@ function test_phase_run_images_not_present() { --platform linux/amd64 \ --orchestrator docker-compose \ --manifests-dir acceptance-tests/data/manifests-1-broken \ - --application-name myapp0a || return 1 - echo "images_not_present: checking install rc" - mender install "$artifact_file" && return 2 # we expect a failure - echo "images_not_present: checking for running containers" - docker ps -q | grep -q . && return 3 # and we expect nothing to be running - return 0 + --application-name myapp0a # we expect a failure here now as we are using docker-compose to pull the images now + + if [ $? -ne 0 ]; then + echo "images_not_present: app-gen artifact generation failed which is expected" + return 0 + else + echo "Unexpected success, we were expecting failure" + return 1 + fi + # echo "images_not_present: checking install rc" + # mender install "$artifact_file" && return 2 # we expect a failure + # echo "images_not_present: checking for running containers" + # docker ps -q | grep -q . && return 3 # and we expect nothing to be running + # return 0 } function test_failed_hook_phase_run_images_not_present() { diff --git a/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh b/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh index 630fb82..4f3fecf 100644 --- a/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh +++ b/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh @@ -65,14 +65,23 @@ function test_phase_run_rollback_on_images_not_present() { --platform linux/amd64 \ --orchestrator docker-compose \ --manifests-dir acceptance-tests/data/manifests-1-broken \ - --application-name myapp0b || return 1 - echo "images_not_present: checking install rc" - mender install "$artifact_file" && return 2 # we expect a failure - sleep $timeout_s - echo "images_not_present: checking for running containers" - docker ps - diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}') - return 0 + --application-name myapp0b # we expect a failure here now as we are using docker-compose to pull the images now + + if [ $? -ne 0 ]; then + echo "images_not_present: app-gen artifact generation failed which is expected" + return 0 + else + echo "Unexpected success, we were expecting failure" + return 1 + fi + + # echo "images_not_present: checking install rc" + # mender install "$artifact_file" && return 2 # we expect a failure + # sleep $timeout_s + # echo "images_not_present: checking for running containers" + # docker ps + # diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}') + # return 0 } function test_failed_hook_phase_run_rollback_on_images_not_present() { diff --git a/gen/app-gen b/gen/app-gen index 542fd16..bdf9ebf 100755 --- a/gen/app-gen +++ b/gen/app-gen @@ -212,12 +212,11 @@ if [ ${#images[@]} -lt 1 ]; then if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then echo "No specific images specified. Will try to extract from docker-compose.yaml file." >&2 - while read -r img; do - images+=("$img") - images+=("$img") - done < <( docker compose --project-directory "$manifests_dir" config --images) + # Check the docker compose has atleast 1 image specified. + img_count=$(docker compose --project-directory "$manifests_dir" config --images | wc -l) - if [ ${#images[@]} -lt 1 ]; then + # If img_count + if [ "$img_count" -lt 1 ]; then echo "Image extraction from docker-compose.yaml failed. Aborting." >&2 show_help_and_exit_error fi @@ -260,12 +259,14 @@ get_image_sha() { pull_image() { local -r url="$1" + # use the provided platform or fall back to global + local platform_arg="${2:-$platform}" [[ "$url" == "" ]] && { return 0; } if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then - docker pull "$url" --platform "$platform" + docker pull "$url" --platform "$platform_arg" || exit 2 else - ctr image pull "$url" --platform "$platform" + ctr image pull "$url" --platform "$platform_arg" || exit 2 fi } @@ -491,32 +492,43 @@ prepare_images() { local output_dir local services - # Extract services from the Docker Compose file - services=$(docker compose --project-directory "$manifests_dir" config --services) - for service in $services; do - # Convert Docker Compose config to JSON - config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json) - # Check if the service has a build context - build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty") - if [ -n "$build_context" ]; then - echo "Building Images from Project" - # Build the image locally using Docker Compose - docker compose --project-directory "$manifests_dir" build "$service" - image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") - images+=("$image_name") - images+=("$image_name") - else - echo "Pulling Images from Directory" - # Get the image name from the Docker Compose file - image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") - images+=("$image_name") - images+=("$image_name") - # Pull the image from the registry - pull_image "$image_name" - fi - # images_shas+=($(get_image_sha "$image_name")) + # Pull images specified in args + for ((i = 0; i < ${#images[@]}; i++)); do + pull_image "${images[${i}]}" done + # Extract services from the Docker Compose file + if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then + echo "Using Docker Compose" + services=$(docker compose --project-directory "$manifests_dir" config --services) + for service in $services; do + # Convert Docker Compose config to JSON + config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json) + # Check if the service has a build context + build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty") + if [ -n "$build_context" ]; then + echo "Building Image from Project" + # Build the image locally using Docker Compose + docker compose --project-directory "$manifests_dir" build "$service" + image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") + images+=("$image_name") + images+=("$image_name") + else + echo "Pulling Image from Directory" + # Get the image name from the Docker Compose file + image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") + images+=("$image_name") + images+=("$image_name") + + # Extract platform from Docker Compose config (if specified) + service_platform=$(echo "$config_json" | jq -r ".services.\"$service\".platform // empty") + # Pull the image from the registry, using the extracted platform if available + pull_image "$image_name" "$service_platform" + fi + # images_shas+=($(get_image_sha "$image_name")) + done + fi + for ((i = 0; i < ${#images[@]}; i++)); do images_shas+=($(get_image_sha "${images[${i}]}")) done @@ -592,7 +604,7 @@ generate_metadata() { prepare_images "${temp_dir}/images" ( cd "${temp_dir}" && tar czvf images.tar.gz images ) mkdir "${temp_dir}/manifests" -for file in "${manifests_dir}/docker-compose.{yaml,yml}"; do +for file in "${manifests_dir}/docker-compose.yaml" "${manifests_dir}/docker-compose.yml"; do if [ -e "$file" ]; then cp -a "$file" "${temp_dir}/manifests/docker-compose.yaml" break # Exit the loop after copying the first found file diff --git a/tests/gen/data/docker/manifests/docker-compose.yml b/tests/gen/data/docker/manifests/docker-compose.yml index a34877b..aba4a13 100644 --- a/tests/gen/data/docker/manifests/docker-compose.yml +++ b/tests/gen/data/docker/manifests/docker-compose.yml @@ -12,6 +12,8 @@ services: - "65536" db: image: docker.io/library/memcached:1.6.18-alpine + # this images doesn't have the support for linux/arm/v7, test case failes for as we are passing platform 01_deep_delta_generate.run.sh + platform: linux/amd64 networks: - local_net environment: