From 3fd1f5e0904153a154437e7859493ac6323fb9d1 Mon Sep 17 00:00:00 2001 From: Dineshkumar J Date: Fri, 4 Oct 2024 18:55:13 +0530 Subject: [PATCH] ci: added yaml to json converter package This will modify the - pull images passed as args - image_not_present test case updated - ci config to install the yq package before testing - update the test script to use debian version instead of the alpine version due to the platform conflict. Signed-off-by: Dineshkumar J --- .gitlab-ci.yml | 3 + .../03.basic.failed.images.not.present.sh | 13 ++-- .../06.rollback.on.images.not.present.sh | 17 ++--- gen/app-gen | 62 +++++++++++-------- .../data/docker/manifests/docker-compose.yml | 2 +- tests/gen/scenarios/00_basic_generate.run.sh | 2 +- 6 files changed, 58 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3cc6dd..5f36163 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,6 +73,9 @@ test:smoketests: alias: docker before_script: - apk --update --no-cache add bash gawk curl aws-cli tree xz jq wget xdelta3 openssl1.1-compat gcompat + - wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O yq + - chmod +x yq + - mv yq /usr/local/bin script: - ./tests/app/run.sh - ./tests/gen/run.sh diff --git a/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh b/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh index 1ac2001..4d5b0c7 100644 --- a/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh +++ b/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh @@ -51,12 +51,15 @@ function test_phase_run_images_not_present() { --platform linux/amd64 \ --orchestrator docker-compose \ --manifests-dir acceptance-tests/data/manifests-1-broken \ - --application-name myapp0a || return 1 - echo "images_not_present: checking install rc" - mender install "$artifact_file" && return 2 # we expect a failure - echo "images_not_present: checking for running containers" - docker ps -q | grep -q . && return 3 # and we expect nothing to be running + --application-name myapp0a && return 2 # we expect a failure here now as we are using docker-compose to pull the images now + echo "images_not_present: app-gen artifact generation failed which is expected" return 0 + + # echo "images_not_present: checking install rc" + # mender install "$artifact_file" && return 2 # we expect a failure + # echo "images_not_present: checking for running containers" + # docker ps -q | grep -q . && return 3 # and we expect nothing to be running + # return 0 } function test_failed_hook_phase_run_images_not_present() { diff --git a/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh b/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh index 630fb82..b4fca95 100644 --- a/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh +++ b/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh @@ -65,14 +65,17 @@ function test_phase_run_rollback_on_images_not_present() { --platform linux/amd64 \ --orchestrator docker-compose \ --manifests-dir acceptance-tests/data/manifests-1-broken \ - --application-name myapp0b || return 1 - echo "images_not_present: checking install rc" - mender install "$artifact_file" && return 2 # we expect a failure - sleep $timeout_s - echo "images_not_present: checking for running containers" - docker ps - diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}') + --application-name myapp0b && return 2 # we expect a failure here now as we are using docker-compose to pull the images now + echo "images_not_present: app-gen artifact generation failed which is expected" return 0 + + # echo "images_not_present: checking install rc" + # mender install "$artifact_file" && return 2 # we expect a failure + # sleep $timeout_s + # echo "images_not_present: checking for running containers" + # docker ps + # diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}') + # return 0 } function test_failed_hook_phase_run_rollback_on_images_not_present() { diff --git a/gen/app-gen b/gen/app-gen index 542fd16..d56db22 100755 --- a/gen/app-gen +++ b/gen/app-gen @@ -263,9 +263,9 @@ pull_image() { [[ "$url" == "" ]] && { return 0; } if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then - docker pull "$url" --platform "$platform" + docker pull "$url" --platform "$platform" || exit 2 else - ctr image pull "$url" --platform "$platform" + ctr image pull "$url" --platform "$platform" || exit 2 fi } @@ -491,32 +491,40 @@ prepare_images() { local output_dir local services - # Extract services from the Docker Compose file - services=$(docker compose --project-directory "$manifests_dir" config --services) - for service in $services; do - # Convert Docker Compose config to JSON - config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json) - # Check if the service has a build context - build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty") - if [ -n "$build_context" ]; then - echo "Building Images from Project" - # Build the image locally using Docker Compose - docker compose --project-directory "$manifests_dir" build "$service" - image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") - images+=("$image_name") - images+=("$image_name") - else - echo "Pulling Images from Directory" - # Get the image name from the Docker Compose file - image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") - images+=("$image_name") - images+=("$image_name") - # Pull the image from the registry - pull_image "$image_name" - fi - # images_shas+=($(get_image_sha "$image_name")) + # Pull images specified in args + for ((i = 0; i < ${#images[@]}; i++)); do + pull_image "${images[${i}]}" done + # Extract services from the Docker Compose file + if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then + echo "Using Docker Compose" + services=$(docker compose --project-directory "$manifests_dir" config --services) + for service in $services; do + # Convert Docker Compose config to JSON + config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json) + # Check if the service has a build context + build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty") + if [ -n "$build_context" ]; then + echo "Building Image from Project" + # Build the image locally using Docker Compose + docker compose --project-directory "$manifests_dir" build "$service" + image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") + images+=("$image_name") + images+=("$image_name") + else + echo "Pulling Image from Directory" + # Get the image name from the Docker Compose file + image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image") + images+=("$image_name") + images+=("$image_name") + # Pull the image from the registry + pull_image "$image_name" + fi + # images_shas+=($(get_image_sha "$image_name")) + done + fi + for ((i = 0; i < ${#images[@]}; i++)); do images_shas+=($(get_image_sha "${images[${i}]}")) done @@ -592,7 +600,7 @@ generate_metadata() { prepare_images "${temp_dir}/images" ( cd "${temp_dir}" && tar czvf images.tar.gz images ) mkdir "${temp_dir}/manifests" -for file in "${manifests_dir}/docker-compose.{yaml,yml}"; do +for file in "${manifests_dir}/docker-compose.yaml" "${manifests_dir}/docker-compose.yml"; do if [ -e "$file" ]; then cp -a "$file" "${temp_dir}/manifests/docker-compose.yaml" break # Exit the loop after copying the first found file diff --git a/tests/gen/data/docker/manifests/docker-compose.yml b/tests/gen/data/docker/manifests/docker-compose.yml index a34877b..3edfc59 100644 --- a/tests/gen/data/docker/manifests/docker-compose.yml +++ b/tests/gen/data/docker/manifests/docker-compose.yml @@ -11,7 +11,7 @@ services: - /bin/sleep - "65536" db: - image: docker.io/library/memcached:1.6.18-alpine + image: docker.io/library/memcached:1.6.18 networks: - local_net environment: diff --git a/tests/gen/scenarios/00_basic_generate.run.sh b/tests/gen/scenarios/00_basic_generate.run.sh index 541c90c..98f7faa 100755 --- a/tests/gen/scenarios/00_basic_generate.run.sh +++ b/tests/gen/scenarios/00_basic_generate.run.sh @@ -30,7 +30,7 @@ artifact_name=$(basename "$temp_dir") # for the test purposes we will get the shasums of both images, # and save them in the expected metadata file image1=docker.io/library/alpine:3.14 -image2=docker.io/library/memcached:1.6.18-alpine +image2=docker.io/library/memcached:1.6.18 docker pull "${image1}" docker pull "${image2}" sha1=$(docker inspect "${image1}" --format '{{.Id}}' | cut -f2 -d:)