ci: added yaml to json converter package

This will modify the - pull images passed as args - image_not_present test case updated - ci config to install the yq package before testing - update the test script to use debian version instead of the alpine version due to the platform conflict. Signed-off-by: Dineshkumar J <[email protected]>
mendersoftware · Oct 10, 2024 · 3fd1f5e · 3fd1f5e
1 parent b88be99
commit 3fd1f5e
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 41 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -73,6 +73,9 @@ test:smoketests:
       alias: docker
   before_script:
     - apk --update --no-cache add bash gawk curl aws-cli tree xz jq wget xdelta3 openssl1.1-compat gcompat
+    - wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O yq
+    - chmod +x yq
+    - mv yq /usr/local/bin
   script:
     - ./tests/app/run.sh
     - ./tests/gen/run.sh

diff --git a/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh b/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh
@@ -51,12 +51,15 @@ function test_phase_run_images_not_present() {
         --platform linux/amd64 \
         --orchestrator docker-compose \
         --manifests-dir acceptance-tests/data/manifests-1-broken \
-        --application-name myapp0a || return 1
-    echo "images_not_present: checking install rc"
-    mender install "$artifact_file" && return 2 # we expect a failure
-    echo "images_not_present: checking for running containers"
-    docker ps -q | grep -q . && return 3 # and we expect nothing to be running
+        --application-name myapp0a && return 2 # we expect a failure here now as we are using docker-compose to pull the images now
+    echo "images_not_present: app-gen artifact generation failed which is expected"
     return 0
+
+    # echo "images_not_present: checking install rc"
+    # mender install "$artifact_file" && return 2 # we expect a failure
+    # echo "images_not_present: checking for running containers"
+    # docker ps -q | grep -q . && return 3 # and we expect nothing to be running
+    # return 0
 }
 
 function test_failed_hook_phase_run_images_not_present() {

diff --git a/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh b/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh
@@ -65,14 +65,17 @@ function test_phase_run_rollback_on_images_not_present() {
         --platform linux/amd64 \
         --orchestrator docker-compose \
         --manifests-dir acceptance-tests/data/manifests-1-broken \
-        --application-name myapp0b || return 1
-    echo "images_not_present: checking install rc"
-    mender install "$artifact_file" && return 2 # we expect a failure
-    sleep $timeout_s
-    echo "images_not_present: checking for running containers"
-    docker ps
-    diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}')
+        --application-name myapp0b && return 2 # we expect a failure here now as we are using docker-compose to pull the images now
+    echo "images_not_present: app-gen artifact generation failed which is expected"
     return 0
+
+    # echo "images_not_present: checking install rc"
+    # mender install "$artifact_file" && return 2 # we expect a failure
+    # sleep $timeout_s
+    # echo "images_not_present: checking for running containers"
+    # docker ps
+    # diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}')
+    # return 0
 }
 
 function test_failed_hook_phase_run_rollback_on_images_not_present() {

diff --git a/gen/app-gen b/gen/app-gen
@@ -263,9 +263,9 @@ pull_image() {
 
     [[ "$url" == "" ]] && { return 0; }
     if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then
-        docker pull "$url" --platform "$platform"
+        docker pull "$url" --platform "$platform" || exit 2
     else
-        ctr image pull "$url" --platform "$platform"
+        ctr image pull "$url" --platform "$platform" || exit 2
     fi
 }
 
@@ -491,32 +491,40 @@ prepare_images() {
     local output_dir
     local services
 
-    # Extract services from the Docker Compose file
-    services=$(docker compose --project-directory "$manifests_dir" config --services)
-    for service in $services; do
-        # Convert Docker Compose config to JSON
-        config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json)
-        # Check if the service has a build context
-        build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty")
-        if [ -n "$build_context" ]; then
-            echo "Building Images from Project"
-            # Build the image locally using Docker Compose
-            docker compose --project-directory "$manifests_dir" build "$service"
-            image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
-            images+=("$image_name")
-            images+=("$image_name")
-        else
-            echo "Pulling Images from Directory"
-            # Get the image name from the Docker Compose file
-            image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
-            images+=("$image_name")
-            images+=("$image_name")
-            # Pull the image from the registry
-            pull_image "$image_name"
-        fi
-        # images_shas+=($(get_image_sha "$image_name"))
+    # Pull images specified in args
+    for ((i = 0; i < ${#images[@]}; i++)); do
+        pull_image "${images[${i}]}"
     done
 
+    # Extract services from the Docker Compose file
+    if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then
+        echo "Using Docker Compose"
+        services=$(docker compose --project-directory "$manifests_dir" config --services)
+        for service in $services; do
+            # Convert Docker Compose config to JSON
+            config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json)
+            # Check if the service has a build context
+            build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty")
+            if [ -n "$build_context" ]; then
+                echo "Building Image from Project"
+                # Build the image locally using Docker Compose
+                docker compose --project-directory "$manifests_dir" build "$service"
+                image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
+                images+=("$image_name")
+                images+=("$image_name")
+            else
+                echo "Pulling Image from Directory"
+                # Get the image name from the Docker Compose file
+                image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
+                images+=("$image_name")
+                images+=("$image_name")
+                # Pull the image from the registry
+                pull_image "$image_name"
+            fi
+            # images_shas+=($(get_image_sha "$image_name"))
+        done
+    fi
+
     for ((i = 0; i < ${#images[@]}; i++)); do
         images_shas+=($(get_image_sha "${images[${i}]}"))
     done
@@ -592,7 +600,7 @@ generate_metadata() {
 prepare_images "${temp_dir}/images"
 ( cd "${temp_dir}" && tar czvf images.tar.gz images  )
 mkdir "${temp_dir}/manifests"
-for file in "${manifests_dir}/docker-compose.{yaml,yml}"; do
+for file in "${manifests_dir}/docker-compose.yaml" "${manifests_dir}/docker-compose.yml"; do
     if [ -e "$file" ]; then
         cp -a "$file" "${temp_dir}/manifests/docker-compose.yaml"
         break  # Exit the loop after copying the first found file

diff --git a/tests/gen/data/docker/manifests/docker-compose.yml b/tests/gen/data/docker/manifests/docker-compose.yml
@@ -11,7 +11,7 @@ services:
             - /bin/sleep
             - "65536"
     db:
-        image: docker.io/library/memcached:1.6.18-alpine
+        image: docker.io/library/memcached:1.6.18
         networks:
             - local_net
         environment:

diff --git a/tests/gen/scenarios/00_basic_generate.run.sh b/tests/gen/scenarios/00_basic_generate.run.sh
@@ -30,7 +30,7 @@ artifact_name=$(basename "$temp_dir")
 # for the test purposes we will get the shasums of both images,
 # and save them in the expected metadata file
 image1=docker.io/library/alpine:3.14
-image2=docker.io/library/memcached:1.6.18-alpine
+image2=docker.io/library/memcached:1.6.18
 docker pull "${image1}"
 docker pull "${image2}"
 sha1=$(docker inspect "${image1}" --format '{{.Id}}' | cut -f2 -d:)