ci: added yaml to json converter package

This will modify the - pull images passed as args - image_not_present test case updated - ci config to install the yq package before testing - update the test script to use debian version instead of the alpine version due to the platform conflict. Signed-off-by: Dineshkumar J <[email protected]>
mendersoftware · Oct 11, 2024 · a9b3515 · a9b3515
1 parent b88be99
commit a9b3515
Show file tree

Hide file tree

Showing 5 changed files with 80 additions and 46 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -73,6 +73,9 @@ test:smoketests:
       alias: docker
   before_script:
     - apk --update --no-cache add bash gawk curl aws-cli tree xz jq wget xdelta3 openssl1.1-compat gcompat
+    - wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O yq
+    - chmod +x yq
+    - mv yq /usr/local/bin
   script:
     - ./tests/app/run.sh
     - ./tests/gen/run.sh

diff --git a/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh b/acceptance-tests/scenarios.d/03.basic.failed.images.not.present.sh
@@ -51,12 +51,20 @@ function test_phase_run_images_not_present() {
         --platform linux/amd64 \
         --orchestrator docker-compose \
         --manifests-dir acceptance-tests/data/manifests-1-broken \
-        --application-name myapp0a || return 1
-    echo "images_not_present: checking install rc"
-    mender install "$artifact_file" && return 2 # we expect a failure
-    echo "images_not_present: checking for running containers"
-    docker ps -q | grep -q . && return 3 # and we expect nothing to be running
-    return 0
+        --application-name myapp0a # we expect a failure here now as we are using docker-compose to pull the images now
+
+    if [ $? -ne 0 ]; then
+        echo "images_not_present: app-gen artifact generation failed which is expected"
+        return 0
+    else
+        echo "Unexpected success, we were expecting failure"
+        return 1
+    fi
+    # echo "images_not_present: checking install rc"
+    # mender install "$artifact_file" && return 2 # we expect a failure
+    # echo "images_not_present: checking for running containers"
+    # docker ps -q | grep -q . && return 3 # and we expect nothing to be running
+    # return 0
 }
 
 function test_failed_hook_phase_run_images_not_present() {

diff --git a/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh b/acceptance-tests/scenarios.d/06.rollback.on.images.not.present.sh
@@ -65,14 +65,23 @@ function test_phase_run_rollback_on_images_not_present() {
         --platform linux/amd64 \
         --orchestrator docker-compose \
         --manifests-dir acceptance-tests/data/manifests-1-broken \
-        --application-name myapp0b || return 1
-    echo "images_not_present: checking install rc"
-    mender install "$artifact_file" && return 2 # we expect a failure
-    sleep $timeout_s
-    echo "images_not_present: checking for running containers"
-    docker ps
-    diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}')
-    return 0
+        --application-name myapp0b # we expect a failure here now as we are using docker-compose to pull the images now
+
+    if [ $? -ne 0 ]; then
+        echo "images_not_present: app-gen artifact generation failed which is expected"
+        return 0
+    else
+        echo "Unexpected success, we were expecting failure"
+        return 1
+    fi
+
+    # echo "images_not_present: checking install rc"
+    # mender install "$artifact_file" && return 2 # we expect a failure
+    # sleep $timeout_s
+    # echo "images_not_present: checking for running containers"
+    # docker ps
+    # diff "${temp_dir}/before-rollback-$$" <(docker ps --format '{{.Image}}')
+    # return 0
 }
 
 function test_failed_hook_phase_run_rollback_on_images_not_present() {

diff --git a/gen/app-gen b/gen/app-gen
@@ -212,12 +212,11 @@ if [ ${#images[@]} -lt 1 ]; then
     if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then
         echo "No specific images specified. Will try to extract from docker-compose.yaml file." >&2
 
-        while read -r img; do
-            images+=("$img")
-            images+=("$img")
-        done < <( docker compose --project-directory "$manifests_dir" config --images)
+        # Check the docker compose has atleast 1 image specified.
+        img_count=$(docker compose --project-directory "$manifests_dir" config --images | wc -l)
 
-        if [ ${#images[@]} -lt 1 ]; then
+        # If img_count
+        if [ "$img_count" -lt 1 ]; then
             echo "Image extraction from docker-compose.yaml failed. Aborting." >&2
             show_help_and_exit_error
         fi
@@ -260,12 +259,14 @@ get_image_sha() {
 
 pull_image() {
     local -r url="$1"
+    # use the provided platform or fall back to global
+    local platform_arg="${2:-$platform}"
 
     [[ "$url" == "" ]] && { return 0; }
     if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then
-        docker pull "$url" --platform "$platform"
+        docker pull "$url" --platform "$platform_arg" || exit 2
     else
-        ctr image pull "$url" --platform "$platform"
+        ctr image pull "$url" --platform "$platform_arg" || exit 2
     fi
 }
 
@@ -491,32 +492,43 @@ prepare_images() {
     local output_dir
     local services
 
-    # Extract services from the Docker Compose file
-    services=$(docker compose --project-directory "$manifests_dir" config --services)
-    for service in $services; do
-        # Convert Docker Compose config to JSON
-        config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json)
-        # Check if the service has a build context
-        build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty")
-        if [ -n "$build_context" ]; then
-            echo "Building Images from Project"
-            # Build the image locally using Docker Compose
-            docker compose --project-directory "$manifests_dir" build "$service"
-            image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
-            images+=("$image_name")
-            images+=("$image_name")
-        else
-            echo "Pulling Images from Directory"
-            # Get the image name from the Docker Compose file
-            image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
-            images+=("$image_name")
-            images+=("$image_name")
-            # Pull the image from the registry
-            pull_image "$image_name"
-        fi
-        # images_shas+=($(get_image_sha "$image_name"))
+    # Pull images specified in args
+    for ((i = 0; i < ${#images[@]}; i++)); do
+        pull_image "${images[${i}]}"
     done
 
+    # Extract services from the Docker Compose file
+    if [[ "${orchestrator}" == "${DOCKER_COMPOSE_ORCHESTRATOR}" ]]; then
+        echo "Using Docker Compose"
+        services=$(docker compose --project-directory "$manifests_dir" config --services)
+        for service in $services; do
+            # Convert Docker Compose config to JSON
+            config_json=$(docker compose --project-directory "$manifests_dir" config | yq eval -o=json)
+            # Check if the service has a build context
+            build_context=$(echo "$config_json" | jq -r ".services.\"$service\".build.context // empty")
+            if [ -n "$build_context" ]; then
+                echo "Building Image from Project"
+                # Build the image locally using Docker Compose
+                docker compose --project-directory "$manifests_dir" build "$service"
+                image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
+                images+=("$image_name")
+                images+=("$image_name")
+            else
+                echo "Pulling Image from Directory"
+                # Get the image name from the Docker Compose file
+                image_name=$(echo "$config_json" | jq -r ".services.\"$service\".image")
+                images+=("$image_name")
+                images+=("$image_name")
+
+                # Extract platform from Docker Compose config (if specified)
+                service_platform=$(echo "$config_json" | jq -r ".services.\"$service\".platform // empty")
+                # Pull the image from the registry, using the extracted platform if available
+                pull_image "$image_name" "$service_platform"
+            fi
+            # images_shas+=($(get_image_sha "$image_name"))
+        done
+    fi
+
     for ((i = 0; i < ${#images[@]}; i++)); do
         images_shas+=($(get_image_sha "${images[${i}]}"))
     done
@@ -592,7 +604,7 @@ generate_metadata() {
 prepare_images "${temp_dir}/images"
 ( cd "${temp_dir}" && tar czvf images.tar.gz images  )
 mkdir "${temp_dir}/manifests"
-for file in "${manifests_dir}/docker-compose.{yaml,yml}"; do
+for file in "${manifests_dir}/docker-compose.yaml" "${manifests_dir}/docker-compose.yml"; do
     if [ -e "$file" ]; then
         cp -a "$file" "${temp_dir}/manifests/docker-compose.yaml"
         break  # Exit the loop after copying the first found file

diff --git a/tests/gen/data/docker/manifests/docker-compose.yml b/tests/gen/data/docker/manifests/docker-compose.yml
@@ -12,6 +12,8 @@ services:
             - "65536"
     db:
         image: docker.io/library/memcached:1.6.18-alpine
+        # this images doesn't have the support for linux/arm/v7, test case failes for as we are passing platform 01_deep_delta_generate.run.sh
+        platform: linux/amd64
         networks:
             - local_net
         environment: