github: extract test file as element of matrix

Before:

  test.sh execute multiple test files.
  This behavior make it hard to distinct failure test cases.

After:

  * Use test.sh as launch script. It doesn't execute multiple test files.
  This change was introduced to make it easy to distinct each test
  cases on error.
  * Use grouping log lines feature
    See
    https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines

Signed-off-by: Kentaro Hayashi <[email protected]>

.github/workflows/apt.yml

@@ -109,6 +109,13 @@ jobs:
           - Debian GNU/Linux bookworm amd64
           - Ubuntu Focal amd64
           - Ubuntu Jammy amd64
+        test-file:
+          - "update-from-v4.sh"
+          - "update-to-next-version-with-backward-compat-for-v4.sh"
+          - "downgrade-to-v4.sh"
+          - "install-newly.sh local"
+          - "install-newly.sh v5"
+          - "install-newly.sh lts"
         include:
           - label: Debian GNU/Linux bullseye amd64
             rake-job: debian-bullseye
@@ -122,6 +129,13 @@ jobs:
           - label: Ubuntu Jammy amd64
             rake-job: ubuntu-jammy
             test-lxc-image: ubuntu:22.04
+        exclude:
+          - label: Debian GNU/Linux bookworm amd64
+            test-file: update-from-v4.sh
+          - label: Debian GNU/Linux bookworm amd64
+            test-file: update-to-next-version-with-backward-compat-for-v4.sh
+          - label: Debian GNU/Linux bookworm amd64
+            test-file: downgrade-to-v4.sh
     steps:
       - uses: actions/checkout@master
       - uses: actions/download-artifact@v4
@@ -131,5 +145,5 @@ jobs:
         with:
           name: packages-apt-source-${{ matrix.rake-job }}
       - uses: canonical/[email protected]
-      - name: Run Test 
-        run: fluent-package/apt/systemd-test/test.sh ${{ matrix.test-lxc-image }}
+      - name: Run Test  ${{ matrix.test-file }} on ${{ matrix.test-lxc-image }}
+        run: fluent-package/apt/systemd-test/test.sh ${{ matrix.test-lxc-image }} ${{ matrix.test-file }}

.github/workflows/yum.yml

@@ -90,7 +90,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/yum/binstubs-test.sh
   v1test:
-    name: Test with CGroup V1
+    name: Test (CGroup V1)
     needs: build
     runs-on: ubuntu-20.04
     strategy:
@@ -100,6 +100,13 @@ jobs:
           - CentOS 7 x86_64
           - AmazonLinux 2 x86_64
           - AmazonLinux 2023 x86_64
+        test-file:
+          - "update-from-v4.sh"
+          - "update-to-next-version-with-backward-compat-for-v4.sh"
+          - "downgrade-to-v4.sh"
+          - "install-newly.sh local"
+          - "install-newly.sh v5"
+          - "install-newly.sh lts"
         include:
           - label: CentOS 7 x86_64
             rake-job: centos-7
@@ -110,17 +117,24 @@ jobs:
           - label: AmazonLinux 2023 x86_64
             rake-job: amazonlinux-2023
             test-lxc-image: images:amazonlinux/2023
+        exclude:
+          - label: AmazonLinux 2023 x86_64
+            test-file: update-from-v4.sh
+          - label: AmazonLinux 2023 x86_64
+            test-file: update-to-next-version-with-backward-compat-for-v4.sh
+          - label: AmazonLinux 2023 x86_64
+            test-file: downgrade-to-v4.sh
     steps:
       - uses: actions/checkout@master
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
       - uses: canonical/[email protected]
-      - name: Run Test
-        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }}
+      - name: Run Test ${{ matrix.test-file }} on ${{ matrix.test-lxc-image }}
+        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }} ${{ matrix.test-file }}
 
   v2test:
-    name: Test with CGroup V2
+    name: Test (CGroup V2)
     needs: build
     runs-on: ubuntu-latest
     strategy:
@@ -129,6 +143,13 @@ jobs:
         label:
           - RockyLinux 8 x86_64
           - AlmaLinux 9 x86_64
+        test-file:
+          - "update-from-v4.sh"
+          - "update-to-next-version-with-backward-compat-for-v4.sh"
+          - "downgrade-to-v4.sh"
+          - "install-newly.sh local"
+          - "install-newly.sh v5"
+          - "install-newly.sh lts"
         include:
           - label: RockyLinux 8 x86_64
             rake-job: rockylinux-8
@@ -142,5 +163,5 @@ jobs:
         with:
           name: packages-${{ matrix.rake-job }}
       - uses: canonical/[email protected]
-      - name: Run Test
-        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }}
+      - name: Run Test ${{ matrix.test-file }} on ${{ matrix.test-lxc-image }}
+        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }} ${{ matrix.test-file }}

fluent-package/apt/systemd-test/test.sh

@@ -2,50 +2,73 @@
 
 if [ -z $1 ]; then
     echo "Error: Need to specify lxc image name."
-    echo "Ex.) $ ./test.sh ubuntu:20.04"
+    echo "Ex.) $ ./test.sh ubuntu:20.04 "
+    echo "Ex. CI) $ ./test.sh ubuntu:20.04 install-newly.sh local"
     exit 1
 fi
 
 image=$1
+test_file=$2
+apt_repo_type=$3
 dir="/host/fluent-package/apt/systemd-test"
 
 set -eux
 
-test_filenames=(
-    update-to-next-version.sh
-)
-
-if [ ! $image = "images:debian/12" ]; then
-    # As no bookworm package for v4, so execute upgrade test for other code name.
-    test_filenames+=(
-        update-from-v4.sh
-        update-to-next-version-with-backward-compat-for-v4.sh
-        downgrade-to-v4.sh
-    )
-fi
-
-for apt_repo_type in local v5 lts; do
-    echo -e "\nRun test: $apt_repo_type\n"
+if [ $CI = "true" ]; then
+    echo -e "::group::Run test: launch $image"
     lxc launch $image target
     sleep 5
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: configure $image"
     lxc config device add target host disk source=$PWD path=/host
     lxc list
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: setup $image"
     lxc exec target -- $dir/setup.sh
-    lxc exec target -- $dir/install-newly.sh $apt_repo_type
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: $test_file $apt_repo_type on $image"
+    lxc exec target -- $dir/$test_file $apt_repo_type
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: cleanup $image"
     lxc stop target
     lxc delete target
-done
+    echo -e "::endgroup::"
+else
+    test_filenames=(
+	update-to-next-version.sh
+    )
 
-for test_filename in ${test_filenames[@]}; do
-    echo -e "\nRun test: $test_filename\n"
-    lxc launch $image target
-    sleep 5
-    lxc config device add target host disk source=$PWD path=/host
-    lxc list
-    lxc exec target -- $dir/setup.sh
-    lxc exec target -- $dir/$test_filename
-    lxc stop target
-    lxc delete target
-done
+    if [ ! $image = "images:debian/12" ]; then
+	# As no bookworm package for v4, so execute upgrade test for other code name.
+	test_filenames+=(
+            update-from-v4.sh
+            update-to-next-version-with-backward-compat-for-v4.sh
+            downgrade-to-v4.sh
+	)
+    fi
+
+    for apt_repo_type in local v5 lts; do
+	echo -e "\nRun test: $apt_repo_type\n"
+	lxc launch $image target
+	sleep 5
+	lxc config device add target host disk source=$PWD path=/host
+	lxc list
+	lxc exec target -- $dir/setup.sh
+	lxc exec target -- $dir/install-newly.sh $apt_repo_type
+	lxc stop target
+	lxc delete target
+    done
 
+    for test_filename in ${test_filenames[@]}; do
+	echo -e "\nRun test: $test_filename\n"
+	lxc launch $image target
+	sleep 5
+	lxc config device add target host disk source=$PWD path=/host
+	lxc list
+	lxc exec target -- $dir/setup.sh
+	lxc exec target -- $dir/$test_filename
+	lxc stop target
+	lxc delete target
+    done
+fi
 echo -e "\nAll Success!\n"

fluent-package/yum/systemd-test/test.sh

@@ -3,47 +3,66 @@
 if [ -z $1 ]; then
     echo "Error: Need to specify distribution name."
     echo "Ex.) $ ./test.sh centos-7"
+    echo "Ex.CI) $ ./test.sh centos-7 install-newly.sh local"
     exit 1
 fi
 
 image=$1
+test_file=$2
+yum_repo_type=$3
 dir="/host/fluent-package/yum/systemd-test"
-
 set -eux
 
-test_filenames=(
-    update-to-next-version.sh
-)
-
-if [ ! $image = "images:amazonlinux/2023" ]; then
-    # As no AmazonLinux/2023 package for v4, so execute upgrade test for other images.
-    test_filenames+=(
-        update-from-v4.sh
-        update-to-next-version-with-backward-compat-for-v4.sh
-        downgrade-to-v4.sh
-    )
-fi
-
-for yum_repo_type in local v5 lts; do
-    echo -e "\nRun test: $yum_repo_type\n"
+if [ $CI = "true" ]; then
+    echo -e "::group::Run test: launch $image"
     lxc launch $image target
     sleep 5
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: configure $image"
     lxc config device add target host disk source=$PWD path=/host
     lxc list
-    lxc exec target -- $dir/install-newly.sh $yum_repo_type
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: $test_file $yum_repo_type on $image"
+    lxc exec target -- $dir/$test_file $yum_repo_type
+    echo -e "::endgroup::"
+    echo -e "::group::Run test: cleanup $image"
     lxc stop target
     lxc delete target
-done
+    echo -e "::endgroup::"
+else
+    test_filenames=(
+	update-to-next-version.sh
+    )
 
-for test_filename in ${test_filenames[@]}; do
-    echo -e "\nRun test: $test_filename\n"
-    lxc launch $image target
-    sleep 5
-    lxc config device add target host disk source=$PWD path=/host
-    lxc list
-    lxc exec target -- $dir/$test_filename
-    lxc stop target
-    lxc delete target
-done
+    if [ ! $image = "images:amazonlinux/2023" ]; then
+	# As no AmazonLinux/2023 package for v4, so execute upgrade test for other images.
+	test_filenames+=(
+            update-from-v4.sh
+            update-to-next-version-with-backward-compat-for-v4.sh
+            downgrade-to-v4.sh
+	)
+    fi
+
+    for yum_repo_type in local v5 lts; do
+	echo -e "\nRun test: $yum_repo_type\n"
+	lxc launch $image target
+	sleep 5
+	lxc config device add target host disk source=$PWD path=/host
+	lxc list
+	lxc exec target -- $dir/install-newly.sh $yum_repo_type
+	lxc stop target
+	lxc delete target
+    done
 
+    for test_filename in ${test_filenames[@]}; do
+	echo -e "\nRun test: $test_filename\n"
+	lxc launch $image target
+	sleep 5
+	lxc config device add target host disk source=$PWD path=/host
+	lxc list
+	lxc exec target -- $dir/$test_filename
+	lxc stop target
+	lxc delete target
+    done
+fi
 echo -e "\nAll Success!\n"