From 7dec774e4e833950882207131b97d583eb4de00d Mon Sep 17 00:00:00 2001
From: Kentaro Hayashi <hayashi@clear-code.com>
Date: Fri, 13 Dec 2024 10:05:24 +0900
Subject: [PATCH] Enable parallel download artifacts (#768)

In the previous versions, it download artifacts from GitHub in serial
not parallel, so it takes huge amount of times.

In this version, it tries to download them in parallel, so it fixes
above issue.

Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>
---
 fluent-package/manage-fluent-repositories.sh | 47 ++++++++++++++++----
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/fluent-package/manage-fluent-repositories.sh b/fluent-package/manage-fluent-repositories.sh
index 635329b96..96538151e 100755
--- a/fluent-package/manage-fluent-repositories.sh
+++ b/fluent-package/manage-fluent-repositories.sh
@@ -196,15 +196,17 @@ EOF
 	     https://api.github.com/repos/fluent/fluent-package-builder/pulls/$PULL_NUMBER | jq --raw-output '.head | .ref + " " + .sha')
 	head_branch=$(echo $response | cut -d' ' -f1)
 	head_sha=$(echo $response | cut -d' ' -f2)
+	rm -f dl.list
 	curl --silent --location \
 	     -H "Accept: application/vnd.github+json" \
 	     -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" \
 	     -H "X-GitHub-Api-Version: 2022-11-28" \
 	     "https://api.github.com/repos/fluent/fluent-package-builder/actions/artifacts?per_page=100&page=$d" | \
-	    jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + .archive_download_url' | while read line
+	    jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + (.size_in_bytes|tostring) + " " + .archive_download_url' > dl.list
+	while read line
 	do
 	    package=$(echo $line | cut -d' ' -f1)
-	    download_url=$(echo $line | cut -d' ' -f2)
+	    download_url=$(echo $line | cut -d' ' -f3)
 	    echo "Downloading $package.zip from $download_url"
 	    case $package in
 		*debian*|*ubuntu*)
@@ -212,23 +214,52 @@ EOF
 		    (cd apt/repositories &&
 			 rm -f $package.zip &&
 			 curl --silent --location --output $package.zip \
-			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &&
-			 unzip -u -o $package.zip)
+			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
 		    ;;
 		*centos*|*rockylinux*|*almalinux*|*amazonlinux*)
 		    mkdir -p yum/repositories
 		    (cd yum/repositories &&
 			 rm -f $package.zip &&
 			 curl --silent --location --output $package.zip \
-			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &&
-			 unzip -u -o $package.zip)
+			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
 		    ;;
 		*)
 		    curl --silent --location --output $package.zip \
-			 -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url
+			 -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &
 		    ;;
 	    esac
-	done
+	done < dl.list
+	wait
+
+	verified=1
+	while read line
+	do
+	    package=$(echo $line | cut -d' ' -f1)
+	    download_size=$(echo $line | cut -d' ' -f2)
+	    case $package in
+		*debian*|*ubuntu*)
+		    actual_size=$(stat --format="%s" apt/repositories/$package.zip)
+		    ;;
+		*rockylinux*|*almalinux*|*amazonlinux*)
+		    actual_size=$(stat --format="%s" yum/repositories/$package.zip)
+		    ;;
+		*)
+		    actual_size=$(stat --format="%s" $package.zip)
+		    ;;
+	    esac
+	    if [ $download_size = "$actual_size" ]; then
+		echo -e "[\e[32m\e[40mOK\e[0m] Verify apt/repositories/$package.zip"
+	    else
+		echo -e "[\e[31m\e[40mNG\e[0m] Verify apt/repositories/$package.zip (expected: $download_size actual: $actual_size)"
+		verified=0
+	    fi
+	done < dl.list
+	if [ $verified -eq 0 ]; then
+	    echo "Downloaded artifacts were corrupted! Check the downloaded artifacts."
+	    exit 1
+	fi
+	(cd apt/repositories && find . -name '*.zip' -exec unzip -u -o {} \;)
+	(cd yum/repositories && find . -name '*.zip' -exec unzip -u -o {} \;)
 	;;
     *)
 	usage