From 18380d927f235260053e3afcf094e77580db0c7a Mon Sep 17 00:00:00 2001
From: Wenxin Zhang <wenxin.zhang@intel.com>
Date: Tue, 14 May 2024 09:41:25 +0800
Subject: [PATCH] update

Signed-off-by: Wenxin Zhang <wenxin.zhang@intel.com>
---
 .github/workflows/model_test.yml              | 16 ++++++++++-----
 .../workflows/scripts/models/collect_log.sh   |  5 +++--
 .../workflows/scripts/models/model_test.sh    | 20 ++++++++++++-------
 3 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/model_test.yml b/.github/workflows/model_test.yml
index 4f18c630..6dd62f9a 100644
--- a/.github/workflows/model_test.yml
+++ b/.github/workflows/model_test.yml
@@ -36,7 +36,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - modelName: "facebook/opt-125m"
+          - modelName: "opt-125m"
             datasets: "piqa"
             device: "cpu"
             tasks: "text-generation"
@@ -45,6 +45,11 @@ jobs:
     steps:
       - name: Clean Up Working Directory
         run: sudo rm -rf ${{github.workspace}}/*
+      
+      - name: Load environment variables
+        if: ${{ matrix.device }} == "cpu"
+        run:
+          cat ~/actions-runner4/.env >> $GITHUB_ENV
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -54,7 +59,7 @@ jobs:
     # We need this because GitHub needs to clone the branch to pipeline
       - name: Docker Build
         run: |
-          docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} .
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile --build-arg http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" --build-arg https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} .
 
       - name: Docker Run
         run: |
@@ -64,6 +69,7 @@ jobs:
           fi
           docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
           -v ${{ github.workspace }}:/GenAIEval \
+          -e http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" -e https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" \
           ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }}
 
       - name: Binary build
@@ -102,7 +108,7 @@ jobs:
             && bash -x collect_log.sh --model=${{ matrix.modelName }} \
              --device=${{ matrix.device }} \
              --datasets=${{ matrix.datasets }} \
-             --tasks=${{ matrix.tasks }}
+             --tasks=${{ matrix.tasks }}"
 
       - name: Publish pipeline artifact
         uses: actions/upload-artifact@v4
@@ -110,7 +116,7 @@ jobs:
         with:
           name: ${{ matrix.device }}-${{ matrix.modelName }}
           path: |
-            ${{ github.workspace }}/${{ matrix.device }}/${{ matrix.modelName }}
-            ${{ github.workspace }}/.summary.log
+            ${{ github.workspace }}/${{ matrix.device }}-${{ matrix.tasks }}-${{ matrix.modelName }}-${{ matrix.datasets }}.log
+            ${{ github.workspace }}/summary.log
           if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
           retention-days: 60 # 1 <= retention-days <= 90
diff --git a/.github/workflows/scripts/models/collect_log.sh b/.github/workflows/scripts/models/collect_log.sh
index a197393a..0cc6db4a 100644
--- a/.github/workflows/scripts/models/collect_log.sh
+++ b/.github/workflows/scripts/models/collect_log.sh
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 set -eo pipefail
+set -x
 source /GenAIEval/.github/workflows/scripts/change_color
 WORKSPACE="/GenAIEval"
 # get parameters
@@ -34,7 +35,7 @@ for i in "$@"; do
     esac
 done
 
-log_file="/GenAIEval/${device}/${model}/${device}-${model}-${tasks}-${datasets}.log"
+log_file="/log/${device}-${tasks}-${model}-${datasets}.log"
 $BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET
 
 echo "working in"
@@ -42,6 +43,6 @@ pwd
 if [[ ! -f ${log_file} ]]; then
     echo "${device};${model};${tasks};${datasets};;${logfile}" >> ${WORKSPACE}/summary.log
 else
-    acc=$(grep -Po "Accuracy .* is:\\s+(\\d+(\\.\\d+)?)" ${log_file} | head -n 1 | sed 's/.*://;s/[^0-9.]//g')
+    acc=$(grep -Po "acc .*(\d+(\.\d+)?)" ${log_file} | awk -F "|" '{print $2}' | head -n 1 | sed 's/.*://;s/[^0-9.]//g')
     echo "${device};${model};${tasks};${datasets};${acc};${logfile}" >> ${WORKSPACE}/summary.log
 fi
diff --git a/.github/workflows/scripts/models/model_test.sh b/.github/workflows/scripts/models/model_test.sh
index fb420086..8edd9926 100644
--- a/.github/workflows/scripts/models/model_test.sh
+++ b/.github/workflows/scripts/models/model_test.sh
@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -eo pipefail
+set -o pipefail
+set -x
 source /GenAIEval/.github/workflows/scripts/change_color
 
 # get parameters
@@ -34,11 +35,7 @@ for i in "$@"; do
     esac
 done
 
-log_dir="/GenAIEval/${device}/${model}"
-mkdir -p ${log_dir}
 working_dir=""
-$BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET
-
 main() {
     case ${tasks} in
         "text-generation")
@@ -48,7 +45,16 @@ main() {
         *)
             echo "Not suppotted task"; exit 1;;
     esac
+    if [[ ${model} == *"opt"* ]]; then
+        pretrained="facebook/${model}"
+    else
+        pretrained="${model}"
+    fi
+    log_dir="/log/${device}/${model}"
+    mkdir -p ${log_dir}
+    $BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET
     run_benchmark
+    cp ${log_dir}/${device}-${tasks}-${model}-${datasets}.log /GenAIEval/
 }
 
 function prepare() {
@@ -69,10 +75,10 @@ function prepare() {
 
 function run_benchmark() {
     cd ${working_dir}
-    overall_log="${log_dir}/${device}-${model}-${tasks}-${datasets}.log"
+    overall_log="${log_dir}/${device}-${tasks}-${model}-${datasets}.log"
     python main.py \
         --model hf \
-        --model_args pretrained=${model} \
+        --model_args pretrained=${pretrained} \
         --tasks ${datasets} \
         --device ${device} \
         --batch_size 112