update

Signed-off-by: Wenxin Zhang <[email protected]>
opea-project · May 15, 2024 · 6adbcd9 · 6adbcd9
1 parent 87b5445
commit 6adbcd9
Show file tree

Hide file tree

Showing 5 changed files with 517 additions and 42 deletions.
diff --git a/.github/workflows/model_test_cpu.yml b/.github/workflows/model_test_cpu.yml
@@ -0,0 +1,165 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Model Test on CPU
+
+on:
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+permissions: write-all
+env:
+  OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/scripts/models
+  SCRIPT_PATH: /GenAIEval/.github/workflows/scripts
+  DOCKER_NAME: "genaieval"
+  DOCKER_TAG: "latest"
+  CONTAINER_NAME: "modelTest"
+
+
+jobs:
+  Evaluation-Workflow:
+    runs-on: aise-cluster-cpu
+    strategy:
+      matrix:
+        include:
+          - modelName: "opt-125m"
+            datasets: "piqa"
+            device: "cpu"
+            tasks: "text-generation"
+      fail-fast: true
+
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Load environment variables
+        run:
+          cat ~/actions-runner4/.env >> $GITHUB_ENV
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+          fetch-tags: true
+    # We need this because GitHub needs to clone the branch to pipeline
+      - name: Docker Build
+        run: |
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile --build-arg http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" --build-arg https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} .
+
+      - name: Docker Run
+        run: |
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+            docker stop ${{ env.CONTAINER_NAME }}
+            docker rm -vf ${{ env.CONTAINER_NAME }} || true
+          fi
+          docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
+          -v ${{ github.workspace }}:/GenAIEval \
+          -e http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" -e https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" \
+          ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }}
+
+      - name: Binary build
+        run: |
+            docker exec ${{ env.CONTAINER_NAME }} \
+            bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install"
+
+      - name: Evaluation
+        run: |
+            docker exec ${{ env.CONTAINER_NAME }} \
+            bash -c "cd /GenAIEval/.github/workflows/scripts/models \
+            && bash model_test.sh --model=${{ matrix.modelName }} --device=${{ matrix.device }} --datasets=${{ matrix.datasets }} --tasks=${{ matrix.tasks }}"
+
+      - name: Collect Log
+        run: |
+            docker exec ${{ env.CONTAINER_NAME }} \
+            bash -c "cd /GenAIEval/.github/workflows/scripts/models \
+            && bash -x collect_log.sh --model=${{ matrix.modelName }} \
+             --device=${{ matrix.device }} \
+             --datasets=${{ matrix.datasets }} \
+             --tasks=${{ matrix.tasks }}"
+
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ${{ matrix.device }}-${{ matrix.tasks }}-${{ matrix.modelName }}-${{ matrix.datasets }}
+          path: |
+            ${{ github.workspace }}/${{ matrix.device }}-${{ matrix.tasks }}-${{ matrix.modelName }}-${{ matrix.datasets }}.log
+            ${{ github.workspace }}/summary.log
+          if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
+          retention-days: 60 # 1 <= retention-days <= 90
+
+  Genreate-Report:
+    runs-on: ubuntu-latest
+    needs: [Evaluation-Workflow]
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Download Summary Log
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.OUT_SCRIPT_PATH }}/log
+      - name: Display structure of downloaded files
+        run: ls -R
+      - name: Analysis Summary
+        run: |
+            cd ${{ env.OUT_SCRIPT_PATH }}
+            ls -R
+
+      - name: Download Reference Artifact
+        id: download-artifact
+        uses: dawidd6/[email protected]
+        with:
+          workflow: model-test.yml
+          name: FinalReport
+          run_id: ${{ vars.ModelTest_CPU_REF_ID }}
+          path: ${{ env.OUT_SCRIPT_PATH }}
+          name_is_regexp: true
+          repo: ${{ github.repository }}
+          check_artifacts: false
+          search_artifacts: false
+          skip_unpack: false
+          if_no_artifact_found: warn
+
+      - name: Display structure of downloaded files
+        run: cd ${{ env.OUT_SCRIPT_PATH }}/log && ls -R
+
+      - name: Generate report
+        run: |
+          echo "------ Generating final report.html ------"
+          cd ${{ env.OUT_SCRIPT_PATH }}
+          mkdir -p generated
+          /usr/bin/bash generate_report.sh
+        env:
+          RUN_DISPLAY_URL: https://https://github.com/opea-project/GenAIEval/actions/runs/${{ github.run_id }}
+          BUILD_NUMBER: ${{ github.run_id }}
+          JOB_STATUS: succeed
+
+      - name: Publish Report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: FinalReport
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated
+
+      - name: Specify performance regression
+        if: ${{ !cancelled() }}
+        run: |
+          if [ ${{ env.is_perf_reg }} == 'true' ]; then
+            echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
+            exit 1
+          fi
diff --git a/.github/workflows/model_test.yml → .github/workflows/model_test_hpu.yml b/.github/workflows/model_test.yml → .github/workflows/model_test_hpu.yml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Model Test
+name: Model Test on HPU
 
 on:
   workflow_dispatch:
@@ -32,13 +32,13 @@ env:
 
 jobs:
   Evaluation-Workflow:
-    runs-on: aise-cluster
+    runs-on: aise-cluster-hpu
     strategy:
       matrix:
         include:
-          - modelName: "facebook/opt-125m"
+          - modelName: "opt-125m"
             datasets: "piqa"
-            device: "cpu"
+            device: "hpu"
             tasks: "text-generation"
       fail-fast: true
 
@@ -54,7 +54,7 @@ jobs:
     # We need this because GitHub needs to clone the branch to pipeline
       - name: Docker Build
         run: |
-          docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} .
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile --build-arg http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" --build-arg https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} .
 
       - name: Docker Run
         run: |
@@ -64,31 +64,14 @@ jobs:
           fi
           docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
           -v ${{ github.workspace }}:/GenAIEval \
+          -e http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" -e https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" \
           ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }}
 
       - name: Binary build
         run: |
             docker exec ${{ env.CONTAINER_NAME }} \
             bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install"
 
-      #- name: Download Reference Artifact
-      #  id: download-artifact
-      #  uses: dawidd6/[email protected]
-      #  with:
-      #    workflow: model_test.yml
-      #    name: ${{ matrix.device }}-${{ matrix.modelName }}
-      #    run_id: ${{ vars.ModelTest_REF_ID }}
-      #    path: ${{ github.workspace }}/${{ matrix.device }}_${{ matrix.modelName }}_refer_log
-      #    name_is_regexp: true
-      #    repo: ${{ github.repository }}
-      #    check_artifacts: false
-      #    search_artifacts: false
-      #    skip_unpack: false
-      #    if_no_artifact_found: warn
-
-      #- name: Display structure of downloaded files
-      #  run: ls -R
-
       - name: Evaluation
         run: |
             docker exec ${{ env.CONTAINER_NAME }} \
@@ -102,15 +85,77 @@ jobs:
             && bash -x collect_log.sh --model=${{ matrix.modelName }} \
              --device=${{ matrix.device }} \
              --datasets=${{ matrix.datasets }} \
-             --tasks=${{ matrix.tasks }}
+             --tasks=${{ matrix.tasks }}"
 
       - name: Publish pipeline artifact
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
         with:
-          name: ${{ matrix.device }}-${{ matrix.modelName }}
+          name: ${{ matrix.device }}-${{ matrix.tasks }}-${{ matrix.modelName }}-${{ matrix.datasets }}
           path: |
-            ${{ github.workspace }}/${{ matrix.device }}/${{ matrix.modelName }}
-            ${{ github.workspace }}/.summary.log
+            ${{ github.workspace }}/${{ matrix.device }}-${{ matrix.tasks }}-${{ matrix.modelName }}-${{ matrix.datasets }}.log
+            ${{ github.workspace }}/summary.log
           if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
           retention-days: 60 # 1 <= retention-days <= 90
+
+  Genreate-Report:
+    runs-on: ubuntu-latest
+    needs: [Evaluation-Workflow]
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Download Summary Log
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.OUT_SCRIPT_PATH }}/log
+      - name: Display structure of downloaded files
+        run: ls -R
+      - name: Analysis Summary
+        run: |
+            cd ${{ env.OUT_SCRIPT_PATH }}
+            ls -R
+
+      - name: Download Reference Artifact
+        id: download-artifact
+        uses: dawidd6/[email protected]
+        with:
+          workflow: model-test.yml
+          name: FinalReport
+          run_id: ${{ vars.ModelTest_HPU_REF_ID }}
+          path: ${{ env.OUT_SCRIPT_PATH }}
+          name_is_regexp: true
+          repo: ${{ github.repository }}
+          check_artifacts: false
+          search_artifacts: false
+          skip_unpack: false
+          if_no_artifact_found: warn
+
+      - name: Display structure of downloaded files
+        run: cd ${{ env.OUT_SCRIPT_PATH }}/log && ls -R
+
+      - name: Generate report
+        run: |
+          echo "------ Generating final report.html ------"
+          cd ${{ env.OUT_SCRIPT_PATH }}
+          mkdir -p generated
+          /usr/bin/bash generate_report.sh
+        env:
+          RUN_DISPLAY_URL: https://https://github.com/opea-project/GenAIEval/actions/runs/${{ github.run_id }}
+          BUILD_NUMBER: ${{ github.run_id }}
+          JOB_STATUS: succeed
+
+      - name: Publish Report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: FinalReport
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated
+
+      - name: Specify performance regression
+        if: ${{ !cancelled() }}
+        run: |
+          if [ ${{ env.is_perf_reg }} == 'true' ]; then
+            echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
+            exit 1
+          fi
diff --git a/.github/workflows/scripts/models/collect_log.sh b/.github/workflows/scripts/models/collect_log.sh
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 set -eo pipefail
+set -x
 source /GenAIEval/.github/workflows/scripts/change_color
 WORKSPACE="/GenAIEval"
 # get parameters
@@ -34,14 +35,14 @@ for i in "$@"; do
     esac
 done
 
-log_file="/GenAIEval/${device}/${model}/${device}-${model}-${tasks}-${datasets}.log"
+log_file="/log/${device}/${model}/${device}-${tasks}-${model}-${datasets}.log"
 $BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET
 
 echo "working in"
 pwd
 if [[ ! -f ${log_file} ]]; then
-    echo "${device};${model};${tasks};${datasets};;${logfile}" >> ${WORKSPACE}/summary.log
+    echo "${device};${model};${tasks};${datasets};;" >> ${WORKSPACE}/summary.log
 else
-    acc=$(grep -Po "Accuracy .* is:\\s+(\\d+(\\.\\d+)?)" ${log_file} | head -n 1 | sed 's/.*://;s/[^0-9.]//g')
-    echo "${device};${model};${tasks};${datasets};${acc};${logfile}" >> ${WORKSPACE}/summary.log
+    acc=$(grep -Po "acc .*(\d+(\.\d+)?)" ${log_file} | awk -F "|" '{print $2}' | head -n 1 | sed 's/.*://;s/[^0-9.]//g')
+    echo "${device};${model};${tasks};${datasets};${acc};" >> ${WORKSPACE}/summary.log
 fi