add ut (#8)

Add unit tests

Signed-off-by: Wenxin Zhang <[email protected]>

.github/workflows/code_scan.yml

@@ -1,5 +1,19 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 name: Code Scan
 

.github/workflows/docker/ut.dockerfile

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG UBUNTU_VER=22.04
+FROM ubuntu:${UBUNTU_VER} as devel
+
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+    aspell \
+    aspell-en \
+    build-essential \
+    python3 \
+    python3-pip \
+    python3-dev \
+    python3-distutils \
+    git \
+    vim \
+    wget
+
+RUN ln -sf $(which python3) /usr/bin/python
+RUN python -m pip install --no-cache-dir pytest
+
+WORKDIR /

.github/workflows/model_test.yml

@@ -0,0 +1,115 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Model Test
+
+on:
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+permissions: write-all
+env:
+  OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/scripts/models
+  SCRIPT_PATH: /GenAIEval/.github/workflows/scripts
+  REPO_NAME: "GenAIEval"
+  DOCKER_TAG: "latest"
+  DOCKER_FILE_NAME: "model.dockerfile"
+  CONTAINER_NAME: "modelTest"
+
+
+jobs:
+  Evaluation-Workflow:
+    runs-on: aise-cluster
+    strategy:
+      matrix:
+        include:
+          - modelName: "EleutherAI/gpt-j-6B"
+            task: "hellaswag"
+            device: "cpu"
+      fail-fast: true
+
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+          fetch-tags: true
+    # We need this because GitHub needs to clone the branch to pipeline
+      - name: Docker Build
+        run: |
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }} -t ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }} .
+
+      - name: Docker Run
+        run: |
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+            docker stop ${{ env.CONTAINER_NAME }}
+            docker rm -vf ${{ env.CONTAINER_NAME }} || true
+          fi
+          docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
+          -v ${{ github.workspace }}:/GenAIEval \
+          ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }}
+
+      - name: Binary build
+        run: |
+            docker exec ${{ env.CONTAINER_NAME }} \
+            bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install"
+
+      #- name: Download Reference Artifact
+      #  id: download-artifact
+      #  uses: dawidd6/[email protected]
+      #  with:
+      #    workflow: model_test.yml
+      #    name: ${{ matrix.device }}-${{ matrix.modelName }}
+      #    run_id: ${{ vars.ModelTest_REF_ID }}
+      #    path: ${{ github.workspace }}/${{ matrix.device }}_${{ matrix.modelName }}_refer_log
+      #    name_is_regexp: true
+      #    repo: ${{ github.repository }}
+      #    check_artifacts: false
+      #    search_artifacts: false
+      #    skip_unpack: false
+      #    if_no_artifact_found: warn
+
+      #- name: Display structure of downloaded files
+      #  run: ls -R
+
+      - name: Evaluation
+        run: |
+            docker exec ${{ env.CONTAINER_NAME }} \
+            bash -c "cd /GenAIEval/.github/workflows/scripts/models \
+            && bash model_test.sh --model=${{ matrix.modelName }} --device=${{ matrix.device }} --tasks=${{ matrix.task }}"
+
+      - name: Collect Log
+        run: |
+            docker exec ${{ env.CONTAINER_NAME }} \
+            bash -c "cd /GenAIEval/.github/workflows/scripts/models \
+            && bash -x collect_log.sh --model=${{ matrix.modelName }} \
+             --device=${{ matrix.device }} \
+             --task=${{ matrix.task }}
+
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ${{ matrix.device }}-${{ matrix.modelName }}
+          path: |
+            ${{ github.workspace }}/${{ matrix.device }}/${{ matrix.modelName }}
+            ${{ github.workspace }}/.summary.log
+          if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
+          retention-days: 60 # 1 <= retention-days <= 90

.github/workflows/scripts/models/collect_log.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+source /GenAIEval/.github/workflows/script/change_color.sh
+WORKSPACE="/GenAIEval"
+# get parameters
+PATTERN='[-a-zA-Z0-9_]*='
+PERF_STABLE_CHECK=true
+for i in "$@"; do
+    case $i in
+        --device=*)
+            device=`echo $i | sed "s/${PATTERN}//"`;;
+        --model=*)
+            model=`echo $i | sed "s/${PATTERN}//"`;;
+        --task=*)
+            task=`echo $i | sed "s/${PATTERN}//"`;;
+        *)
+            echo "Parameter $i not recognized."; exit 1;;
+    esac
+done
+
+output_file="/GenAIEval/${device}/${model}/${device}-${model}-${task}.log"
+$BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET
+
+echo "working in"
+pwd
+if [[ ! -f ${output_file} ]]; then
+    echo "${device};${model};${task};;${logfile}" >> ${WORKSPACE}/summary.log
+else
+    acc=$(grep -Po "Accuracy .* is:\\s+(\\d+(\\.\\d+)?)" ${acc_log_name} | head -n 1 | sed 's/.*://;s/[^0-9.]//g')
+    echo "${device};${model};${task};${acc};${logfile}" >> ${WORKSPACE}/summary.log
+fi

.github/workflows/scripts/models/model_test.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+source /GenAIEval/.github/workflows/script/change_color.sh
+
+# get parameters
+PATTERN='[-a-zA-Z0-9_]*='
+PERF_STABLE_CHECK=true
+for i in "$@"; do
+    case $i in
+        --device=*)
+            device=`echo $i | sed "s/${PATTERN}//"`;;
+        --model=*)
+            model=`echo $i | sed "s/${PATTERN}//"`;;
+        --task=*)
+            task=`echo $i | sed "s/${PATTERN}//"`;;
+        *)
+            echo "Parameter $i not recognized."; exit 1;;
+    esac
+done
+
+log_dir="/GenAIEval/${device}/${model}"
+mkdir -p ${log_dir}
+
+$BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET
+
+main() {
+    #prepare
+    run_benchmark
+}
+
+function prepare() {
+    ## prepare env
+    working_dir="/GenAIEval"
+    cd ${working_dir}
+    echo "Working in ${working_dir}"
+    echo -e "\nInstalling model requirements..."
+    if [ -f "requirements.txt" ]; then
+        python -m pip install -r requirements.txt
+        pip list
+    else
+        echo "Not found requirements.txt file."
+    fi
+}
+
+function run_benchmark() {
+    cd ${working_dir}
+    pip install --upgrade-strategy eager optimum[habana]
+    overall_log="${log_dir}/${device}-${model}-${task}.log"
+    python main.py \
+        --model hf \
+        --model_args pretrained=${model} \
+        --tasks ${task} \
+        --device ${device} \
+        --batch_size 8
+        2>&1 | tee ${overall_log}
+
+    status=$?
+    if [ ${status} != 0 ]; then
+        echo "Evaluation process returned non-zero exit code."
+        exit 1
+    fi
+}
+
+main