diff --git a/.github/workflows/code_scan.yml b/.github/workflows/code_scan.yml index 19af9e19..fce9c13e 100644 --- a/.github/workflows/code_scan.yml +++ b/.github/workflows/code_scan.yml @@ -1,5 +1,19 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. name: Code Scan diff --git a/.github/workflows/docker/ut.dockerfile b/.github/workflows/docker/ut.dockerfile new file mode 100644 index 00000000..843885d3 --- /dev/null +++ b/.github/workflows/docker/ut.dockerfile @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG UBUNTU_VER=22.04 +FROM ubuntu:${UBUNTU_VER} as devel + +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + aspell \ + aspell-en \ + build-essential \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + git \ + vim \ + wget + +RUN ln -sf $(which python3) /usr/bin/python +RUN python -m pip install --no-cache-dir pytest + +WORKDIR / \ No newline at end of file diff --git a/.github/workflows/model_test.yml b/.github/workflows/model_test.yml new file mode 100644 index 00000000..7a481c8b --- /dev/null +++ b/.github/workflows/model_test.yml @@ -0,0 +1,115 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Model Test + +on: + workflow_dispatch: + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +permissions: write-all +env: + OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/scripts/models + SCRIPT_PATH: /GenAIEval/.github/workflows/scripts + REPO_NAME: "GenAIEval" + DOCKER_TAG: "latest" + DOCKER_FILE_NAME: "model.dockerfile" + CONTAINER_NAME: "modelTest" + + +jobs: + Evaluation-Workflow: + runs-on: aise-cluster + strategy: + matrix: + include: + - modelName: "EleutherAI/gpt-j-6B" + task: "hellaswag" + device: "cpu" + fail-fast: true + + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + submodules: "recursive" + fetch-tags: true + # We need this because GitHub needs to clone the branch to pipeline + - name: Docker Build + run: | + docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }} -t ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }} . + + - name: Docker Run + run: | + if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then + docker stop ${{ env.CONTAINER_NAME }} + docker rm -vf ${{ env.CONTAINER_NAME }} || true + fi + docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \ + -v ${{ github.workspace }}:/GenAIEval \ + ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }} + + - name: Binary build + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install" + + #- name: Download Reference Artifact + # id: download-artifact + # uses: dawidd6/action-download-artifact@v3.1.2 + # with: + # workflow: model_test.yml + # name: ${{ matrix.device }}-${{ matrix.modelName }} + # run_id: ${{ vars.ModelTest_REF_ID }} + # path: ${{ github.workspace }}/${{ matrix.device }}_${{ matrix.modelName }}_refer_log + # name_is_regexp: true + # repo: ${{ github.repository }} + # check_artifacts: false + # search_artifacts: false + # skip_unpack: false + # if_no_artifact_found: warn + + #- name: Display structure of downloaded files + # run: ls -R + + - name: Evaluation + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "cd /GenAIEval/.github/workflows/scripts/models \ + && bash model_test.sh --model=${{ matrix.modelName }} --device=${{ matrix.device }} --tasks=${{ matrix.task }}" + + - name: Collect Log + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "cd /GenAIEval/.github/workflows/scripts/models \ + && bash -x collect_log.sh --model=${{ matrix.modelName }} \ + --device=${{ matrix.device }} \ + --task=${{ matrix.task }} + + - name: Publish pipeline artifact + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ${{ matrix.device }}-${{ matrix.modelName }} + path: | + ${{ github.workspace }}/${{ matrix.device }}/${{ matrix.modelName }} + ${{ github.workspace }}/.summary.log + if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn` + retention-days: 60 # 1 <= retention-days <= 90 diff --git a/.github/workflows/scripts/models/collect_log.sh b/.github/workflows/scripts/models/collect_log.sh new file mode 100644 index 00000000..f6a68330 --- /dev/null +++ b/.github/workflows/scripts/models/collect_log.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail +source /GenAIEval/.github/workflows/script/change_color.sh +WORKSPACE="/GenAIEval" +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' +PERF_STABLE_CHECK=true +for i in "$@"; do + case $i in + --device=*) + device=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --task=*) + task=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +output_file="/GenAIEval/${device}/${model}/${device}-${model}-${task}.log" +$BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET + +echo "working in" +pwd +if [[ ! -f ${output_file} ]]; then + echo "${device};${model};${task};;${logfile}" >> ${WORKSPACE}/summary.log +else + acc=$(grep -Po "Accuracy .* is:\\s+(\\d+(\\.\\d+)?)" ${acc_log_name} | head -n 1 | sed 's/.*://;s/[^0-9.]//g') + echo "${device};${model};${task};${acc};${logfile}" >> ${WORKSPACE}/summary.log +fi diff --git a/.github/workflows/scripts/models/model_test.sh b/.github/workflows/scripts/models/model_test.sh new file mode 100644 index 00000000..fae21add --- /dev/null +++ b/.github/workflows/scripts/models/model_test.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail +source /GenAIEval/.github/workflows/script/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' +PERF_STABLE_CHECK=true +for i in "$@"; do + case $i in + --device=*) + device=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --task=*) + task=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +log_dir="/GenAIEval/${device}/${model}" +mkdir -p ${log_dir} + +$BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET + +main() { + #prepare + run_benchmark +} + +function prepare() { + ## prepare env + working_dir="/GenAIEval" + cd ${working_dir} + echo "Working in ${working_dir}" + echo -e "\nInstalling model requirements..." + if [ -f "requirements.txt" ]; then + python -m pip install -r requirements.txt + pip list + else + echo "Not found requirements.txt file." + fi +} + +function run_benchmark() { + cd ${working_dir} + pip install --upgrade-strategy eager optimum[habana] + overall_log="${log_dir}/${device}-${model}-${task}.log" + python main.py \ + --model hf \ + --model_args pretrained=${model} \ + --tasks ${task} \ + --device ${device} \ + --batch_size 8 + 2>&1 | tee ${overall_log} + + status=$? + if [ ${status} != 0 ]; then + echo "Evaluation process returned non-zero exit code." + exit 1 + fi +} + +main diff --git a/.github/workflows/scripts/unittest/calc_coverage.sh b/.github/workflows/scripts/unittest/calc_coverage.sh new file mode 100644 index 00000000..e00df60d --- /dev/null +++ b/.github/workflows/scripts/unittest/calc_coverage.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ../../change_color.sh +LOG_DIR=$1 +coverage_compare="${LOG_DIR}/coverage_compare.html" +coverage_log_pr="${LOG_DIR}/UnitTestPR-test/coverage_pr" +coverage_log_base="${LOG_DIR}/UnitTestbaseline/coverage_base" + +function get_coverage_data() { + # Input argument + local coverage_xml="$1" + + # Get coverage data + local coverage_data=$(python3 -c "import xml.etree.ElementTree as ET; root = ET.parse('$coverage_xml').getroot(); print(ET.tostring(root).decode())") + if [[ -z "$coverage_data" ]]; then + echo "Failed to get coverage data from $coverage_xml." + exit 1 + fi + + # Get lines coverage + local lines_covered=$(echo "$coverage_data" | grep -o 'lines-covered="[0-9]*"' | cut -d '"' -f 2) + local lines_valid=$(echo "$coverage_data" | grep -o 'lines-valid="[0-9]*"' | cut -d '"' -f 2) + if [ $lines_valid == 0 ]; then + local lines_coverage=0 + else + local lines_coverage=$(awk "BEGIN {printf \"%.3f\", 100 * $lines_covered / $lines_valid}") + fi + + # Get branches coverage + local branches_covered=$(echo "$coverage_data" | grep -o 'branches-covered="[0-9]*"' | cut -d '"' -f 2) + local branches_valid=$(echo "$coverage_data" | grep -o 'branches-valid="[0-9]*"' | cut -d '"' -f 2) + if [ $branches_valid == 0 ]; then + local branches_coverage=0 + else + local branches_coverage=$(awk "BEGIN {printf \"%.3f\", 100 * $branches_covered/$branches_valid}") + fi + + # Return values + echo "$lines_covered $lines_valid $lines_coverage $branches_covered $branches_valid $branches_coverage" +} + +function compare_coverage() { + $BOLD_YELLOW && echo "compare coverage" && $RESET + + coverage_PR_xml="${coverage_log_pr}/coverage.xml" + coverage_PR_data=$(get_coverage_data $coverage_PR_xml) + read lines_PR_covered lines_PR_valid coverage_PR_lines_rate branches_PR_covered branches_PR_valid coverage_PR_branches_rate <<<"$coverage_PR_data" + + coverage_base_xml="${coverage_log_base}/coverage.xml" + coverage_base_data=$(get_coverage_data $coverage_base_xml) + read lines_base_covered lines_base_valid coverage_base_lines_rate branches_base_covered branches_base_valid coverage_base_branches_rate <<<"$coverage_base_data" + + $BOLD_BLUE && echo "PR lines coverage: $lines_PR_covered/$lines_PR_valid ($coverage_PR_lines_rate%)" && $RESET + $BOLD_BLUE && echo "PR branches coverage: $branches_PR_covered/$branches_PR_valid ($coverage_PR_branches_rate%)" && $RESET + $BOLD_BLUE && echo "BASE lines coverage: $lines_base_covered/$lines_base_valid ($coverage_base_lines_rate%)" && $RESET + $BOLD_BLUE && echo "BASE branches coverage: $branches_base_covered/$branches_base_valid ($coverage_base_branches_rate%)" && $RESET +} + +function check_coverage_status() { + # Declare an array to hold failed items + declare -a fail_items=() + + if (($(bc -l <<<"${coverage_PR_lines_rate}+0.5 < ${coverage_base_lines_rate}"))); then + fail_items+=("lines") + fi + if (($(bc -l <<<"${coverage_PR_branches_rate}+1 < ${coverage_base_branches_rate}"))); then + fail_items+=("branches") + fi + + if [[ ${#fail_items[@]} -ne 0 ]]; then + fail_items_str=$( + IFS=', ' + echo "${fail_items[*]}" + ) + for item in "${fail_items[@]}"; do + case "$item" in + lines) + decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_lines_rate - $coverage_base_lines_rate" | bc -l))) + ;; + branches) + decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_branches_rate - $coverage_base_branches_rate" | bc -l))) + ;; + *) + echo "Unknown item: $item" + continue + ;; + esac + $BOLD_RED && echo "Unit Test failed with ${item} coverage decrease ${decrease}%" && $RESET + done + $BOLD_RED && echo "compare coverage to give detail info" && $RESET + bash compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "FAILED" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate} + exit 1 + else + $BOLD_GREEN && echo "Unit Test success with coverage lines: ${coverage_PR_lines_rate}%, branches: ${coverage_PR_branches_rate}%" && $RESET + $BOLD_GREEN && echo "compare coverage to give detail info" && $RESET + bash compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "SUCCESS" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate} + fi +} + +function main() { + compare_coverage + check_coverage_status + $BOLD_BLUE && echo "PR lines coverage: $lines_PR_covered/$lines_PR_valid ($coverage_PR_lines_rate%)" && $RESET + $BOLD_BLUE && echo "PR branches coverage: $branches_PR_covered/$branches_PR_valid ($coverage_PR_branches_rate%)" && $RESET + $BOLD_BLUE && echo "BASE lines coverage: $lines_base_covered/$lines_base_valid ($coverage_base_lines_rate%)" && $RESET + $BOLD_BLUE && echo "BASE branches coverage: $branches_base_covered/$branches_base_valid ($coverage_base_branches_rate%)" && $RESET +} + +main diff --git a/.github/workflows/scripts/unittest/compare_coverage.sh b/.github/workflows/scripts/unittest/compare_coverage.sh new file mode 100644 index 00000000..88a4e1e5 --- /dev/null +++ b/.github/workflows/scripts/unittest/compare_coverage.sh @@ -0,0 +1,235 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output_file=$1 +coverage_pr_log=$2 +coverage_base_log=$3 +coverage_status=$4 +coverage_PR_lines_rate=$5 +coverage_base_lines_rate=$6 +coverage_PR_branches_rate=$7 +coverage_base_branches_rate=$8 +module_name="GenAIEval" +[[ ! -f $coverage_pr_log ]] && exit 1 +[[ ! -f $coverage_base_log ]] && exit 1 +file_name="./coverage_compare" +sed -i "s|\/usr.*${module_name}\/||g" $coverage_pr_log +sed -i "s|\/usr.*${module_name}\/||g" $coverage_base_log +diff $coverage_pr_log $coverage_base_log >diff_file +grep -Po "[<,>,\d].*" diff_file | awk '{print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7}' | sed "/Name/d" | sed "/TOTAL/d" | sed "/---/d" >$file_name +[[ ! -s $file_name ]] && exit 0 +[[ -f $output_file ]] && rm -f $output_file +touch $output_file + +function generate_html_head { + + cat >${output_file} < + + + + + + UT coverage + + + +eof +} + +function extract_diff_data() { + local file_name=$1 diff_file=$2 reg=$3 + local file=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $1}') + local miss=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $3}') + local cover=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $6}') + local branch=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $4}') + + echo "$file $miss $cover $branch" +} + +function write_compare_details() { + local file=$1 miss1=$2 branch1=$3 cover1=$4 miss2=$5 branch2=$6 cover2=$7 + echo """ + + PR | BASE + ${file} + ${miss1} | ${miss2} + ${branch1} | ${branch2} + ${cover1} | ${cover2} + + """ >>${output_file} +} + +function get_color() { + local decrease=$1 + if (($(echo "$decrease < 0" | bc -l))); then + local color="#FFD2D2" + else + local color="#90EE90" + fi + echo "$color" +} + +function generate_coverage_summary() { + # generate table head + local Lines_cover_decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_lines_rate - $coverage_base_lines_rate" | bc -l))) + local Branches_cover_decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_branches_rate - $coverage_base_branches_rate" | bc -l))) + + read lines_coverage_color <<<"$(get_color ${Lines_cover_decrease})" + read branches_coverage_color <<<"$(get_color ${Branches_cover_decrease})" + + echo """ + +
+

Coverage Summary : ${coverage_status}

+ + + + + + + + + + + + + + + + + + + +
Base coveragePR coverageDiff
Lines ${coverage_base_lines_rate}% ${coverage_PR_lines_rate}% ${Lines_cover_decrease}%
Branches ${coverage_base_branches_rate}% ${coverage_PR_branches_rate}% ${Branches_cover_decrease}%
+
+ """ >>${output_file} +} + +function generate_coverage_details() { + echo """ +
+

Coverage Detail

+ + + + + + + + + """ >>${output_file} + # generate compare detail + cat ${file_name} | while read line; do + if [[ $(echo $line | grep "[0-9]a[0-9]") ]] && [[ $(grep -A 1 "$line" ${file_name} | grep ">") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep ">") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po ">.*[a-z,A-Z].*.py" | sed "s|>||g") + for diff_file in ${diff_file_name}; do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + diff_coverage_data=$(extract_diff_data ${file_name} ${diff_file} ">") + read file miss cover branch <<<"$diff_coverage_data" + write_compare_details $file "NA" "NA" "NA" $miss $branch $cover + done + elif [[ $(echo $line | grep "[0-9]c[0-9]") ]] && [[ $(cat ${file_name} | grep -A 1 "$line" | grep "<") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g") + for diff_file in ${diff_file_name}; do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + diff_coverage_data1=$(extract_diff_data ${file_name} ${diff_file} "<") + read file1 miss1 cover1 branch1 <<<"$diff_coverage_data1" + diff_coverage_data2=$(extract_diff_data ${file_name} ${diff_file} ">") + read file2 miss2 cover2 branch2 <<<"$diff_coverage_data2" + write_compare_details $file1 $miss1 $branch1 $cover1 $miss2 $branch2 $cover2 + done + elif [[ $(echo $line | grep "[0-9]d[0-9]") ]] && [[ $(cat ${file_name} | grep -A 1 "$line" | grep "<") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g") + for diff_file in ${diff_file_name}; do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + diff_coverage_data=$(extract_diff_data ${file_name} ${diff_file} "<") + read file miss cover branch <<<"$diff_coverage_data" + write_compare_details $file $miss $branch $cover "NA" "NA" "NA" + done + fi + done + # generate table end + echo """ +
CommitFileNameMissBranchCover
+
+ + +""" >>${output_file} +} + +function main { + generate_html_head + generate_coverage_summary + + if [[ ${coverage_status} = "SUCCESS" ]]; then + echo """""" >>${output_file} + echo "coverage PASS, no need to compare difference" + exit 0 + else + generate_coverage_details + fi +} + +main diff --git a/.github/workflows/scripts/unittest/coveragerc b/.github/workflows/scripts/unittest/coveragerc new file mode 100644 index 00000000..f2d68ae8 --- /dev/null +++ b/.github/workflows/scripts/unittest/coveragerc @@ -0,0 +1,9 @@ +[run] +branch = True + +[report] +omit = + */**/fake*yaml + */**/fake.py +exclude_lines = + pragma: no cover diff --git a/.github/workflows/scripts/unittest/unittest.sh b/.github/workflows/scripts/unittest/unittest.sh new file mode 100644 index 00000000..a241c471 --- /dev/null +++ b/.github/workflows/scripts/unittest/unittest.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +source /GenAIEval/.github/workflows/scripts/change_color +export COVERAGE_RCFILE="/GenAIEval/.github/workflows/scripts/unittest/coveragerc" +LOG_DIR=/GenAIEval/log_dir +mkdir -p ${LOG_DIR} +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' +PERF_STABLE_CHECK=true + +for i in "$@"; do + case $i in + --test_name=*) + test_name=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +function pytest() { + local coverage_log_dir="${LOG_DIR}/$1" + mkdir -p ${coverage_log_dir} + ut_log_name="${LOG_DIR}/unit_test_$1.log" + export GLOG_minloglevel=2 + + genaieval_path=$(python -c 'import GenAIEval; import os; print(os.path.dirname(GenAIEval.__file__))') + find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${genaieval_path}"' --append ,g' | sed 's/$/ --verbose/' >run.sh + coverage erase + + # run UT + $BOLD_YELLOW && echo "cat run.sh..." && $RESET + cat run.sh | tee ${ut_log_name} + $BOLD_YELLOW && echo "------UT start-------" && $RESET + bash run.sh 2>&1 | tee -a ${ut_log_name} + $BOLD_YELLOW && echo "------UT end -------" && $RESET + + # run coverage report + coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log_dir}/coverage.log + coverage html -d ${coverage_log_dir}/htmlcov --rcfile=${COVERAGE_RCFILE} + coverage xml -o ${coverage_log_dir}/coverage.xml --rcfile=${COVERAGE_RCFILE} + + # check UT status + if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ]; then + $BOLD_RED && echo "Find errors in UT, please search [FAILED]..." && $RESET + exit 1 + fi + if [ $(grep -c "ModuleNotFoundError:" ${ut_log_name}) != 0 ]; then + $BOLD_RED && echo "Find errors in UT, please search [ModuleNotFoundError:]..." && $RESET + exit 1 + fi + if [ $(grep -c "core dumped" ${ut_log_name}) != 0 ]; then + $BOLD_RED && echo "Find errors in UT, please search [core dumped]..." && $RESET + exit 1 + fi + if [ $(grep -c "OK" ${ut_log_name}) == 0 ]; then + $BOLD_RED && echo "No pass case found, please check the output..." && $RESET + exit 1 + fi + if [ $(grep -c "==ERROR:" ${ut_log_name}) != 0 ]; then + $BOLD_RED && echo "ERROR found in UT, please check the output..." && $RESET + exit 1 + fi + if [ $(grep -c "Segmentation fault" ${ut_log_name}) != 0 ]; then + $BOLD_RED && echo "Segmentation Fault found in UT, please check the output..." && $RESET + exit 1 + fi + if [ $(grep -c "ImportError:" ${ut_log_name}) != 0 ]; then + $BOLD_RED && echo "ImportError found in UT, please check the output..." && $RESET + exit 1 + fi + $BOLD_GREEN && echo "UT finished successfully! " && $RESET +} + +function main() { + cd /GenAIEval/tests || exit 1 + if [ -f "requirements.txt" ]; then + python -m pip install --default-timeout=100 -r requirements.txt + pip list + else + echo "Not found requirements.txt file." + fi + pip install coverage + pip install pytest + echo "test on ${test_name}" + if [[ $test_name == "PR-test" ]]; then + pytest "pr" + elif [[ $test_name == "baseline" ]]; then + pytest "base" + fi +} + +main diff --git a/.github/workflows/trellix.yml b/.github/workflows/trellix.yml index 8779f3b9..256f24ef 100644 --- a/.github/workflows/trellix.yml +++ b/.github/workflows/trellix.yml @@ -1,5 +1,19 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. name: Trellix Command Line Scanner diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 00000000..8562165d --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: UnitTest + +on: + pull_request: + branches: [main] + types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped + paths: + - .github/workflows/unittest.yml + - GenAIEval/** + - setup.py + - tests/** + workflow_dispatch: + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CONTAINER_NAME: "unittest-eval" + DOCKER_NAME: "unittest-eval" + DOCKER_TAG: "latest" + +jobs: + Unit-Test: + strategy: + matrix: + include: + - test_branch: ${{ github.ref }} + test_name: "PR-test" + #- test_branch: "main" + # test_name: "baseline" + runs-on: aise-cluster + name: unit-test-${{ matrix.test_name }} + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + submodules: "recursive" + ref: ${{ matrix.test_branch }} + fetch-tags: true + + - name: Docker Build + run: | + docker build -f ${{ github.workspace }}/.github/workflows/docker/ut.dockerfile -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} . + + - name: Docker Run + run: | + if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then + docker stop ${{ env.CONTAINER_NAME }} + docker rm -vf ${{ env.CONTAINER_NAME }} || true + fi + docker run -dit --memory="4g" --memory-reservation="1g" --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} --shm-size="1g" \ + -v ${{ github.workspace }}:/GenAIEval ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} + + - name: Install Dependencies + run: | + docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install" + + - name: Run UT + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "bash /GenAIEval/.github/workflows/scripts/unittest/unittest.sh --test_name=${{ matrix.test_name }}" + + - name: Publish pipeline artifact + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: "UnitTest${{ matrix.test_name }}" + path: ${{ github.workspace }}/log_dir + + Genreate-UT-Report: + runs-on: ubuntu-latest + needs: [Unit-Test] + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Download UT PR Log + uses: actions/download-artifact@v4 + with: + path: ${{ github.workspace }}/log_dir + + - name: Display structure of downloaded files + run: cd ${{ github.workspace }}/log_dir && ls -R + + - name: Calculate coverage + run: | + cd ${{ github.workspace }}/.github/workflows/scripts/unittest + /usr/bin/bash calc_coverage.sh ${{ github.workspace }}/log_dir + + - name: Publish pipeline artifact + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: Unit Test + path: ${{ github.workspace }}/log_dir + retention-days: 5