refine CI for neuralspeed

Signed-off-by: Wenxin Zhang <[email protected]>
intel · Nov 22, 2023 · 1401b43 · 1401b43
1 parent 468338c
commit 1401b43
Show file tree

Hide file tree

Showing 15 changed files with 56 additions and 328 deletions.
diff --git a/.github/workflows/copyright_check.yml b/.github/workflows/copyright_check.yml
@@ -4,7 +4,7 @@ on:
   pull_request:
     branches: [main]
     paths:
-      - intel_extension_for_transformers/**
+      - neural_speed/**
       - setup.py
       - .github/workflows/format_scan.yml
   workflow_dispatch:
@@ -20,7 +20,7 @@ env:
 
 jobs:
   format-scan:
-    runs-on: itrex-node-spell
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         job_name: ["copyright"]
@@ -32,7 +32,7 @@ jobs:
           if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
               docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
               echo "remove left files through container ..."
-              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /neural-speed && rm -fr /neural-speed/* && rm -fr /neural-speed/.* || true"
           fi
           
       - name: Checkout out Repo
@@ -45,7 +45,7 @@ jobs:
           mkdir -p ${{ env.CODE_SCAN_LOG_PATH }}
           supported_extensions=(py, sh, yaml)
           git fetch
-          git --no-pager diff --name-only remotes/origin/${{ github.base_ref }} ${{ github.workspace }}/intel_extension_for_transformers> ${{ env.CODE_SCAN_LOG_PATH }}/diff.log
+          git --no-pager diff --name-only remotes/origin/${{ github.base_ref }} ${{ github.workspace }}/neural_speed> ${{ env.CODE_SCAN_LOG_PATH }}/diff.log
           files=$(cat ${{ env.CODE_SCAN_LOG_PATH }}/diff.log | awk '!a[$0]++')
           $LIGHT_PURPLE && echo " ----------------- checking ... --------------------------" && $RESET
           if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then 

diff --git a/.github/workflows/cpp-graph-test.yml b/.github/workflows/cpp-graph-test.yml
@@ -6,9 +6,8 @@ on:
     paths:
        - '.github/workflows/cpp-graph-test.yml'
        - '.github/workflows/script/models/cpp_graph_inference.sh'
-       - 'intel_extension_for_transformers/llm/runtime/graph/**'
-       - 'intel_extension_for_transformers/llm/library/jblas/**'
-       - '!intel_extension_for_transformers/llm/runtime/graph/README.md'
+       - 'neural_speed/**'
+       - 'bestla/**'
   workflow_dispatch:
     inputs:
       compiler_version:
@@ -57,15 +56,14 @@ jobs:
           bash ${{ github.workspace }}/.github/workflows/script/prepare_env_with_conda.sh "cpp-graph-test" "3.8"
 
       - name: Binary build
-        # cpp model does not requires itrex package
         if: 0 == 1
         run: |
           cd ${{ github.workspace }}
           conda activate cpp-graph-test || source activate cpp-graph-test
           pip install build --upgrade
           pip install -r requirements.txt
           python setup.py sdist bdist_wheel
-          pip install dist/intel_extension_for_transformers*.whl
+          pip install dist/neuralspeed*.whl
           pip list
 
       - name: BF16 Benchmark
@@ -88,7 +86,7 @@ jobs:
           retention-days: 60 # 1 <= retention-days <= 90
 
   Genreate-Report:
-    runs-on: itrex-node-spell
+    runs-on: ubuntu-latest
     needs: [CPP-Graph-Workflow]
     steps:
       - name: Docker Clean Up
@@ -97,7 +95,7 @@ jobs:
           if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
               docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
               echo "remove left files through container ..."
-              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /neural-speed && rm -fr /neural-speed/* && rm -fr /neural-speed/.* || true"
           fi
       - name: Checkout out Repo
         uses: actions/checkout@v3
@@ -137,7 +135,7 @@ jobs:
           /usr/bin/bash generate_report.sh --workflow=deploy
           sed -n '/<body>/,/<\/body>/p' generated/report.html | sed -r '/^$/d' | sed -r 's/^ +//g' >> $GITHUB_STEP_SUMMARY
         env:
-          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
+          RUN_DISPLAY_URL: https://github.com/neural-speed/actions/runs/${{ github.run_id }}
           BUILD_NUMBER: ${{ github.run_id }}
           JOB_STATUS: succeed
           MR_source_branch: ${{ github.head_ref }}

diff --git a/.github/workflows/format_scan.yml b/.github/workflows/format_scan.yml
@@ -4,9 +4,7 @@ on:
   pull_request:
     branches: [main]
     paths:
-      - intel_extension_for_transformers/**
-      - neural_chat/**
-      - workflows/**
+      - neural_speed/**
       - setup.py
       - .github/workflows/format_scan.yml
       - .github/workflows/script/formatScan/**
@@ -26,7 +24,7 @@ env:
 
 jobs:
   format-scan:
-    runs-on: itrex-node-spell
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         job_name: [
@@ -46,7 +44,7 @@ jobs:
           if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
               docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
               echo "remove left files through container ..."
-              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /neural-speed && rm -fr /neural-speed/* && rm -fr /neural-speed/.* || true"
           fi
 
       - name: Checkout out Repo
@@ -63,18 +61,18 @@ jobs:
             docker rm -vf ${{ env.CONTAINER_NAME }}-${{ runner.name }} || true
           fi
           docker run -dit --memory="4g" --memory-reservation="1g" --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }}-${{ runner.name }} --shm-size="1g" \
-          -v ${{ github.workspace }}:/intel-extension-for-transformers \
+          -v ${{ github.workspace }}:/neural-speed \
           ${{ env.REPO_NAME }}:${{ env.REPO_TAG }}
 
       - name: Env build
         run: |
           docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} \
-          bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
+          bash /neural-speed/.github/workflows/script/prepare_env.sh
 
       - name: Code scan check
         run: |
           docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} \
-          bash -c "bash /intel-extension-for-transformers/.github/workflows/script/formatScan/${{ matrix.job_name }}.sh"
+          bash -c "bash /neural-speed/.github/workflows/script/formatScan/${{ matrix.job_name }}.sh"
 
       - name: Publish pipeline artifact
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/llm-test.yml b/.github/workflows/llm-test.yml
diff --git a/.github/workflows/scripts/formatScan/bandit.sh b/.github/workflows/scripts/formatScan/bandit.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
-source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+source /neural-speed/.github/workflows/script/change_color.sh
 pip install bandit==1.7.4
-log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
-python -m bandit -r -lll -iii /intel-extension-for-transformers >${log_dir}/bandit.log
+log_dir=/neural-speed/.github/workflows/script/formatScan
+python -m bandit -r -lll -iii /neural-speed >${log_dir}/bandit.log
 exit_code=$?
 
 $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"

diff --git a/.github/workflows/scripts/formatScan/clangformat.sh b/.github/workflows/scripts/formatScan/clangformat.sh
@@ -1,14 +1,14 @@
 #!/bin/bash
-source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+source /neural-speed/.github/workflows/script/change_color.sh
 
 pip install clang-format==14.0.0
-log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
+log_dir=/neural-speed/.github/workflows/script/formatScan
 log_path=${log_dir}/clangformat.log
 
-cd /intel-extension-for-transformers
+cd /neural-speed
 git config --global --add safe.directory "*"
 
-cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/graph
+cd /neural-speed/neural_speed
 python scripts/clang-format.py
 
 echo "run git diff"

diff --git a/.github/workflows/scripts/formatScan/cloc.sh b/.github/workflows/scripts/formatScan/cloc.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 
-source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
-log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
-cloc --include-lang=Python --csv --out=${log_dir}/cloc.csv /intel-extension-for-transformers
+source /neural-speed/.github/workflows/script/change_color.sh
+log_dir=/neural-speed/.github/workflows/script/formatScan
+cloc --include-lang=Python --csv --out=${log_dir}/cloc.csv /neural-speed
diff --git a/.github/workflows/scripts/formatScan/cpplint.sh b/.github/workflows/scripts/formatScan/cpplint.sh
@@ -1,14 +1,13 @@
 #!/bin/bash
 
-source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+source /neural-speed/.github/workflows/script/change_color.sh
 
 pip install cpplint
-REPO_DIR=/intel-extension-for-transformers
-log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
+REPO_DIR=/neural-speed
+log_dir=/neural-speed/.github/workflows/script/formatScan
 log_path=${log_dir}/cpplint.log
-cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/llm/runtime/deprecated/compile 2>&1 | tee ${log_path}
-cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/llm/runtime/deprecated/executor 2>&1 | tee -a ${log_path}
-cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/llm/runtime/deprecated/test 2>&1 | tee -a ${log_path}
+cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/neural_speed 2>&1 | tee ${log_path}
+cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/bestla 2>&1 | tee -a ${log_path}
 if [[ ! -f ${log_path} ]] || [[ $(grep -c "Total errors found:" ${log_path}) != 0 ]]; then
     exit 1
 fi

diff --git a/.github/workflows/scripts/formatScan/pydocstyle.sh b/.github/workflows/scripts/formatScan/pydocstyle.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
-source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+source /neural-speed/.github/workflows/script/change_color.sh
 
-REPO_DIR=/intel-extension-for-transformers
-log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
+REPO_DIR=/neural-speed
+log_dir=/neural-speed/.github/workflows/script/formatScan
 pydocstyle --convention=google ${REPO_DIR} >${log_dir}/pydocstyle.log
 exit_code=$?