diff --git a/.github/code_spell_ignore.txt b/.github/code_spell_ignore.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/.github/workflows/code_scan.yml b/.github/workflows/code_scan.yml new file mode 100644 index 00000000..19a112b6 --- /dev/null +++ b/.github/workflows/code_scan.yml @@ -0,0 +1,61 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Code Scan + +on: + pull_request: + branches: [main] + types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped + paths-ignore: + - "**.md" + workflow_dispatch: + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + DOCKER_CONFIG_NAME: "commonDockerConfig" + REPO_NAME: "code-scan" + REPO_TAG: "1.0" + DOCKER_FILE_NAME: "code-scan" + CONTAINER_NAME: "code-scan" + +jobs: + code-scan: + runs-on: ubuntu-latest + strategy: + matrix: + job_name: ["bandit", "hadolint"] + fail-fast: false + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Docker Build + run: | + docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} . + + - name: Docker Run + run: | + if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then + docker stop ${{ env.CONTAINER_NAME }} + docker rm -vf ${{ env.CONTAINER_NAME }} || true + fi + docker run -dit --memory="4g" --memory-reservation="1g" --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} --shm-size="1g" \ + -v ${{ github.workspace }}:/GenAIEval \ + ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} + + - name: Code scan check + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "bash /GenAIEval/.github/workflows/scripts/codeScan/${{ matrix.job_name }}.sh" + + - name: Publish pipeline artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.job_name }} + path: ${{ github.workspace }}/.github/workflows/scripts/codeScan/${{ matrix.job_name }}.* \ No newline at end of file diff --git a/.github/workflows/dockers/code-scan.dockerfile b/.github/workflows/dockers/code-scan.dockerfile new file mode 100644 index 00000000..3e157fec --- /dev/null +++ b/.github/workflows/dockers/code-scan.dockerfile @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG UBUNTU_VER=22.04 +FROM ubuntu:${UBUNTU_VER} as devel + +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + aspell \ + aspell-en \ + build-essential \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + wget + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip install --no-cache-dir bandit==1.7.8 +RUN wget -O /bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 +RUN chmod +x /bin/hadolint + +WORKDIR / \ No newline at end of file diff --git a/.github/workflows/scripts/bandit.sh b/.github/workflows/scripts/bandit.sh new file mode 100644 index 00000000..e2561dbd --- /dev/null +++ b/.github/workflows/scripts/bandit.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /GenAIEval/.github/workflows/scripts/change_color +log_dir=/GenAIEval/.github/workflows/scripts/codeScan +python -m bandit -r -lll -iii /GenAIEval >${log_dir}/bandit.log +exit_code=$? + +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" +cat ${log_dir}/bandit.log +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + +if [ ${exit_code} -ne 0 ]; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET + exit 1 +fi + +$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET +exit 0 \ No newline at end of file diff --git a/.github/workflows/scripts/change_color b/.github/workflows/scripts/change_color new file mode 100644 index 00000000..a9fd74cd --- /dev/null +++ b/.github/workflows/scripts/change_color @@ -0,0 +1,80 @@ +#!/bin/bash + +# -------------- general approach start---------------- + +# 1. import this file: +# source path/change_color.sh +# 2. use COLOR/BG: +# $VARIABLE_NAME && out_put_content && $RESET +# 3. COLOR + BG: +# $COLOR/BG_VARIABLE_NAME && $BG/COLOR_VARIABLE_NAME && out_put_content && $RESET +# 4. custom +# abbreviation(change number) +# txt number range (30, 37) +# bg number range (40, 47) +# special effects number range (1, 7) +# echo -en \\E[number1 + ; + number2 + ; + number3 + m" +# e.g - BG_GRAY+LIGHT_RED = "echo -en \\E[47;31m" + +# -------------- general approach end----------------== + +# general setting +# ------------- light_color start---------------- +# black +LIGHT_BLACK="echo -en \\E[30m" +# red +LIGHT_RED="echo -en \\E[31m" +# green +LIGHT_GREEN="echo -en \\E[32m" +# yellow +LIGHT_YELLOW="echo -en \\E[33m" +# blue +LIGHT_BLUE="echo -en \\E[34m" +# purple +LIGHT_PURPLE="echo -en \\E[35m" +# cyan +LIGHT_CYAN="echo -en \\E[36m" +# gray +LIGHT_GRAY="echo -en \\E[37m" +# ------------- light_color end---------------- + +# ------------- bold_color start---------------- +# black +BOLD_BLACK="echo -en \\E[1;30m" +# red +BOLD_RED="echo -en \\E[1;31m" +# green +BOLD_GREEN="echo -en \\E[1;32m" +# yellow +BOLD_YELLOW="echo -en \\E[1;33m" +# blue +BOLD_BLUE="echo -en \\E[1;34m" +# purple +BOLD_PURPLE="echo -en \\E[1;35m" +# cyan +BOLD_CYAN="echo -en \\E[1;36m" +# gray +BOLD_GRAY="echo -en \\E[1;37m" +# ------------- bold_color end---------------- + +# ------------- background_color start---------------- +# black +BG_BLACK="echo -en \\E[40m" +# red +BG_RED="echo -en \\E[41m" +# green +BG_GREEN="echo -en \\E[42m" +# yellow +BG_YELLOW="echo -en \\E[43m" +# blue +BG_BLUE="echo -en \\E[44m" +# purple +BG_PURPLE="echo -en \\E[45m" +# cyan +BG_CYAN="echo -en \\E[46m" +# gray +BG_GRAY="echo -en \\E[47m" +# ------------- background_color end---------------- + +# close +RESET="echo -en \\E[0m" \ No newline at end of file diff --git a/.github/workflows/scripts/trellix.sh b/.github/workflows/scripts/trellix.sh new file mode 100644 index 00000000..6b9ca6cd --- /dev/null +++ b/.github/workflows/scripts/trellix.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ${workspace}/.github/workflows/scripts/change_color +log_dir=${workspace}/.github/workflows/scripts/codeScan + + +echo "---Updating definition (DAT) files ---" +DEFS_URL=https://update.nai.com/products/commonupdater/current/vscandat1000/dat/0000 +echo "Finding latest defs at $DEFS_URL/avvdat.ini..." \ + && wget -q $DEFS_URL/avvdat.ini \ + && echo "SUCCESS" || fail + +inifile="avvdat.ini" +filename=`awk -F"=" '$2 ~ /avvdat.*zip/ { print $2 } ' $inifile` +filename2="$(echo -e "${filename}" | tr -d '[:space:]')" + +if [ -z "$filename2" ] +then + echo "Cannot get defs information from INI file:" + cat $inifile + fail +fi + +echo "Downloading latest defs from $DEFS_URL/$filename2..." \ + && wget -q $DEFS_URL/$filename2 \ + && echo "SUCCESS" || fail + +echo "Extracting latest defs..." \ + && unzip -o $filename2 -d /usr/local/uvscan \ + && echo "SUCCESS" || fail + +echo "--- Scanning ---" +ENV_SCAN_OPTS="--analyze --mime --program --recursive --unzip --threads 4 --summary --verbose --html=${workspace}/.github/workflows/scripts/codeScan/report.html" +echo "Scan Options: $ENV_SCAN_OPTS" + +rm -r ${workspace}/avvdat* +rm -r ${workspace}/.git +uvscan $ENV_SCAN_OPTS ${workspace} 2>&1 | tee ${log_dir}/trellix.log + + +if [[ $(grep "Possibly Infected" ${log_dir}/trellix.log | sed 's/[^0-9]//g') != 0 ]]; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET + exit 1 +fi + +$BOLD_PURPLE && echo "Congratulations, Trellix Scan passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET +exit 0 \ No newline at end of file diff --git a/.github/workflows/trellix.yml b/.github/workflows/trellix.yml new file mode 100644 index 00000000..3ee2c26b --- /dev/null +++ b/.github/workflows/trellix.yml @@ -0,0 +1,30 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Trellix Command Line Scanner + +on: + workflow_dispatch: + schedule: + - cron: "35 1 * * 6" + +jobs: + Trellix: + runs-on: trellix + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Run Trellix Scanner + env: + workspace: ${{ github.workspace }} + run: bash .github/workflows/scripts/codeScan/trellix.sh + + - name: Publish pipeline artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + path: ${{ github.workspace }}/.github/workflows/scripts/codeScan/report.html \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..56c94e25 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,109 @@ +ci: + autofix_prs: true + autoupdate_schedule: quarterly + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + files: (.*\.(py|md|rst|yaml|yml|json|ts|js|html|svelte|sh))$ + - id: check-json + - id: check-yaml + - id: debug-statements + - id: requirements-txt-fixer + - id: trailing-whitespace + files: (.*\.(py|rst|cmake|yaml|yml|json|ts|js|html|svelte|sh))$ + + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + hooks: + - id: insert-license + files: (.*\.(py|yaml|yml|sh))$ + args: + [ + --license-filepath=.github/license_template.txt, + --use-current-year, + --detect-license-in-X-top-lines=40, + --skip-license-insertion-comment=Copyright, + ] + - id: insert-license + files: (.*\.(ts|js))$ + args: + [ + --license-filepath=.github/license_template.txt, + --use-current-year, + --detect-license-in-X-top-lines=40, + --skip-license-insertion-comment=Copyright, + --comment-style=//, + ] + - id: insert-license + files: (.*\.(html|svelte))$ + args: + [ + --license-filepath=.github/license_template.txt, + --use-current-year, + --detect-license-in-X-top-lines=40, + --skip-license-insertion-comment=Copyright, + --comment-style=, + ] + + - repo: https://github.com/asottile/yesqa + rev: v1.5.0 + hooks: + - id: yesqa + name: Unused noqa + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + + - repo: https://github.com/PyCQA/docformatter + rev: v1.7.5 + hooks: + - id: docformatter + args: [ + --in-place, + --wrap-summaries=0, # 0 means disable wrap + --wrap-descriptions=0, # 0 means disable wrap + --black, + --style=google, + ] + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v4.0.0-alpha.8" # Use the sha / tag you want to point at + hooks: + - id: prettier + args: [--print-width=120] + types_or: [yaml, markdown, html, css, scss, javascript, json, ts, shell, sh] + additional_dependencies: + - prettier@3.2.5 + + - repo: https://github.com/psf/black.git + rev: 24.3.0 + hooks: + - id: black + files: (.*\.py)$ + + - repo: https://github.com/asottile/blacken-docs + rev: 1.16.0 + hooks: + - id: blacken-docs + args: [--line-length=120, --skip-errors] + additional_dependencies: + - black==24.3.0 + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [-w] + additional_dependencies: + - tomli + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.5 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --no-cache] \ No newline at end of file